{ "cls_name": "RBLNGemma4ForConditionalGenerationConfig", "dtype": "float32", "optimum_rbln_version": "0.11.2", "_compile_cfgs": [ { "compiled_model_name": "compiled_model", "input_info": [ [ [ "image_features", [ 1, 280, 1152 ], "float32" ] ] ], "npu": "RBLN-CA25", "num_devices": 1 } ], "batch_size": 8, "vision_tower": { "cls_name": "RBLNGemma4VisionModelConfig", "dtype": "float32", "optimum_rbln_version": "0.11.2", "_compile_cfgs": [ { "compiled_model_name": "compiled_model", "input_info": [ [ [ "inputs_embeds", [ 1, 2520, 1152 ], "float32" ], [ "pixel_position_ids", [ 1, 2520, 2 ], "int64" ], [ "attn_mask", [ 1, 2520 ], "float32" ], [ "padding_positions", [ 1, 2520 ], "bool" ], [ "cos", [ 1, 2520, 72 ], "float32" ], [ "sin", [ 1, 2520, 72 ], "float32" ] ] ], "npu": "RBLN-CA25", "num_devices": 1 } ], "batch_size": 1, "max_soft_tokens": [ 280 ], "pooling_kernel_size": 3, "patch_size": 16, "output_hidden_states": false }, "language_model": { "cls_name": "RBLNGemma4ForCausalLMConfig", "dtype": "float32", "optimum_rbln_version": "0.11.2", "_compile_cfgs": [ { "compiled_model_name": "prefill", "input_info": [ [ "inputs_embeds", [ 1, 128, 5376 ], "float32" ], [ "cache_position", [ 1, 128 ], "int32" ], [ "block_tables", [ 16 ], "int16" ], [ "local_block_tables", [ 1 ], "int16" ], [ "query_position", [], "int16" ], [ "attention_mask", [ 1, 262144 ], "float32" ], [ "position_ids", [ 1, 128 ], "int32" ], [ "past_key_values_0", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_1", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_2", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_3", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_4", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_5", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_6", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_7", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_8", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_9", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_10", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_11", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_12", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_13", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_14", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_15", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_16", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_17", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_18", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_19", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_20", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_21", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_22", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_23", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_24", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_25", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_26", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_27", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_28", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_29", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_30", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_31", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_32", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_33", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_34", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_35", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_36", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_37", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_38", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_39", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_40", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_41", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_42", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_43", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_44", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_45", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_46", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_47", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_48", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_49", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_50", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_51", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_52", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_53", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_54", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_55", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_56", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_57", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_58", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_59", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_60", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_61", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_62", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_63", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_64", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_65", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_66", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_67", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_68", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_69", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_70", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_71", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_72", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_73", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_74", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_75", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_76", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_77", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_78", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_79", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_80", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_81", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_82", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_83", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_84", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_85", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_86", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_87", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_88", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_89", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_90", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_91", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_92", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_93", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_94", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_95", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_96", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_97", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_98", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_99", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_100", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_101", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_102", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_103", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_104", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_105", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_106", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_107", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_108", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_109", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_110", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_111", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_112", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_113", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_114", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_115", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_116", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_117", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_118", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_119", [ 1, 4, 16384, 512 ], "float32" ] ], "npu": "RBLN-CA25", "num_devices": 16 }, { "compiled_model_name": "image_prefill_384", "input_info": [ [ "inputs_embeds", [ 1, 384, 5376 ], "float32" ], [ "cache_position", [ 1, 384 ], "int32" ], [ "block_tables", [ 16 ], "int16" ], [ "local_block_tables", [ 1 ], "int16" ], [ "query_position", [], "int16" ], [ "attention_mask", [ 1, 262144 ], "float32" ], [ "position_ids", [ 1, 384 ], "int32" ], [ "past_key_values_0", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_1", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_2", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_3", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_4", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_5", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_6", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_7", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_8", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_9", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_10", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_11", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_12", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_13", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_14", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_15", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_16", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_17", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_18", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_19", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_20", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_21", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_22", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_23", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_24", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_25", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_26", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_27", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_28", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_29", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_30", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_31", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_32", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_33", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_34", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_35", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_36", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_37", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_38", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_39", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_40", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_41", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_42", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_43", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_44", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_45", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_46", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_47", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_48", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_49", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_50", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_51", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_52", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_53", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_54", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_55", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_56", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_57", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_58", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_59", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_60", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_61", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_62", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_63", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_64", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_65", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_66", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_67", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_68", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_69", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_70", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_71", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_72", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_73", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_74", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_75", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_76", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_77", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_78", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_79", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_80", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_81", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_82", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_83", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_84", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_85", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_86", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_87", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_88", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_89", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_90", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_91", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_92", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_93", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_94", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_95", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_96", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_97", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_98", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_99", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_100", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_101", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_102", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_103", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_104", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_105", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_106", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_107", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_108", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_109", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_110", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_111", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_112", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_113", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_114", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_115", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_116", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_117", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_118", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_119", [ 1, 4, 16384, 512 ], "float32" ] ], "npu": "RBLN-CA25", "num_devices": 16 }, { "compiled_model_name": "decoder_batch_8", "input_info": [ [ "inputs_embeds", [ 8, 1, 5376 ], "float32" ], [ "cache_position", [ 8, 1 ], "int32" ], [ "block_tables", [ 8, 16 ], "int16" ], [ "local_block_tables", [ 8, 1 ], "int16" ], [ "attention_mask", [ 8, 262144 ], "float32" ], [ "position_ids", [ 8, 1 ], "int32" ], [ "past_key_values_0", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_1", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_2", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_3", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_4", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_5", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_6", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_7", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_8", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_9", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_10", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_11", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_12", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_13", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_14", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_15", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_16", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_17", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_18", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_19", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_20", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_21", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_22", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_23", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_24", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_25", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_26", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_27", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_28", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_29", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_30", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_31", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_32", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_33", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_34", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_35", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_36", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_37", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_38", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_39", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_40", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_41", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_42", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_43", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_44", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_45", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_46", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_47", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_48", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_49", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_50", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_51", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_52", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_53", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_54", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_55", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_56", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_57", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_58", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_59", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_60", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_61", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_62", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_63", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_64", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_65", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_66", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_67", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_68", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_69", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_70", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_71", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_72", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_73", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_74", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_75", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_76", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_77", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_78", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_79", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_80", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_81", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_82", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_83", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_84", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_85", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_86", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_87", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_88", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_89", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_90", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_91", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_92", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_93", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_94", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_95", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_96", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_97", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_98", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_99", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_100", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_101", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_102", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_103", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_104", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_105", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_106", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_107", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_108", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_109", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_110", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_111", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_112", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_113", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_114", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_115", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_116", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_117", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_118", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_119", [ 1, 4, 16384, 512 ], "float32" ] ], "npu": "RBLN-CA25", "num_devices": 16 }, { "compiled_model_name": "decoder_batch_4", "input_info": [ [ "inputs_embeds", [ 4, 1, 5376 ], "float32" ], [ "cache_position", [ 4, 1 ], "int32" ], [ "block_tables", [ 4, 16 ], "int16" ], [ "local_block_tables", [ 4, 1 ], "int16" ], [ "attention_mask", [ 4, 262144 ], "float32" ], [ "position_ids", [ 4, 1 ], "int32" ], [ "past_key_values_0", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_1", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_2", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_3", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_4", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_5", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_6", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_7", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_8", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_9", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_10", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_11", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_12", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_13", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_14", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_15", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_16", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_17", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_18", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_19", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_20", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_21", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_22", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_23", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_24", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_25", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_26", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_27", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_28", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_29", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_30", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_31", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_32", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_33", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_34", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_35", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_36", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_37", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_38", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_39", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_40", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_41", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_42", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_43", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_44", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_45", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_46", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_47", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_48", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_49", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_50", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_51", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_52", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_53", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_54", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_55", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_56", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_57", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_58", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_59", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_60", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_61", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_62", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_63", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_64", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_65", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_66", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_67", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_68", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_69", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_70", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_71", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_72", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_73", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_74", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_75", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_76", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_77", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_78", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_79", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_80", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_81", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_82", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_83", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_84", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_85", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_86", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_87", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_88", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_89", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_90", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_91", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_92", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_93", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_94", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_95", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_96", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_97", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_98", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_99", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_100", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_101", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_102", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_103", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_104", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_105", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_106", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_107", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_108", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_109", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_110", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_111", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_112", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_113", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_114", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_115", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_116", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_117", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_118", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_119", [ 1, 4, 16384, 512 ], "float32" ] ], "npu": "RBLN-CA25", "num_devices": 16 }, { "compiled_model_name": "decoder_batch_2", "input_info": [ [ "inputs_embeds", [ 2, 1, 5376 ], "float32" ], [ "cache_position", [ 2, 1 ], "int32" ], [ "block_tables", [ 2, 16 ], "int16" ], [ "local_block_tables", [ 2, 1 ], "int16" ], [ "attention_mask", [ 2, 262144 ], "float32" ], [ "position_ids", [ 2, 1 ], "int32" ], [ "past_key_values_0", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_1", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_2", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_3", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_4", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_5", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_6", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_7", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_8", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_9", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_10", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_11", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_12", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_13", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_14", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_15", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_16", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_17", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_18", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_19", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_20", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_21", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_22", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_23", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_24", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_25", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_26", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_27", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_28", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_29", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_30", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_31", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_32", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_33", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_34", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_35", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_36", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_37", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_38", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_39", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_40", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_41", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_42", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_43", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_44", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_45", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_46", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_47", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_48", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_49", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_50", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_51", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_52", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_53", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_54", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_55", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_56", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_57", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_58", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_59", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_60", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_61", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_62", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_63", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_64", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_65", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_66", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_67", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_68", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_69", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_70", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_71", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_72", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_73", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_74", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_75", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_76", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_77", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_78", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_79", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_80", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_81", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_82", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_83", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_84", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_85", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_86", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_87", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_88", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_89", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_90", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_91", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_92", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_93", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_94", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_95", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_96", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_97", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_98", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_99", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_100", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_101", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_102", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_103", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_104", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_105", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_106", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_107", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_108", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_109", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_110", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_111", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_112", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_113", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_114", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_115", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_116", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_117", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_118", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_119", [ 1, 4, 16384, 512 ], "float32" ] ], "npu": "RBLN-CA25", "num_devices": 16 }, { "compiled_model_name": "decoder_batch_1", "input_info": [ [ "inputs_embeds", [ 1, 1, 5376 ], "float32" ], [ "cache_position", [ 1, 1 ], "int32" ], [ "block_tables", [ 1, 16 ], "int16" ], [ "local_block_tables", [ 1, 1 ], "int16" ], [ "attention_mask", [ 1, 262144 ], "float32" ], [ "position_ids", [ 1, 1 ], "int32" ], [ "past_key_values_0", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_1", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_2", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_3", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_4", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_5", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_6", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_7", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_8", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_9", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_10", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_11", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_12", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_13", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_14", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_15", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_16", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_17", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_18", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_19", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_20", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_21", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_22", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_23", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_24", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_25", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_26", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_27", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_28", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_29", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_30", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_31", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_32", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_33", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_34", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_35", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_36", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_37", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_38", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_39", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_40", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_41", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_42", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_43", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_44", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_45", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_46", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_47", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_48", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_49", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_50", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_51", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_52", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_53", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_54", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_55", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_56", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_57", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_58", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_59", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_60", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_61", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_62", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_63", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_64", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_65", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_66", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_67", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_68", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_69", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_70", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_71", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_72", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_73", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_74", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_75", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_76", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_77", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_78", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_79", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_80", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_81", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_82", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_83", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_84", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_85", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_86", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_87", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_88", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_89", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_90", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_91", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_92", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_93", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_94", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_95", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_96", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_97", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_98", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_99", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_100", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_101", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_102", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_103", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_104", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_105", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_106", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_107", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_108", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_109", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_110", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_111", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_112", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_113", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_114", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_115", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_116", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_117", [ 8, 16, 1024, 256 ], "float32" ], [ "past_key_values_118", [ 1, 4, 16384, 512 ], "float32" ], [ "past_key_values_119", [ 1, 4, 16384, 512 ], "float32" ] ], "npu": "RBLN-CA25", "num_devices": 16 } ], "batch_size": 8, "max_seq_len": 262144, "use_inputs_embeds": true, "use_position_ids": true, "use_attention_mask": true, "quantization": { "format": "rbln", "weights": "int8", "activations": "fp16", "kv_caches": "fp16", "dynamic": false }, "lora_config": null, "attn_impl": "flash_attn", "kvcache_partition_len": 16384, "kvcache_block_size": 16384, "prefill_chunk_size": 128, "kvcache_num_blocks": 36, "cache_impl": "hybrid", "sliding_window": 1024, "sliding_window_layers": [ 0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 36, 37, 38, 39, 40, 42, 43, 44, 45, 46, 48, 49, 50, 51, 52, 54, 55, 56, 57, 58 ], "phases": [ "prefill", "image_prefill", "decode" ], "logits_to_keep": 1, "output_hidden_states": false, "decoder_batch_sizes": [ 8, 4, 2, 1 ], "cache_metas": [ { "name": "past_key_values_0", "layer_index": 0, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_1", "layer_index": 0, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_2", "layer_index": 1, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_3", "layer_index": 1, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_4", "layer_index": 2, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_5", "layer_index": 2, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_6", "layer_index": 3, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_7", "layer_index": 3, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_8", "layer_index": 4, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_9", "layer_index": 4, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_10", "layer_index": 5, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_11", "layer_index": 5, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_12", "layer_index": 6, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_13", "layer_index": 6, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_14", "layer_index": 7, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_15", "layer_index": 7, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_16", "layer_index": 8, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_17", "layer_index": 8, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_18", "layer_index": 9, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_19", "layer_index": 9, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_20", "layer_index": 10, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_21", "layer_index": 10, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_22", "layer_index": 11, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_23", "layer_index": 11, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_24", "layer_index": 12, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_25", "layer_index": 12, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_26", "layer_index": 13, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_27", "layer_index": 13, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_28", "layer_index": 14, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_29", "layer_index": 14, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_30", "layer_index": 15, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_31", "layer_index": 15, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_32", "layer_index": 16, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_33", "layer_index": 16, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_34", "layer_index": 17, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_35", "layer_index": 17, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_36", "layer_index": 18, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_37", "layer_index": 18, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_38", "layer_index": 19, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_39", "layer_index": 19, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_40", "layer_index": 20, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_41", "layer_index": 20, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_42", "layer_index": 21, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_43", "layer_index": 21, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_44", "layer_index": 22, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_45", "layer_index": 22, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_46", "layer_index": 23, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_47", "layer_index": 23, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_48", "layer_index": 24, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_49", "layer_index": 24, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_50", "layer_index": 25, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_51", "layer_index": 25, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_52", "layer_index": 26, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_53", "layer_index": 26, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_54", "layer_index": 27, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_55", "layer_index": 27, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_56", "layer_index": 28, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_57", "layer_index": 28, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_58", "layer_index": 29, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_59", "layer_index": 29, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_60", "layer_index": 30, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_61", "layer_index": 30, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_62", "layer_index": 31, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_63", "layer_index": 31, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_64", "layer_index": 32, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_65", "layer_index": 32, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_66", "layer_index": 33, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_67", "layer_index": 33, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_68", "layer_index": 34, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_69", "layer_index": 34, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_70", "layer_index": 35, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_71", "layer_index": 35, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_72", "layer_index": 36, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_73", "layer_index": 36, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_74", "layer_index": 37, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_75", "layer_index": 37, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_76", "layer_index": 38, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_77", "layer_index": 38, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_78", "layer_index": 39, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_79", "layer_index": 39, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_80", "layer_index": 40, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_81", "layer_index": 40, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_82", "layer_index": 41, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_83", "layer_index": 41, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_84", "layer_index": 42, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_85", "layer_index": 42, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_86", "layer_index": 43, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_87", "layer_index": 43, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_88", "layer_index": 44, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_89", "layer_index": 44, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_90", "layer_index": 45, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_91", "layer_index": 45, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_92", "layer_index": 46, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_93", "layer_index": 46, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_94", "layer_index": 47, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_95", "layer_index": 47, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_96", "layer_index": 48, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_97", "layer_index": 48, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_98", "layer_index": 49, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_99", "layer_index": 49, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_100", "layer_index": 50, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_101", "layer_index": 50, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_102", "layer_index": 51, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_103", "layer_index": 51, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_104", "layer_index": 52, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_105", "layer_index": 52, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_106", "layer_index": 53, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_107", "layer_index": 53, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_108", "layer_index": 54, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_109", "layer_index": 54, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_110", "layer_index": 55, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_111", "layer_index": 55, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_112", "layer_index": 56, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_113", "layer_index": 56, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_114", "layer_index": 57, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_115", "layer_index": 57, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_116", "layer_index": 58, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_117", "layer_index": 58, "shape": [ 8, 16, 1024, 256 ], "layer_type": "sliding_attention", "is_auto": false, "dtype": "float32" }, { "name": "past_key_values_118", "layer_index": 59, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" }, { "name": "past_key_values_119", "layer_index": 59, "shape": [ 128, 4, 16384, 512 ], "layer_type": "full_attention", "is_auto": true, "dtype": "float32" } ], "image_prefill_chunk_size": [ 384 ] } }