diff --git "a/cache/modellayers.0input_layernormMLADFRMSNORM_meta.json" "b/cache/modellayers.0input_layernormMLADFRMSNORM_meta.json" new file mode 100755--- /dev/null +++ "b/cache/modellayers.0input_layernormMLADFRMSNORM_meta.json" @@ -0,0 +1,37447 @@ +{ + "dd_meta_major_version": 1, + "dd_meta_minor_version": 4, + "state_table_updates": [], + "op_list": [ + { + "name": "/model/layers.0/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/embed_tokens/Gather/output_0.out19_14_0" + ], + "const_args": [ + "model.layers.0.input_layernorm.weight", + "eps_19_14_0" + ], + "out_args": [ + "/model/layers.0/input_layernorm/output_0.out19_14_0" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "m_x": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.0/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/input_layernorm/output_0.out19_14_0" + ], + "const_args": [ + "model.layers.0.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.k_proj.Add.bias.preformat", + "model.layers.0.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.0/attn/k_proj/Add/output_0.out19_11_1" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.0/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.0/attn/k_proj/Add/output_0.out19_11_1", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.0.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "0", + "0" + ] + } + } + }, + { + "name": "/model/layers.0/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/input_layernorm/output_0.out19_14_0" + ], + "const_args": [ + "model.layers.0.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.q_proj.Add.bias.preformat", + "model.layers.0.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.0/attn/q_proj/Add/output_0.out19_11_0" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.0/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.0/attn/q_proj/Add/output_0.out19_11_0", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.0/attn/q_rotary/RotaryEmbedding/output_0.out19_3_0_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.0/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/input_layernorm/output_0.out19_14_0" + ], + "const_args": [ + "model.layers.0.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.v_proj.Add.bias.preformat", + "model.layers.0.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.0.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "1", + "1" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.0/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.0/attn/q_rotary/RotaryEmbedding/output_0.out19_3_0_1", + "present.0.key", + "present.0.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.0/attn/GroupQueryAttention/output_0.out19_3_0" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "0", + "0", + "2", + "0", + "1", + "1" + ] + } + } + }, + { + "name": "/model/layers.0/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/attn/GroupQueryAttention/output_0.out19_3_0" + ], + "const_args": [ + "model.layers.0.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.0/attn/o_proj/MatMulNBits/output_0.out19_11_2" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.0/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/embed_tokens/Gather/output_0.out19_14_0", + "/model/layers.0/attn/o_proj/MatMulNBits/output_0.out19_11_2" + ], + "const_args": [], + "out_args": [ + "/model/layers.0/post_attention_layernorm/output_3.out19_15_0" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.0/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.0/post_attention_layernorm/output_3.out19_15_0" + ], + "const_args": [ + "model.layers.0.post_attention_layernorm.weight", + "eps_19_15_0" + ], + "out_args": [ + "/model/layers.0/post_attention_layernorm/output_0.out19_15_0" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.0/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/post_attention_layernorm/output_0.out19_15_0" + ], + "const_args": [ + "model.layers.0.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.0.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.0.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.0.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.0/mlp/gate_proj/MatMulNBits/output_0.out19_11_3" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.0/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/post_attention_layernorm/output_0.out19_15_0" + ], + "const_args": [ + "model.layers.0.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.0.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.0.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.0.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.0/mlp/up_proj/MatMulNBits/output_0.out19_11_4" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.0/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.0/mlp/gate_proj/MatMulNBits/output_0.out19_11_3" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.0/mlp/act_fn/Mul/output_0.in.inter.out19_6_0" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.0/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.0/mlp/act_fn/Mul/output_0.in.inter.out19_6_0", + "/model/layers.0/mlp/up_proj/MatMulNBits/output_0.out19_11_4" + ], + "const_args": [], + "out_args": [ + "/model/layers.0/mlp/Mul/output_0.out19_12_0" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.0/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.0/mlp/Mul/output_0.out19_12_0" + ], + "const_args": [ + "model.layers.0.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.0/mlp/down_proj/MatMulNBits/output_0.out19_11_5" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.1/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.0/post_attention_layernorm/output_3.out19_15_0", + "/model/layers.0/mlp/down_proj/MatMulNBits/output_0.out19_11_5" + ], + "const_args": [], + "out_args": [ + "/model/layers.1/input_layernorm/output_3.out19_15_1" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.1/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.1/input_layernorm/output_3.out19_15_1" + ], + "const_args": [ + "model.layers.1.input_layernorm.weight", + "eps_19_15_1" + ], + "out_args": [ + "/model/layers.1/input_layernorm/output_0.out19_15_1" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.1/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/input_layernorm/output_0.out19_15_1" + ], + "const_args": [ + "model.layers.1.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.k_proj.Add.bias.preformat", + "model.layers.1.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.1/attn/k_proj/Add/output_0.out19_11_7" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.1/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.1/attn/k_proj/Add/output_0.out19_11_7", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.1.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "2", + "2" + ] + } + } + }, + { + "name": "/model/layers.1/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/input_layernorm/output_0.out19_15_1" + ], + "const_args": [ + "model.layers.1.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.q_proj.Add.bias.preformat", + "model.layers.1.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.1/attn/q_proj/Add/output_0.out19_11_6" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.1/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.1/attn/q_proj/Add/output_0.out19_11_6", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.1/attn/q_rotary/RotaryEmbedding/output_0.out19_3_1_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.1/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/input_layernorm/output_0.out19_15_1" + ], + "const_args": [ + "model.layers.1.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.v_proj.Add.bias.preformat", + "model.layers.1.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.1.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "3", + "3" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.1/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.1/attn/q_rotary/RotaryEmbedding/output_0.out19_3_1_1", + "present.1.key", + "present.1.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.1/attn/GroupQueryAttention/output_0.out19_3_1" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "2", + "2", + "2", + "0", + "3", + "3" + ] + } + } + }, + { + "name": "/model/layers.1/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/attn/GroupQueryAttention/output_0.out19_3_1" + ], + "const_args": [ + "model.layers.1.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.1/attn/o_proj/MatMulNBits/output_0.out19_11_8" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.1/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.1/input_layernorm/output_3.out19_15_1", + "/model/layers.1/attn/o_proj/MatMulNBits/output_0.out19_11_8" + ], + "const_args": [], + "out_args": [ + "/model/layers.1/post_attention_layernorm/output_3.out19_15_2" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.1/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.1/post_attention_layernorm/output_3.out19_15_2" + ], + "const_args": [ + "model.layers.1.post_attention_layernorm.weight", + "eps_19_15_2" + ], + "out_args": [ + "/model/layers.1/post_attention_layernorm/output_0.out19_15_2" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.1/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/post_attention_layernorm/output_0.out19_15_2" + ], + "const_args": [ + "model.layers.1.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.1.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.1.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.1.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.1/mlp/gate_proj/MatMulNBits/output_0.out19_11_9" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.1/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/post_attention_layernorm/output_0.out19_15_2" + ], + "const_args": [ + "model.layers.1.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.1.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.1.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.1.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.1/mlp/up_proj/MatMulNBits/output_0.out19_11_10" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.1/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.1/mlp/gate_proj/MatMulNBits/output_0.out19_11_9" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.1/mlp/act_fn/Mul/output_0.in.inter.out19_6_1" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.1/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.1/mlp/act_fn/Mul/output_0.in.inter.out19_6_1", + "/model/layers.1/mlp/up_proj/MatMulNBits/output_0.out19_11_10" + ], + "const_args": [], + "out_args": [ + "/model/layers.1/mlp/Mul/output_0.out19_12_1" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.1/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.1/mlp/Mul/output_0.out19_12_1" + ], + "const_args": [ + "model.layers.1.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.1/mlp/down_proj/MatMulNBits/output_0.out19_11_11" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.2/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.1/post_attention_layernorm/output_3.out19_15_2", + "/model/layers.1/mlp/down_proj/MatMulNBits/output_0.out19_11_11" + ], + "const_args": [], + "out_args": [ + "/model/layers.2/input_layernorm/output_3.out19_15_3" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.2/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.2/input_layernorm/output_3.out19_15_3" + ], + "const_args": [ + "model.layers.2.input_layernorm.weight", + "eps_19_15_3" + ], + "out_args": [ + "/model/layers.2/input_layernorm/output_0.out19_15_3" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.2/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/input_layernorm/output_0.out19_15_3" + ], + "const_args": [ + "model.layers.2.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.k_proj.Add.bias.preformat", + "model.layers.2.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.2/attn/k_proj/Add/output_0.out19_11_13" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.2/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.2/attn/k_proj/Add/output_0.out19_11_13", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.2.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "4", + "4" + ] + } + } + }, + { + "name": "/model/layers.2/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/input_layernorm/output_0.out19_15_3" + ], + "const_args": [ + "model.layers.2.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.q_proj.Add.bias.preformat", + "model.layers.2.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.2/attn/q_proj/Add/output_0.out19_11_12" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.2/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.2/attn/q_proj/Add/output_0.out19_11_12", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.2/attn/q_rotary/RotaryEmbedding/output_0.out19_3_2_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.2/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/input_layernorm/output_0.out19_15_3" + ], + "const_args": [ + "model.layers.2.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.v_proj.Add.bias.preformat", + "model.layers.2.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.2.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "5", + "5" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.2/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.2/attn/q_rotary/RotaryEmbedding/output_0.out19_3_2_1", + "present.2.key", + "present.2.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.2/attn/GroupQueryAttention/output_0.out19_3_2" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "4", + "4", + "2", + "0", + "5", + "5" + ] + } + } + }, + { + "name": "/model/layers.2/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/attn/GroupQueryAttention/output_0.out19_3_2" + ], + "const_args": [ + "model.layers.2.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.2/attn/o_proj/MatMulNBits/output_0.out19_11_14" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.2/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.2/input_layernorm/output_3.out19_15_3", + "/model/layers.2/attn/o_proj/MatMulNBits/output_0.out19_11_14" + ], + "const_args": [], + "out_args": [ + "/model/layers.2/post_attention_layernorm/output_3.out19_15_4" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.2/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.2/post_attention_layernorm/output_3.out19_15_4" + ], + "const_args": [ + "model.layers.2.post_attention_layernorm.weight", + "eps_19_15_4" + ], + "out_args": [ + "/model/layers.2/post_attention_layernorm/output_0.out19_15_4" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.2/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/post_attention_layernorm/output_0.out19_15_4" + ], + "const_args": [ + "model.layers.2.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.2.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.2.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.2.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.2/mlp/gate_proj/MatMulNBits/output_0.out19_11_15" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.2/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/post_attention_layernorm/output_0.out19_15_4" + ], + "const_args": [ + "model.layers.2.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.2.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.2.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.2.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.2/mlp/up_proj/MatMulNBits/output_0.out19_11_16" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.2/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.2/mlp/gate_proj/MatMulNBits/output_0.out19_11_15" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.2/mlp/act_fn/Mul/output_0.in.inter.out19_6_2" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.2/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.2/mlp/act_fn/Mul/output_0.in.inter.out19_6_2", + "/model/layers.2/mlp/up_proj/MatMulNBits/output_0.out19_11_16" + ], + "const_args": [], + "out_args": [ + "/model/layers.2/mlp/Mul/output_0.out19_12_2" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.2/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.2/mlp/Mul/output_0.out19_12_2" + ], + "const_args": [ + "model.layers.2.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.2/mlp/down_proj/MatMulNBits/output_0.out19_11_17" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.3/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.2/post_attention_layernorm/output_3.out19_15_4", + "/model/layers.2/mlp/down_proj/MatMulNBits/output_0.out19_11_17" + ], + "const_args": [], + "out_args": [ + "/model/layers.3/input_layernorm/output_3.out19_15_5" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.3/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.3/input_layernorm/output_3.out19_15_5" + ], + "const_args": [ + "model.layers.3.input_layernorm.weight", + "eps_19_15_5" + ], + "out_args": [ + "/model/layers.3/input_layernorm/output_0.out19_15_5" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.3/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/input_layernorm/output_0.out19_15_5" + ], + "const_args": [ + "model.layers.3.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.k_proj.Add.bias.preformat", + "model.layers.3.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.3/attn/k_proj/Add/output_0.out19_11_19" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.3/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.3/attn/k_proj/Add/output_0.out19_11_19", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.3.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "6", + "6" + ] + } + } + }, + { + "name": "/model/layers.3/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/input_layernorm/output_0.out19_15_5" + ], + "const_args": [ + "model.layers.3.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.q_proj.Add.bias.preformat", + "model.layers.3.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.3/attn/q_proj/Add/output_0.out19_11_18" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.3/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.3/attn/q_proj/Add/output_0.out19_11_18", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.3/attn/q_rotary/RotaryEmbedding/output_0.out19_3_3_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.3/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/input_layernorm/output_0.out19_15_5" + ], + "const_args": [ + "model.layers.3.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.v_proj.Add.bias.preformat", + "model.layers.3.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.3.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "7", + "7" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.3/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.3/attn/q_rotary/RotaryEmbedding/output_0.out19_3_3_1", + "present.3.key", + "present.3.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.3/attn/GroupQueryAttention/output_0.out19_3_3" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "6", + "6", + "2", + "0", + "7", + "7" + ] + } + } + }, + { + "name": "/model/layers.3/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/attn/GroupQueryAttention/output_0.out19_3_3" + ], + "const_args": [ + "model.layers.3.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.3/attn/o_proj/MatMulNBits/output_0.out19_11_20" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.3/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.3/input_layernorm/output_3.out19_15_5", + "/model/layers.3/attn/o_proj/MatMulNBits/output_0.out19_11_20" + ], + "const_args": [], + "out_args": [ + "/model/layers.3/post_attention_layernorm/output_3.out19_15_6" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.3/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.3/post_attention_layernorm/output_3.out19_15_6" + ], + "const_args": [ + "model.layers.3.post_attention_layernorm.weight", + "eps_19_15_6" + ], + "out_args": [ + "/model/layers.3/post_attention_layernorm/output_0.out19_15_6" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.3/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/post_attention_layernorm/output_0.out19_15_6" + ], + "const_args": [ + "model.layers.3.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.3.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.3.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.3.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.3/mlp/gate_proj/MatMulNBits/output_0.out19_11_21" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.3/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/post_attention_layernorm/output_0.out19_15_6" + ], + "const_args": [ + "model.layers.3.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.3.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.3.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.3.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.3/mlp/up_proj/MatMulNBits/output_0.out19_11_22" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.3/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.3/mlp/gate_proj/MatMulNBits/output_0.out19_11_21" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.3/mlp/act_fn/Mul/output_0.in.inter.out19_6_3" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.3/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.3/mlp/act_fn/Mul/output_0.in.inter.out19_6_3", + "/model/layers.3/mlp/up_proj/MatMulNBits/output_0.out19_11_22" + ], + "const_args": [], + "out_args": [ + "/model/layers.3/mlp/Mul/output_0.out19_12_3" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.3/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.3/mlp/Mul/output_0.out19_12_3" + ], + "const_args": [ + "model.layers.3.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.3/mlp/down_proj/MatMulNBits/output_0.out19_11_23" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.4/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.3/post_attention_layernorm/output_3.out19_15_6", + "/model/layers.3/mlp/down_proj/MatMulNBits/output_0.out19_11_23" + ], + "const_args": [], + "out_args": [ + "/model/layers.4/input_layernorm/output_3.out19_15_7" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.4/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.4/input_layernorm/output_3.out19_15_7" + ], + "const_args": [ + "model.layers.4.input_layernorm.weight", + "eps_19_15_7" + ], + "out_args": [ + "/model/layers.4/input_layernorm/output_0.out19_15_7" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.4/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/input_layernorm/output_0.out19_15_7" + ], + "const_args": [ + "model.layers.4.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.k_proj.Add.bias.preformat", + "model.layers.4.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.4/attn/k_proj/Add/output_0.out19_11_25" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.4/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.4/attn/k_proj/Add/output_0.out19_11_25", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.4.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "8", + "8" + ] + } + } + }, + { + "name": "/model/layers.4/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/input_layernorm/output_0.out19_15_7" + ], + "const_args": [ + "model.layers.4.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.q_proj.Add.bias.preformat", + "model.layers.4.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.4/attn/q_proj/Add/output_0.out19_11_24" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.4/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.4/attn/q_proj/Add/output_0.out19_11_24", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.4/attn/q_rotary/RotaryEmbedding/output_0.out19_3_4_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.4/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/input_layernorm/output_0.out19_15_7" + ], + "const_args": [ + "model.layers.4.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.v_proj.Add.bias.preformat", + "model.layers.4.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.4.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "9", + "9" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.4/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.4/attn/q_rotary/RotaryEmbedding/output_0.out19_3_4_1", + "present.4.key", + "present.4.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.4/attn/GroupQueryAttention/output_0.out19_3_4" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "8", + "8", + "2", + "0", + "9", + "9" + ] + } + } + }, + { + "name": "/model/layers.4/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/attn/GroupQueryAttention/output_0.out19_3_4" + ], + "const_args": [ + "model.layers.4.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.4/attn/o_proj/MatMulNBits/output_0.out19_11_26" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.4/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.4/input_layernorm/output_3.out19_15_7", + "/model/layers.4/attn/o_proj/MatMulNBits/output_0.out19_11_26" + ], + "const_args": [], + "out_args": [ + "/model/layers.4/post_attention_layernorm/output_3.out19_15_8" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.4/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.4/post_attention_layernorm/output_3.out19_15_8" + ], + "const_args": [ + "model.layers.4.post_attention_layernorm.weight", + "eps_19_15_8" + ], + "out_args": [ + "/model/layers.4/post_attention_layernorm/output_0.out19_15_8" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.4/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/post_attention_layernorm/output_0.out19_15_8" + ], + "const_args": [ + "model.layers.4.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.4.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.4.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.4.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.4/mlp/gate_proj/MatMulNBits/output_0.out19_11_27" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.4/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/post_attention_layernorm/output_0.out19_15_8" + ], + "const_args": [ + "model.layers.4.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.4.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.4.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.4.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.4/mlp/up_proj/MatMulNBits/output_0.out19_11_28" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.4/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.4/mlp/gate_proj/MatMulNBits/output_0.out19_11_27" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.4/mlp/act_fn/Mul/output_0.in.inter.out19_6_4" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.4/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.4/mlp/act_fn/Mul/output_0.in.inter.out19_6_4", + "/model/layers.4/mlp/up_proj/MatMulNBits/output_0.out19_11_28" + ], + "const_args": [], + "out_args": [ + "/model/layers.4/mlp/Mul/output_0.out19_12_4" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.4/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.4/mlp/Mul/output_0.out19_12_4" + ], + "const_args": [ + "model.layers.4.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.4/mlp/down_proj/MatMulNBits/output_0.out19_11_29" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.5/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.4/post_attention_layernorm/output_3.out19_15_8", + "/model/layers.4/mlp/down_proj/MatMulNBits/output_0.out19_11_29" + ], + "const_args": [], + "out_args": [ + "/model/layers.5/input_layernorm/output_3.out19_15_9" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.5/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.5/input_layernorm/output_3.out19_15_9" + ], + "const_args": [ + "model.layers.5.input_layernorm.weight", + "eps_19_15_9" + ], + "out_args": [ + "/model/layers.5/input_layernorm/output_0.out19_15_9" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.5/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/input_layernorm/output_0.out19_15_9" + ], + "const_args": [ + "model.layers.5.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.k_proj.Add.bias.preformat", + "model.layers.5.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.5/attn/k_proj/Add/output_0.out19_11_31" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.5/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.5/attn/k_proj/Add/output_0.out19_11_31", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.5.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "10", + "10" + ] + } + } + }, + { + "name": "/model/layers.5/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/input_layernorm/output_0.out19_15_9" + ], + "const_args": [ + "model.layers.5.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.q_proj.Add.bias.preformat", + "model.layers.5.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.5/attn/q_proj/Add/output_0.out19_11_30" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.5/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.5/attn/q_proj/Add/output_0.out19_11_30", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.5/attn/q_rotary/RotaryEmbedding/output_0.out19_3_5_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.5/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/input_layernorm/output_0.out19_15_9" + ], + "const_args": [ + "model.layers.5.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.v_proj.Add.bias.preformat", + "model.layers.5.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.5.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "11", + "11" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.5/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.5/attn/q_rotary/RotaryEmbedding/output_0.out19_3_5_1", + "present.5.key", + "present.5.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.5/attn/GroupQueryAttention/output_0.out19_3_5" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "10", + "10", + "2", + "0", + "11", + "11" + ] + } + } + }, + { + "name": "/model/layers.5/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/attn/GroupQueryAttention/output_0.out19_3_5" + ], + "const_args": [ + "model.layers.5.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.5/attn/o_proj/MatMulNBits/output_0.out19_11_32" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.5/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.5/input_layernorm/output_3.out19_15_9", + "/model/layers.5/attn/o_proj/MatMulNBits/output_0.out19_11_32" + ], + "const_args": [], + "out_args": [ + "/model/layers.5/post_attention_layernorm/output_3.out19_15_10" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.5/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.5/post_attention_layernorm/output_3.out19_15_10" + ], + "const_args": [ + "model.layers.5.post_attention_layernorm.weight", + "eps_19_15_10" + ], + "out_args": [ + "/model/layers.5/post_attention_layernorm/output_0.out19_15_10" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.5/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/post_attention_layernorm/output_0.out19_15_10" + ], + "const_args": [ + "model.layers.5.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.5.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.5.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.5.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.5/mlp/gate_proj/MatMulNBits/output_0.out19_11_33" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.5/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/post_attention_layernorm/output_0.out19_15_10" + ], + "const_args": [ + "model.layers.5.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.5.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.5.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.5.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.5/mlp/up_proj/MatMulNBits/output_0.out19_11_34" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.5/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.5/mlp/gate_proj/MatMulNBits/output_0.out19_11_33" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.5/mlp/act_fn/Mul/output_0.in.inter.out19_6_5" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.5/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.5/mlp/act_fn/Mul/output_0.in.inter.out19_6_5", + "/model/layers.5/mlp/up_proj/MatMulNBits/output_0.out19_11_34" + ], + "const_args": [], + "out_args": [ + "/model/layers.5/mlp/Mul/output_0.out19_12_5" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.5/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.5/mlp/Mul/output_0.out19_12_5" + ], + "const_args": [ + "model.layers.5.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.5/mlp/down_proj/MatMulNBits/output_0.out19_11_35" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.6/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.5/post_attention_layernorm/output_3.out19_15_10", + "/model/layers.5/mlp/down_proj/MatMulNBits/output_0.out19_11_35" + ], + "const_args": [], + "out_args": [ + "/model/layers.6/input_layernorm/output_3.out19_15_11" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.6/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.6/input_layernorm/output_3.out19_15_11" + ], + "const_args": [ + "model.layers.6.input_layernorm.weight", + "eps_19_15_11" + ], + "out_args": [ + "/model/layers.6/input_layernorm/output_0.out19_15_11" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.6/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/input_layernorm/output_0.out19_15_11" + ], + "const_args": [ + "model.layers.6.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.k_proj.Add.bias.preformat", + "model.layers.6.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.6/attn/k_proj/Add/output_0.out19_11_37" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.6/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.6/attn/k_proj/Add/output_0.out19_11_37", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.6.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "12", + "12" + ] + } + } + }, + { + "name": "/model/layers.6/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/input_layernorm/output_0.out19_15_11" + ], + "const_args": [ + "model.layers.6.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.q_proj.Add.bias.preformat", + "model.layers.6.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.6/attn/q_proj/Add/output_0.out19_11_36" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.6/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.6/attn/q_proj/Add/output_0.out19_11_36", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.6/attn/q_rotary/RotaryEmbedding/output_0.out19_3_6_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.6/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/input_layernorm/output_0.out19_15_11" + ], + "const_args": [ + "model.layers.6.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.v_proj.Add.bias.preformat", + "model.layers.6.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.6.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "13", + "13" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.6/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.6/attn/q_rotary/RotaryEmbedding/output_0.out19_3_6_1", + "present.6.key", + "present.6.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.6/attn/GroupQueryAttention/output_0.out19_3_6" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "12", + "12", + "2", + "0", + "13", + "13" + ] + } + } + }, + { + "name": "/model/layers.6/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/attn/GroupQueryAttention/output_0.out19_3_6" + ], + "const_args": [ + "model.layers.6.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.6/attn/o_proj/MatMulNBits/output_0.out19_11_38" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.6/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.6/input_layernorm/output_3.out19_15_11", + "/model/layers.6/attn/o_proj/MatMulNBits/output_0.out19_11_38" + ], + "const_args": [], + "out_args": [ + "/model/layers.6/post_attention_layernorm/output_3.out19_15_12" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.6/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.6/post_attention_layernorm/output_3.out19_15_12" + ], + "const_args": [ + "model.layers.6.post_attention_layernorm.weight", + "eps_19_15_12" + ], + "out_args": [ + "/model/layers.6/post_attention_layernorm/output_0.out19_15_12" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.6/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/post_attention_layernorm/output_0.out19_15_12" + ], + "const_args": [ + "model.layers.6.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.6.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.6.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.6.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.6/mlp/gate_proj/MatMulNBits/output_0.out19_11_39" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.6/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/post_attention_layernorm/output_0.out19_15_12" + ], + "const_args": [ + "model.layers.6.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.6.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.6.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.6.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.6/mlp/up_proj/MatMulNBits/output_0.out19_11_40" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.6/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.6/mlp/gate_proj/MatMulNBits/output_0.out19_11_39" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.6/mlp/act_fn/Mul/output_0.in.inter.out19_6_6" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.6/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.6/mlp/act_fn/Mul/output_0.in.inter.out19_6_6", + "/model/layers.6/mlp/up_proj/MatMulNBits/output_0.out19_11_40" + ], + "const_args": [], + "out_args": [ + "/model/layers.6/mlp/Mul/output_0.out19_12_6" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.6/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.6/mlp/Mul/output_0.out19_12_6" + ], + "const_args": [ + "model.layers.6.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.6/mlp/down_proj/MatMulNBits/output_0.out19_11_41" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.7/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.6/post_attention_layernorm/output_3.out19_15_12", + "/model/layers.6/mlp/down_proj/MatMulNBits/output_0.out19_11_41" + ], + "const_args": [], + "out_args": [ + "/model/layers.7/input_layernorm/output_3.out19_15_13" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.7/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.7/input_layernorm/output_3.out19_15_13" + ], + "const_args": [ + "model.layers.7.input_layernorm.weight", + "eps_19_15_13" + ], + "out_args": [ + "/model/layers.7/input_layernorm/output_0.out19_15_13" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.7/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/input_layernorm/output_0.out19_15_13" + ], + "const_args": [ + "model.layers.7.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.k_proj.Add.bias.preformat", + "model.layers.7.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.7/attn/k_proj/Add/output_0.out19_11_43" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.7/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.7/attn/k_proj/Add/output_0.out19_11_43", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.7.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "14", + "14" + ] + } + } + }, + { + "name": "/model/layers.7/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/input_layernorm/output_0.out19_15_13" + ], + "const_args": [ + "model.layers.7.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.q_proj.Add.bias.preformat", + "model.layers.7.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.7/attn/q_proj/Add/output_0.out19_11_42" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.7/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.7/attn/q_proj/Add/output_0.out19_11_42", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.7/attn/q_rotary/RotaryEmbedding/output_0.out19_3_7_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.7/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/input_layernorm/output_0.out19_15_13" + ], + "const_args": [ + "model.layers.7.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.v_proj.Add.bias.preformat", + "model.layers.7.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.7.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "15", + "15" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.7/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.7/attn/q_rotary/RotaryEmbedding/output_0.out19_3_7_1", + "present.7.key", + "present.7.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.7/attn/GroupQueryAttention/output_0.out19_3_7" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "14", + "14", + "2", + "0", + "15", + "15" + ] + } + } + }, + { + "name": "/model/layers.7/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/attn/GroupQueryAttention/output_0.out19_3_7" + ], + "const_args": [ + "model.layers.7.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.7/attn/o_proj/MatMulNBits/output_0.out19_11_44" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.7/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.7/input_layernorm/output_3.out19_15_13", + "/model/layers.7/attn/o_proj/MatMulNBits/output_0.out19_11_44" + ], + "const_args": [], + "out_args": [ + "/model/layers.7/post_attention_layernorm/output_3.out19_15_14" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.7/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.7/post_attention_layernorm/output_3.out19_15_14" + ], + "const_args": [ + "model.layers.7.post_attention_layernorm.weight", + "eps_19_15_14" + ], + "out_args": [ + "/model/layers.7/post_attention_layernorm/output_0.out19_15_14" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.7/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/post_attention_layernorm/output_0.out19_15_14" + ], + "const_args": [ + "model.layers.7.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.7.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.7.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.7.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.7/mlp/gate_proj/MatMulNBits/output_0.out19_11_45" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.7/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/post_attention_layernorm/output_0.out19_15_14" + ], + "const_args": [ + "model.layers.7.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.7.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.7.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.7.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.7/mlp/up_proj/MatMulNBits/output_0.out19_11_46" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.7/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.7/mlp/gate_proj/MatMulNBits/output_0.out19_11_45" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.7/mlp/act_fn/Mul/output_0.in.inter.out19_6_7" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.7/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.7/mlp/act_fn/Mul/output_0.in.inter.out19_6_7", + "/model/layers.7/mlp/up_proj/MatMulNBits/output_0.out19_11_46" + ], + "const_args": [], + "out_args": [ + "/model/layers.7/mlp/Mul/output_0.out19_12_7" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.7/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.7/mlp/Mul/output_0.out19_12_7" + ], + "const_args": [ + "model.layers.7.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.7/mlp/down_proj/MatMulNBits/output_0.out19_11_47" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.8/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.7/post_attention_layernorm/output_3.out19_15_14", + "/model/layers.7/mlp/down_proj/MatMulNBits/output_0.out19_11_47" + ], + "const_args": [], + "out_args": [ + "/model/layers.8/input_layernorm/output_3.out19_15_15" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.8/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.8/input_layernorm/output_3.out19_15_15" + ], + "const_args": [ + "model.layers.8.input_layernorm.weight", + "eps_19_15_15" + ], + "out_args": [ + "/model/layers.8/input_layernorm/output_0.out19_15_15" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.8/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/input_layernorm/output_0.out19_15_15" + ], + "const_args": [ + "model.layers.8.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.k_proj.Add.bias.preformat", + "model.layers.8.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.8/attn/k_proj/Add/output_0.out19_11_49" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.8/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.8/attn/k_proj/Add/output_0.out19_11_49", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.8.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "16", + "16" + ] + } + } + }, + { + "name": "/model/layers.8/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/input_layernorm/output_0.out19_15_15" + ], + "const_args": [ + "model.layers.8.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.q_proj.Add.bias.preformat", + "model.layers.8.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.8/attn/q_proj/Add/output_0.out19_11_48" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.8/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.8/attn/q_proj/Add/output_0.out19_11_48", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.8/attn/q_rotary/RotaryEmbedding/output_0.out19_3_8_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.8/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/input_layernorm/output_0.out19_15_15" + ], + "const_args": [ + "model.layers.8.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.v_proj.Add.bias.preformat", + "model.layers.8.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.8.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "17", + "17" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.8/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.8/attn/q_rotary/RotaryEmbedding/output_0.out19_3_8_1", + "present.8.key", + "present.8.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.8/attn/GroupQueryAttention/output_0.out19_3_8" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "16", + "16", + "2", + "0", + "17", + "17" + ] + } + } + }, + { + "name": "/model/layers.8/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/attn/GroupQueryAttention/output_0.out19_3_8" + ], + "const_args": [ + "model.layers.8.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.8/attn/o_proj/MatMulNBits/output_0.out19_11_50" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.8/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.8/input_layernorm/output_3.out19_15_15", + "/model/layers.8/attn/o_proj/MatMulNBits/output_0.out19_11_50" + ], + "const_args": [], + "out_args": [ + "/model/layers.8/post_attention_layernorm/output_3.out19_15_16" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.8/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.8/post_attention_layernorm/output_3.out19_15_16" + ], + "const_args": [ + "model.layers.8.post_attention_layernorm.weight", + "eps_19_15_16" + ], + "out_args": [ + "/model/layers.8/post_attention_layernorm/output_0.out19_15_16" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.8/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/post_attention_layernorm/output_0.out19_15_16" + ], + "const_args": [ + "model.layers.8.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.8.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.8.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.8.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.8/mlp/gate_proj/MatMulNBits/output_0.out19_11_51" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.8/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/post_attention_layernorm/output_0.out19_15_16" + ], + "const_args": [ + "model.layers.8.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.8.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.8.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.8.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.8/mlp/up_proj/MatMulNBits/output_0.out19_11_52" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.8/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.8/mlp/gate_proj/MatMulNBits/output_0.out19_11_51" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.8/mlp/act_fn/Mul/output_0.in.inter.out19_6_8" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.8/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.8/mlp/act_fn/Mul/output_0.in.inter.out19_6_8", + "/model/layers.8/mlp/up_proj/MatMulNBits/output_0.out19_11_52" + ], + "const_args": [], + "out_args": [ + "/model/layers.8/mlp/Mul/output_0.out19_12_8" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.8/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.8/mlp/Mul/output_0.out19_12_8" + ], + "const_args": [ + "model.layers.8.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.8/mlp/down_proj/MatMulNBits/output_0.out19_11_53" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.9/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.8/post_attention_layernorm/output_3.out19_15_16", + "/model/layers.8/mlp/down_proj/MatMulNBits/output_0.out19_11_53" + ], + "const_args": [], + "out_args": [ + "/model/layers.9/input_layernorm/output_3.out19_15_17" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.9/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.9/input_layernorm/output_3.out19_15_17" + ], + "const_args": [ + "model.layers.9.input_layernorm.weight", + "eps_19_15_17" + ], + "out_args": [ + "/model/layers.9/input_layernorm/output_0.out19_15_17" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.9/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/input_layernorm/output_0.out19_15_17" + ], + "const_args": [ + "model.layers.9.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.k_proj.Add.bias.preformat", + "model.layers.9.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.9/attn/k_proj/Add/output_0.out19_11_55" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.9/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.9/attn/k_proj/Add/output_0.out19_11_55", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.9.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "18", + "18" + ] + } + } + }, + { + "name": "/model/layers.9/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/input_layernorm/output_0.out19_15_17" + ], + "const_args": [ + "model.layers.9.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.q_proj.Add.bias.preformat", + "model.layers.9.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.9/attn/q_proj/Add/output_0.out19_11_54" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.9/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.9/attn/q_proj/Add/output_0.out19_11_54", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.9/attn/q_rotary/RotaryEmbedding/output_0.out19_3_9_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.9/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/input_layernorm/output_0.out19_15_17" + ], + "const_args": [ + "model.layers.9.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.v_proj.Add.bias.preformat", + "model.layers.9.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.9.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "19", + "19" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.9/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.9/attn/q_rotary/RotaryEmbedding/output_0.out19_3_9_1", + "present.9.key", + "present.9.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.9/attn/GroupQueryAttention/output_0.out19_3_9" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "18", + "18", + "2", + "0", + "19", + "19" + ] + } + } + }, + { + "name": "/model/layers.9/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/attn/GroupQueryAttention/output_0.out19_3_9" + ], + "const_args": [ + "model.layers.9.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.9/attn/o_proj/MatMulNBits/output_0.out19_11_56" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.9/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.9/input_layernorm/output_3.out19_15_17", + "/model/layers.9/attn/o_proj/MatMulNBits/output_0.out19_11_56" + ], + "const_args": [], + "out_args": [ + "/model/layers.9/post_attention_layernorm/output_3.out19_15_18" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.9/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.9/post_attention_layernorm/output_3.out19_15_18" + ], + "const_args": [ + "model.layers.9.post_attention_layernorm.weight", + "eps_19_15_18" + ], + "out_args": [ + "/model/layers.9/post_attention_layernorm/output_0.out19_15_18" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.9/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/post_attention_layernorm/output_0.out19_15_18" + ], + "const_args": [ + "model.layers.9.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.9.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.9.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.9.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.9/mlp/gate_proj/MatMulNBits/output_0.out19_11_57" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.9/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/post_attention_layernorm/output_0.out19_15_18" + ], + "const_args": [ + "model.layers.9.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.9.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.9.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.9.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.9/mlp/up_proj/MatMulNBits/output_0.out19_11_58" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.9/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.9/mlp/gate_proj/MatMulNBits/output_0.out19_11_57" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.9/mlp/act_fn/Mul/output_0.in.inter.out19_6_9" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.9/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.9/mlp/act_fn/Mul/output_0.in.inter.out19_6_9", + "/model/layers.9/mlp/up_proj/MatMulNBits/output_0.out19_11_58" + ], + "const_args": [], + "out_args": [ + "/model/layers.9/mlp/Mul/output_0.out19_12_9" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.9/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.9/mlp/Mul/output_0.out19_12_9" + ], + "const_args": [ + "model.layers.9.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.9/mlp/down_proj/MatMulNBits/output_0.out19_11_59" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.10/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.9/post_attention_layernorm/output_3.out19_15_18", + "/model/layers.9/mlp/down_proj/MatMulNBits/output_0.out19_11_59" + ], + "const_args": [], + "out_args": [ + "/model/layers.10/input_layernorm/output_3.out19_15_19" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.10/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.10/input_layernorm/output_3.out19_15_19" + ], + "const_args": [ + "model.layers.10.input_layernorm.weight", + "eps_19_15_19" + ], + "out_args": [ + "/model/layers.10/input_layernorm/output_0.out19_15_19" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.10/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/input_layernorm/output_0.out19_15_19" + ], + "const_args": [ + "model.layers.10.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.k_proj.Add.bias.preformat", + "model.layers.10.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.10/attn/k_proj/Add/output_0.out19_11_61" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.10/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.10/attn/k_proj/Add/output_0.out19_11_61", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.10.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "20", + "20" + ] + } + } + }, + { + "name": "/model/layers.10/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/input_layernorm/output_0.out19_15_19" + ], + "const_args": [ + "model.layers.10.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.q_proj.Add.bias.preformat", + "model.layers.10.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.10/attn/q_proj/Add/output_0.out19_11_60" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.10/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.10/attn/q_proj/Add/output_0.out19_11_60", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.10/attn/q_rotary/RotaryEmbedding/output_0.out19_3_10_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.10/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/input_layernorm/output_0.out19_15_19" + ], + "const_args": [ + "model.layers.10.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.v_proj.Add.bias.preformat", + "model.layers.10.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.10.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "21", + "21" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.10/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.10/attn/q_rotary/RotaryEmbedding/output_0.out19_3_10_1", + "present.10.key", + "present.10.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.10/attn/GroupQueryAttention/output_0.out19_3_10" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "20", + "20", + "2", + "0", + "21", + "21" + ] + } + } + }, + { + "name": "/model/layers.10/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/attn/GroupQueryAttention/output_0.out19_3_10" + ], + "const_args": [ + "model.layers.10.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.10/attn/o_proj/MatMulNBits/output_0.out19_11_62" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.10/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.10/input_layernorm/output_3.out19_15_19", + "/model/layers.10/attn/o_proj/MatMulNBits/output_0.out19_11_62" + ], + "const_args": [], + "out_args": [ + "/model/layers.10/post_attention_layernorm/output_3.out19_15_20" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.10/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.10/post_attention_layernorm/output_3.out19_15_20" + ], + "const_args": [ + "model.layers.10.post_attention_layernorm.weight", + "eps_19_15_20" + ], + "out_args": [ + "/model/layers.10/post_attention_layernorm/output_0.out19_15_20" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.10/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/post_attention_layernorm/output_0.out19_15_20" + ], + "const_args": [ + "model.layers.10.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.10.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.10.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.10.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.10/mlp/gate_proj/MatMulNBits/output_0.out19_11_63" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.10/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/post_attention_layernorm/output_0.out19_15_20" + ], + "const_args": [ + "model.layers.10.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.10.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.10.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.10.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.10/mlp/up_proj/MatMulNBits/output_0.out19_11_64" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.10/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.10/mlp/gate_proj/MatMulNBits/output_0.out19_11_63" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.10/mlp/act_fn/Mul/output_0.in.inter.out19_6_10" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.10/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.10/mlp/act_fn/Mul/output_0.in.inter.out19_6_10", + "/model/layers.10/mlp/up_proj/MatMulNBits/output_0.out19_11_64" + ], + "const_args": [], + "out_args": [ + "/model/layers.10/mlp/Mul/output_0.out19_12_10" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.10/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.10/mlp/Mul/output_0.out19_12_10" + ], + "const_args": [ + "model.layers.10.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.10/mlp/down_proj/MatMulNBits/output_0.out19_11_65" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.11/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.10/post_attention_layernorm/output_3.out19_15_20", + "/model/layers.10/mlp/down_proj/MatMulNBits/output_0.out19_11_65" + ], + "const_args": [], + "out_args": [ + "/model/layers.11/input_layernorm/output_3.out19_15_21" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.11/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.11/input_layernorm/output_3.out19_15_21" + ], + "const_args": [ + "model.layers.11.input_layernorm.weight", + "eps_19_15_21" + ], + "out_args": [ + "/model/layers.11/input_layernorm/output_0.out19_15_21" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.11/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/input_layernorm/output_0.out19_15_21" + ], + "const_args": [ + "model.layers.11.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.k_proj.Add.bias.preformat", + "model.layers.11.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.11/attn/k_proj/Add/output_0.out19_11_67" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.11/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.11/attn/k_proj/Add/output_0.out19_11_67", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.11.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "22", + "22" + ] + } + } + }, + { + "name": "/model/layers.11/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/input_layernorm/output_0.out19_15_21" + ], + "const_args": [ + "model.layers.11.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.q_proj.Add.bias.preformat", + "model.layers.11.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.11/attn/q_proj/Add/output_0.out19_11_66" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.11/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.11/attn/q_proj/Add/output_0.out19_11_66", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.11/attn/q_rotary/RotaryEmbedding/output_0.out19_3_11_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.11/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/input_layernorm/output_0.out19_15_21" + ], + "const_args": [ + "model.layers.11.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.v_proj.Add.bias.preformat", + "model.layers.11.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.11.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "23", + "23" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.11/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.11/attn/q_rotary/RotaryEmbedding/output_0.out19_3_11_1", + "present.11.key", + "present.11.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.11/attn/GroupQueryAttention/output_0.out19_3_11" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "22", + "22", + "2", + "0", + "23", + "23" + ] + } + } + }, + { + "name": "/model/layers.11/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/attn/GroupQueryAttention/output_0.out19_3_11" + ], + "const_args": [ + "model.layers.11.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.11/attn/o_proj/MatMulNBits/output_0.out19_11_68" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.11/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.11/input_layernorm/output_3.out19_15_21", + "/model/layers.11/attn/o_proj/MatMulNBits/output_0.out19_11_68" + ], + "const_args": [], + "out_args": [ + "/model/layers.11/post_attention_layernorm/output_3.out19_15_22" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.11/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.11/post_attention_layernorm/output_3.out19_15_22" + ], + "const_args": [ + "model.layers.11.post_attention_layernorm.weight", + "eps_19_15_22" + ], + "out_args": [ + "/model/layers.11/post_attention_layernorm/output_0.out19_15_22" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.11/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/post_attention_layernorm/output_0.out19_15_22" + ], + "const_args": [ + "model.layers.11.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.11.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.11.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.11.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.11/mlp/gate_proj/MatMulNBits/output_0.out19_11_69" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.11/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/post_attention_layernorm/output_0.out19_15_22" + ], + "const_args": [ + "model.layers.11.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.11.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.11.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.11.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.11/mlp/up_proj/MatMulNBits/output_0.out19_11_70" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.11/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.11/mlp/gate_proj/MatMulNBits/output_0.out19_11_69" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.11/mlp/act_fn/Mul/output_0.in.inter.out19_6_11" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.11/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.11/mlp/act_fn/Mul/output_0.in.inter.out19_6_11", + "/model/layers.11/mlp/up_proj/MatMulNBits/output_0.out19_11_70" + ], + "const_args": [], + "out_args": [ + "/model/layers.11/mlp/Mul/output_0.out19_12_11" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.11/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.11/mlp/Mul/output_0.out19_12_11" + ], + "const_args": [ + "model.layers.11.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.11/mlp/down_proj/MatMulNBits/output_0.out19_11_71" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.12/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.11/post_attention_layernorm/output_3.out19_15_22", + "/model/layers.11/mlp/down_proj/MatMulNBits/output_0.out19_11_71" + ], + "const_args": [], + "out_args": [ + "/model/layers.12/input_layernorm/output_3.out19_15_23" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.12/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.12/input_layernorm/output_3.out19_15_23" + ], + "const_args": [ + "model.layers.12.input_layernorm.weight", + "eps_19_15_23" + ], + "out_args": [ + "/model/layers.12/input_layernorm/output_0.out19_15_23" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.12/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/input_layernorm/output_0.out19_15_23" + ], + "const_args": [ + "model.layers.12.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.k_proj.Add.bias.preformat", + "model.layers.12.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.12/attn/k_proj/Add/output_0.out19_11_73" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.12/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.12/attn/k_proj/Add/output_0.out19_11_73", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.12.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "24", + "24" + ] + } + } + }, + { + "name": "/model/layers.12/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/input_layernorm/output_0.out19_15_23" + ], + "const_args": [ + "model.layers.12.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.q_proj.Add.bias.preformat", + "model.layers.12.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.12/attn/q_proj/Add/output_0.out19_11_72" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.12/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.12/attn/q_proj/Add/output_0.out19_11_72", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.12/attn/q_rotary/RotaryEmbedding/output_0.out19_3_12_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.12/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/input_layernorm/output_0.out19_15_23" + ], + "const_args": [ + "model.layers.12.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.v_proj.Add.bias.preformat", + "model.layers.12.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.12.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "25", + "25" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.12/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.12/attn/q_rotary/RotaryEmbedding/output_0.out19_3_12_1", + "present.12.key", + "present.12.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.12/attn/GroupQueryAttention/output_0.out19_3_12" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "24", + "24", + "2", + "0", + "25", + "25" + ] + } + } + }, + { + "name": "/model/layers.12/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/attn/GroupQueryAttention/output_0.out19_3_12" + ], + "const_args": [ + "model.layers.12.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.12/attn/o_proj/MatMulNBits/output_0.out19_11_74" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.12/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.12/input_layernorm/output_3.out19_15_23", + "/model/layers.12/attn/o_proj/MatMulNBits/output_0.out19_11_74" + ], + "const_args": [], + "out_args": [ + "/model/layers.12/post_attention_layernorm/output_3.out19_15_24" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.12/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.12/post_attention_layernorm/output_3.out19_15_24" + ], + "const_args": [ + "model.layers.12.post_attention_layernorm.weight", + "eps_19_15_24" + ], + "out_args": [ + "/model/layers.12/post_attention_layernorm/output_0.out19_15_24" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.12/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/post_attention_layernorm/output_0.out19_15_24" + ], + "const_args": [ + "model.layers.12.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.12.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.12.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.12.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.12/mlp/gate_proj/MatMulNBits/output_0.out19_11_75" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.12/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/post_attention_layernorm/output_0.out19_15_24" + ], + "const_args": [ + "model.layers.12.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.12.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.12.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.12.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.12/mlp/up_proj/MatMulNBits/output_0.out19_11_76" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.12/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.12/mlp/gate_proj/MatMulNBits/output_0.out19_11_75" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.12/mlp/act_fn/Mul/output_0.in.inter.out19_6_12" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.12/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.12/mlp/act_fn/Mul/output_0.in.inter.out19_6_12", + "/model/layers.12/mlp/up_proj/MatMulNBits/output_0.out19_11_76" + ], + "const_args": [], + "out_args": [ + "/model/layers.12/mlp/Mul/output_0.out19_12_12" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.12/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.12/mlp/Mul/output_0.out19_12_12" + ], + "const_args": [ + "model.layers.12.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.12/mlp/down_proj/MatMulNBits/output_0.out19_11_77" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.13/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.12/post_attention_layernorm/output_3.out19_15_24", + "/model/layers.12/mlp/down_proj/MatMulNBits/output_0.out19_11_77" + ], + "const_args": [], + "out_args": [ + "/model/layers.13/input_layernorm/output_3.out19_15_25" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.13/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.13/input_layernorm/output_3.out19_15_25" + ], + "const_args": [ + "model.layers.13.input_layernorm.weight", + "eps_19_15_25" + ], + "out_args": [ + "/model/layers.13/input_layernorm/output_0.out19_15_25" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.13/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/input_layernorm/output_0.out19_15_25" + ], + "const_args": [ + "model.layers.13.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.k_proj.Add.bias.preformat", + "model.layers.13.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.13/attn/k_proj/Add/output_0.out19_11_79" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.13/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.13/attn/k_proj/Add/output_0.out19_11_79", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.13.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "26", + "26" + ] + } + } + }, + { + "name": "/model/layers.13/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/input_layernorm/output_0.out19_15_25" + ], + "const_args": [ + "model.layers.13.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.q_proj.Add.bias.preformat", + "model.layers.13.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.13/attn/q_proj/Add/output_0.out19_11_78" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.13/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.13/attn/q_proj/Add/output_0.out19_11_78", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.13/attn/q_rotary/RotaryEmbedding/output_0.out19_3_13_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.13/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/input_layernorm/output_0.out19_15_25" + ], + "const_args": [ + "model.layers.13.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.v_proj.Add.bias.preformat", + "model.layers.13.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.13.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "27", + "27" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.13/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.13/attn/q_rotary/RotaryEmbedding/output_0.out19_3_13_1", + "present.13.key", + "present.13.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.13/attn/GroupQueryAttention/output_0.out19_3_13" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "26", + "26", + "2", + "0", + "27", + "27" + ] + } + } + }, + { + "name": "/model/layers.13/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/attn/GroupQueryAttention/output_0.out19_3_13" + ], + "const_args": [ + "model.layers.13.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.13/attn/o_proj/MatMulNBits/output_0.out19_11_80" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.13/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.13/input_layernorm/output_3.out19_15_25", + "/model/layers.13/attn/o_proj/MatMulNBits/output_0.out19_11_80" + ], + "const_args": [], + "out_args": [ + "/model/layers.13/post_attention_layernorm/output_3.out19_15_26" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.13/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.13/post_attention_layernorm/output_3.out19_15_26" + ], + "const_args": [ + "model.layers.13.post_attention_layernorm.weight", + "eps_19_15_26" + ], + "out_args": [ + "/model/layers.13/post_attention_layernorm/output_0.out19_15_26" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.13/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/post_attention_layernorm/output_0.out19_15_26" + ], + "const_args": [ + "model.layers.13.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.13.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.13.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.13.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.13/mlp/gate_proj/MatMulNBits/output_0.out19_11_81" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.13/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/post_attention_layernorm/output_0.out19_15_26" + ], + "const_args": [ + "model.layers.13.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.13.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.13.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.13.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.13/mlp/up_proj/MatMulNBits/output_0.out19_11_82" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.13/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.13/mlp/gate_proj/MatMulNBits/output_0.out19_11_81" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.13/mlp/act_fn/Mul/output_0.in.inter.out19_6_13" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.13/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.13/mlp/act_fn/Mul/output_0.in.inter.out19_6_13", + "/model/layers.13/mlp/up_proj/MatMulNBits/output_0.out19_11_82" + ], + "const_args": [], + "out_args": [ + "/model/layers.13/mlp/Mul/output_0.out19_12_13" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.13/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.13/mlp/Mul/output_0.out19_12_13" + ], + "const_args": [ + "model.layers.13.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.13/mlp/down_proj/MatMulNBits/output_0.out19_11_83" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.14/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.13/post_attention_layernorm/output_3.out19_15_26", + "/model/layers.13/mlp/down_proj/MatMulNBits/output_0.out19_11_83" + ], + "const_args": [], + "out_args": [ + "/model/layers.14/input_layernorm/output_3.out19_15_27" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.14/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.14/input_layernorm/output_3.out19_15_27" + ], + "const_args": [ + "model.layers.14.input_layernorm.weight", + "eps_19_15_27" + ], + "out_args": [ + "/model/layers.14/input_layernorm/output_0.out19_15_27" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.14/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/input_layernorm/output_0.out19_15_27" + ], + "const_args": [ + "model.layers.14.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.k_proj.Add.bias.preformat", + "model.layers.14.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.14/attn/k_proj/Add/output_0.out19_11_85" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.14/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.14/attn/k_proj/Add/output_0.out19_11_85", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.14.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "28", + "28" + ] + } + } + }, + { + "name": "/model/layers.14/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/input_layernorm/output_0.out19_15_27" + ], + "const_args": [ + "model.layers.14.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.q_proj.Add.bias.preformat", + "model.layers.14.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.14/attn/q_proj/Add/output_0.out19_11_84" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.14/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.14/attn/q_proj/Add/output_0.out19_11_84", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.14/attn/q_rotary/RotaryEmbedding/output_0.out19_3_14_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.14/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/input_layernorm/output_0.out19_15_27" + ], + "const_args": [ + "model.layers.14.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.v_proj.Add.bias.preformat", + "model.layers.14.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.14.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "29", + "29" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.14/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.14/attn/q_rotary/RotaryEmbedding/output_0.out19_3_14_1", + "present.14.key", + "present.14.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.14/attn/GroupQueryAttention/output_0.out19_3_14" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "28", + "28", + "2", + "0", + "29", + "29" + ] + } + } + }, + { + "name": "/model/layers.14/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/attn/GroupQueryAttention/output_0.out19_3_14" + ], + "const_args": [ + "model.layers.14.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.14/attn/o_proj/MatMulNBits/output_0.out19_11_86" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.14/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.14/input_layernorm/output_3.out19_15_27", + "/model/layers.14/attn/o_proj/MatMulNBits/output_0.out19_11_86" + ], + "const_args": [], + "out_args": [ + "/model/layers.14/post_attention_layernorm/output_3.out19_15_28" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.14/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.14/post_attention_layernorm/output_3.out19_15_28" + ], + "const_args": [ + "model.layers.14.post_attention_layernorm.weight", + "eps_19_15_28" + ], + "out_args": [ + "/model/layers.14/post_attention_layernorm/output_0.out19_15_28" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.14/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/post_attention_layernorm/output_0.out19_15_28" + ], + "const_args": [ + "model.layers.14.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.14.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.14.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.14.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.14/mlp/gate_proj/MatMulNBits/output_0.out19_11_87" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.14/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/post_attention_layernorm/output_0.out19_15_28" + ], + "const_args": [ + "model.layers.14.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.14.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.14.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.14.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.14/mlp/up_proj/MatMulNBits/output_0.out19_11_88" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.14/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.14/mlp/gate_proj/MatMulNBits/output_0.out19_11_87" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.14/mlp/act_fn/Mul/output_0.in.inter.out19_6_14" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.14/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.14/mlp/act_fn/Mul/output_0.in.inter.out19_6_14", + "/model/layers.14/mlp/up_proj/MatMulNBits/output_0.out19_11_88" + ], + "const_args": [], + "out_args": [ + "/model/layers.14/mlp/Mul/output_0.out19_12_14" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.14/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.14/mlp/Mul/output_0.out19_12_14" + ], + "const_args": [ + "model.layers.14.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.14/mlp/down_proj/MatMulNBits/output_0.out19_11_89" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.15/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.14/post_attention_layernorm/output_3.out19_15_28", + "/model/layers.14/mlp/down_proj/MatMulNBits/output_0.out19_11_89" + ], + "const_args": [], + "out_args": [ + "/model/layers.15/input_layernorm/output_3.out19_15_29" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.15/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.15/input_layernorm/output_3.out19_15_29" + ], + "const_args": [ + "model.layers.15.input_layernorm.weight", + "eps_19_15_29" + ], + "out_args": [ + "/model/layers.15/input_layernorm/output_0.out19_15_29" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.15/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/input_layernorm/output_0.out19_15_29" + ], + "const_args": [ + "model.layers.15.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.k_proj.Add.bias.preformat", + "model.layers.15.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.15/attn/k_proj/Add/output_0.out19_11_91" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.15/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.15/attn/k_proj/Add/output_0.out19_11_91", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.15.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "30", + "30" + ] + } + } + }, + { + "name": "/model/layers.15/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/input_layernorm/output_0.out19_15_29" + ], + "const_args": [ + "model.layers.15.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.q_proj.Add.bias.preformat", + "model.layers.15.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.15/attn/q_proj/Add/output_0.out19_11_90" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.15/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.15/attn/q_proj/Add/output_0.out19_11_90", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.15/attn/q_rotary/RotaryEmbedding/output_0.out19_3_15_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.15/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/input_layernorm/output_0.out19_15_29" + ], + "const_args": [ + "model.layers.15.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.v_proj.Add.bias.preformat", + "model.layers.15.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.15.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "31", + "31" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.15/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.15/attn/q_rotary/RotaryEmbedding/output_0.out19_3_15_1", + "present.15.key", + "present.15.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.15/attn/GroupQueryAttention/output_0.out19_3_15" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "30", + "30", + "2", + "0", + "31", + "31" + ] + } + } + }, + { + "name": "/model/layers.15/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/attn/GroupQueryAttention/output_0.out19_3_15" + ], + "const_args": [ + "model.layers.15.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.15/attn/o_proj/MatMulNBits/output_0.out19_11_92" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.15/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.15/input_layernorm/output_3.out19_15_29", + "/model/layers.15/attn/o_proj/MatMulNBits/output_0.out19_11_92" + ], + "const_args": [], + "out_args": [ + "/model/layers.15/post_attention_layernorm/output_3.out19_15_30" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.15/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.15/post_attention_layernorm/output_3.out19_15_30" + ], + "const_args": [ + "model.layers.15.post_attention_layernorm.weight", + "eps_19_15_30" + ], + "out_args": [ + "/model/layers.15/post_attention_layernorm/output_0.out19_15_30" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.15/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/post_attention_layernorm/output_0.out19_15_30" + ], + "const_args": [ + "model.layers.15.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.15.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.15.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.15.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.15/mlp/gate_proj/MatMulNBits/output_0.out19_11_93" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.15/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/post_attention_layernorm/output_0.out19_15_30" + ], + "const_args": [ + "model.layers.15.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.15.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.15.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.15.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.15/mlp/up_proj/MatMulNBits/output_0.out19_11_94" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.15/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.15/mlp/gate_proj/MatMulNBits/output_0.out19_11_93" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.15/mlp/act_fn/Mul/output_0.in.inter.out19_6_15" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.15/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.15/mlp/act_fn/Mul/output_0.in.inter.out19_6_15", + "/model/layers.15/mlp/up_proj/MatMulNBits/output_0.out19_11_94" + ], + "const_args": [], + "out_args": [ + "/model/layers.15/mlp/Mul/output_0.out19_12_15" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.15/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.15/mlp/Mul/output_0.out19_12_15" + ], + "const_args": [ + "model.layers.15.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.15/mlp/down_proj/MatMulNBits/output_0.out19_11_95" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.16/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.15/post_attention_layernorm/output_3.out19_15_30", + "/model/layers.15/mlp/down_proj/MatMulNBits/output_0.out19_11_95" + ], + "const_args": [], + "out_args": [ + "/model/layers.16/input_layernorm/output_3.out19_15_31" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.16/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.16/input_layernorm/output_3.out19_15_31" + ], + "const_args": [ + "model.layers.16.input_layernorm.weight", + "eps_19_15_31" + ], + "out_args": [ + "/model/layers.16/input_layernorm/output_0.out19_15_31" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.16/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/input_layernorm/output_0.out19_15_31" + ], + "const_args": [ + "model.layers.16.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.k_proj.Add.bias.preformat", + "model.layers.16.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.16/attn/k_proj/Add/output_0.out19_11_97" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.16/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.16/attn/k_proj/Add/output_0.out19_11_97", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.16.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "32", + "32" + ] + } + } + }, + { + "name": "/model/layers.16/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/input_layernorm/output_0.out19_15_31" + ], + "const_args": [ + "model.layers.16.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.q_proj.Add.bias.preformat", + "model.layers.16.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.16/attn/q_proj/Add/output_0.out19_11_96" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.16/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.16/attn/q_proj/Add/output_0.out19_11_96", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.16/attn/q_rotary/RotaryEmbedding/output_0.out19_3_16_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.16/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/input_layernorm/output_0.out19_15_31" + ], + "const_args": [ + "model.layers.16.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.v_proj.Add.bias.preformat", + "model.layers.16.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.16.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "33", + "33" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.16/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.16/attn/q_rotary/RotaryEmbedding/output_0.out19_3_16_1", + "present.16.key", + "present.16.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.16/attn/GroupQueryAttention/output_0.out19_3_16" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "32", + "32", + "2", + "0", + "33", + "33" + ] + } + } + }, + { + "name": "/model/layers.16/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/attn/GroupQueryAttention/output_0.out19_3_16" + ], + "const_args": [ + "model.layers.16.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.16/attn/o_proj/MatMulNBits/output_0.out19_11_98" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.16/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.16/input_layernorm/output_3.out19_15_31", + "/model/layers.16/attn/o_proj/MatMulNBits/output_0.out19_11_98" + ], + "const_args": [], + "out_args": [ + "/model/layers.16/post_attention_layernorm/output_3.out19_15_32" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.16/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.16/post_attention_layernorm/output_3.out19_15_32" + ], + "const_args": [ + "model.layers.16.post_attention_layernorm.weight", + "eps_19_15_32" + ], + "out_args": [ + "/model/layers.16/post_attention_layernorm/output_0.out19_15_32" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.16/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/post_attention_layernorm/output_0.out19_15_32" + ], + "const_args": [ + "model.layers.16.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.16.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.16.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.16.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.16/mlp/gate_proj/MatMulNBits/output_0.out19_11_99" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.16/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/post_attention_layernorm/output_0.out19_15_32" + ], + "const_args": [ + "model.layers.16.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.16.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.16.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.16.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.16/mlp/up_proj/MatMulNBits/output_0.out19_11_100" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.16/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.16/mlp/gate_proj/MatMulNBits/output_0.out19_11_99" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.16/mlp/act_fn/Mul/output_0.in.inter.out19_6_16" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.16/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.16/mlp/act_fn/Mul/output_0.in.inter.out19_6_16", + "/model/layers.16/mlp/up_proj/MatMulNBits/output_0.out19_11_100" + ], + "const_args": [], + "out_args": [ + "/model/layers.16/mlp/Mul/output_0.out19_12_16" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.16/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.16/mlp/Mul/output_0.out19_12_16" + ], + "const_args": [ + "model.layers.16.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.16/mlp/down_proj/MatMulNBits/output_0.out19_11_101" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.17/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.16/post_attention_layernorm/output_3.out19_15_32", + "/model/layers.16/mlp/down_proj/MatMulNBits/output_0.out19_11_101" + ], + "const_args": [], + "out_args": [ + "/model/layers.17/input_layernorm/output_3.out19_15_33" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.17/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.17/input_layernorm/output_3.out19_15_33" + ], + "const_args": [ + "model.layers.17.input_layernorm.weight", + "eps_19_15_33" + ], + "out_args": [ + "/model/layers.17/input_layernorm/output_0.out19_15_33" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.17/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/input_layernorm/output_0.out19_15_33" + ], + "const_args": [ + "model.layers.17.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.k_proj.Add.bias.preformat", + "model.layers.17.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.17/attn/k_proj/Add/output_0.out19_11_103" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.17/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.17/attn/k_proj/Add/output_0.out19_11_103", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.17.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "34", + "34" + ] + } + } + }, + { + "name": "/model/layers.17/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/input_layernorm/output_0.out19_15_33" + ], + "const_args": [ + "model.layers.17.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.q_proj.Add.bias.preformat", + "model.layers.17.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.17/attn/q_proj/Add/output_0.out19_11_102" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.17/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.17/attn/q_proj/Add/output_0.out19_11_102", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.17/attn/q_rotary/RotaryEmbedding/output_0.out19_3_17_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.17/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/input_layernorm/output_0.out19_15_33" + ], + "const_args": [ + "model.layers.17.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.v_proj.Add.bias.preformat", + "model.layers.17.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.17.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "35", + "35" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.17/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.17/attn/q_rotary/RotaryEmbedding/output_0.out19_3_17_1", + "present.17.key", + "present.17.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.17/attn/GroupQueryAttention/output_0.out19_3_17" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "34", + "34", + "2", + "0", + "35", + "35" + ] + } + } + }, + { + "name": "/model/layers.17/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/attn/GroupQueryAttention/output_0.out19_3_17" + ], + "const_args": [ + "model.layers.17.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.17/attn/o_proj/MatMulNBits/output_0.out19_11_104" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.17/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.17/input_layernorm/output_3.out19_15_33", + "/model/layers.17/attn/o_proj/MatMulNBits/output_0.out19_11_104" + ], + "const_args": [], + "out_args": [ + "/model/layers.17/post_attention_layernorm/output_3.out19_15_34" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.17/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.17/post_attention_layernorm/output_3.out19_15_34" + ], + "const_args": [ + "model.layers.17.post_attention_layernorm.weight", + "eps_19_15_34" + ], + "out_args": [ + "/model/layers.17/post_attention_layernorm/output_0.out19_15_34" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.17/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/post_attention_layernorm/output_0.out19_15_34" + ], + "const_args": [ + "model.layers.17.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.17.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.17.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.17.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.17/mlp/gate_proj/MatMulNBits/output_0.out19_11_105" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.17/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/post_attention_layernorm/output_0.out19_15_34" + ], + "const_args": [ + "model.layers.17.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.17.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.17.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.17.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.17/mlp/up_proj/MatMulNBits/output_0.out19_11_106" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.17/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.17/mlp/gate_proj/MatMulNBits/output_0.out19_11_105" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.17/mlp/act_fn/Mul/output_0.in.inter.out19_6_17" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.17/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.17/mlp/act_fn/Mul/output_0.in.inter.out19_6_17", + "/model/layers.17/mlp/up_proj/MatMulNBits/output_0.out19_11_106" + ], + "const_args": [], + "out_args": [ + "/model/layers.17/mlp/Mul/output_0.out19_12_17" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.17/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.17/mlp/Mul/output_0.out19_12_17" + ], + "const_args": [ + "model.layers.17.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.17/mlp/down_proj/MatMulNBits/output_0.out19_11_107" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.18/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.17/post_attention_layernorm/output_3.out19_15_34", + "/model/layers.17/mlp/down_proj/MatMulNBits/output_0.out19_11_107" + ], + "const_args": [], + "out_args": [ + "/model/layers.18/input_layernorm/output_3.out19_15_35" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.18/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.18/input_layernorm/output_3.out19_15_35" + ], + "const_args": [ + "model.layers.18.input_layernorm.weight", + "eps_19_15_35" + ], + "out_args": [ + "/model/layers.18/input_layernorm/output_0.out19_15_35" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.18/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/input_layernorm/output_0.out19_15_35" + ], + "const_args": [ + "model.layers.18.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.k_proj.Add.bias.preformat", + "model.layers.18.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.18/attn/k_proj/Add/output_0.out19_11_109" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.18/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.18/attn/k_proj/Add/output_0.out19_11_109", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.18.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "36", + "36" + ] + } + } + }, + { + "name": "/model/layers.18/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/input_layernorm/output_0.out19_15_35" + ], + "const_args": [ + "model.layers.18.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.q_proj.Add.bias.preformat", + "model.layers.18.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.18/attn/q_proj/Add/output_0.out19_11_108" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.18/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.18/attn/q_proj/Add/output_0.out19_11_108", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.18/attn/q_rotary/RotaryEmbedding/output_0.out19_3_18_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.18/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/input_layernorm/output_0.out19_15_35" + ], + "const_args": [ + "model.layers.18.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.v_proj.Add.bias.preformat", + "model.layers.18.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.18.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "37", + "37" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.18/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.18/attn/q_rotary/RotaryEmbedding/output_0.out19_3_18_1", + "present.18.key", + "present.18.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.18/attn/GroupQueryAttention/output_0.out19_3_18" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "36", + "36", + "2", + "0", + "37", + "37" + ] + } + } + }, + { + "name": "/model/layers.18/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/attn/GroupQueryAttention/output_0.out19_3_18" + ], + "const_args": [ + "model.layers.18.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.18/attn/o_proj/MatMulNBits/output_0.out19_11_110" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.18/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.18/input_layernorm/output_3.out19_15_35", + "/model/layers.18/attn/o_proj/MatMulNBits/output_0.out19_11_110" + ], + "const_args": [], + "out_args": [ + "/model/layers.18/post_attention_layernorm/output_3.out19_15_36" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.18/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.18/post_attention_layernorm/output_3.out19_15_36" + ], + "const_args": [ + "model.layers.18.post_attention_layernorm.weight", + "eps_19_15_36" + ], + "out_args": [ + "/model/layers.18/post_attention_layernorm/output_0.out19_15_36" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.18/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/post_attention_layernorm/output_0.out19_15_36" + ], + "const_args": [ + "model.layers.18.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.18.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.18.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.18.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.18/mlp/gate_proj/MatMulNBits/output_0.out19_11_111" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.18/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/post_attention_layernorm/output_0.out19_15_36" + ], + "const_args": [ + "model.layers.18.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.18.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.18.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.18.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.18/mlp/up_proj/MatMulNBits/output_0.out19_11_112" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.18/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.18/mlp/gate_proj/MatMulNBits/output_0.out19_11_111" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.18/mlp/act_fn/Mul/output_0.in.inter.out19_6_18" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.18/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.18/mlp/act_fn/Mul/output_0.in.inter.out19_6_18", + "/model/layers.18/mlp/up_proj/MatMulNBits/output_0.out19_11_112" + ], + "const_args": [], + "out_args": [ + "/model/layers.18/mlp/Mul/output_0.out19_12_18" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.18/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.18/mlp/Mul/output_0.out19_12_18" + ], + "const_args": [ + "model.layers.18.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.18/mlp/down_proj/MatMulNBits/output_0.out19_11_113" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.19/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.18/post_attention_layernorm/output_3.out19_15_36", + "/model/layers.18/mlp/down_proj/MatMulNBits/output_0.out19_11_113" + ], + "const_args": [], + "out_args": [ + "/model/layers.19/input_layernorm/output_3.out19_15_37" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.19/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.19/input_layernorm/output_3.out19_15_37" + ], + "const_args": [ + "model.layers.19.input_layernorm.weight", + "eps_19_15_37" + ], + "out_args": [ + "/model/layers.19/input_layernorm/output_0.out19_15_37" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.19/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/input_layernorm/output_0.out19_15_37" + ], + "const_args": [ + "model.layers.19.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.k_proj.Add.bias.preformat", + "model.layers.19.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.19/attn/k_proj/Add/output_0.out19_11_115" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.19/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.19/attn/k_proj/Add/output_0.out19_11_115", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.19.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "38", + "38" + ] + } + } + }, + { + "name": "/model/layers.19/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/input_layernorm/output_0.out19_15_37" + ], + "const_args": [ + "model.layers.19.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.q_proj.Add.bias.preformat", + "model.layers.19.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.19/attn/q_proj/Add/output_0.out19_11_114" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.19/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.19/attn/q_proj/Add/output_0.out19_11_114", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.19/attn/q_rotary/RotaryEmbedding/output_0.out19_3_19_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.19/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/input_layernorm/output_0.out19_15_37" + ], + "const_args": [ + "model.layers.19.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.v_proj.Add.bias.preformat", + "model.layers.19.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.19.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "39", + "39" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.19/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.19/attn/q_rotary/RotaryEmbedding/output_0.out19_3_19_1", + "present.19.key", + "present.19.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.19/attn/GroupQueryAttention/output_0.out19_3_19" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "38", + "38", + "2", + "0", + "39", + "39" + ] + } + } + }, + { + "name": "/model/layers.19/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/attn/GroupQueryAttention/output_0.out19_3_19" + ], + "const_args": [ + "model.layers.19.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.19/attn/o_proj/MatMulNBits/output_0.out19_11_116" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.19/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.19/input_layernorm/output_3.out19_15_37", + "/model/layers.19/attn/o_proj/MatMulNBits/output_0.out19_11_116" + ], + "const_args": [], + "out_args": [ + "/model/layers.19/post_attention_layernorm/output_3.out19_15_38" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.19/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.19/post_attention_layernorm/output_3.out19_15_38" + ], + "const_args": [ + "model.layers.19.post_attention_layernorm.weight", + "eps_19_15_38" + ], + "out_args": [ + "/model/layers.19/post_attention_layernorm/output_0.out19_15_38" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.19/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/post_attention_layernorm/output_0.out19_15_38" + ], + "const_args": [ + "model.layers.19.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.19.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.19.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.19.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.19/mlp/gate_proj/MatMulNBits/output_0.out19_11_117" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.19/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/post_attention_layernorm/output_0.out19_15_38" + ], + "const_args": [ + "model.layers.19.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.19.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.19.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.19.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.19/mlp/up_proj/MatMulNBits/output_0.out19_11_118" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.19/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.19/mlp/gate_proj/MatMulNBits/output_0.out19_11_117" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.19/mlp/act_fn/Mul/output_0.in.inter.out19_6_19" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.19/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.19/mlp/act_fn/Mul/output_0.in.inter.out19_6_19", + "/model/layers.19/mlp/up_proj/MatMulNBits/output_0.out19_11_118" + ], + "const_args": [], + "out_args": [ + "/model/layers.19/mlp/Mul/output_0.out19_12_19" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.19/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.19/mlp/Mul/output_0.out19_12_19" + ], + "const_args": [ + "model.layers.19.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.19/mlp/down_proj/MatMulNBits/output_0.out19_11_119" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.20/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.19/post_attention_layernorm/output_3.out19_15_38", + "/model/layers.19/mlp/down_proj/MatMulNBits/output_0.out19_11_119" + ], + "const_args": [], + "out_args": [ + "/model/layers.20/input_layernorm/output_3.out19_15_39" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.20/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.20/input_layernorm/output_3.out19_15_39" + ], + "const_args": [ + "model.layers.20.input_layernorm.weight", + "eps_19_15_39" + ], + "out_args": [ + "/model/layers.20/input_layernorm/output_0.out19_15_39" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.20/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/input_layernorm/output_0.out19_15_39" + ], + "const_args": [ + "model.layers.20.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.k_proj.Add.bias.preformat", + "model.layers.20.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.20/attn/k_proj/Add/output_0.out19_11_121" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.20/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.20/attn/k_proj/Add/output_0.out19_11_121", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.20.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "40", + "40" + ] + } + } + }, + { + "name": "/model/layers.20/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/input_layernorm/output_0.out19_15_39" + ], + "const_args": [ + "model.layers.20.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.q_proj.Add.bias.preformat", + "model.layers.20.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.20/attn/q_proj/Add/output_0.out19_11_120" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.20/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.20/attn/q_proj/Add/output_0.out19_11_120", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.20/attn/q_rotary/RotaryEmbedding/output_0.out19_3_20_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.20/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/input_layernorm/output_0.out19_15_39" + ], + "const_args": [ + "model.layers.20.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.v_proj.Add.bias.preformat", + "model.layers.20.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.20.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "41", + "41" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.20/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.20/attn/q_rotary/RotaryEmbedding/output_0.out19_3_20_1", + "present.20.key", + "present.20.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.20/attn/GroupQueryAttention/output_0.out19_3_20" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "40", + "40", + "2", + "0", + "41", + "41" + ] + } + } + }, + { + "name": "/model/layers.20/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/attn/GroupQueryAttention/output_0.out19_3_20" + ], + "const_args": [ + "model.layers.20.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.20/attn/o_proj/MatMulNBits/output_0.out19_11_122" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.20/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.20/input_layernorm/output_3.out19_15_39", + "/model/layers.20/attn/o_proj/MatMulNBits/output_0.out19_11_122" + ], + "const_args": [], + "out_args": [ + "/model/layers.20/post_attention_layernorm/output_3.out19_15_40" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.20/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.20/post_attention_layernorm/output_3.out19_15_40" + ], + "const_args": [ + "model.layers.20.post_attention_layernorm.weight", + "eps_19_15_40" + ], + "out_args": [ + "/model/layers.20/post_attention_layernorm/output_0.out19_15_40" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.20/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/post_attention_layernorm/output_0.out19_15_40" + ], + "const_args": [ + "model.layers.20.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.20.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.20.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.20.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.20/mlp/gate_proj/MatMulNBits/output_0.out19_11_123" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.20/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/post_attention_layernorm/output_0.out19_15_40" + ], + "const_args": [ + "model.layers.20.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.20.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.20.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.20.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.20/mlp/up_proj/MatMulNBits/output_0.out19_11_124" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.20/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.20/mlp/gate_proj/MatMulNBits/output_0.out19_11_123" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.20/mlp/act_fn/Mul/output_0.in.inter.out19_6_20" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.20/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.20/mlp/act_fn/Mul/output_0.in.inter.out19_6_20", + "/model/layers.20/mlp/up_proj/MatMulNBits/output_0.out19_11_124" + ], + "const_args": [], + "out_args": [ + "/model/layers.20/mlp/Mul/output_0.out19_12_20" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.20/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.20/mlp/Mul/output_0.out19_12_20" + ], + "const_args": [ + "model.layers.20.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.20/mlp/down_proj/MatMulNBits/output_0.out19_11_125" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.21/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.20/post_attention_layernorm/output_3.out19_15_40", + "/model/layers.20/mlp/down_proj/MatMulNBits/output_0.out19_11_125" + ], + "const_args": [], + "out_args": [ + "/model/layers.21/input_layernorm/output_3.out19_15_41" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.21/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.21/input_layernorm/output_3.out19_15_41" + ], + "const_args": [ + "model.layers.21.input_layernorm.weight", + "eps_19_15_41" + ], + "out_args": [ + "/model/layers.21/input_layernorm/output_0.out19_15_41" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.21/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/input_layernorm/output_0.out19_15_41" + ], + "const_args": [ + "model.layers.21.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.k_proj.Add.bias.preformat", + "model.layers.21.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.21/attn/k_proj/Add/output_0.out19_11_127" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.21/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.21/attn/k_proj/Add/output_0.out19_11_127", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.21.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "42", + "42" + ] + } + } + }, + { + "name": "/model/layers.21/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/input_layernorm/output_0.out19_15_41" + ], + "const_args": [ + "model.layers.21.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.q_proj.Add.bias.preformat", + "model.layers.21.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.21/attn/q_proj/Add/output_0.out19_11_126" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.21/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.21/attn/q_proj/Add/output_0.out19_11_126", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.21/attn/q_rotary/RotaryEmbedding/output_0.out19_3_21_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.21/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/input_layernorm/output_0.out19_15_41" + ], + "const_args": [ + "model.layers.21.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.v_proj.Add.bias.preformat", + "model.layers.21.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.21.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "43", + "43" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.21/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.21/attn/q_rotary/RotaryEmbedding/output_0.out19_3_21_1", + "present.21.key", + "present.21.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.21/attn/GroupQueryAttention/output_0.out19_3_21" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "42", + "42", + "2", + "0", + "43", + "43" + ] + } + } + }, + { + "name": "/model/layers.21/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/attn/GroupQueryAttention/output_0.out19_3_21" + ], + "const_args": [ + "model.layers.21.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.21/attn/o_proj/MatMulNBits/output_0.out19_11_128" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.21/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.21/input_layernorm/output_3.out19_15_41", + "/model/layers.21/attn/o_proj/MatMulNBits/output_0.out19_11_128" + ], + "const_args": [], + "out_args": [ + "/model/layers.21/post_attention_layernorm/output_3.out19_15_42" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.21/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.21/post_attention_layernorm/output_3.out19_15_42" + ], + "const_args": [ + "model.layers.21.post_attention_layernorm.weight", + "eps_19_15_42" + ], + "out_args": [ + "/model/layers.21/post_attention_layernorm/output_0.out19_15_42" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.21/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/post_attention_layernorm/output_0.out19_15_42" + ], + "const_args": [ + "model.layers.21.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.21.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.21.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.21.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.21/mlp/gate_proj/MatMulNBits/output_0.out19_11_129" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.21/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/post_attention_layernorm/output_0.out19_15_42" + ], + "const_args": [ + "model.layers.21.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.21.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.21.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.21.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.21/mlp/up_proj/MatMulNBits/output_0.out19_11_130" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.21/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.21/mlp/gate_proj/MatMulNBits/output_0.out19_11_129" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.21/mlp/act_fn/Mul/output_0.in.inter.out19_6_21" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.21/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.21/mlp/act_fn/Mul/output_0.in.inter.out19_6_21", + "/model/layers.21/mlp/up_proj/MatMulNBits/output_0.out19_11_130" + ], + "const_args": [], + "out_args": [ + "/model/layers.21/mlp/Mul/output_0.out19_12_21" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.21/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.21/mlp/Mul/output_0.out19_12_21" + ], + "const_args": [ + "model.layers.21.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.21/mlp/down_proj/MatMulNBits/output_0.out19_11_131" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.22/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.21/post_attention_layernorm/output_3.out19_15_42", + "/model/layers.21/mlp/down_proj/MatMulNBits/output_0.out19_11_131" + ], + "const_args": [], + "out_args": [ + "/model/layers.22/input_layernorm/output_3.out19_15_43" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.22/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.22/input_layernorm/output_3.out19_15_43" + ], + "const_args": [ + "model.layers.22.input_layernorm.weight", + "eps_19_15_43" + ], + "out_args": [ + "/model/layers.22/input_layernorm/output_0.out19_15_43" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.22/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/input_layernorm/output_0.out19_15_43" + ], + "const_args": [ + "model.layers.22.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.k_proj.Add.bias.preformat", + "model.layers.22.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.22/attn/k_proj/Add/output_0.out19_11_133" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.22/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.22/attn/k_proj/Add/output_0.out19_11_133", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.22.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "44", + "44" + ] + } + } + }, + { + "name": "/model/layers.22/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/input_layernorm/output_0.out19_15_43" + ], + "const_args": [ + "model.layers.22.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.q_proj.Add.bias.preformat", + "model.layers.22.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.22/attn/q_proj/Add/output_0.out19_11_132" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.22/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.22/attn/q_proj/Add/output_0.out19_11_132", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.22/attn/q_rotary/RotaryEmbedding/output_0.out19_3_22_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.22/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/input_layernorm/output_0.out19_15_43" + ], + "const_args": [ + "model.layers.22.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.v_proj.Add.bias.preformat", + "model.layers.22.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.22.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "45", + "45" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.22/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.22/attn/q_rotary/RotaryEmbedding/output_0.out19_3_22_1", + "present.22.key", + "present.22.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.22/attn/GroupQueryAttention/output_0.out19_3_22" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "44", + "44", + "2", + "0", + "45", + "45" + ] + } + } + }, + { + "name": "/model/layers.22/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/attn/GroupQueryAttention/output_0.out19_3_22" + ], + "const_args": [ + "model.layers.22.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.22/attn/o_proj/MatMulNBits/output_0.out19_11_134" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.22/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.22/input_layernorm/output_3.out19_15_43", + "/model/layers.22/attn/o_proj/MatMulNBits/output_0.out19_11_134" + ], + "const_args": [], + "out_args": [ + "/model/layers.22/post_attention_layernorm/output_3.out19_15_44" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.22/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.22/post_attention_layernorm/output_3.out19_15_44" + ], + "const_args": [ + "model.layers.22.post_attention_layernorm.weight", + "eps_19_15_44" + ], + "out_args": [ + "/model/layers.22/post_attention_layernorm/output_0.out19_15_44" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.22/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/post_attention_layernorm/output_0.out19_15_44" + ], + "const_args": [ + "model.layers.22.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.22.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.22.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.22.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.22/mlp/gate_proj/MatMulNBits/output_0.out19_11_135" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.22/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/post_attention_layernorm/output_0.out19_15_44" + ], + "const_args": [ + "model.layers.22.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.22.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.22.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.22.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.22/mlp/up_proj/MatMulNBits/output_0.out19_11_136" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.22/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.22/mlp/gate_proj/MatMulNBits/output_0.out19_11_135" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.22/mlp/act_fn/Mul/output_0.in.inter.out19_6_22" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.22/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.22/mlp/act_fn/Mul/output_0.in.inter.out19_6_22", + "/model/layers.22/mlp/up_proj/MatMulNBits/output_0.out19_11_136" + ], + "const_args": [], + "out_args": [ + "/model/layers.22/mlp/Mul/output_0.out19_12_22" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.22/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.22/mlp/Mul/output_0.out19_12_22" + ], + "const_args": [ + "model.layers.22.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.22/mlp/down_proj/MatMulNBits/output_0.out19_11_137" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.23/input_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.22/post_attention_layernorm/output_3.out19_15_44", + "/model/layers.22/mlp/down_proj/MatMulNBits/output_0.out19_11_137" + ], + "const_args": [], + "out_args": [ + "/model/layers.23/input_layernorm/output_3.out19_15_45" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.23/input_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.23/input_layernorm/output_3.out19_15_45" + ], + "const_args": [ + "model.layers.23.input_layernorm.weight", + "eps_19_15_45" + ], + "out_args": [ + "/model/layers.23/input_layernorm/output_0.out19_15_45" + ], + "attrs": { + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.23/attn/k_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/input_layernorm/output_0.out19_15_45" + ], + "const_args": [ + "model.layers.23.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.k_proj.Add.bias.preformat", + "model.layers.23.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.k_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.23/attn/k_proj/Add/output_0.out19_11_139" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.23/attn/k_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.23/attn/k_proj/Add/output_0.out19_11_139", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "present.23.key" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "2", + "0", + "46", + "46" + ] + } + } + }, + { + "name": "/model/layers.23/attn/q_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/input_layernorm/output_0.out19_15_45" + ], + "const_args": [ + "model.layers.23.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.q_proj.Add.bias.preformat", + "model.layers.23.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.q_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.23/attn/q_proj/Add/output_0.out19_11_138" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.23/attn/q_rotary/MLADFMHAROPE", + "type": "MLADFMHAROPE", + "in_args": [ + "/model/layers.23/attn/q_proj/Add/output_0.out19_11_138", + "sin_cache_cos_cache_prefill_slice" + ], + "const_args": [], + "out_args": [ + "/model/layers.23/attn/q_rotary/RotaryEmbedding/output_0.out19_3_23_1" + ], + "attrs": { + "transpose": { + "type": "str", + "value": [ + "input" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "head_dim": { + "type": "int", + "value": [ + "64" + ] + } + } + }, + { + "name": "/model/layers.23/attn/v_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/input_layernorm/output_0.out19_15_45" + ], + "const_args": [ + "model.layers.23.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.v_proj.Add.bias.preformat", + "model.layers.23.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.v_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "present.23.value" + ], + "attrs": { + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "128" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "split_ext_buf": { + "type": "int", + "value": [ + "0" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + }, + "total_seq_len": { + "type": "int", + "value": [ + "4096" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "5", + "0", + "47", + "47" + ] + }, + "head_num": { + "type": "int", + "value": [ + "2" + ] + }, + "use_gm": { + "type": "int", + "value": [ + "1" + ] + } + } + }, + { + "name": "/model/layers.23/attn/FLATMHATTFT", + "type": "FLATMHATTFT", + "in_args": [ + "/model/layers.23/attn/q_rotary/RotaryEmbedding/output_0.out19_3_23_1", + "present.23.key", + "present.23.value" + ], + "const_args": [], + "out_args": [ + "/model/layers.23/attn/GroupQueryAttention/output_0.out19_3_23" + ], + "attrs": { + "input_shape": { + "type": "str", + "value": [ + "14", + "2", + "sequence_length_padded", + "4096", + "64" + ] + }, + "use_gm": { + "type": "str", + "value": [ + "v2" + ] + }, + "external_buffers": { + "type": "int", + "value": [ + "1", + "0", + "46", + "46", + "2", + "0", + "47", + "47" + ] + } + } + }, + { + "name": "/model/layers.23/attn/o_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/attn/GroupQueryAttention/output_0.out19_3_23" + ], + "const_args": [ + "model.layers.23.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.23/attn/o_proj/MatMulNBits/output_0.out19_11_140" + ], + "attrs": { + "prune": { + "type": "int", + "value": [ + "1" + ] + }, + "dynamic_count": { + "type": "int", + "value": [ + "10" + ] + }, + "dynamic_shapes": { + "type": "int", + "value": [ + "128", + "256", + "512", + "768", + "1024", + "1536", + "2048", + "2304", + "3072", + "4096" + ] + }, + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "prune_enable": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.23/post_attention_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.23/input_layernorm/output_3.out19_15_45", + "/model/layers.23/attn/o_proj/MatMulNBits/output_0.out19_11_140" + ], + "const_args": [], + "out_args": [ + "/model/layers.23/post_attention_layernorm/output_3.out19_15_46" + ], + "attrs": { + "prune": { + "type": "int", + "value": [ + "1" + ] + }, + "dynamic_count": { + "type": "int", + "value": [ + "10" + ] + }, + "dynamic_shapes": { + "type": "int", + "value": [ + "128", + "256", + "512", + "768", + "1024", + "1536", + "2048", + "2304", + "3072", + "4096" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.23/post_attention_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "/model/layers.23/post_attention_layernorm/output_3.out19_15_46" + ], + "const_args": [ + "model.layers.23.post_attention_layernorm.weight", + "eps_19_15_46" + ], + "out_args": [ + "/model/layers.23/post_attention_layernorm/output_0.out19_15_46" + ], + "attrs": { + "prune": { + "type": "int", + "value": [ + "1" + ] + }, + "dynamic_count": { + "type": "int", + "value": [ + "10" + ] + }, + "dynamic_shapes": { + "type": "int", + "value": [ + "128", + "256", + "512", + "768", + "1024", + "1536", + "2048", + "2304", + "3072", + "4096" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.23/mlp/gate_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/post_attention_layernorm/output_0.out19_15_46" + ], + "const_args": [ + "model.layers.23.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.23.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.23.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.23.mlp.gate_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.23/mlp/gate_proj/MatMulNBits/output_0.out19_11_141" + ], + "attrs": { + "prune": { + "type": "int", + "value": [ + "1" + ] + }, + "dynamic_count": { + "type": "int", + "value": [ + "10" + ] + }, + "dynamic_shapes": { + "type": "int", + "value": [ + "128", + "256", + "512", + "768", + "1024", + "1536", + "2048", + "2304", + "3072", + "4096" + ] + }, + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "prune_enable": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.23/mlp/up_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/post_attention_layernorm/output_0.out19_15_46" + ], + "const_args": [ + "model.layers.23.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.23.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.23.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.23.mlp.up_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.23/mlp/up_proj/MatMulNBits/output_0.out19_11_142" + ], + "attrs": { + "prune": { + "type": "int", + "value": [ + "1" + ] + }, + "dynamic_count": { + "type": "int", + "value": [ + "10" + ] + }, + "dynamic_shapes": { + "type": "int", + "value": [ + "128", + "256", + "512", + "768", + "1024", + "1536", + "2048", + "2304", + "3072", + "4096" + ] + }, + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "4864" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "896" + ] + }, + "prune_enable": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.23/mlp/act_fn/SILU", + "type": "SILU", + "in_args": [ + "/model/layers.23/mlp/gate_proj/MatMulNBits/output_0.out19_11_141" + ], + "const_args": [], + "out_args": [ + "InsertedPrecisionFreeCast_/model/layers.23/mlp/act_fn/Mul/output_0.in.inter.out19_6_23" + ], + "attrs": { + "prune": { + "type": "int", + "value": [ + "1" + ] + }, + "dynamic_count": { + "type": "int", + "value": [ + "10" + ] + }, + "dynamic_shapes": { + "type": "int", + "value": [ + "128", + "256", + "512", + "768", + "1024", + "1536", + "2048", + "2304", + "3072", + "4096" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.23/mlp/ELWMUL", + "type": "ELWMUL", + "in_args": [ + "InsertedPrecisionFreeCast_/model/layers.23/mlp/act_fn/Mul/output_0.in.inter.out19_6_23", + "/model/layers.23/mlp/up_proj/MatMulNBits/output_0.out19_11_142" + ], + "const_args": [], + "out_args": [ + "/model/layers.23/mlp/Mul/output_0.out19_12_23" + ], + "attrs": { + "prune": { + "type": "int", + "value": [ + "1" + ] + }, + "dynamic_count": { + "type": "int", + "value": [ + "10" + ] + }, + "dynamic_shapes": { + "type": "int", + "value": [ + "128", + "256", + "512", + "768", + "1024", + "1536", + "2048", + "2304", + "3072", + "4096" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.23/mlp/down_proj/MatMulNBits", + "type": "MladfMatMul", + "in_args": [ + "/model/layers.23/mlp/Mul/output_0.out19_12_23" + ], + "const_args": [ + "model.layers.23.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.qzeros.preformat" + ], + "out_args": [ + "/model/layers.23/mlp/down_proj/MatMulNBits/output_0.out19_11_143" + ], + "attrs": { + "prune": { + "type": "int", + "value": [ + "1" + ] + }, + "dynamic_count": { + "type": "int", + "value": [ + "10" + ] + }, + "dynamic_shapes": { + "type": "int", + "value": [ + "128", + "256", + "512", + "768", + "1024", + "1536", + "2048", + "2304", + "3072", + "4096" + ] + }, + "accuracy_level": { + "type": "int", + "value": [ + "0" + ] + }, + "bits": { + "type": "int", + "value": [ + "4" + ] + }, + "N": { + "type": "int", + "value": [ + "896" + ] + }, + "block_size": { + "type": "int", + "value": [ + "128" + ] + }, + "K": { + "type": "int", + "value": [ + "4864" + ] + }, + "prune_enable": { + "type": "int", + "value": [ + "1" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + }, + "default_shape": { + "type": "int", + "value": [ + "1" + ] + }, + "offload_npu": { + "type": "int", + "value": [ + "1" + ] + }, + "group_size": { + "type": "int", + "value": [ + "128" + ] + } + } + }, + { + "name": "/model/layers.24/final_norm_layernorm/MLADFADD", + "type": "MLADFADD", + "in_args": [ + "/model/layers.23/post_attention_layernorm/output_3.out19_15_46", + "/model/layers.23/mlp/down_proj/MatMulNBits/output_0.out19_11_143" + ], + "const_args": [], + "out_args": [ + "dummy_19_15_47" + ], + "attrs": { + "prune": { + "type": "int", + "value": [ + "1" + ] + }, + "dynamic_count": { + "type": "int", + "value": [ + "10" + ] + }, + "dynamic_shapes": { + "type": "int", + "value": [ + "128", + "256", + "512", + "768", + "1024", + "1536", + "2048", + "2304", + "3072", + "4096" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + }, + { + "name": "/model/layers.24/final_norm_layernorm/MLADFRMSNORM", + "type": "MLADFRMSNORM", + "in_args": [ + "dummy_19_15_47" + ], + "const_args": [ + "model.layers.24.final_norm_layernorm.weight", + "eps_19_15_47" + ], + "out_args": [ + "/model/layers.24/final_norm_layernorm/output_0.out19_15_47" + ], + "attrs": { + "prune": { + "type": "int", + "value": [ + "1" + ] + }, + "dynamic_count": { + "type": "int", + "value": [ + "10" + ] + }, + "dynamic_shapes": { + "type": "int", + "value": [ + "128", + "256", + "512", + "768", + "1024", + "1536", + "2048", + "2304", + "3072", + "4096" + ] + }, + "op_version": { + "type": "str", + "value": [ + "v2" + ] + } + } + } + ], + "fused_tensors": { + "in": { + "buffer_size": 2304, + "xrt_arg_id": 0, + "packed_tensors": [ + "/model/embed_tokens/Gather/output_0.out19_14_0", + "sin_cache_cos_cache_prefill_slice" + ] + }, + "out": { + "buffer_size": 1792, + "xrt_arg_id": 1, + "packed_tensors": [ + "/model/layers.24/final_norm_layernorm/output_0.out19_15_47" + ] + }, + "scratch": { + "buffer_size": 1327104, + "xrt_arg_id": 2, + "packed_tensors": [ + "/model/layers.0/input_layernorm/output_0.out19_14_0", + "/model/layers.0/attn/k_proj/Add/output_0.out19_11_1", + "/model/layers.0/attn/q_proj/Add/output_0.out19_11_0", + "/model/layers.0/attn/q_rotary/RotaryEmbedding/output_0.out19_3_0_1", + "/model/layers.0/attn/GroupQueryAttention/output_0.out19_3_0", + "/model/layers.0/attn/o_proj/MatMulNBits/output_0.out19_11_2", + "/model/layers.0/post_attention_layernorm/output_3.out19_15_0", + "/model/layers.0/post_attention_layernorm/output_0.out19_15_0", + "/model/layers.0/mlp/gate_proj/MatMulNBits/output_0.out19_11_3", + "/model/layers.0/mlp/up_proj/MatMulNBits/output_0.out19_11_4", + "InsertedPrecisionFreeCast_/model/layers.0/mlp/act_fn/Mul/output_0.in.inter.out19_6_0", + "/model/layers.0/mlp/Mul/output_0.out19_12_0", + "/model/layers.0/mlp/down_proj/MatMulNBits/output_0.out19_11_5", + "/model/layers.1/input_layernorm/output_3.out19_15_1", + "/model/layers.1/input_layernorm/output_0.out19_15_1", + "/model/layers.1/attn/k_proj/Add/output_0.out19_11_7", + "/model/layers.1/attn/q_proj/Add/output_0.out19_11_6", + "/model/layers.1/attn/q_rotary/RotaryEmbedding/output_0.out19_3_1_1", + "/model/layers.1/attn/GroupQueryAttention/output_0.out19_3_1", + "/model/layers.1/attn/o_proj/MatMulNBits/output_0.out19_11_8", + "/model/layers.1/post_attention_layernorm/output_3.out19_15_2", + "/model/layers.1/post_attention_layernorm/output_0.out19_15_2", + "/model/layers.1/mlp/gate_proj/MatMulNBits/output_0.out19_11_9", + "/model/layers.1/mlp/up_proj/MatMulNBits/output_0.out19_11_10", + "InsertedPrecisionFreeCast_/model/layers.1/mlp/act_fn/Mul/output_0.in.inter.out19_6_1", + "/model/layers.1/mlp/Mul/output_0.out19_12_1", + "/model/layers.1/mlp/down_proj/MatMulNBits/output_0.out19_11_11", + "/model/layers.2/input_layernorm/output_3.out19_15_3", + "/model/layers.2/input_layernorm/output_0.out19_15_3", + "/model/layers.2/attn/k_proj/Add/output_0.out19_11_13", + "/model/layers.2/attn/q_proj/Add/output_0.out19_11_12", + "/model/layers.2/attn/q_rotary/RotaryEmbedding/output_0.out19_3_2_1", + "/model/layers.2/attn/GroupQueryAttention/output_0.out19_3_2", + "/model/layers.2/attn/o_proj/MatMulNBits/output_0.out19_11_14", + "/model/layers.2/post_attention_layernorm/output_3.out19_15_4", + "/model/layers.2/post_attention_layernorm/output_0.out19_15_4", + "/model/layers.2/mlp/gate_proj/MatMulNBits/output_0.out19_11_15", + "/model/layers.2/mlp/up_proj/MatMulNBits/output_0.out19_11_16", + "InsertedPrecisionFreeCast_/model/layers.2/mlp/act_fn/Mul/output_0.in.inter.out19_6_2", + "/model/layers.2/mlp/Mul/output_0.out19_12_2", + "/model/layers.2/mlp/down_proj/MatMulNBits/output_0.out19_11_17", + "/model/layers.3/input_layernorm/output_3.out19_15_5", + "/model/layers.3/input_layernorm/output_0.out19_15_5", + "/model/layers.3/attn/k_proj/Add/output_0.out19_11_19", + "/model/layers.3/attn/q_proj/Add/output_0.out19_11_18", + "/model/layers.3/attn/q_rotary/RotaryEmbedding/output_0.out19_3_3_1", + "/model/layers.3/attn/GroupQueryAttention/output_0.out19_3_3", + "/model/layers.3/attn/o_proj/MatMulNBits/output_0.out19_11_20", + "/model/layers.3/post_attention_layernorm/output_3.out19_15_6", + "/model/layers.3/post_attention_layernorm/output_0.out19_15_6", + "/model/layers.3/mlp/gate_proj/MatMulNBits/output_0.out19_11_21", + "/model/layers.3/mlp/up_proj/MatMulNBits/output_0.out19_11_22", + "InsertedPrecisionFreeCast_/model/layers.3/mlp/act_fn/Mul/output_0.in.inter.out19_6_3", + "/model/layers.3/mlp/Mul/output_0.out19_12_3", + "/model/layers.3/mlp/down_proj/MatMulNBits/output_0.out19_11_23", + "/model/layers.4/input_layernorm/output_3.out19_15_7", + "/model/layers.4/input_layernorm/output_0.out19_15_7", + "/model/layers.4/attn/k_proj/Add/output_0.out19_11_25", + "/model/layers.4/attn/q_proj/Add/output_0.out19_11_24", + "/model/layers.4/attn/q_rotary/RotaryEmbedding/output_0.out19_3_4_1", + "/model/layers.4/attn/GroupQueryAttention/output_0.out19_3_4", + "/model/layers.4/attn/o_proj/MatMulNBits/output_0.out19_11_26", + "/model/layers.4/post_attention_layernorm/output_3.out19_15_8", + "/model/layers.4/post_attention_layernorm/output_0.out19_15_8", + "/model/layers.4/mlp/gate_proj/MatMulNBits/output_0.out19_11_27", + "/model/layers.4/mlp/up_proj/MatMulNBits/output_0.out19_11_28", + "InsertedPrecisionFreeCast_/model/layers.4/mlp/act_fn/Mul/output_0.in.inter.out19_6_4", + "/model/layers.4/mlp/Mul/output_0.out19_12_4", + "/model/layers.4/mlp/down_proj/MatMulNBits/output_0.out19_11_29", + "/model/layers.5/input_layernorm/output_3.out19_15_9", + "/model/layers.5/input_layernorm/output_0.out19_15_9", + "/model/layers.5/attn/k_proj/Add/output_0.out19_11_31", + "/model/layers.5/attn/q_proj/Add/output_0.out19_11_30", + "/model/layers.5/attn/q_rotary/RotaryEmbedding/output_0.out19_3_5_1", + "/model/layers.5/attn/GroupQueryAttention/output_0.out19_3_5", + "/model/layers.5/attn/o_proj/MatMulNBits/output_0.out19_11_32", + "/model/layers.5/post_attention_layernorm/output_3.out19_15_10", + "/model/layers.5/post_attention_layernorm/output_0.out19_15_10", + "/model/layers.5/mlp/gate_proj/MatMulNBits/output_0.out19_11_33", + "/model/layers.5/mlp/up_proj/MatMulNBits/output_0.out19_11_34", + "InsertedPrecisionFreeCast_/model/layers.5/mlp/act_fn/Mul/output_0.in.inter.out19_6_5", + "/model/layers.5/mlp/Mul/output_0.out19_12_5", + "/model/layers.5/mlp/down_proj/MatMulNBits/output_0.out19_11_35", + "/model/layers.6/input_layernorm/output_3.out19_15_11", + "/model/layers.6/input_layernorm/output_0.out19_15_11", + "/model/layers.6/attn/k_proj/Add/output_0.out19_11_37", + "/model/layers.6/attn/q_proj/Add/output_0.out19_11_36", + "/model/layers.6/attn/q_rotary/RotaryEmbedding/output_0.out19_3_6_1", + "/model/layers.6/attn/GroupQueryAttention/output_0.out19_3_6", + "/model/layers.6/attn/o_proj/MatMulNBits/output_0.out19_11_38", + "/model/layers.6/post_attention_layernorm/output_3.out19_15_12", + "/model/layers.6/post_attention_layernorm/output_0.out19_15_12", + "/model/layers.6/mlp/gate_proj/MatMulNBits/output_0.out19_11_39", + "/model/layers.6/mlp/up_proj/MatMulNBits/output_0.out19_11_40", + "InsertedPrecisionFreeCast_/model/layers.6/mlp/act_fn/Mul/output_0.in.inter.out19_6_6", + "/model/layers.6/mlp/Mul/output_0.out19_12_6", + "/model/layers.6/mlp/down_proj/MatMulNBits/output_0.out19_11_41", + "/model/layers.7/input_layernorm/output_3.out19_15_13", + "/model/layers.7/input_layernorm/output_0.out19_15_13", + "/model/layers.7/attn/k_proj/Add/output_0.out19_11_43", + "/model/layers.7/attn/q_proj/Add/output_0.out19_11_42", + "/model/layers.7/attn/q_rotary/RotaryEmbedding/output_0.out19_3_7_1", + "/model/layers.7/attn/GroupQueryAttention/output_0.out19_3_7", + "/model/layers.7/attn/o_proj/MatMulNBits/output_0.out19_11_44", + "/model/layers.7/post_attention_layernorm/output_3.out19_15_14", + "/model/layers.7/post_attention_layernorm/output_0.out19_15_14", + "/model/layers.7/mlp/gate_proj/MatMulNBits/output_0.out19_11_45", + "/model/layers.7/mlp/up_proj/MatMulNBits/output_0.out19_11_46", + "InsertedPrecisionFreeCast_/model/layers.7/mlp/act_fn/Mul/output_0.in.inter.out19_6_7", + "/model/layers.7/mlp/Mul/output_0.out19_12_7", + "/model/layers.7/mlp/down_proj/MatMulNBits/output_0.out19_11_47", + "/model/layers.8/input_layernorm/output_3.out19_15_15", + "/model/layers.8/input_layernorm/output_0.out19_15_15", + "/model/layers.8/attn/k_proj/Add/output_0.out19_11_49", + "/model/layers.8/attn/q_proj/Add/output_0.out19_11_48", + "/model/layers.8/attn/q_rotary/RotaryEmbedding/output_0.out19_3_8_1", + "/model/layers.8/attn/GroupQueryAttention/output_0.out19_3_8", + "/model/layers.8/attn/o_proj/MatMulNBits/output_0.out19_11_50", + "/model/layers.8/post_attention_layernorm/output_3.out19_15_16", + "/model/layers.8/post_attention_layernorm/output_0.out19_15_16", + "/model/layers.8/mlp/gate_proj/MatMulNBits/output_0.out19_11_51", + "/model/layers.8/mlp/up_proj/MatMulNBits/output_0.out19_11_52", + "InsertedPrecisionFreeCast_/model/layers.8/mlp/act_fn/Mul/output_0.in.inter.out19_6_8", + "/model/layers.8/mlp/Mul/output_0.out19_12_8", + "/model/layers.8/mlp/down_proj/MatMulNBits/output_0.out19_11_53", + "/model/layers.9/input_layernorm/output_3.out19_15_17", + "/model/layers.9/input_layernorm/output_0.out19_15_17", + "/model/layers.9/attn/k_proj/Add/output_0.out19_11_55", + "/model/layers.9/attn/q_proj/Add/output_0.out19_11_54", + "/model/layers.9/attn/q_rotary/RotaryEmbedding/output_0.out19_3_9_1", + "/model/layers.9/attn/GroupQueryAttention/output_0.out19_3_9", + "/model/layers.9/attn/o_proj/MatMulNBits/output_0.out19_11_56", + "/model/layers.9/post_attention_layernorm/output_3.out19_15_18", + "/model/layers.9/post_attention_layernorm/output_0.out19_15_18", + "/model/layers.9/mlp/gate_proj/MatMulNBits/output_0.out19_11_57", + "/model/layers.9/mlp/up_proj/MatMulNBits/output_0.out19_11_58", + "InsertedPrecisionFreeCast_/model/layers.9/mlp/act_fn/Mul/output_0.in.inter.out19_6_9", + "/model/layers.9/mlp/Mul/output_0.out19_12_9", + "/model/layers.9/mlp/down_proj/MatMulNBits/output_0.out19_11_59", + "/model/layers.10/input_layernorm/output_3.out19_15_19", + "/model/layers.10/input_layernorm/output_0.out19_15_19", + "/model/layers.10/attn/k_proj/Add/output_0.out19_11_61", + "/model/layers.10/attn/q_proj/Add/output_0.out19_11_60", + "/model/layers.10/attn/q_rotary/RotaryEmbedding/output_0.out19_3_10_1", + "/model/layers.10/attn/GroupQueryAttention/output_0.out19_3_10", + "/model/layers.10/attn/o_proj/MatMulNBits/output_0.out19_11_62", + "/model/layers.10/post_attention_layernorm/output_3.out19_15_20", + "/model/layers.10/post_attention_layernorm/output_0.out19_15_20", + "/model/layers.10/mlp/gate_proj/MatMulNBits/output_0.out19_11_63", + "/model/layers.10/mlp/up_proj/MatMulNBits/output_0.out19_11_64", + "InsertedPrecisionFreeCast_/model/layers.10/mlp/act_fn/Mul/output_0.in.inter.out19_6_10", + "/model/layers.10/mlp/Mul/output_0.out19_12_10", + "/model/layers.10/mlp/down_proj/MatMulNBits/output_0.out19_11_65", + "/model/layers.11/input_layernorm/output_3.out19_15_21", + "/model/layers.11/input_layernorm/output_0.out19_15_21", + "/model/layers.11/attn/k_proj/Add/output_0.out19_11_67", + "/model/layers.11/attn/q_proj/Add/output_0.out19_11_66", + "/model/layers.11/attn/q_rotary/RotaryEmbedding/output_0.out19_3_11_1", + "/model/layers.11/attn/GroupQueryAttention/output_0.out19_3_11", + "/model/layers.11/attn/o_proj/MatMulNBits/output_0.out19_11_68", + "/model/layers.11/post_attention_layernorm/output_3.out19_15_22", + "/model/layers.11/post_attention_layernorm/output_0.out19_15_22", + "/model/layers.11/mlp/gate_proj/MatMulNBits/output_0.out19_11_69", + "/model/layers.11/mlp/up_proj/MatMulNBits/output_0.out19_11_70", + "InsertedPrecisionFreeCast_/model/layers.11/mlp/act_fn/Mul/output_0.in.inter.out19_6_11", + "/model/layers.11/mlp/Mul/output_0.out19_12_11", + "/model/layers.11/mlp/down_proj/MatMulNBits/output_0.out19_11_71", + "/model/layers.12/input_layernorm/output_3.out19_15_23", + "/model/layers.12/input_layernorm/output_0.out19_15_23", + "/model/layers.12/attn/k_proj/Add/output_0.out19_11_73", + "/model/layers.12/attn/q_proj/Add/output_0.out19_11_72", + "/model/layers.12/attn/q_rotary/RotaryEmbedding/output_0.out19_3_12_1", + "/model/layers.12/attn/GroupQueryAttention/output_0.out19_3_12", + "/model/layers.12/attn/o_proj/MatMulNBits/output_0.out19_11_74", + "/model/layers.12/post_attention_layernorm/output_3.out19_15_24", + "/model/layers.12/post_attention_layernorm/output_0.out19_15_24", + "/model/layers.12/mlp/gate_proj/MatMulNBits/output_0.out19_11_75", + "/model/layers.12/mlp/up_proj/MatMulNBits/output_0.out19_11_76", + "InsertedPrecisionFreeCast_/model/layers.12/mlp/act_fn/Mul/output_0.in.inter.out19_6_12", + "/model/layers.12/mlp/Mul/output_0.out19_12_12", + "/model/layers.12/mlp/down_proj/MatMulNBits/output_0.out19_11_77", + "/model/layers.13/input_layernorm/output_3.out19_15_25", + "/model/layers.13/input_layernorm/output_0.out19_15_25", + "/model/layers.13/attn/k_proj/Add/output_0.out19_11_79", + "/model/layers.13/attn/q_proj/Add/output_0.out19_11_78", + "/model/layers.13/attn/q_rotary/RotaryEmbedding/output_0.out19_3_13_1", + "/model/layers.13/attn/GroupQueryAttention/output_0.out19_3_13", + "/model/layers.13/attn/o_proj/MatMulNBits/output_0.out19_11_80", + "/model/layers.13/post_attention_layernorm/output_3.out19_15_26", + "/model/layers.13/post_attention_layernorm/output_0.out19_15_26", + "/model/layers.13/mlp/gate_proj/MatMulNBits/output_0.out19_11_81", + "/model/layers.13/mlp/up_proj/MatMulNBits/output_0.out19_11_82", + "InsertedPrecisionFreeCast_/model/layers.13/mlp/act_fn/Mul/output_0.in.inter.out19_6_13", + "/model/layers.13/mlp/Mul/output_0.out19_12_13", + "/model/layers.13/mlp/down_proj/MatMulNBits/output_0.out19_11_83", + "/model/layers.14/input_layernorm/output_3.out19_15_27", + "/model/layers.14/input_layernorm/output_0.out19_15_27", + "/model/layers.14/attn/k_proj/Add/output_0.out19_11_85", + "/model/layers.14/attn/q_proj/Add/output_0.out19_11_84", + "/model/layers.14/attn/q_rotary/RotaryEmbedding/output_0.out19_3_14_1", + "/model/layers.14/attn/GroupQueryAttention/output_0.out19_3_14", + "/model/layers.14/attn/o_proj/MatMulNBits/output_0.out19_11_86", + "/model/layers.14/post_attention_layernorm/output_3.out19_15_28", + "/model/layers.14/post_attention_layernorm/output_0.out19_15_28", + "/model/layers.14/mlp/gate_proj/MatMulNBits/output_0.out19_11_87", + "/model/layers.14/mlp/up_proj/MatMulNBits/output_0.out19_11_88", + "InsertedPrecisionFreeCast_/model/layers.14/mlp/act_fn/Mul/output_0.in.inter.out19_6_14", + "/model/layers.14/mlp/Mul/output_0.out19_12_14", + "/model/layers.14/mlp/down_proj/MatMulNBits/output_0.out19_11_89", + "/model/layers.15/input_layernorm/output_3.out19_15_29", + "/model/layers.15/input_layernorm/output_0.out19_15_29", + "/model/layers.15/attn/k_proj/Add/output_0.out19_11_91", + "/model/layers.15/attn/q_proj/Add/output_0.out19_11_90", + "/model/layers.15/attn/q_rotary/RotaryEmbedding/output_0.out19_3_15_1", + "/model/layers.15/attn/GroupQueryAttention/output_0.out19_3_15", + "/model/layers.15/attn/o_proj/MatMulNBits/output_0.out19_11_92", + "/model/layers.15/post_attention_layernorm/output_3.out19_15_30", + "/model/layers.15/post_attention_layernorm/output_0.out19_15_30", + "/model/layers.15/mlp/gate_proj/MatMulNBits/output_0.out19_11_93", + "/model/layers.15/mlp/up_proj/MatMulNBits/output_0.out19_11_94", + "InsertedPrecisionFreeCast_/model/layers.15/mlp/act_fn/Mul/output_0.in.inter.out19_6_15", + "/model/layers.15/mlp/Mul/output_0.out19_12_15", + "/model/layers.15/mlp/down_proj/MatMulNBits/output_0.out19_11_95", + "/model/layers.16/input_layernorm/output_3.out19_15_31", + "/model/layers.16/input_layernorm/output_0.out19_15_31", + "/model/layers.16/attn/k_proj/Add/output_0.out19_11_97", + "/model/layers.16/attn/q_proj/Add/output_0.out19_11_96", + "/model/layers.16/attn/q_rotary/RotaryEmbedding/output_0.out19_3_16_1", + "/model/layers.16/attn/GroupQueryAttention/output_0.out19_3_16", + "/model/layers.16/attn/o_proj/MatMulNBits/output_0.out19_11_98", + "/model/layers.16/post_attention_layernorm/output_3.out19_15_32", + "/model/layers.16/post_attention_layernorm/output_0.out19_15_32", + "/model/layers.16/mlp/gate_proj/MatMulNBits/output_0.out19_11_99", + "/model/layers.16/mlp/up_proj/MatMulNBits/output_0.out19_11_100", + "InsertedPrecisionFreeCast_/model/layers.16/mlp/act_fn/Mul/output_0.in.inter.out19_6_16", + "/model/layers.16/mlp/Mul/output_0.out19_12_16", + "/model/layers.16/mlp/down_proj/MatMulNBits/output_0.out19_11_101", + "/model/layers.17/input_layernorm/output_3.out19_15_33", + "/model/layers.17/input_layernorm/output_0.out19_15_33", + "/model/layers.17/attn/k_proj/Add/output_0.out19_11_103", + "/model/layers.17/attn/q_proj/Add/output_0.out19_11_102", + "/model/layers.17/attn/q_rotary/RotaryEmbedding/output_0.out19_3_17_1", + "/model/layers.17/attn/GroupQueryAttention/output_0.out19_3_17", + "/model/layers.17/attn/o_proj/MatMulNBits/output_0.out19_11_104", + "/model/layers.17/post_attention_layernorm/output_3.out19_15_34", + "/model/layers.17/post_attention_layernorm/output_0.out19_15_34", + "/model/layers.17/mlp/gate_proj/MatMulNBits/output_0.out19_11_105", + "/model/layers.17/mlp/up_proj/MatMulNBits/output_0.out19_11_106", + "InsertedPrecisionFreeCast_/model/layers.17/mlp/act_fn/Mul/output_0.in.inter.out19_6_17", + "/model/layers.17/mlp/Mul/output_0.out19_12_17", + "/model/layers.17/mlp/down_proj/MatMulNBits/output_0.out19_11_107", + "/model/layers.18/input_layernorm/output_3.out19_15_35", + "/model/layers.18/input_layernorm/output_0.out19_15_35", + "/model/layers.18/attn/k_proj/Add/output_0.out19_11_109", + "/model/layers.18/attn/q_proj/Add/output_0.out19_11_108", + "/model/layers.18/attn/q_rotary/RotaryEmbedding/output_0.out19_3_18_1", + "/model/layers.18/attn/GroupQueryAttention/output_0.out19_3_18", + "/model/layers.18/attn/o_proj/MatMulNBits/output_0.out19_11_110", + "/model/layers.18/post_attention_layernorm/output_3.out19_15_36", + "/model/layers.18/post_attention_layernorm/output_0.out19_15_36", + "/model/layers.18/mlp/gate_proj/MatMulNBits/output_0.out19_11_111", + "/model/layers.18/mlp/up_proj/MatMulNBits/output_0.out19_11_112", + "InsertedPrecisionFreeCast_/model/layers.18/mlp/act_fn/Mul/output_0.in.inter.out19_6_18", + "/model/layers.18/mlp/Mul/output_0.out19_12_18", + "/model/layers.18/mlp/down_proj/MatMulNBits/output_0.out19_11_113", + "/model/layers.19/input_layernorm/output_3.out19_15_37", + "/model/layers.19/input_layernorm/output_0.out19_15_37", + "/model/layers.19/attn/k_proj/Add/output_0.out19_11_115", + "/model/layers.19/attn/q_proj/Add/output_0.out19_11_114", + "/model/layers.19/attn/q_rotary/RotaryEmbedding/output_0.out19_3_19_1", + "/model/layers.19/attn/GroupQueryAttention/output_0.out19_3_19", + "/model/layers.19/attn/o_proj/MatMulNBits/output_0.out19_11_116", + "/model/layers.19/post_attention_layernorm/output_3.out19_15_38", + "/model/layers.19/post_attention_layernorm/output_0.out19_15_38", + "/model/layers.19/mlp/gate_proj/MatMulNBits/output_0.out19_11_117", + "/model/layers.19/mlp/up_proj/MatMulNBits/output_0.out19_11_118", + "InsertedPrecisionFreeCast_/model/layers.19/mlp/act_fn/Mul/output_0.in.inter.out19_6_19", + "/model/layers.19/mlp/Mul/output_0.out19_12_19", + "/model/layers.19/mlp/down_proj/MatMulNBits/output_0.out19_11_119", + "/model/layers.20/input_layernorm/output_3.out19_15_39", + "/model/layers.20/input_layernorm/output_0.out19_15_39", + "/model/layers.20/attn/k_proj/Add/output_0.out19_11_121", + "/model/layers.20/attn/q_proj/Add/output_0.out19_11_120", + "/model/layers.20/attn/q_rotary/RotaryEmbedding/output_0.out19_3_20_1", + "/model/layers.20/attn/GroupQueryAttention/output_0.out19_3_20", + "/model/layers.20/attn/o_proj/MatMulNBits/output_0.out19_11_122", + "/model/layers.20/post_attention_layernorm/output_3.out19_15_40", + "/model/layers.20/post_attention_layernorm/output_0.out19_15_40", + "/model/layers.20/mlp/gate_proj/MatMulNBits/output_0.out19_11_123", + "/model/layers.20/mlp/up_proj/MatMulNBits/output_0.out19_11_124", + "InsertedPrecisionFreeCast_/model/layers.20/mlp/act_fn/Mul/output_0.in.inter.out19_6_20", + "/model/layers.20/mlp/Mul/output_0.out19_12_20", + "/model/layers.20/mlp/down_proj/MatMulNBits/output_0.out19_11_125", + "/model/layers.21/input_layernorm/output_3.out19_15_41", + "/model/layers.21/input_layernorm/output_0.out19_15_41", + "/model/layers.21/attn/k_proj/Add/output_0.out19_11_127", + "/model/layers.21/attn/q_proj/Add/output_0.out19_11_126", + "/model/layers.21/attn/q_rotary/RotaryEmbedding/output_0.out19_3_21_1", + "/model/layers.21/attn/GroupQueryAttention/output_0.out19_3_21", + "/model/layers.21/attn/o_proj/MatMulNBits/output_0.out19_11_128", + "/model/layers.21/post_attention_layernorm/output_3.out19_15_42", + "/model/layers.21/post_attention_layernorm/output_0.out19_15_42", + "/model/layers.21/mlp/gate_proj/MatMulNBits/output_0.out19_11_129", + "/model/layers.21/mlp/up_proj/MatMulNBits/output_0.out19_11_130", + "InsertedPrecisionFreeCast_/model/layers.21/mlp/act_fn/Mul/output_0.in.inter.out19_6_21", + "/model/layers.21/mlp/Mul/output_0.out19_12_21", + "/model/layers.21/mlp/down_proj/MatMulNBits/output_0.out19_11_131", + "/model/layers.22/input_layernorm/output_3.out19_15_43", + "/model/layers.22/input_layernorm/output_0.out19_15_43", + "/model/layers.22/attn/k_proj/Add/output_0.out19_11_133", + "/model/layers.22/attn/q_proj/Add/output_0.out19_11_132", + "/model/layers.22/attn/q_rotary/RotaryEmbedding/output_0.out19_3_22_1", + "/model/layers.22/attn/GroupQueryAttention/output_0.out19_3_22", + "/model/layers.22/attn/o_proj/MatMulNBits/output_0.out19_11_134", + "/model/layers.22/post_attention_layernorm/output_3.out19_15_44", + "/model/layers.22/post_attention_layernorm/output_0.out19_15_44", + "/model/layers.22/mlp/gate_proj/MatMulNBits/output_0.out19_11_135", + "/model/layers.22/mlp/up_proj/MatMulNBits/output_0.out19_11_136", + "InsertedPrecisionFreeCast_/model/layers.22/mlp/act_fn/Mul/output_0.in.inter.out19_6_22", + "/model/layers.22/mlp/Mul/output_0.out19_12_22", + "/model/layers.22/mlp/down_proj/MatMulNBits/output_0.out19_11_137", + "/model/layers.23/input_layernorm/output_3.out19_15_45", + "/model/layers.23/input_layernorm/output_0.out19_15_45", + "/model/layers.23/attn/k_proj/Add/output_0.out19_11_139", + "/model/layers.23/attn/q_proj/Add/output_0.out19_11_138", + "/model/layers.23/attn/q_rotary/RotaryEmbedding/output_0.out19_3_23_1", + "/model/layers.23/attn/GroupQueryAttention/output_0.out19_3_23", + "/model/layers.23/attn/o_proj/MatMulNBits/output_0.out19_11_140", + "/model/layers.23/post_attention_layernorm/output_3.out19_15_46", + "/model/layers.23/post_attention_layernorm/output_0.out19_15_46", + "/model/layers.23/mlp/gate_proj/MatMulNBits/output_0.out19_11_141", + "/model/layers.23/mlp/up_proj/MatMulNBits/output_0.out19_11_142", + "InsertedPrecisionFreeCast_/model/layers.23/mlp/act_fn/Mul/output_0.in.inter.out19_6_23", + "/model/layers.23/mlp/Mul/output_0.out19_12_23", + "/model/layers.23/mlp/down_proj/MatMulNBits/output_0.out19_11_143", + "dummy_19_15_47" + ] + }, + "const": { + "buffer_size": 373108676, + "xrt_arg_id": 3, + "packed_tensors": [ + "model.layers.0.input_layernorm.weight", + "eps_19_14_0", + "model.layers.0.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.k_proj.Add.bias.preformat", + "model.layers.0.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.0.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.q_proj.Add.bias.preformat", + "model.layers.0.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.0.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.v_proj.Add.bias.preformat", + "model.layers.0.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.0.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.0.post_attention_layernorm.weight", + "eps_19_15_0", + "model.layers.0.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.0.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.0.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.0.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.0.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.0.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.0.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.0.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.0.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.input_layernorm.weight", + "eps_19_15_1", + "model.layers.1.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.k_proj.Add.bias.preformat", + "model.layers.1.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.q_proj.Add.bias.preformat", + "model.layers.1.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.v_proj.Add.bias.preformat", + "model.layers.1.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.1.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.post_attention_layernorm.weight", + "eps_19_15_2", + "model.layers.1.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.1.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.1.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.1.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.1.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.1.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.1.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.1.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.input_layernorm.weight", + "eps_19_15_3", + "model.layers.2.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.k_proj.Add.bias.preformat", + "model.layers.2.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.q_proj.Add.bias.preformat", + "model.layers.2.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.v_proj.Add.bias.preformat", + "model.layers.2.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.2.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.post_attention_layernorm.weight", + "eps_19_15_4", + "model.layers.2.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.2.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.2.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.2.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.2.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.2.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.2.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.2.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.input_layernorm.weight", + "eps_19_15_5", + "model.layers.3.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.k_proj.Add.bias.preformat", + "model.layers.3.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.q_proj.Add.bias.preformat", + "model.layers.3.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.v_proj.Add.bias.preformat", + "model.layers.3.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.3.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.post_attention_layernorm.weight", + "eps_19_15_6", + "model.layers.3.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.3.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.3.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.3.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.3.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.3.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.3.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.3.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.input_layernorm.weight", + "eps_19_15_7", + "model.layers.4.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.k_proj.Add.bias.preformat", + "model.layers.4.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.q_proj.Add.bias.preformat", + "model.layers.4.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.v_proj.Add.bias.preformat", + "model.layers.4.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.4.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.post_attention_layernorm.weight", + "eps_19_15_8", + "model.layers.4.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.4.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.4.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.4.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.4.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.4.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.4.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.4.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.input_layernorm.weight", + "eps_19_15_9", + "model.layers.5.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.k_proj.Add.bias.preformat", + "model.layers.5.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.q_proj.Add.bias.preformat", + "model.layers.5.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.v_proj.Add.bias.preformat", + "model.layers.5.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.5.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.post_attention_layernorm.weight", + "eps_19_15_10", + "model.layers.5.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.5.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.5.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.5.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.5.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.5.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.5.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.5.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.input_layernorm.weight", + "eps_19_15_11", + "model.layers.6.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.k_proj.Add.bias.preformat", + "model.layers.6.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.q_proj.Add.bias.preformat", + "model.layers.6.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.v_proj.Add.bias.preformat", + "model.layers.6.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.6.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.post_attention_layernorm.weight", + "eps_19_15_12", + "model.layers.6.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.6.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.6.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.6.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.6.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.6.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.6.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.6.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.input_layernorm.weight", + "eps_19_15_13", + "model.layers.7.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.k_proj.Add.bias.preformat", + "model.layers.7.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.q_proj.Add.bias.preformat", + "model.layers.7.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.v_proj.Add.bias.preformat", + "model.layers.7.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.7.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.post_attention_layernorm.weight", + "eps_19_15_14", + "model.layers.7.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.7.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.7.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.7.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.7.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.7.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.7.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.7.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.input_layernorm.weight", + "eps_19_15_15", + "model.layers.8.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.k_proj.Add.bias.preformat", + "model.layers.8.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.q_proj.Add.bias.preformat", + "model.layers.8.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.v_proj.Add.bias.preformat", + "model.layers.8.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.8.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.post_attention_layernorm.weight", + "eps_19_15_16", + "model.layers.8.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.8.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.8.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.8.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.8.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.8.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.8.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.8.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.input_layernorm.weight", + "eps_19_15_17", + "model.layers.9.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.k_proj.Add.bias.preformat", + "model.layers.9.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.q_proj.Add.bias.preformat", + "model.layers.9.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.v_proj.Add.bias.preformat", + "model.layers.9.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.9.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.post_attention_layernorm.weight", + "eps_19_15_18", + "model.layers.9.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.9.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.9.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.9.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.9.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.9.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.9.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.9.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.input_layernorm.weight", + "eps_19_15_19", + "model.layers.10.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.k_proj.Add.bias.preformat", + "model.layers.10.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.q_proj.Add.bias.preformat", + "model.layers.10.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.v_proj.Add.bias.preformat", + "model.layers.10.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.10.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.post_attention_layernorm.weight", + "eps_19_15_20", + "model.layers.10.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.10.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.10.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.10.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.10.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.10.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.10.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.10.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.input_layernorm.weight", + "eps_19_15_21", + "model.layers.11.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.k_proj.Add.bias.preformat", + "model.layers.11.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.q_proj.Add.bias.preformat", + "model.layers.11.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.v_proj.Add.bias.preformat", + "model.layers.11.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.11.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.post_attention_layernorm.weight", + "eps_19_15_22", + "model.layers.11.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.11.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.11.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.11.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.11.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.11.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.11.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.11.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.input_layernorm.weight", + "eps_19_15_23", + "model.layers.12.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.k_proj.Add.bias.preformat", + "model.layers.12.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.q_proj.Add.bias.preformat", + "model.layers.12.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.v_proj.Add.bias.preformat", + "model.layers.12.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.12.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.post_attention_layernorm.weight", + "eps_19_15_24", + "model.layers.12.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.12.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.12.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.12.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.12.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.12.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.12.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.12.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.input_layernorm.weight", + "eps_19_15_25", + "model.layers.13.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.k_proj.Add.bias.preformat", + "model.layers.13.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.q_proj.Add.bias.preformat", + "model.layers.13.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.v_proj.Add.bias.preformat", + "model.layers.13.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.13.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.post_attention_layernorm.weight", + "eps_19_15_26", + "model.layers.13.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.13.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.13.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.13.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.13.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.13.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.13.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.13.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.input_layernorm.weight", + "eps_19_15_27", + "model.layers.14.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.k_proj.Add.bias.preformat", + "model.layers.14.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.q_proj.Add.bias.preformat", + "model.layers.14.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.v_proj.Add.bias.preformat", + "model.layers.14.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.14.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.post_attention_layernorm.weight", + "eps_19_15_28", + "model.layers.14.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.14.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.14.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.14.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.14.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.14.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.14.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.14.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.input_layernorm.weight", + "eps_19_15_29", + "model.layers.15.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.k_proj.Add.bias.preformat", + "model.layers.15.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.q_proj.Add.bias.preformat", + "model.layers.15.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.v_proj.Add.bias.preformat", + "model.layers.15.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.15.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.post_attention_layernorm.weight", + "eps_19_15_30", + "model.layers.15.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.15.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.15.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.15.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.15.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.15.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.15.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.15.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.input_layernorm.weight", + "eps_19_15_31", + "model.layers.16.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.k_proj.Add.bias.preformat", + "model.layers.16.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.q_proj.Add.bias.preformat", + "model.layers.16.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.v_proj.Add.bias.preformat", + "model.layers.16.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.16.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.post_attention_layernorm.weight", + "eps_19_15_32", + "model.layers.16.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.16.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.16.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.16.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.16.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.16.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.16.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.16.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.input_layernorm.weight", + "eps_19_15_33", + "model.layers.17.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.k_proj.Add.bias.preformat", + "model.layers.17.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.q_proj.Add.bias.preformat", + "model.layers.17.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.v_proj.Add.bias.preformat", + "model.layers.17.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.17.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.post_attention_layernorm.weight", + "eps_19_15_34", + "model.layers.17.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.17.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.17.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.17.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.17.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.17.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.17.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.17.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.input_layernorm.weight", + "eps_19_15_35", + "model.layers.18.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.k_proj.Add.bias.preformat", + "model.layers.18.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.q_proj.Add.bias.preformat", + "model.layers.18.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.v_proj.Add.bias.preformat", + "model.layers.18.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.18.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.post_attention_layernorm.weight", + "eps_19_15_36", + "model.layers.18.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.18.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.18.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.18.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.18.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.18.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.18.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.18.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.input_layernorm.weight", + "eps_19_15_37", + "model.layers.19.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.k_proj.Add.bias.preformat", + "model.layers.19.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.q_proj.Add.bias.preformat", + "model.layers.19.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.v_proj.Add.bias.preformat", + "model.layers.19.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.19.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.post_attention_layernorm.weight", + "eps_19_15_38", + "model.layers.19.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.19.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.19.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.19.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.19.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.19.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.19.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.19.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.input_layernorm.weight", + "eps_19_15_39", + "model.layers.20.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.k_proj.Add.bias.preformat", + "model.layers.20.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.q_proj.Add.bias.preformat", + "model.layers.20.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.v_proj.Add.bias.preformat", + "model.layers.20.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.20.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.post_attention_layernorm.weight", + "eps_19_15_40", + "model.layers.20.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.20.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.20.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.20.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.20.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.20.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.20.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.20.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.input_layernorm.weight", + "eps_19_15_41", + "model.layers.21.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.k_proj.Add.bias.preformat", + "model.layers.21.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.q_proj.Add.bias.preformat", + "model.layers.21.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.v_proj.Add.bias.preformat", + "model.layers.21.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.21.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.post_attention_layernorm.weight", + "eps_19_15_42", + "model.layers.21.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.21.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.21.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.21.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.21.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.21.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.21.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.21.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.input_layernorm.weight", + "eps_19_15_43", + "model.layers.22.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.k_proj.Add.bias.preformat", + "model.layers.22.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.q_proj.Add.bias.preformat", + "model.layers.22.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.v_proj.Add.bias.preformat", + "model.layers.22.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.22.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.post_attention_layernorm.weight", + "eps_19_15_44", + "model.layers.22.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.22.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.22.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.22.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.22.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.22.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.22.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.22.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.input_layernorm.weight", + "eps_19_15_45", + "model.layers.23.attn.k_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.k_proj.Add.bias.preformat", + "model.layers.23.attn.k_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.k_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.attn.q_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.q_proj.Add.bias.preformat", + "model.layers.23.attn.q_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.q_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.attn.v_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.v_proj.Add.bias.preformat", + "model.layers.23.attn.v_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.v_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.qweight.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.bias.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.scales.preformat", + "model.layers.23.attn.o_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.post_attention_layernorm.weight", + "eps_19_15_46", + "model.layers.23.mlp.gate_proj.MatMulNBits.qweight.preformat", + "model.layers.23.mlp.gate_proj.MatMulNBits.bias.preformat", + "model.layers.23.mlp.gate_proj.MatMulNBits.scales.preformat", + "model.layers.23.mlp.gate_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.mlp.up_proj.MatMulNBits.qweight.preformat", + "model.layers.23.mlp.up_proj.MatMulNBits.bias.preformat", + "model.layers.23.mlp.up_proj.MatMulNBits.scales.preformat", + "model.layers.23.mlp.up_proj.MatMulNBits.qzeros.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.qweight.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.bias.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.scales.preformat", + "model.layers.23.mlp.down_proj.MatMulNBits.qzeros.preformat", + "model.layers.24.final_norm_layernorm.weight", + "eps_19_15_47" + ] + }, + "super_instr": { + "buffer_size": 0, + "xrt_arg_id": 4, + "packed_tensors": [] + }, + "ext_buf_0": { + "buffer_size": 50331648, + "xrt_arg_id": 5, + "packed_tensors": [ + "present.0.key", + "present.0.value", + "present.1.key", + "present.1.value", + "present.2.key", + "present.2.value", + "present.3.key", + "present.3.value", + "present.4.key", + "present.4.value", + "present.5.key", + "present.5.value", + "present.6.key", + "present.6.value", + "present.7.key", + "present.7.value", + "present.8.key", + "present.8.value", + "present.9.key", + "present.9.value", + "present.10.key", + "present.10.value", + "present.11.key", + "present.11.value", + "present.12.key", + "present.12.value", + "present.13.key", + "present.13.value", + "present.14.key", + "present.14.value", + "present.15.key", + "present.15.value", + "present.16.key", + "present.16.value", + "present.17.key", + "present.17.value", + "present.18.key", + "present.18.value", + "present.19.key", + "present.19.value", + "present.20.key", + "present.20.value", + "present.21.key", + "present.21.value", + "present.22.key", + "present.22.value", + "present.23.key", + "present.23.value" + ] + } + }, + "tensor_map": { + "/model/embed_tokens/Gather/output_0.out19_14_0": { + "packed_buffer_label": "in", + "xrt_arg_id": 0, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 0 + }, + "sin_cache_cos_cache_prefill_slice": { + "packed_buffer_label": "in", + "xrt_arg_id": 0, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1792 + }, + "/model/layers.24/final_norm_layernorm/output_0.out19_15_47": { + "packed_buffer_label": "out", + "xrt_arg_id": 1, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 0 + }, + "/model/layers.0/input_layernorm/output_0.out19_14_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 0 + }, + "/model/layers.0/attn/k_proj/Add/output_0.out19_11_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1792 + }, + "/model/layers.0/attn/q_proj/Add/output_0.out19_11_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 2048 + }, + "/model/layers.0/attn/q_rotary/RotaryEmbedding/output_0.out19_3_0_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 3840 + }, + "/model/layers.0/attn/GroupQueryAttention/output_0.out19_3_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 5632 + }, + "/model/layers.0/attn/o_proj/MatMulNBits/output_0.out19_11_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 7424 + }, + "/model/layers.0/post_attention_layernorm/output_3.out19_15_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 9216 + }, + "/model/layers.0/post_attention_layernorm/output_0.out19_15_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 11008 + }, + "/model/layers.0/mlp/gate_proj/MatMulNBits/output_0.out19_11_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 12800 + }, + "/model/layers.0/mlp/up_proj/MatMulNBits/output_0.out19_11_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 22528 + }, + "InsertedPrecisionFreeCast_/model/layers.0/mlp/act_fn/Mul/output_0.in.inter.out19_6_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 32256 + }, + "/model/layers.0/mlp/Mul/output_0.out19_12_0": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 41984 + }, + "/model/layers.0/mlp/down_proj/MatMulNBits/output_0.out19_11_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 51712 + }, + "/model/layers.1/input_layernorm/output_3.out19_15_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 53504 + }, + "/model/layers.1/input_layernorm/output_0.out19_15_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 55296 + }, + "/model/layers.1/attn/k_proj/Add/output_0.out19_11_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 57088 + }, + "/model/layers.1/attn/q_proj/Add/output_0.out19_11_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 57344 + }, + "/model/layers.1/attn/q_rotary/RotaryEmbedding/output_0.out19_3_1_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 59136 + }, + "/model/layers.1/attn/GroupQueryAttention/output_0.out19_3_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 60928 + }, + "/model/layers.1/attn/o_proj/MatMulNBits/output_0.out19_11_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 62720 + }, + "/model/layers.1/post_attention_layernorm/output_3.out19_15_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 64512 + }, + "/model/layers.1/post_attention_layernorm/output_0.out19_15_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 66304 + }, + "/model/layers.1/mlp/gate_proj/MatMulNBits/output_0.out19_11_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 68096 + }, + "/model/layers.1/mlp/up_proj/MatMulNBits/output_0.out19_11_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 77824 + }, + "InsertedPrecisionFreeCast_/model/layers.1/mlp/act_fn/Mul/output_0.in.inter.out19_6_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 87552 + }, + "/model/layers.1/mlp/Mul/output_0.out19_12_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 97280 + }, + "/model/layers.1/mlp/down_proj/MatMulNBits/output_0.out19_11_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 107008 + }, + "/model/layers.2/input_layernorm/output_3.out19_15_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 108800 + }, + "/model/layers.2/input_layernorm/output_0.out19_15_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 110592 + }, + "/model/layers.2/attn/k_proj/Add/output_0.out19_11_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 112384 + }, + "/model/layers.2/attn/q_proj/Add/output_0.out19_11_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 112640 + }, + "/model/layers.2/attn/q_rotary/RotaryEmbedding/output_0.out19_3_2_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 114432 + }, + "/model/layers.2/attn/GroupQueryAttention/output_0.out19_3_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 116224 + }, + "/model/layers.2/attn/o_proj/MatMulNBits/output_0.out19_11_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 118016 + }, + "/model/layers.2/post_attention_layernorm/output_3.out19_15_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 119808 + }, + "/model/layers.2/post_attention_layernorm/output_0.out19_15_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 121600 + }, + "/model/layers.2/mlp/gate_proj/MatMulNBits/output_0.out19_11_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 123392 + }, + "/model/layers.2/mlp/up_proj/MatMulNBits/output_0.out19_11_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 133120 + }, + "InsertedPrecisionFreeCast_/model/layers.2/mlp/act_fn/Mul/output_0.in.inter.out19_6_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 142848 + }, + "/model/layers.2/mlp/Mul/output_0.out19_12_2": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 152576 + }, + "/model/layers.2/mlp/down_proj/MatMulNBits/output_0.out19_11_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 162304 + }, + "/model/layers.3/input_layernorm/output_3.out19_15_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 164096 + }, + "/model/layers.3/input_layernorm/output_0.out19_15_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 165888 + }, + "/model/layers.3/attn/k_proj/Add/output_0.out19_11_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 167680 + }, + "/model/layers.3/attn/q_proj/Add/output_0.out19_11_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 167936 + }, + "/model/layers.3/attn/q_rotary/RotaryEmbedding/output_0.out19_3_3_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 169728 + }, + "/model/layers.3/attn/GroupQueryAttention/output_0.out19_3_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 171520 + }, + "/model/layers.3/attn/o_proj/MatMulNBits/output_0.out19_11_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 173312 + }, + "/model/layers.3/post_attention_layernorm/output_3.out19_15_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 175104 + }, + "/model/layers.3/post_attention_layernorm/output_0.out19_15_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 176896 + }, + "/model/layers.3/mlp/gate_proj/MatMulNBits/output_0.out19_11_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 178688 + }, + "/model/layers.3/mlp/up_proj/MatMulNBits/output_0.out19_11_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 188416 + }, + "InsertedPrecisionFreeCast_/model/layers.3/mlp/act_fn/Mul/output_0.in.inter.out19_6_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 198144 + }, + "/model/layers.3/mlp/Mul/output_0.out19_12_3": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 207872 + }, + "/model/layers.3/mlp/down_proj/MatMulNBits/output_0.out19_11_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 217600 + }, + "/model/layers.4/input_layernorm/output_3.out19_15_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 219392 + }, + "/model/layers.4/input_layernorm/output_0.out19_15_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 221184 + }, + "/model/layers.4/attn/k_proj/Add/output_0.out19_11_25": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 222976 + }, + "/model/layers.4/attn/q_proj/Add/output_0.out19_11_24": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 223232 + }, + "/model/layers.4/attn/q_rotary/RotaryEmbedding/output_0.out19_3_4_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 225024 + }, + "/model/layers.4/attn/GroupQueryAttention/output_0.out19_3_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 226816 + }, + "/model/layers.4/attn/o_proj/MatMulNBits/output_0.out19_11_26": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 228608 + }, + "/model/layers.4/post_attention_layernorm/output_3.out19_15_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 230400 + }, + "/model/layers.4/post_attention_layernorm/output_0.out19_15_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 232192 + }, + "/model/layers.4/mlp/gate_proj/MatMulNBits/output_0.out19_11_27": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 233984 + }, + "/model/layers.4/mlp/up_proj/MatMulNBits/output_0.out19_11_28": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 243712 + }, + "InsertedPrecisionFreeCast_/model/layers.4/mlp/act_fn/Mul/output_0.in.inter.out19_6_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 253440 + }, + "/model/layers.4/mlp/Mul/output_0.out19_12_4": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 263168 + }, + "/model/layers.4/mlp/down_proj/MatMulNBits/output_0.out19_11_29": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 272896 + }, + "/model/layers.5/input_layernorm/output_3.out19_15_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 274688 + }, + "/model/layers.5/input_layernorm/output_0.out19_15_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 276480 + }, + "/model/layers.5/attn/k_proj/Add/output_0.out19_11_31": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 278272 + }, + "/model/layers.5/attn/q_proj/Add/output_0.out19_11_30": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 278528 + }, + "/model/layers.5/attn/q_rotary/RotaryEmbedding/output_0.out19_3_5_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 280320 + }, + "/model/layers.5/attn/GroupQueryAttention/output_0.out19_3_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 282112 + }, + "/model/layers.5/attn/o_proj/MatMulNBits/output_0.out19_11_32": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 283904 + }, + "/model/layers.5/post_attention_layernorm/output_3.out19_15_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 285696 + }, + "/model/layers.5/post_attention_layernorm/output_0.out19_15_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 287488 + }, + "/model/layers.5/mlp/gate_proj/MatMulNBits/output_0.out19_11_33": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 289280 + }, + "/model/layers.5/mlp/up_proj/MatMulNBits/output_0.out19_11_34": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 299008 + }, + "InsertedPrecisionFreeCast_/model/layers.5/mlp/act_fn/Mul/output_0.in.inter.out19_6_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 308736 + }, + "/model/layers.5/mlp/Mul/output_0.out19_12_5": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 318464 + }, + "/model/layers.5/mlp/down_proj/MatMulNBits/output_0.out19_11_35": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 328192 + }, + "/model/layers.6/input_layernorm/output_3.out19_15_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 329984 + }, + "/model/layers.6/input_layernorm/output_0.out19_15_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 331776 + }, + "/model/layers.6/attn/k_proj/Add/output_0.out19_11_37": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 333568 + }, + "/model/layers.6/attn/q_proj/Add/output_0.out19_11_36": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 333824 + }, + "/model/layers.6/attn/q_rotary/RotaryEmbedding/output_0.out19_3_6_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 335616 + }, + "/model/layers.6/attn/GroupQueryAttention/output_0.out19_3_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 337408 + }, + "/model/layers.6/attn/o_proj/MatMulNBits/output_0.out19_11_38": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 339200 + }, + "/model/layers.6/post_attention_layernorm/output_3.out19_15_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 340992 + }, + "/model/layers.6/post_attention_layernorm/output_0.out19_15_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 342784 + }, + "/model/layers.6/mlp/gate_proj/MatMulNBits/output_0.out19_11_39": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 344576 + }, + "/model/layers.6/mlp/up_proj/MatMulNBits/output_0.out19_11_40": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 354304 + }, + "InsertedPrecisionFreeCast_/model/layers.6/mlp/act_fn/Mul/output_0.in.inter.out19_6_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 364032 + }, + "/model/layers.6/mlp/Mul/output_0.out19_12_6": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 373760 + }, + "/model/layers.6/mlp/down_proj/MatMulNBits/output_0.out19_11_41": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 383488 + }, + "/model/layers.7/input_layernorm/output_3.out19_15_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 385280 + }, + "/model/layers.7/input_layernorm/output_0.out19_15_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 387072 + }, + "/model/layers.7/attn/k_proj/Add/output_0.out19_11_43": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 388864 + }, + "/model/layers.7/attn/q_proj/Add/output_0.out19_11_42": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 389120 + }, + "/model/layers.7/attn/q_rotary/RotaryEmbedding/output_0.out19_3_7_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 390912 + }, + "/model/layers.7/attn/GroupQueryAttention/output_0.out19_3_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 392704 + }, + "/model/layers.7/attn/o_proj/MatMulNBits/output_0.out19_11_44": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 394496 + }, + "/model/layers.7/post_attention_layernorm/output_3.out19_15_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 396288 + }, + "/model/layers.7/post_attention_layernorm/output_0.out19_15_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 398080 + }, + "/model/layers.7/mlp/gate_proj/MatMulNBits/output_0.out19_11_45": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 399872 + }, + "/model/layers.7/mlp/up_proj/MatMulNBits/output_0.out19_11_46": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 409600 + }, + "InsertedPrecisionFreeCast_/model/layers.7/mlp/act_fn/Mul/output_0.in.inter.out19_6_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 419328 + }, + "/model/layers.7/mlp/Mul/output_0.out19_12_7": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 429056 + }, + "/model/layers.7/mlp/down_proj/MatMulNBits/output_0.out19_11_47": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 438784 + }, + "/model/layers.8/input_layernorm/output_3.out19_15_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 440576 + }, + "/model/layers.8/input_layernorm/output_0.out19_15_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 442368 + }, + "/model/layers.8/attn/k_proj/Add/output_0.out19_11_49": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 444160 + }, + "/model/layers.8/attn/q_proj/Add/output_0.out19_11_48": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 444416 + }, + "/model/layers.8/attn/q_rotary/RotaryEmbedding/output_0.out19_3_8_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 446208 + }, + "/model/layers.8/attn/GroupQueryAttention/output_0.out19_3_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 448000 + }, + "/model/layers.8/attn/o_proj/MatMulNBits/output_0.out19_11_50": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 449792 + }, + "/model/layers.8/post_attention_layernorm/output_3.out19_15_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 451584 + }, + "/model/layers.8/post_attention_layernorm/output_0.out19_15_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 453376 + }, + "/model/layers.8/mlp/gate_proj/MatMulNBits/output_0.out19_11_51": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 455168 + }, + "/model/layers.8/mlp/up_proj/MatMulNBits/output_0.out19_11_52": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 464896 + }, + "InsertedPrecisionFreeCast_/model/layers.8/mlp/act_fn/Mul/output_0.in.inter.out19_6_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 474624 + }, + "/model/layers.8/mlp/Mul/output_0.out19_12_8": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 484352 + }, + "/model/layers.8/mlp/down_proj/MatMulNBits/output_0.out19_11_53": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 494080 + }, + "/model/layers.9/input_layernorm/output_3.out19_15_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 495872 + }, + "/model/layers.9/input_layernorm/output_0.out19_15_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 497664 + }, + "/model/layers.9/attn/k_proj/Add/output_0.out19_11_55": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 499456 + }, + "/model/layers.9/attn/q_proj/Add/output_0.out19_11_54": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 499712 + }, + "/model/layers.9/attn/q_rotary/RotaryEmbedding/output_0.out19_3_9_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 501504 + }, + "/model/layers.9/attn/GroupQueryAttention/output_0.out19_3_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 503296 + }, + "/model/layers.9/attn/o_proj/MatMulNBits/output_0.out19_11_56": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 505088 + }, + "/model/layers.9/post_attention_layernorm/output_3.out19_15_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 506880 + }, + "/model/layers.9/post_attention_layernorm/output_0.out19_15_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 508672 + }, + "/model/layers.9/mlp/gate_proj/MatMulNBits/output_0.out19_11_57": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 510464 + }, + "/model/layers.9/mlp/up_proj/MatMulNBits/output_0.out19_11_58": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 520192 + }, + "InsertedPrecisionFreeCast_/model/layers.9/mlp/act_fn/Mul/output_0.in.inter.out19_6_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 529920 + }, + "/model/layers.9/mlp/Mul/output_0.out19_12_9": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 539648 + }, + "/model/layers.9/mlp/down_proj/MatMulNBits/output_0.out19_11_59": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 549376 + }, + "/model/layers.10/input_layernorm/output_3.out19_15_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 551168 + }, + "/model/layers.10/input_layernorm/output_0.out19_15_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 552960 + }, + "/model/layers.10/attn/k_proj/Add/output_0.out19_11_61": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 554752 + }, + "/model/layers.10/attn/q_proj/Add/output_0.out19_11_60": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 555008 + }, + "/model/layers.10/attn/q_rotary/RotaryEmbedding/output_0.out19_3_10_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 556800 + }, + "/model/layers.10/attn/GroupQueryAttention/output_0.out19_3_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 558592 + }, + "/model/layers.10/attn/o_proj/MatMulNBits/output_0.out19_11_62": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 560384 + }, + "/model/layers.10/post_attention_layernorm/output_3.out19_15_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 562176 + }, + "/model/layers.10/post_attention_layernorm/output_0.out19_15_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 563968 + }, + "/model/layers.10/mlp/gate_proj/MatMulNBits/output_0.out19_11_63": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 565760 + }, + "/model/layers.10/mlp/up_proj/MatMulNBits/output_0.out19_11_64": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 575488 + }, + "InsertedPrecisionFreeCast_/model/layers.10/mlp/act_fn/Mul/output_0.in.inter.out19_6_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 585216 + }, + "/model/layers.10/mlp/Mul/output_0.out19_12_10": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 594944 + }, + "/model/layers.10/mlp/down_proj/MatMulNBits/output_0.out19_11_65": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 604672 + }, + "/model/layers.11/input_layernorm/output_3.out19_15_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 606464 + }, + "/model/layers.11/input_layernorm/output_0.out19_15_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 608256 + }, + "/model/layers.11/attn/k_proj/Add/output_0.out19_11_67": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 610048 + }, + "/model/layers.11/attn/q_proj/Add/output_0.out19_11_66": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 610304 + }, + "/model/layers.11/attn/q_rotary/RotaryEmbedding/output_0.out19_3_11_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 612096 + }, + "/model/layers.11/attn/GroupQueryAttention/output_0.out19_3_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 613888 + }, + "/model/layers.11/attn/o_proj/MatMulNBits/output_0.out19_11_68": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 615680 + }, + "/model/layers.11/post_attention_layernorm/output_3.out19_15_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 617472 + }, + "/model/layers.11/post_attention_layernorm/output_0.out19_15_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 619264 + }, + "/model/layers.11/mlp/gate_proj/MatMulNBits/output_0.out19_11_69": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 621056 + }, + "/model/layers.11/mlp/up_proj/MatMulNBits/output_0.out19_11_70": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 630784 + }, + "InsertedPrecisionFreeCast_/model/layers.11/mlp/act_fn/Mul/output_0.in.inter.out19_6_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 640512 + }, + "/model/layers.11/mlp/Mul/output_0.out19_12_11": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 650240 + }, + "/model/layers.11/mlp/down_proj/MatMulNBits/output_0.out19_11_71": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 659968 + }, + "/model/layers.12/input_layernorm/output_3.out19_15_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 661760 + }, + "/model/layers.12/input_layernorm/output_0.out19_15_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 663552 + }, + "/model/layers.12/attn/k_proj/Add/output_0.out19_11_73": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 665344 + }, + "/model/layers.12/attn/q_proj/Add/output_0.out19_11_72": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 665600 + }, + "/model/layers.12/attn/q_rotary/RotaryEmbedding/output_0.out19_3_12_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 667392 + }, + "/model/layers.12/attn/GroupQueryAttention/output_0.out19_3_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 669184 + }, + "/model/layers.12/attn/o_proj/MatMulNBits/output_0.out19_11_74": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 670976 + }, + "/model/layers.12/post_attention_layernorm/output_3.out19_15_24": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 672768 + }, + "/model/layers.12/post_attention_layernorm/output_0.out19_15_24": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 674560 + }, + "/model/layers.12/mlp/gate_proj/MatMulNBits/output_0.out19_11_75": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 676352 + }, + "/model/layers.12/mlp/up_proj/MatMulNBits/output_0.out19_11_76": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 686080 + }, + "InsertedPrecisionFreeCast_/model/layers.12/mlp/act_fn/Mul/output_0.in.inter.out19_6_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 695808 + }, + "/model/layers.12/mlp/Mul/output_0.out19_12_12": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 705536 + }, + "/model/layers.12/mlp/down_proj/MatMulNBits/output_0.out19_11_77": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 715264 + }, + "/model/layers.13/input_layernorm/output_3.out19_15_25": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 717056 + }, + "/model/layers.13/input_layernorm/output_0.out19_15_25": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 718848 + }, + "/model/layers.13/attn/k_proj/Add/output_0.out19_11_79": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 720640 + }, + "/model/layers.13/attn/q_proj/Add/output_0.out19_11_78": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 720896 + }, + "/model/layers.13/attn/q_rotary/RotaryEmbedding/output_0.out19_3_13_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 722688 + }, + "/model/layers.13/attn/GroupQueryAttention/output_0.out19_3_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 724480 + }, + "/model/layers.13/attn/o_proj/MatMulNBits/output_0.out19_11_80": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 726272 + }, + "/model/layers.13/post_attention_layernorm/output_3.out19_15_26": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 728064 + }, + "/model/layers.13/post_attention_layernorm/output_0.out19_15_26": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 729856 + }, + "/model/layers.13/mlp/gate_proj/MatMulNBits/output_0.out19_11_81": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 731648 + }, + "/model/layers.13/mlp/up_proj/MatMulNBits/output_0.out19_11_82": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 741376 + }, + "InsertedPrecisionFreeCast_/model/layers.13/mlp/act_fn/Mul/output_0.in.inter.out19_6_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 751104 + }, + "/model/layers.13/mlp/Mul/output_0.out19_12_13": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 760832 + }, + "/model/layers.13/mlp/down_proj/MatMulNBits/output_0.out19_11_83": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 770560 + }, + "/model/layers.14/input_layernorm/output_3.out19_15_27": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 772352 + }, + "/model/layers.14/input_layernorm/output_0.out19_15_27": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 774144 + }, + "/model/layers.14/attn/k_proj/Add/output_0.out19_11_85": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 775936 + }, + "/model/layers.14/attn/q_proj/Add/output_0.out19_11_84": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 776192 + }, + "/model/layers.14/attn/q_rotary/RotaryEmbedding/output_0.out19_3_14_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 777984 + }, + "/model/layers.14/attn/GroupQueryAttention/output_0.out19_3_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 779776 + }, + "/model/layers.14/attn/o_proj/MatMulNBits/output_0.out19_11_86": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 781568 + }, + "/model/layers.14/post_attention_layernorm/output_3.out19_15_28": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 783360 + }, + "/model/layers.14/post_attention_layernorm/output_0.out19_15_28": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 785152 + }, + "/model/layers.14/mlp/gate_proj/MatMulNBits/output_0.out19_11_87": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 786944 + }, + "/model/layers.14/mlp/up_proj/MatMulNBits/output_0.out19_11_88": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 796672 + }, + "InsertedPrecisionFreeCast_/model/layers.14/mlp/act_fn/Mul/output_0.in.inter.out19_6_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 806400 + }, + "/model/layers.14/mlp/Mul/output_0.out19_12_14": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 816128 + }, + "/model/layers.14/mlp/down_proj/MatMulNBits/output_0.out19_11_89": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 825856 + }, + "/model/layers.15/input_layernorm/output_3.out19_15_29": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 827648 + }, + "/model/layers.15/input_layernorm/output_0.out19_15_29": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 829440 + }, + "/model/layers.15/attn/k_proj/Add/output_0.out19_11_91": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 831232 + }, + "/model/layers.15/attn/q_proj/Add/output_0.out19_11_90": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 831488 + }, + "/model/layers.15/attn/q_rotary/RotaryEmbedding/output_0.out19_3_15_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 833280 + }, + "/model/layers.15/attn/GroupQueryAttention/output_0.out19_3_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 835072 + }, + "/model/layers.15/attn/o_proj/MatMulNBits/output_0.out19_11_92": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 836864 + }, + "/model/layers.15/post_attention_layernorm/output_3.out19_15_30": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 838656 + }, + "/model/layers.15/post_attention_layernorm/output_0.out19_15_30": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 840448 + }, + "/model/layers.15/mlp/gate_proj/MatMulNBits/output_0.out19_11_93": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 842240 + }, + "/model/layers.15/mlp/up_proj/MatMulNBits/output_0.out19_11_94": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 851968 + }, + "InsertedPrecisionFreeCast_/model/layers.15/mlp/act_fn/Mul/output_0.in.inter.out19_6_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 861696 + }, + "/model/layers.15/mlp/Mul/output_0.out19_12_15": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 871424 + }, + "/model/layers.15/mlp/down_proj/MatMulNBits/output_0.out19_11_95": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 881152 + }, + "/model/layers.16/input_layernorm/output_3.out19_15_31": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 882944 + }, + "/model/layers.16/input_layernorm/output_0.out19_15_31": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 884736 + }, + "/model/layers.16/attn/k_proj/Add/output_0.out19_11_97": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 886528 + }, + "/model/layers.16/attn/q_proj/Add/output_0.out19_11_96": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 886784 + }, + "/model/layers.16/attn/q_rotary/RotaryEmbedding/output_0.out19_3_16_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 888576 + }, + "/model/layers.16/attn/GroupQueryAttention/output_0.out19_3_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 890368 + }, + "/model/layers.16/attn/o_proj/MatMulNBits/output_0.out19_11_98": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 892160 + }, + "/model/layers.16/post_attention_layernorm/output_3.out19_15_32": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 893952 + }, + "/model/layers.16/post_attention_layernorm/output_0.out19_15_32": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 895744 + }, + "/model/layers.16/mlp/gate_proj/MatMulNBits/output_0.out19_11_99": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 897536 + }, + "/model/layers.16/mlp/up_proj/MatMulNBits/output_0.out19_11_100": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 907264 + }, + "InsertedPrecisionFreeCast_/model/layers.16/mlp/act_fn/Mul/output_0.in.inter.out19_6_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 916992 + }, + "/model/layers.16/mlp/Mul/output_0.out19_12_16": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 926720 + }, + "/model/layers.16/mlp/down_proj/MatMulNBits/output_0.out19_11_101": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 936448 + }, + "/model/layers.17/input_layernorm/output_3.out19_15_33": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 938240 + }, + "/model/layers.17/input_layernorm/output_0.out19_15_33": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 940032 + }, + "/model/layers.17/attn/k_proj/Add/output_0.out19_11_103": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 941824 + }, + "/model/layers.17/attn/q_proj/Add/output_0.out19_11_102": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 942080 + }, + "/model/layers.17/attn/q_rotary/RotaryEmbedding/output_0.out19_3_17_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 943872 + }, + "/model/layers.17/attn/GroupQueryAttention/output_0.out19_3_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 945664 + }, + "/model/layers.17/attn/o_proj/MatMulNBits/output_0.out19_11_104": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 947456 + }, + "/model/layers.17/post_attention_layernorm/output_3.out19_15_34": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 949248 + }, + "/model/layers.17/post_attention_layernorm/output_0.out19_15_34": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 951040 + }, + "/model/layers.17/mlp/gate_proj/MatMulNBits/output_0.out19_11_105": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 952832 + }, + "/model/layers.17/mlp/up_proj/MatMulNBits/output_0.out19_11_106": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 962560 + }, + "InsertedPrecisionFreeCast_/model/layers.17/mlp/act_fn/Mul/output_0.in.inter.out19_6_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 972288 + }, + "/model/layers.17/mlp/Mul/output_0.out19_12_17": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 982016 + }, + "/model/layers.17/mlp/down_proj/MatMulNBits/output_0.out19_11_107": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 991744 + }, + "/model/layers.18/input_layernorm/output_3.out19_15_35": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 993536 + }, + "/model/layers.18/input_layernorm/output_0.out19_15_35": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 995328 + }, + "/model/layers.18/attn/k_proj/Add/output_0.out19_11_109": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 997120 + }, + "/model/layers.18/attn/q_proj/Add/output_0.out19_11_108": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 997376 + }, + "/model/layers.18/attn/q_rotary/RotaryEmbedding/output_0.out19_3_18_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 999168 + }, + "/model/layers.18/attn/GroupQueryAttention/output_0.out19_3_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1000960 + }, + "/model/layers.18/attn/o_proj/MatMulNBits/output_0.out19_11_110": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1002752 + }, + "/model/layers.18/post_attention_layernorm/output_3.out19_15_36": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1004544 + }, + "/model/layers.18/post_attention_layernorm/output_0.out19_15_36": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1006336 + }, + "/model/layers.18/mlp/gate_proj/MatMulNBits/output_0.out19_11_111": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1008128 + }, + "/model/layers.18/mlp/up_proj/MatMulNBits/output_0.out19_11_112": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1017856 + }, + "InsertedPrecisionFreeCast_/model/layers.18/mlp/act_fn/Mul/output_0.in.inter.out19_6_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1027584 + }, + "/model/layers.18/mlp/Mul/output_0.out19_12_18": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1037312 + }, + "/model/layers.18/mlp/down_proj/MatMulNBits/output_0.out19_11_113": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1047040 + }, + "/model/layers.19/input_layernorm/output_3.out19_15_37": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1048832 + }, + "/model/layers.19/input_layernorm/output_0.out19_15_37": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1050624 + }, + "/model/layers.19/attn/k_proj/Add/output_0.out19_11_115": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1052416 + }, + "/model/layers.19/attn/q_proj/Add/output_0.out19_11_114": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1052672 + }, + "/model/layers.19/attn/q_rotary/RotaryEmbedding/output_0.out19_3_19_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1054464 + }, + "/model/layers.19/attn/GroupQueryAttention/output_0.out19_3_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1056256 + }, + "/model/layers.19/attn/o_proj/MatMulNBits/output_0.out19_11_116": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1058048 + }, + "/model/layers.19/post_attention_layernorm/output_3.out19_15_38": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1059840 + }, + "/model/layers.19/post_attention_layernorm/output_0.out19_15_38": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1061632 + }, + "/model/layers.19/mlp/gate_proj/MatMulNBits/output_0.out19_11_117": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1063424 + }, + "/model/layers.19/mlp/up_proj/MatMulNBits/output_0.out19_11_118": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1073152 + }, + "InsertedPrecisionFreeCast_/model/layers.19/mlp/act_fn/Mul/output_0.in.inter.out19_6_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1082880 + }, + "/model/layers.19/mlp/Mul/output_0.out19_12_19": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1092608 + }, + "/model/layers.19/mlp/down_proj/MatMulNBits/output_0.out19_11_119": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1102336 + }, + "/model/layers.20/input_layernorm/output_3.out19_15_39": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1104128 + }, + "/model/layers.20/input_layernorm/output_0.out19_15_39": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1105920 + }, + "/model/layers.20/attn/k_proj/Add/output_0.out19_11_121": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1107712 + }, + "/model/layers.20/attn/q_proj/Add/output_0.out19_11_120": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1107968 + }, + "/model/layers.20/attn/q_rotary/RotaryEmbedding/output_0.out19_3_20_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1109760 + }, + "/model/layers.20/attn/GroupQueryAttention/output_0.out19_3_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1111552 + }, + "/model/layers.20/attn/o_proj/MatMulNBits/output_0.out19_11_122": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1113344 + }, + "/model/layers.20/post_attention_layernorm/output_3.out19_15_40": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1115136 + }, + "/model/layers.20/post_attention_layernorm/output_0.out19_15_40": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1116928 + }, + "/model/layers.20/mlp/gate_proj/MatMulNBits/output_0.out19_11_123": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1118720 + }, + "/model/layers.20/mlp/up_proj/MatMulNBits/output_0.out19_11_124": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1128448 + }, + "InsertedPrecisionFreeCast_/model/layers.20/mlp/act_fn/Mul/output_0.in.inter.out19_6_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1138176 + }, + "/model/layers.20/mlp/Mul/output_0.out19_12_20": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1147904 + }, + "/model/layers.20/mlp/down_proj/MatMulNBits/output_0.out19_11_125": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1157632 + }, + "/model/layers.21/input_layernorm/output_3.out19_15_41": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1159424 + }, + "/model/layers.21/input_layernorm/output_0.out19_15_41": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1161216 + }, + "/model/layers.21/attn/k_proj/Add/output_0.out19_11_127": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1163008 + }, + "/model/layers.21/attn/q_proj/Add/output_0.out19_11_126": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1163264 + }, + "/model/layers.21/attn/q_rotary/RotaryEmbedding/output_0.out19_3_21_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1165056 + }, + "/model/layers.21/attn/GroupQueryAttention/output_0.out19_3_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1166848 + }, + "/model/layers.21/attn/o_proj/MatMulNBits/output_0.out19_11_128": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1168640 + }, + "/model/layers.21/post_attention_layernorm/output_3.out19_15_42": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1170432 + }, + "/model/layers.21/post_attention_layernorm/output_0.out19_15_42": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1172224 + }, + "/model/layers.21/mlp/gate_proj/MatMulNBits/output_0.out19_11_129": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1174016 + }, + "/model/layers.21/mlp/up_proj/MatMulNBits/output_0.out19_11_130": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1183744 + }, + "InsertedPrecisionFreeCast_/model/layers.21/mlp/act_fn/Mul/output_0.in.inter.out19_6_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1193472 + }, + "/model/layers.21/mlp/Mul/output_0.out19_12_21": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1203200 + }, + "/model/layers.21/mlp/down_proj/MatMulNBits/output_0.out19_11_131": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1212928 + }, + "/model/layers.22/input_layernorm/output_3.out19_15_43": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1214720 + }, + "/model/layers.22/input_layernorm/output_0.out19_15_43": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1216512 + }, + "/model/layers.22/attn/k_proj/Add/output_0.out19_11_133": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1218304 + }, + "/model/layers.22/attn/q_proj/Add/output_0.out19_11_132": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1218560 + }, + "/model/layers.22/attn/q_rotary/RotaryEmbedding/output_0.out19_3_22_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1220352 + }, + "/model/layers.22/attn/GroupQueryAttention/output_0.out19_3_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1222144 + }, + "/model/layers.22/attn/o_proj/MatMulNBits/output_0.out19_11_134": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1223936 + }, + "/model/layers.22/post_attention_layernorm/output_3.out19_15_44": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1225728 + }, + "/model/layers.22/post_attention_layernorm/output_0.out19_15_44": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1227520 + }, + "/model/layers.22/mlp/gate_proj/MatMulNBits/output_0.out19_11_135": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1229312 + }, + "/model/layers.22/mlp/up_proj/MatMulNBits/output_0.out19_11_136": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1239040 + }, + "InsertedPrecisionFreeCast_/model/layers.22/mlp/act_fn/Mul/output_0.in.inter.out19_6_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1248768 + }, + "/model/layers.22/mlp/Mul/output_0.out19_12_22": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1258496 + }, + "/model/layers.22/mlp/down_proj/MatMulNBits/output_0.out19_11_137": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1268224 + }, + "/model/layers.23/input_layernorm/output_3.out19_15_45": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1270016 + }, + "/model/layers.23/input_layernorm/output_0.out19_15_45": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1271808 + }, + "/model/layers.23/attn/k_proj/Add/output_0.out19_11_139": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 2, + 1, + 64 + ], + "size_in_bytes": 256, + "op_tensor_size": 256, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1273600 + }, + "/model/layers.23/attn/q_proj/Add/output_0.out19_11_138": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1273856 + }, + "/model/layers.23/attn/q_rotary/RotaryEmbedding/output_0.out19_3_23_1": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 14, + 1, + 64 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1275648 + }, + "/model/layers.23/attn/GroupQueryAttention/output_0.out19_3_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1277440 + }, + "/model/layers.23/attn/o_proj/MatMulNBits/output_0.out19_11_140": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1279232 + }, + "/model/layers.23/post_attention_layernorm/output_3.out19_15_46": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1281024 + }, + "/model/layers.23/post_attention_layernorm/output_0.out19_15_46": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1282816 + }, + "/model/layers.23/mlp/gate_proj/MatMulNBits/output_0.out19_11_141": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1284608 + }, + "/model/layers.23/mlp/up_proj/MatMulNBits/output_0.out19_11_142": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1294336 + }, + "InsertedPrecisionFreeCast_/model/layers.23/mlp/act_fn/Mul/output_0.in.inter.out19_6_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1304064 + }, + "/model/layers.23/mlp/Mul/output_0.out19_12_23": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 4864 + ], + "size_in_bytes": 9728, + "op_tensor_size": 9728, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1313792 + }, + "/model/layers.23/mlp/down_proj/MatMulNBits/output_0.out19_11_143": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1323520 + }, + "dummy_19_15_47": { + "packed_buffer_label": "scratch", + "xrt_arg_id": 2, + "dtype": "bfloat16", + "shape": [ + 1, + 1, + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "dynamic_shapes": [ + "False", + "sequence_length_padded", + "False" + ], + "offset": 1325312 + }, + "model.layers.0.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 0, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_0.const", + "file_size": 1792 + }, + "eps_19_14_0": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 1792, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_1.const", + "file_size": 2 + }, + "model.layers.0.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 1796, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_2.const", + "file_size": 114688 + }, + "model.layers.0.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 116484, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_3.const", + "file_size": 512 + }, + "model.layers.0.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 116996, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_4.const", + "file_size": 3584 + }, + "model.layers.0.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 120580, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_5.const", + "file_size": 896 + }, + "model.layers.0.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 121476, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_6.const", + "file_size": 802816 + }, + "model.layers.0.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 924292, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_7.const", + "file_size": 3584 + }, + "model.layers.0.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 927876, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_8.const", + "file_size": 25088 + }, + "model.layers.0.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 952964, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_9.const", + "file_size": 6272 + }, + "model.layers.0.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 959236, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_10.const", + "file_size": 114688 + }, + "model.layers.0.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 1073924, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_11.const", + "file_size": 512 + }, + "model.layers.0.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 1074436, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_12.const", + "file_size": 3584 + }, + "model.layers.0.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 1078020, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_13.const", + "file_size": 896 + }, + "model.layers.0.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 1078916, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_14.const", + "file_size": 802816 + }, + "model.layers.0.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 1881732, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_15.const", + "file_size": 3584 + }, + "model.layers.0.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 1885316, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_16.const", + "file_size": 25088 + }, + "model.layers.0.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 1910404, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_17.const", + "file_size": 6272 + }, + "model.layers.0.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 1916676, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_18.const", + "file_size": 1792 + }, + "eps_19_15_0": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 1918468, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_19.const", + "file_size": 2 + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 1918472, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_20.const", + "file_size": 4358144 + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 6276616, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_21.const", + "file_size": 19456 + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 6296072, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_22.const", + "file_size": 136192 + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 6432264, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_23.const", + "file_size": 34048 + }, + "model.layers.0.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 6466312, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_24.const", + "file_size": 4358144 + }, + "model.layers.0.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 10824456, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_25.const", + "file_size": 19456 + }, + "model.layers.0.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 10843912, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_26.const", + "file_size": 136192 + }, + "model.layers.0.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 10980104, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_27.const", + "file_size": 34048 + }, + "model.layers.0.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 11014152, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_28.const", + "file_size": 4358144 + }, + "model.layers.0.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 15372296, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_29.const", + "file_size": 3584 + }, + "model.layers.0.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 15375880, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_30.const", + "file_size": 136192 + }, + "model.layers.0.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 15512072, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_31.const", + "file_size": 34048 + }, + "model.layers.1.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 15546120, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_32.const", + "file_size": 1792 + }, + "eps_19_15_1": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 15547912, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_33.const", + "file_size": 2 + }, + "model.layers.1.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 15547916, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_34.const", + "file_size": 114688 + }, + "model.layers.1.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 15662604, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_35.const", + "file_size": 512 + }, + "model.layers.1.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 15663116, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_36.const", + "file_size": 3584 + }, + "model.layers.1.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 15666700, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_37.const", + "file_size": 896 + }, + "model.layers.1.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 15667596, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_38.const", + "file_size": 802816 + }, + "model.layers.1.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 16470412, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_39.const", + "file_size": 3584 + }, + "model.layers.1.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 16473996, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_40.const", + "file_size": 25088 + }, + "model.layers.1.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 16499084, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_41.const", + "file_size": 6272 + }, + "model.layers.1.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 16505356, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_42.const", + "file_size": 114688 + }, + "model.layers.1.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 16620044, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_43.const", + "file_size": 512 + }, + "model.layers.1.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 16620556, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_44.const", + "file_size": 3584 + }, + "model.layers.1.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 16624140, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_45.const", + "file_size": 896 + }, + "model.layers.1.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 16625036, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_46.const", + "file_size": 802816 + }, + "model.layers.1.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 17427852, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_47.const", + "file_size": 3584 + }, + "model.layers.1.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 17431436, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_48.const", + "file_size": 25088 + }, + "model.layers.1.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 17456524, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_49.const", + "file_size": 6272 + }, + "model.layers.1.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 17462796, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_50.const", + "file_size": 1792 + }, + "eps_19_15_2": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 17464588, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_51.const", + "file_size": 2 + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 17464592, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_52.const", + "file_size": 4358144 + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 21822736, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_53.const", + "file_size": 19456 + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 21842192, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_54.const", + "file_size": 136192 + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 21978384, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_55.const", + "file_size": 34048 + }, + "model.layers.1.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 22012432, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_56.const", + "file_size": 4358144 + }, + "model.layers.1.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 26370576, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_57.const", + "file_size": 19456 + }, + "model.layers.1.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 26390032, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_58.const", + "file_size": 136192 + }, + "model.layers.1.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 26526224, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_59.const", + "file_size": 34048 + }, + "model.layers.1.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 26560272, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_60.const", + "file_size": 4358144 + }, + "model.layers.1.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 30918416, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_61.const", + "file_size": 3584 + }, + "model.layers.1.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 30922000, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_62.const", + "file_size": 136192 + }, + "model.layers.1.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 31058192, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_63.const", + "file_size": 34048 + }, + "model.layers.2.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 31092240, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_64.const", + "file_size": 1792 + }, + "eps_19_15_3": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 31094032, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_65.const", + "file_size": 2 + }, + "model.layers.2.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 31094036, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_66.const", + "file_size": 114688 + }, + "model.layers.2.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 31208724, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_67.const", + "file_size": 512 + }, + "model.layers.2.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 31209236, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_68.const", + "file_size": 3584 + }, + "model.layers.2.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 31212820, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_69.const", + "file_size": 896 + }, + "model.layers.2.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 31213716, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_70.const", + "file_size": 802816 + }, + "model.layers.2.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 32016532, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_71.const", + "file_size": 3584 + }, + "model.layers.2.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 32020116, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_72.const", + "file_size": 25088 + }, + "model.layers.2.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 32045204, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_73.const", + "file_size": 6272 + }, + "model.layers.2.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 32051476, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_74.const", + "file_size": 114688 + }, + "model.layers.2.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 32166164, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_75.const", + "file_size": 512 + }, + "model.layers.2.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 32166676, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_76.const", + "file_size": 3584 + }, + "model.layers.2.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 32170260, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_77.const", + "file_size": 896 + }, + "model.layers.2.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 32171156, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_78.const", + "file_size": 802816 + }, + "model.layers.2.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 32973972, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_79.const", + "file_size": 3584 + }, + "model.layers.2.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 32977556, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_80.const", + "file_size": 25088 + }, + "model.layers.2.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 33002644, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_81.const", + "file_size": 6272 + }, + "model.layers.2.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 33008916, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_82.const", + "file_size": 1792 + }, + "eps_19_15_4": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 33010708, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_83.const", + "file_size": 2 + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 33010712, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_84.const", + "file_size": 4358144 + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 37368856, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_85.const", + "file_size": 19456 + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 37388312, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_86.const", + "file_size": 136192 + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 37524504, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_87.const", + "file_size": 34048 + }, + "model.layers.2.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 37558552, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_88.const", + "file_size": 4358144 + }, + "model.layers.2.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 41916696, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_89.const", + "file_size": 19456 + }, + "model.layers.2.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 41936152, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_90.const", + "file_size": 136192 + }, + "model.layers.2.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 42072344, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_91.const", + "file_size": 34048 + }, + "model.layers.2.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 42106392, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_92.const", + "file_size": 4358144 + }, + "model.layers.2.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 46464536, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_93.const", + "file_size": 3584 + }, + "model.layers.2.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 46468120, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_94.const", + "file_size": 136192 + }, + "model.layers.2.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 46604312, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_95.const", + "file_size": 34048 + }, + "model.layers.3.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 46638360, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_96.const", + "file_size": 1792 + }, + "eps_19_15_5": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 46640152, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_97.const", + "file_size": 2 + }, + "model.layers.3.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 46640156, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_98.const", + "file_size": 114688 + }, + "model.layers.3.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 46754844, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_99.const", + "file_size": 512 + }, + "model.layers.3.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 46755356, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_100.const", + "file_size": 3584 + }, + "model.layers.3.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 46758940, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_101.const", + "file_size": 896 + }, + "model.layers.3.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 46759836, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_102.const", + "file_size": 802816 + }, + "model.layers.3.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 47562652, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_103.const", + "file_size": 3584 + }, + "model.layers.3.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 47566236, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_104.const", + "file_size": 25088 + }, + "model.layers.3.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 47591324, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_105.const", + "file_size": 6272 + }, + "model.layers.3.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 47597596, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_106.const", + "file_size": 114688 + }, + "model.layers.3.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 47712284, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_107.const", + "file_size": 512 + }, + "model.layers.3.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 47712796, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_108.const", + "file_size": 3584 + }, + "model.layers.3.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 47716380, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_109.const", + "file_size": 896 + }, + "model.layers.3.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 47717276, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_110.const", + "file_size": 802816 + }, + "model.layers.3.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 48520092, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_111.const", + "file_size": 3584 + }, + "model.layers.3.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 48523676, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_112.const", + "file_size": 25088 + }, + "model.layers.3.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 48548764, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_113.const", + "file_size": 6272 + }, + "model.layers.3.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 48555036, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_114.const", + "file_size": 1792 + }, + "eps_19_15_6": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 48556828, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_115.const", + "file_size": 2 + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 48556832, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_116.const", + "file_size": 4358144 + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 52914976, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_117.const", + "file_size": 19456 + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 52934432, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_118.const", + "file_size": 136192 + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 53070624, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_119.const", + "file_size": 34048 + }, + "model.layers.3.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 53104672, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_120.const", + "file_size": 4358144 + }, + "model.layers.3.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 57462816, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_121.const", + "file_size": 19456 + }, + "model.layers.3.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 57482272, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_122.const", + "file_size": 136192 + }, + "model.layers.3.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 57618464, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_123.const", + "file_size": 34048 + }, + "model.layers.3.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 57652512, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_124.const", + "file_size": 4358144 + }, + "model.layers.3.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 62010656, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_125.const", + "file_size": 3584 + }, + "model.layers.3.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 62014240, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_126.const", + "file_size": 136192 + }, + "model.layers.3.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 62150432, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_127.const", + "file_size": 34048 + }, + "model.layers.4.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 62184480, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_128.const", + "file_size": 1792 + }, + "eps_19_15_7": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 62186272, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_129.const", + "file_size": 2 + }, + "model.layers.4.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 62186276, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_130.const", + "file_size": 114688 + }, + "model.layers.4.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 62300964, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_131.const", + "file_size": 512 + }, + "model.layers.4.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 62301476, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_132.const", + "file_size": 3584 + }, + "model.layers.4.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 62305060, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_133.const", + "file_size": 896 + }, + "model.layers.4.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 62305956, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_134.const", + "file_size": 802816 + }, + "model.layers.4.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 63108772, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_135.const", + "file_size": 3584 + }, + "model.layers.4.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 63112356, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_136.const", + "file_size": 25088 + }, + "model.layers.4.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 63137444, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_137.const", + "file_size": 6272 + }, + "model.layers.4.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 63143716, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_138.const", + "file_size": 114688 + }, + "model.layers.4.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 63258404, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_139.const", + "file_size": 512 + }, + "model.layers.4.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 63258916, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_140.const", + "file_size": 3584 + }, + "model.layers.4.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 63262500, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_141.const", + "file_size": 896 + }, + "model.layers.4.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 63263396, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_142.const", + "file_size": 802816 + }, + "model.layers.4.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 64066212, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_143.const", + "file_size": 3584 + }, + "model.layers.4.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 64069796, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_144.const", + "file_size": 25088 + }, + "model.layers.4.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 64094884, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_145.const", + "file_size": 6272 + }, + "model.layers.4.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 64101156, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_146.const", + "file_size": 1792 + }, + "eps_19_15_8": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 64102948, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_147.const", + "file_size": 2 + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 64102952, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_148.const", + "file_size": 4358144 + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 68461096, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_149.const", + "file_size": 19456 + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 68480552, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_150.const", + "file_size": 136192 + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 68616744, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_151.const", + "file_size": 34048 + }, + "model.layers.4.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 68650792, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_152.const", + "file_size": 4358144 + }, + "model.layers.4.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 73008936, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_153.const", + "file_size": 19456 + }, + "model.layers.4.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 73028392, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_154.const", + "file_size": 136192 + }, + "model.layers.4.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 73164584, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_155.const", + "file_size": 34048 + }, + "model.layers.4.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 73198632, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_156.const", + "file_size": 4358144 + }, + "model.layers.4.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 77556776, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_157.const", + "file_size": 3584 + }, + "model.layers.4.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 77560360, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_158.const", + "file_size": 136192 + }, + "model.layers.4.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 77696552, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_159.const", + "file_size": 34048 + }, + "model.layers.5.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 77730600, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_160.const", + "file_size": 1792 + }, + "eps_19_15_9": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 77732392, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_161.const", + "file_size": 2 + }, + "model.layers.5.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 77732396, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_162.const", + "file_size": 114688 + }, + "model.layers.5.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 77847084, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_163.const", + "file_size": 512 + }, + "model.layers.5.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 77847596, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_164.const", + "file_size": 3584 + }, + "model.layers.5.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 77851180, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_165.const", + "file_size": 896 + }, + "model.layers.5.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 77852076, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_166.const", + "file_size": 802816 + }, + "model.layers.5.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 78654892, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_167.const", + "file_size": 3584 + }, + "model.layers.5.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 78658476, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_168.const", + "file_size": 25088 + }, + "model.layers.5.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 78683564, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_169.const", + "file_size": 6272 + }, + "model.layers.5.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 78689836, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_170.const", + "file_size": 114688 + }, + "model.layers.5.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 78804524, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_171.const", + "file_size": 512 + }, + "model.layers.5.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 78805036, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_172.const", + "file_size": 3584 + }, + "model.layers.5.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 78808620, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_173.const", + "file_size": 896 + }, + "model.layers.5.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 78809516, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_174.const", + "file_size": 802816 + }, + "model.layers.5.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 79612332, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_175.const", + "file_size": 3584 + }, + "model.layers.5.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 79615916, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_176.const", + "file_size": 25088 + }, + "model.layers.5.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 79641004, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_177.const", + "file_size": 6272 + }, + "model.layers.5.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 79647276, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_178.const", + "file_size": 1792 + }, + "eps_19_15_10": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 79649068, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_179.const", + "file_size": 2 + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 79649072, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_180.const", + "file_size": 4358144 + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 84007216, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_181.const", + "file_size": 19456 + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 84026672, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_182.const", + "file_size": 136192 + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 84162864, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_183.const", + "file_size": 34048 + }, + "model.layers.5.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 84196912, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_184.const", + "file_size": 4358144 + }, + "model.layers.5.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 88555056, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_185.const", + "file_size": 19456 + }, + "model.layers.5.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 88574512, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_186.const", + "file_size": 136192 + }, + "model.layers.5.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 88710704, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_187.const", + "file_size": 34048 + }, + "model.layers.5.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 88744752, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_188.const", + "file_size": 4358144 + }, + "model.layers.5.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 93102896, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_189.const", + "file_size": 3584 + }, + "model.layers.5.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 93106480, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_190.const", + "file_size": 136192 + }, + "model.layers.5.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 93242672, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_191.const", + "file_size": 34048 + }, + "model.layers.6.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 93276720, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_192.const", + "file_size": 1792 + }, + "eps_19_15_11": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 93278512, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_193.const", + "file_size": 2 + }, + "model.layers.6.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 93278516, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_194.const", + "file_size": 114688 + }, + "model.layers.6.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 93393204, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_195.const", + "file_size": 512 + }, + "model.layers.6.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 93393716, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_196.const", + "file_size": 3584 + }, + "model.layers.6.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 93397300, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_197.const", + "file_size": 896 + }, + "model.layers.6.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 93398196, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_198.const", + "file_size": 802816 + }, + "model.layers.6.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 94201012, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_199.const", + "file_size": 3584 + }, + "model.layers.6.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 94204596, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_200.const", + "file_size": 25088 + }, + "model.layers.6.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 94229684, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_201.const", + "file_size": 6272 + }, + "model.layers.6.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 94235956, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_202.const", + "file_size": 114688 + }, + "model.layers.6.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 94350644, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_203.const", + "file_size": 512 + }, + "model.layers.6.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 94351156, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_204.const", + "file_size": 3584 + }, + "model.layers.6.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 94354740, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_205.const", + "file_size": 896 + }, + "model.layers.6.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 94355636, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_206.const", + "file_size": 802816 + }, + "model.layers.6.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 95158452, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_207.const", + "file_size": 3584 + }, + "model.layers.6.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 95162036, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_208.const", + "file_size": 25088 + }, + "model.layers.6.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 95187124, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_209.const", + "file_size": 6272 + }, + "model.layers.6.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 95193396, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_210.const", + "file_size": 1792 + }, + "eps_19_15_12": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 95195188, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_211.const", + "file_size": 2 + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 95195192, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_212.const", + "file_size": 4358144 + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 99553336, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_213.const", + "file_size": 19456 + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 99572792, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_214.const", + "file_size": 136192 + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 99708984, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_215.const", + "file_size": 34048 + }, + "model.layers.6.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 99743032, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_216.const", + "file_size": 4358144 + }, + "model.layers.6.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 104101176, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_217.const", + "file_size": 19456 + }, + "model.layers.6.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 104120632, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_218.const", + "file_size": 136192 + }, + "model.layers.6.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 104256824, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_219.const", + "file_size": 34048 + }, + "model.layers.6.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 104290872, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_220.const", + "file_size": 4358144 + }, + "model.layers.6.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 108649016, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_221.const", + "file_size": 3584 + }, + "model.layers.6.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 108652600, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_222.const", + "file_size": 136192 + }, + "model.layers.6.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 108788792, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_223.const", + "file_size": 34048 + }, + "model.layers.7.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 108822840, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_224.const", + "file_size": 1792 + }, + "eps_19_15_13": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 108824632, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_225.const", + "file_size": 2 + }, + "model.layers.7.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 108824636, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_226.const", + "file_size": 114688 + }, + "model.layers.7.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 108939324, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_227.const", + "file_size": 512 + }, + "model.layers.7.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 108939836, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_228.const", + "file_size": 3584 + }, + "model.layers.7.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 108943420, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_229.const", + "file_size": 896 + }, + "model.layers.7.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 108944316, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_230.const", + "file_size": 802816 + }, + "model.layers.7.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 109747132, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_231.const", + "file_size": 3584 + }, + "model.layers.7.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 109750716, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_232.const", + "file_size": 25088 + }, + "model.layers.7.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 109775804, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_233.const", + "file_size": 6272 + }, + "model.layers.7.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 109782076, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_234.const", + "file_size": 114688 + }, + "model.layers.7.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 109896764, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_235.const", + "file_size": 512 + }, + "model.layers.7.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 109897276, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_236.const", + "file_size": 3584 + }, + "model.layers.7.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 109900860, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_237.const", + "file_size": 896 + }, + "model.layers.7.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 109901756, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_238.const", + "file_size": 802816 + }, + "model.layers.7.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 110704572, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_239.const", + "file_size": 3584 + }, + "model.layers.7.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 110708156, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_240.const", + "file_size": 25088 + }, + "model.layers.7.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 110733244, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_241.const", + "file_size": 6272 + }, + "model.layers.7.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 110739516, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_242.const", + "file_size": 1792 + }, + "eps_19_15_14": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 110741308, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_243.const", + "file_size": 2 + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 110741312, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_244.const", + "file_size": 4358144 + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 115099456, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_245.const", + "file_size": 19456 + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 115118912, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_246.const", + "file_size": 136192 + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 115255104, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_247.const", + "file_size": 34048 + }, + "model.layers.7.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 115289152, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_248.const", + "file_size": 4358144 + }, + "model.layers.7.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 119647296, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_249.const", + "file_size": 19456 + }, + "model.layers.7.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 119666752, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_250.const", + "file_size": 136192 + }, + "model.layers.7.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 119802944, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_251.const", + "file_size": 34048 + }, + "model.layers.7.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 119836992, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_252.const", + "file_size": 4358144 + }, + "model.layers.7.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 124195136, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_253.const", + "file_size": 3584 + }, + "model.layers.7.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 124198720, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_254.const", + "file_size": 136192 + }, + "model.layers.7.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 124334912, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_255.const", + "file_size": 34048 + }, + "model.layers.8.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 124368960, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_256.const", + "file_size": 1792 + }, + "eps_19_15_15": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 124370752, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_257.const", + "file_size": 2 + }, + "model.layers.8.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 124370756, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_258.const", + "file_size": 114688 + }, + "model.layers.8.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 124485444, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_259.const", + "file_size": 512 + }, + "model.layers.8.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 124485956, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_260.const", + "file_size": 3584 + }, + "model.layers.8.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 124489540, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_261.const", + "file_size": 896 + }, + "model.layers.8.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 124490436, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_262.const", + "file_size": 802816 + }, + "model.layers.8.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 125293252, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_263.const", + "file_size": 3584 + }, + "model.layers.8.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 125296836, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_264.const", + "file_size": 25088 + }, + "model.layers.8.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 125321924, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_265.const", + "file_size": 6272 + }, + "model.layers.8.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 125328196, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_266.const", + "file_size": 114688 + }, + "model.layers.8.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 125442884, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_267.const", + "file_size": 512 + }, + "model.layers.8.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 125443396, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_268.const", + "file_size": 3584 + }, + "model.layers.8.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 125446980, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_269.const", + "file_size": 896 + }, + "model.layers.8.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 125447876, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_270.const", + "file_size": 802816 + }, + "model.layers.8.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 126250692, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_271.const", + "file_size": 3584 + }, + "model.layers.8.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 126254276, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_272.const", + "file_size": 25088 + }, + "model.layers.8.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 126279364, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_273.const", + "file_size": 6272 + }, + "model.layers.8.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 126285636, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_274.const", + "file_size": 1792 + }, + "eps_19_15_16": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 126287428, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_275.const", + "file_size": 2 + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 126287432, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_276.const", + "file_size": 4358144 + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 130645576, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_277.const", + "file_size": 19456 + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 130665032, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_278.const", + "file_size": 136192 + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 130801224, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_279.const", + "file_size": 34048 + }, + "model.layers.8.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 130835272, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_280.const", + "file_size": 4358144 + }, + "model.layers.8.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 135193416, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_281.const", + "file_size": 19456 + }, + "model.layers.8.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 135212872, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_282.const", + "file_size": 136192 + }, + "model.layers.8.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 135349064, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_283.const", + "file_size": 34048 + }, + "model.layers.8.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 135383112, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_284.const", + "file_size": 4358144 + }, + "model.layers.8.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 139741256, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_285.const", + "file_size": 3584 + }, + "model.layers.8.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 139744840, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_286.const", + "file_size": 136192 + }, + "model.layers.8.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 139881032, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_287.const", + "file_size": 34048 + }, + "model.layers.9.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 139915080, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_288.const", + "file_size": 1792 + }, + "eps_19_15_17": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 139916872, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_289.const", + "file_size": 2 + }, + "model.layers.9.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 139916876, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_290.const", + "file_size": 114688 + }, + "model.layers.9.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 140031564, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_291.const", + "file_size": 512 + }, + "model.layers.9.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 140032076, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_292.const", + "file_size": 3584 + }, + "model.layers.9.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 140035660, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_293.const", + "file_size": 896 + }, + "model.layers.9.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 140036556, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_294.const", + "file_size": 802816 + }, + "model.layers.9.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 140839372, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_295.const", + "file_size": 3584 + }, + "model.layers.9.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 140842956, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_296.const", + "file_size": 25088 + }, + "model.layers.9.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 140868044, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_297.const", + "file_size": 6272 + }, + "model.layers.9.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 140874316, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_298.const", + "file_size": 114688 + }, + "model.layers.9.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 140989004, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_299.const", + "file_size": 512 + }, + "model.layers.9.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 140989516, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_300.const", + "file_size": 3584 + }, + "model.layers.9.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 140993100, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_301.const", + "file_size": 896 + }, + "model.layers.9.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 140993996, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_302.const", + "file_size": 802816 + }, + "model.layers.9.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 141796812, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_303.const", + "file_size": 3584 + }, + "model.layers.9.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 141800396, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_304.const", + "file_size": 25088 + }, + "model.layers.9.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 141825484, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_305.const", + "file_size": 6272 + }, + "model.layers.9.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 141831756, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_306.const", + "file_size": 1792 + }, + "eps_19_15_18": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 141833548, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_307.const", + "file_size": 2 + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 141833552, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_308.const", + "file_size": 4358144 + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 146191696, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_309.const", + "file_size": 19456 + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 146211152, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_310.const", + "file_size": 136192 + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 146347344, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_311.const", + "file_size": 34048 + }, + "model.layers.9.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 146381392, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_312.const", + "file_size": 4358144 + }, + "model.layers.9.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 150739536, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_313.const", + "file_size": 19456 + }, + "model.layers.9.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 150758992, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_314.const", + "file_size": 136192 + }, + "model.layers.9.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 150895184, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_315.const", + "file_size": 34048 + }, + "model.layers.9.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 150929232, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_316.const", + "file_size": 4358144 + }, + "model.layers.9.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 155287376, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_317.const", + "file_size": 3584 + }, + "model.layers.9.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 155290960, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_318.const", + "file_size": 136192 + }, + "model.layers.9.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 155427152, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_319.const", + "file_size": 34048 + }, + "model.layers.10.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 155461200, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_320.const", + "file_size": 1792 + }, + "eps_19_15_19": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 155462992, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_321.const", + "file_size": 2 + }, + "model.layers.10.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 155462996, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_322.const", + "file_size": 114688 + }, + "model.layers.10.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 155577684, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_323.const", + "file_size": 512 + }, + "model.layers.10.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 155578196, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_324.const", + "file_size": 3584 + }, + "model.layers.10.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 155581780, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_325.const", + "file_size": 896 + }, + "model.layers.10.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 155582676, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_326.const", + "file_size": 802816 + }, + "model.layers.10.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 156385492, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_327.const", + "file_size": 3584 + }, + "model.layers.10.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 156389076, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_328.const", + "file_size": 25088 + }, + "model.layers.10.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 156414164, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_329.const", + "file_size": 6272 + }, + "model.layers.10.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 156420436, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_330.const", + "file_size": 114688 + }, + "model.layers.10.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 156535124, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_331.const", + "file_size": 512 + }, + "model.layers.10.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 156535636, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_332.const", + "file_size": 3584 + }, + "model.layers.10.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 156539220, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_333.const", + "file_size": 896 + }, + "model.layers.10.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 156540116, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_334.const", + "file_size": 802816 + }, + "model.layers.10.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 157342932, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_335.const", + "file_size": 3584 + }, + "model.layers.10.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 157346516, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_336.const", + "file_size": 25088 + }, + "model.layers.10.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 157371604, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_337.const", + "file_size": 6272 + }, + "model.layers.10.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 157377876, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_338.const", + "file_size": 1792 + }, + "eps_19_15_20": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 157379668, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_339.const", + "file_size": 2 + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 157379672, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_340.const", + "file_size": 4358144 + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 161737816, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_341.const", + "file_size": 19456 + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 161757272, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_342.const", + "file_size": 136192 + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 161893464, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_343.const", + "file_size": 34048 + }, + "model.layers.10.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 161927512, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_344.const", + "file_size": 4358144 + }, + "model.layers.10.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 166285656, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_345.const", + "file_size": 19456 + }, + "model.layers.10.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 166305112, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_346.const", + "file_size": 136192 + }, + "model.layers.10.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 166441304, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_347.const", + "file_size": 34048 + }, + "model.layers.10.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 166475352, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_348.const", + "file_size": 4358144 + }, + "model.layers.10.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 170833496, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_349.const", + "file_size": 3584 + }, + "model.layers.10.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 170837080, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_350.const", + "file_size": 136192 + }, + "model.layers.10.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 170973272, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_351.const", + "file_size": 34048 + }, + "model.layers.11.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 171007320, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_352.const", + "file_size": 1792 + }, + "eps_19_15_21": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 171009112, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_353.const", + "file_size": 2 + }, + "model.layers.11.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 171009116, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_354.const", + "file_size": 114688 + }, + "model.layers.11.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 171123804, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_355.const", + "file_size": 512 + }, + "model.layers.11.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 171124316, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_356.const", + "file_size": 3584 + }, + "model.layers.11.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 171127900, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_357.const", + "file_size": 896 + }, + "model.layers.11.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 171128796, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_358.const", + "file_size": 802816 + }, + "model.layers.11.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 171931612, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_359.const", + "file_size": 3584 + }, + "model.layers.11.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 171935196, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_360.const", + "file_size": 25088 + }, + "model.layers.11.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 171960284, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_361.const", + "file_size": 6272 + }, + "model.layers.11.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 171966556, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_362.const", + "file_size": 114688 + }, + "model.layers.11.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 172081244, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_363.const", + "file_size": 512 + }, + "model.layers.11.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 172081756, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_364.const", + "file_size": 3584 + }, + "model.layers.11.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 172085340, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_365.const", + "file_size": 896 + }, + "model.layers.11.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 172086236, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_366.const", + "file_size": 802816 + }, + "model.layers.11.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 172889052, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_367.const", + "file_size": 3584 + }, + "model.layers.11.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 172892636, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_368.const", + "file_size": 25088 + }, + "model.layers.11.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 172917724, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_369.const", + "file_size": 6272 + }, + "model.layers.11.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 172923996, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_370.const", + "file_size": 1792 + }, + "eps_19_15_22": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 172925788, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_371.const", + "file_size": 2 + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 172925792, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_372.const", + "file_size": 4358144 + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 177283936, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_373.const", + "file_size": 19456 + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 177303392, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_374.const", + "file_size": 136192 + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 177439584, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_375.const", + "file_size": 34048 + }, + "model.layers.11.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 177473632, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_376.const", + "file_size": 4358144 + }, + "model.layers.11.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 181831776, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_377.const", + "file_size": 19456 + }, + "model.layers.11.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 181851232, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_378.const", + "file_size": 136192 + }, + "model.layers.11.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 181987424, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_379.const", + "file_size": 34048 + }, + "model.layers.11.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 182021472, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_380.const", + "file_size": 4358144 + }, + "model.layers.11.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 186379616, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_381.const", + "file_size": 3584 + }, + "model.layers.11.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 186383200, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_382.const", + "file_size": 136192 + }, + "model.layers.11.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 186519392, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_383.const", + "file_size": 34048 + }, + "model.layers.12.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 186553440, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_384.const", + "file_size": 1792 + }, + "eps_19_15_23": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 186555232, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_385.const", + "file_size": 2 + }, + "model.layers.12.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 186555236, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_386.const", + "file_size": 114688 + }, + "model.layers.12.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 186669924, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_387.const", + "file_size": 512 + }, + "model.layers.12.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 186670436, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_388.const", + "file_size": 3584 + }, + "model.layers.12.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 186674020, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_389.const", + "file_size": 896 + }, + "model.layers.12.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 186674916, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_390.const", + "file_size": 802816 + }, + "model.layers.12.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 187477732, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_391.const", + "file_size": 3584 + }, + "model.layers.12.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 187481316, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_392.const", + "file_size": 25088 + }, + "model.layers.12.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 187506404, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_393.const", + "file_size": 6272 + }, + "model.layers.12.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 187512676, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_394.const", + "file_size": 114688 + }, + "model.layers.12.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 187627364, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_395.const", + "file_size": 512 + }, + "model.layers.12.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 187627876, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_396.const", + "file_size": 3584 + }, + "model.layers.12.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 187631460, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_397.const", + "file_size": 896 + }, + "model.layers.12.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 187632356, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_398.const", + "file_size": 802816 + }, + "model.layers.12.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 188435172, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_399.const", + "file_size": 3584 + }, + "model.layers.12.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 188438756, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_400.const", + "file_size": 25088 + }, + "model.layers.12.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 188463844, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_401.const", + "file_size": 6272 + }, + "model.layers.12.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 188470116, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_402.const", + "file_size": 1792 + }, + "eps_19_15_24": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 188471908, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_403.const", + "file_size": 2 + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 188471912, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_404.const", + "file_size": 4358144 + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 192830056, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_405.const", + "file_size": 19456 + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 192849512, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_406.const", + "file_size": 136192 + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 192985704, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_407.const", + "file_size": 34048 + }, + "model.layers.12.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 193019752, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_408.const", + "file_size": 4358144 + }, + "model.layers.12.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 197377896, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_409.const", + "file_size": 19456 + }, + "model.layers.12.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 197397352, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_410.const", + "file_size": 136192 + }, + "model.layers.12.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 197533544, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_411.const", + "file_size": 34048 + }, + "model.layers.12.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 197567592, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_412.const", + "file_size": 4358144 + }, + "model.layers.12.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 201925736, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_413.const", + "file_size": 3584 + }, + "model.layers.12.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 201929320, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_414.const", + "file_size": 136192 + }, + "model.layers.12.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 202065512, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_415.const", + "file_size": 34048 + }, + "model.layers.13.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 202099560, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_416.const", + "file_size": 1792 + }, + "eps_19_15_25": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 202101352, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_417.const", + "file_size": 2 + }, + "model.layers.13.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 202101356, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_418.const", + "file_size": 114688 + }, + "model.layers.13.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 202216044, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_419.const", + "file_size": 512 + }, + "model.layers.13.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 202216556, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_420.const", + "file_size": 3584 + }, + "model.layers.13.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 202220140, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_421.const", + "file_size": 896 + }, + "model.layers.13.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 202221036, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_422.const", + "file_size": 802816 + }, + "model.layers.13.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 203023852, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_423.const", + "file_size": 3584 + }, + "model.layers.13.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 203027436, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_424.const", + "file_size": 25088 + }, + "model.layers.13.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 203052524, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_425.const", + "file_size": 6272 + }, + "model.layers.13.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 203058796, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_426.const", + "file_size": 114688 + }, + "model.layers.13.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 203173484, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_427.const", + "file_size": 512 + }, + "model.layers.13.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 203173996, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_428.const", + "file_size": 3584 + }, + "model.layers.13.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 203177580, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_429.const", + "file_size": 896 + }, + "model.layers.13.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 203178476, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_430.const", + "file_size": 802816 + }, + "model.layers.13.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 203981292, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_431.const", + "file_size": 3584 + }, + "model.layers.13.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 203984876, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_432.const", + "file_size": 25088 + }, + "model.layers.13.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 204009964, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_433.const", + "file_size": 6272 + }, + "model.layers.13.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 204016236, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_434.const", + "file_size": 1792 + }, + "eps_19_15_26": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 204018028, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_435.const", + "file_size": 2 + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 204018032, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_436.const", + "file_size": 4358144 + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 208376176, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_437.const", + "file_size": 19456 + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 208395632, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_438.const", + "file_size": 136192 + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 208531824, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_439.const", + "file_size": 34048 + }, + "model.layers.13.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 208565872, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_440.const", + "file_size": 4358144 + }, + "model.layers.13.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 212924016, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_441.const", + "file_size": 19456 + }, + "model.layers.13.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 212943472, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_442.const", + "file_size": 136192 + }, + "model.layers.13.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 213079664, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_443.const", + "file_size": 34048 + }, + "model.layers.13.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 213113712, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_444.const", + "file_size": 4358144 + }, + "model.layers.13.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 217471856, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_445.const", + "file_size": 3584 + }, + "model.layers.13.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 217475440, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_446.const", + "file_size": 136192 + }, + "model.layers.13.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 217611632, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_447.const", + "file_size": 34048 + }, + "model.layers.14.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 217645680, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_448.const", + "file_size": 1792 + }, + "eps_19_15_27": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 217647472, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_449.const", + "file_size": 2 + }, + "model.layers.14.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 217647476, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_450.const", + "file_size": 114688 + }, + "model.layers.14.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 217762164, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_451.const", + "file_size": 512 + }, + "model.layers.14.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 217762676, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_452.const", + "file_size": 3584 + }, + "model.layers.14.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 217766260, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_453.const", + "file_size": 896 + }, + "model.layers.14.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 217767156, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_454.const", + "file_size": 802816 + }, + "model.layers.14.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 218569972, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_455.const", + "file_size": 3584 + }, + "model.layers.14.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 218573556, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_456.const", + "file_size": 25088 + }, + "model.layers.14.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 218598644, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_457.const", + "file_size": 6272 + }, + "model.layers.14.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 218604916, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_458.const", + "file_size": 114688 + }, + "model.layers.14.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 218719604, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_459.const", + "file_size": 512 + }, + "model.layers.14.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 218720116, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_460.const", + "file_size": 3584 + }, + "model.layers.14.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 218723700, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_461.const", + "file_size": 896 + }, + "model.layers.14.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 218724596, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_462.const", + "file_size": 802816 + }, + "model.layers.14.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 219527412, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_463.const", + "file_size": 3584 + }, + "model.layers.14.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 219530996, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_464.const", + "file_size": 25088 + }, + "model.layers.14.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 219556084, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_465.const", + "file_size": 6272 + }, + "model.layers.14.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 219562356, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_466.const", + "file_size": 1792 + }, + "eps_19_15_28": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 219564148, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_467.const", + "file_size": 2 + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 219564152, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_468.const", + "file_size": 4358144 + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 223922296, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_469.const", + "file_size": 19456 + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 223941752, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_470.const", + "file_size": 136192 + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 224077944, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_471.const", + "file_size": 34048 + }, + "model.layers.14.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 224111992, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_472.const", + "file_size": 4358144 + }, + "model.layers.14.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 228470136, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_473.const", + "file_size": 19456 + }, + "model.layers.14.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 228489592, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_474.const", + "file_size": 136192 + }, + "model.layers.14.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 228625784, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_475.const", + "file_size": 34048 + }, + "model.layers.14.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 228659832, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_476.const", + "file_size": 4358144 + }, + "model.layers.14.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 233017976, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_477.const", + "file_size": 3584 + }, + "model.layers.14.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 233021560, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_478.const", + "file_size": 136192 + }, + "model.layers.14.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 233157752, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_479.const", + "file_size": 34048 + }, + "model.layers.15.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 233191800, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_480.const", + "file_size": 1792 + }, + "eps_19_15_29": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 233193592, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_481.const", + "file_size": 2 + }, + "model.layers.15.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 233193596, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_482.const", + "file_size": 114688 + }, + "model.layers.15.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 233308284, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_483.const", + "file_size": 512 + }, + "model.layers.15.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 233308796, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_484.const", + "file_size": 3584 + }, + "model.layers.15.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 233312380, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_485.const", + "file_size": 896 + }, + "model.layers.15.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 233313276, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_486.const", + "file_size": 802816 + }, + "model.layers.15.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 234116092, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_487.const", + "file_size": 3584 + }, + "model.layers.15.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 234119676, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_488.const", + "file_size": 25088 + }, + "model.layers.15.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 234144764, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_489.const", + "file_size": 6272 + }, + "model.layers.15.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 234151036, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_490.const", + "file_size": 114688 + }, + "model.layers.15.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 234265724, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_491.const", + "file_size": 512 + }, + "model.layers.15.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 234266236, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_492.const", + "file_size": 3584 + }, + "model.layers.15.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 234269820, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_493.const", + "file_size": 896 + }, + "model.layers.15.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 234270716, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_494.const", + "file_size": 802816 + }, + "model.layers.15.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 235073532, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_495.const", + "file_size": 3584 + }, + "model.layers.15.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 235077116, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_496.const", + "file_size": 25088 + }, + "model.layers.15.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 235102204, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_497.const", + "file_size": 6272 + }, + "model.layers.15.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 235108476, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_498.const", + "file_size": 1792 + }, + "eps_19_15_30": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 235110268, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_499.const", + "file_size": 2 + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 235110272, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_500.const", + "file_size": 4358144 + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 239468416, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_501.const", + "file_size": 19456 + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 239487872, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_502.const", + "file_size": 136192 + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 239624064, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_503.const", + "file_size": 34048 + }, + "model.layers.15.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 239658112, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_504.const", + "file_size": 4358144 + }, + "model.layers.15.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 244016256, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_505.const", + "file_size": 19456 + }, + "model.layers.15.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 244035712, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_506.const", + "file_size": 136192 + }, + "model.layers.15.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 244171904, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_507.const", + "file_size": 34048 + }, + "model.layers.15.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 244205952, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_508.const", + "file_size": 4358144 + }, + "model.layers.15.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 248564096, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_509.const", + "file_size": 3584 + }, + "model.layers.15.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 248567680, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_510.const", + "file_size": 136192 + }, + "model.layers.15.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 248703872, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_511.const", + "file_size": 34048 + }, + "model.layers.16.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 248737920, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_512.const", + "file_size": 1792 + }, + "eps_19_15_31": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 248739712, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_513.const", + "file_size": 2 + }, + "model.layers.16.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 248739716, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_514.const", + "file_size": 114688 + }, + "model.layers.16.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 248854404, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_515.const", + "file_size": 512 + }, + "model.layers.16.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 248854916, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_516.const", + "file_size": 3584 + }, + "model.layers.16.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 248858500, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_517.const", + "file_size": 896 + }, + "model.layers.16.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 248859396, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_518.const", + "file_size": 802816 + }, + "model.layers.16.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 249662212, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_519.const", + "file_size": 3584 + }, + "model.layers.16.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 249665796, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_520.const", + "file_size": 25088 + }, + "model.layers.16.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 249690884, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_521.const", + "file_size": 6272 + }, + "model.layers.16.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 249697156, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_522.const", + "file_size": 114688 + }, + "model.layers.16.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 249811844, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_523.const", + "file_size": 512 + }, + "model.layers.16.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 249812356, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_524.const", + "file_size": 3584 + }, + "model.layers.16.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 249815940, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_525.const", + "file_size": 896 + }, + "model.layers.16.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 249816836, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_526.const", + "file_size": 802816 + }, + "model.layers.16.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 250619652, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_527.const", + "file_size": 3584 + }, + "model.layers.16.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 250623236, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_528.const", + "file_size": 25088 + }, + "model.layers.16.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 250648324, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_529.const", + "file_size": 6272 + }, + "model.layers.16.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 250654596, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_530.const", + "file_size": 1792 + }, + "eps_19_15_32": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 250656388, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_531.const", + "file_size": 2 + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 250656392, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_532.const", + "file_size": 4358144 + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 255014536, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_533.const", + "file_size": 19456 + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 255033992, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_534.const", + "file_size": 136192 + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 255170184, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_535.const", + "file_size": 34048 + }, + "model.layers.16.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 255204232, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_536.const", + "file_size": 4358144 + }, + "model.layers.16.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 259562376, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_537.const", + "file_size": 19456 + }, + "model.layers.16.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 259581832, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_538.const", + "file_size": 136192 + }, + "model.layers.16.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 259718024, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_539.const", + "file_size": 34048 + }, + "model.layers.16.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 259752072, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_540.const", + "file_size": 4358144 + }, + "model.layers.16.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 264110216, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_541.const", + "file_size": 3584 + }, + "model.layers.16.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 264113800, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_542.const", + "file_size": 136192 + }, + "model.layers.16.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 264249992, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_543.const", + "file_size": 34048 + }, + "model.layers.17.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 264284040, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_544.const", + "file_size": 1792 + }, + "eps_19_15_33": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 264285832, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_545.const", + "file_size": 2 + }, + "model.layers.17.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 264285836, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_546.const", + "file_size": 114688 + }, + "model.layers.17.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 264400524, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_547.const", + "file_size": 512 + }, + "model.layers.17.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 264401036, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_548.const", + "file_size": 3584 + }, + "model.layers.17.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 264404620, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_549.const", + "file_size": 896 + }, + "model.layers.17.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 264405516, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_550.const", + "file_size": 802816 + }, + "model.layers.17.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 265208332, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_551.const", + "file_size": 3584 + }, + "model.layers.17.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 265211916, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_552.const", + "file_size": 25088 + }, + "model.layers.17.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 265237004, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_553.const", + "file_size": 6272 + }, + "model.layers.17.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 265243276, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_554.const", + "file_size": 114688 + }, + "model.layers.17.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 265357964, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_555.const", + "file_size": 512 + }, + "model.layers.17.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 265358476, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_556.const", + "file_size": 3584 + }, + "model.layers.17.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 265362060, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_557.const", + "file_size": 896 + }, + "model.layers.17.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 265362956, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_558.const", + "file_size": 802816 + }, + "model.layers.17.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 266165772, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_559.const", + "file_size": 3584 + }, + "model.layers.17.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 266169356, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_560.const", + "file_size": 25088 + }, + "model.layers.17.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 266194444, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_561.const", + "file_size": 6272 + }, + "model.layers.17.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 266200716, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_562.const", + "file_size": 1792 + }, + "eps_19_15_34": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 266202508, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_563.const", + "file_size": 2 + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 266202512, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_564.const", + "file_size": 4358144 + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 270560656, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_565.const", + "file_size": 19456 + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 270580112, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_566.const", + "file_size": 136192 + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 270716304, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_567.const", + "file_size": 34048 + }, + "model.layers.17.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 270750352, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_568.const", + "file_size": 4358144 + }, + "model.layers.17.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 275108496, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_569.const", + "file_size": 19456 + }, + "model.layers.17.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 275127952, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_570.const", + "file_size": 136192 + }, + "model.layers.17.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 275264144, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_571.const", + "file_size": 34048 + }, + "model.layers.17.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 275298192, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_572.const", + "file_size": 4358144 + }, + "model.layers.17.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 279656336, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_573.const", + "file_size": 3584 + }, + "model.layers.17.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 279659920, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_574.const", + "file_size": 136192 + }, + "model.layers.17.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 279796112, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_575.const", + "file_size": 34048 + }, + "model.layers.18.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 279830160, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_576.const", + "file_size": 1792 + }, + "eps_19_15_35": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 279831952, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_577.const", + "file_size": 2 + }, + "model.layers.18.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 279831956, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_578.const", + "file_size": 114688 + }, + "model.layers.18.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 279946644, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_579.const", + "file_size": 512 + }, + "model.layers.18.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 279947156, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_580.const", + "file_size": 3584 + }, + "model.layers.18.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 279950740, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_581.const", + "file_size": 896 + }, + "model.layers.18.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 279951636, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_582.const", + "file_size": 802816 + }, + "model.layers.18.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 280754452, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_583.const", + "file_size": 3584 + }, + "model.layers.18.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 280758036, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_584.const", + "file_size": 25088 + }, + "model.layers.18.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 280783124, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_585.const", + "file_size": 6272 + }, + "model.layers.18.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 280789396, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_586.const", + "file_size": 114688 + }, + "model.layers.18.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 280904084, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_587.const", + "file_size": 512 + }, + "model.layers.18.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 280904596, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_588.const", + "file_size": 3584 + }, + "model.layers.18.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 280908180, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_589.const", + "file_size": 896 + }, + "model.layers.18.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 280909076, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_590.const", + "file_size": 802816 + }, + "model.layers.18.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 281711892, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_591.const", + "file_size": 3584 + }, + "model.layers.18.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 281715476, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_592.const", + "file_size": 25088 + }, + "model.layers.18.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 281740564, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_593.const", + "file_size": 6272 + }, + "model.layers.18.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 281746836, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_594.const", + "file_size": 1792 + }, + "eps_19_15_36": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 281748628, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_595.const", + "file_size": 2 + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 281748632, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_596.const", + "file_size": 4358144 + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 286106776, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_597.const", + "file_size": 19456 + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 286126232, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_598.const", + "file_size": 136192 + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 286262424, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_599.const", + "file_size": 34048 + }, + "model.layers.18.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 286296472, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_600.const", + "file_size": 4358144 + }, + "model.layers.18.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 290654616, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_601.const", + "file_size": 19456 + }, + "model.layers.18.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 290674072, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_602.const", + "file_size": 136192 + }, + "model.layers.18.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 290810264, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_603.const", + "file_size": 34048 + }, + "model.layers.18.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 290844312, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_604.const", + "file_size": 4358144 + }, + "model.layers.18.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 295202456, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_605.const", + "file_size": 3584 + }, + "model.layers.18.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 295206040, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_606.const", + "file_size": 136192 + }, + "model.layers.18.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 295342232, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_607.const", + "file_size": 34048 + }, + "model.layers.19.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 295376280, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_608.const", + "file_size": 1792 + }, + "eps_19_15_37": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 295378072, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_609.const", + "file_size": 2 + }, + "model.layers.19.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 295378076, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_610.const", + "file_size": 114688 + }, + "model.layers.19.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 295492764, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_611.const", + "file_size": 512 + }, + "model.layers.19.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 295493276, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_612.const", + "file_size": 3584 + }, + "model.layers.19.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 295496860, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_613.const", + "file_size": 896 + }, + "model.layers.19.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 295497756, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_614.const", + "file_size": 802816 + }, + "model.layers.19.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 296300572, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_615.const", + "file_size": 3584 + }, + "model.layers.19.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 296304156, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_616.const", + "file_size": 25088 + }, + "model.layers.19.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 296329244, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_617.const", + "file_size": 6272 + }, + "model.layers.19.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 296335516, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_618.const", + "file_size": 114688 + }, + "model.layers.19.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 296450204, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_619.const", + "file_size": 512 + }, + "model.layers.19.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 296450716, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_620.const", + "file_size": 3584 + }, + "model.layers.19.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 296454300, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_621.const", + "file_size": 896 + }, + "model.layers.19.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 296455196, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_622.const", + "file_size": 802816 + }, + "model.layers.19.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 297258012, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_623.const", + "file_size": 3584 + }, + "model.layers.19.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 297261596, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_624.const", + "file_size": 25088 + }, + "model.layers.19.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 297286684, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_625.const", + "file_size": 6272 + }, + "model.layers.19.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 297292956, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_626.const", + "file_size": 1792 + }, + "eps_19_15_38": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 297294748, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_627.const", + "file_size": 2 + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 297294752, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_628.const", + "file_size": 4358144 + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 301652896, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_629.const", + "file_size": 19456 + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 301672352, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_630.const", + "file_size": 136192 + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 301808544, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_631.const", + "file_size": 34048 + }, + "model.layers.19.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 301842592, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_632.const", + "file_size": 4358144 + }, + "model.layers.19.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 306200736, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_633.const", + "file_size": 19456 + }, + "model.layers.19.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 306220192, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_634.const", + "file_size": 136192 + }, + "model.layers.19.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 306356384, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_635.const", + "file_size": 34048 + }, + "model.layers.19.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 306390432, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_636.const", + "file_size": 4358144 + }, + "model.layers.19.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 310748576, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_637.const", + "file_size": 3584 + }, + "model.layers.19.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 310752160, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_638.const", + "file_size": 136192 + }, + "model.layers.19.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 310888352, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_639.const", + "file_size": 34048 + }, + "model.layers.20.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 310922400, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_640.const", + "file_size": 1792 + }, + "eps_19_15_39": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 310924192, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_641.const", + "file_size": 2 + }, + "model.layers.20.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 310924196, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_642.const", + "file_size": 114688 + }, + "model.layers.20.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 311038884, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_643.const", + "file_size": 512 + }, + "model.layers.20.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 311039396, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_644.const", + "file_size": 3584 + }, + "model.layers.20.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 311042980, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_645.const", + "file_size": 896 + }, + "model.layers.20.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 311043876, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_646.const", + "file_size": 802816 + }, + "model.layers.20.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 311846692, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_647.const", + "file_size": 3584 + }, + "model.layers.20.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 311850276, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_648.const", + "file_size": 25088 + }, + "model.layers.20.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 311875364, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_649.const", + "file_size": 6272 + }, + "model.layers.20.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 311881636, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_650.const", + "file_size": 114688 + }, + "model.layers.20.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 311996324, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_651.const", + "file_size": 512 + }, + "model.layers.20.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 311996836, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_652.const", + "file_size": 3584 + }, + "model.layers.20.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 312000420, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_653.const", + "file_size": 896 + }, + "model.layers.20.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 312001316, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_654.const", + "file_size": 802816 + }, + "model.layers.20.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 312804132, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_655.const", + "file_size": 3584 + }, + "model.layers.20.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 312807716, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_656.const", + "file_size": 25088 + }, + "model.layers.20.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 312832804, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_657.const", + "file_size": 6272 + }, + "model.layers.20.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 312839076, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_658.const", + "file_size": 1792 + }, + "eps_19_15_40": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 312840868, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_659.const", + "file_size": 2 + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 312840872, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_660.const", + "file_size": 4358144 + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 317199016, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_661.const", + "file_size": 19456 + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 317218472, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_662.const", + "file_size": 136192 + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 317354664, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_663.const", + "file_size": 34048 + }, + "model.layers.20.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 317388712, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_664.const", + "file_size": 4358144 + }, + "model.layers.20.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 321746856, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_665.const", + "file_size": 19456 + }, + "model.layers.20.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 321766312, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_666.const", + "file_size": 136192 + }, + "model.layers.20.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 321902504, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_667.const", + "file_size": 34048 + }, + "model.layers.20.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 321936552, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_668.const", + "file_size": 4358144 + }, + "model.layers.20.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 326294696, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_669.const", + "file_size": 3584 + }, + "model.layers.20.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 326298280, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_670.const", + "file_size": 136192 + }, + "model.layers.20.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 326434472, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_671.const", + "file_size": 34048 + }, + "model.layers.21.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 326468520, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_672.const", + "file_size": 1792 + }, + "eps_19_15_41": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 326470312, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_673.const", + "file_size": 2 + }, + "model.layers.21.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 326470316, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_674.const", + "file_size": 114688 + }, + "model.layers.21.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 326585004, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_675.const", + "file_size": 512 + }, + "model.layers.21.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 326585516, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_676.const", + "file_size": 3584 + }, + "model.layers.21.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 326589100, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_677.const", + "file_size": 896 + }, + "model.layers.21.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 326589996, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_678.const", + "file_size": 802816 + }, + "model.layers.21.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 327392812, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_679.const", + "file_size": 3584 + }, + "model.layers.21.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 327396396, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_680.const", + "file_size": 25088 + }, + "model.layers.21.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 327421484, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_681.const", + "file_size": 6272 + }, + "model.layers.21.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 327427756, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_682.const", + "file_size": 114688 + }, + "model.layers.21.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 327542444, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_683.const", + "file_size": 512 + }, + "model.layers.21.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 327542956, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_684.const", + "file_size": 3584 + }, + "model.layers.21.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 327546540, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_685.const", + "file_size": 896 + }, + "model.layers.21.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 327547436, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_686.const", + "file_size": 802816 + }, + "model.layers.21.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 328350252, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_687.const", + "file_size": 3584 + }, + "model.layers.21.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 328353836, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_688.const", + "file_size": 25088 + }, + "model.layers.21.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 328378924, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_689.const", + "file_size": 6272 + }, + "model.layers.21.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 328385196, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_690.const", + "file_size": 1792 + }, + "eps_19_15_42": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 328386988, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_691.const", + "file_size": 2 + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 328386992, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_692.const", + "file_size": 4358144 + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 332745136, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_693.const", + "file_size": 19456 + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 332764592, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_694.const", + "file_size": 136192 + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 332900784, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_695.const", + "file_size": 34048 + }, + "model.layers.21.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 332934832, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_696.const", + "file_size": 4358144 + }, + "model.layers.21.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 337292976, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_697.const", + "file_size": 19456 + }, + "model.layers.21.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 337312432, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_698.const", + "file_size": 136192 + }, + "model.layers.21.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 337448624, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_699.const", + "file_size": 34048 + }, + "model.layers.21.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 337482672, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_700.const", + "file_size": 4358144 + }, + "model.layers.21.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 341840816, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_701.const", + "file_size": 3584 + }, + "model.layers.21.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 341844400, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_702.const", + "file_size": 136192 + }, + "model.layers.21.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 341980592, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_703.const", + "file_size": 34048 + }, + "model.layers.22.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 342014640, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_704.const", + "file_size": 1792 + }, + "eps_19_15_43": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 342016432, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_705.const", + "file_size": 2 + }, + "model.layers.22.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 342016436, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_706.const", + "file_size": 114688 + }, + "model.layers.22.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 342131124, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_707.const", + "file_size": 512 + }, + "model.layers.22.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 342131636, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_708.const", + "file_size": 3584 + }, + "model.layers.22.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 342135220, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_709.const", + "file_size": 896 + }, + "model.layers.22.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 342136116, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_710.const", + "file_size": 802816 + }, + "model.layers.22.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 342938932, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_711.const", + "file_size": 3584 + }, + "model.layers.22.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 342942516, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_712.const", + "file_size": 25088 + }, + "model.layers.22.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 342967604, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_713.const", + "file_size": 6272 + }, + "model.layers.22.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 342973876, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_714.const", + "file_size": 114688 + }, + "model.layers.22.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 343088564, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_715.const", + "file_size": 512 + }, + "model.layers.22.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 343089076, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_716.const", + "file_size": 3584 + }, + "model.layers.22.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 343092660, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_717.const", + "file_size": 896 + }, + "model.layers.22.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 343093556, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_718.const", + "file_size": 802816 + }, + "model.layers.22.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 343896372, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_719.const", + "file_size": 3584 + }, + "model.layers.22.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 343899956, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_720.const", + "file_size": 25088 + }, + "model.layers.22.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 343925044, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_721.const", + "file_size": 6272 + }, + "model.layers.22.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 343931316, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_722.const", + "file_size": 1792 + }, + "eps_19_15_44": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 343933108, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_723.const", + "file_size": 2 + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 343933112, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_724.const", + "file_size": 4358144 + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 348291256, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_725.const", + "file_size": 19456 + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 348310712, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_726.const", + "file_size": 136192 + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 348446904, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_727.const", + "file_size": 34048 + }, + "model.layers.22.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 348480952, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_728.const", + "file_size": 4358144 + }, + "model.layers.22.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 352839096, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_729.const", + "file_size": 19456 + }, + "model.layers.22.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 352858552, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_730.const", + "file_size": 136192 + }, + "model.layers.22.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 352994744, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_731.const", + "file_size": 34048 + }, + "model.layers.22.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 353028792, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_732.const", + "file_size": 4358144 + }, + "model.layers.22.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 357386936, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_733.const", + "file_size": 3584 + }, + "model.layers.22.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 357390520, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_734.const", + "file_size": 136192 + }, + "model.layers.22.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 357526712, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_735.const", + "file_size": 34048 + }, + "model.layers.23.input_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 357560760, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_736.const", + "file_size": 1792 + }, + "eps_19_15_45": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 357562552, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_737.const", + "file_size": 2 + }, + "model.layers.23.attn.k_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 357562556, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_738.const", + "file_size": 114688 + }, + "model.layers.23.attn.k_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 357677244, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_739.const", + "file_size": 512 + }, + "model.layers.23.attn.k_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 357677756, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_740.const", + "file_size": 3584 + }, + "model.layers.23.attn.k_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 357681340, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_741.const", + "file_size": 896 + }, + "model.layers.23.attn.q_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 357682236, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_742.const", + "file_size": 802816 + }, + "model.layers.23.attn.q_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 358485052, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_743.const", + "file_size": 3584 + }, + "model.layers.23.attn.q_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 358488636, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_744.const", + "file_size": 25088 + }, + "model.layers.23.attn.q_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 358513724, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_745.const", + "file_size": 6272 + }, + "model.layers.23.attn.v_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 128 + ], + "size_in_bytes": 114688, + "op_tensor_size": 114688, + "offset": 358519996, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_746.const", + "file_size": 114688 + }, + "model.layers.23.attn.v_proj.Add.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 128 + ], + "size_in_bytes": 512, + "op_tensor_size": 512, + "offset": 358634684, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_747.const", + "file_size": 512 + }, + "model.layers.23.attn.v_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 358635196, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_748.const", + "file_size": 3584 + }, + "model.layers.23.attn.v_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896 + ], + "size_in_bytes": 896, + "op_tensor_size": 896, + "offset": 358638780, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_749.const", + "file_size": 896 + }, + "model.layers.23.attn.o_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 896 + ], + "size_in_bytes": 802816, + "op_tensor_size": 802816, + "offset": 358639676, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_750.const", + "file_size": 802816 + }, + "model.layers.23.attn.o_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 359442492, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_751.const", + "file_size": 3584 + }, + "model.layers.23.attn.o_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 6272 + ], + "size_in_bytes": 25088, + "op_tensor_size": 25088, + "offset": 359446076, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_752.const", + "file_size": 25088 + }, + "model.layers.23.attn.o_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 6272 + ], + "size_in_bytes": 6272, + "op_tensor_size": 6272, + "offset": 359471164, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_753.const", + "file_size": 6272 + }, + "model.layers.23.post_attention_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 359477436, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_754.const", + "file_size": 1792 + }, + "eps_19_15_46": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 359479228, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_755.const", + "file_size": 2 + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 359479232, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_756.const", + "file_size": 4358144 + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 363837376, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_757.const", + "file_size": 19456 + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 363856832, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_758.const", + "file_size": 136192 + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 363993024, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_759.const", + "file_size": 34048 + }, + "model.layers.23.mlp.up_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 896, + 4864 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 364027072, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_760.const", + "file_size": 4358144 + }, + "model.layers.23.mlp.up_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 4864 + ], + "size_in_bytes": 19456, + "op_tensor_size": 19456, + "offset": 368385216, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_761.const", + "file_size": 19456 + }, + "model.layers.23.mlp.up_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 368404672, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_762.const", + "file_size": 136192 + }, + "model.layers.23.mlp.up_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 368540864, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_763.const", + "file_size": 34048 + }, + "model.layers.23.mlp.down_proj.MatMulNBits.qweight.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 4864, + 896 + ], + "size_in_bytes": 4358144, + "op_tensor_size": 4358144, + "offset": 368574912, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_764.const", + "file_size": 4358144 + }, + "model.layers.23.mlp.down_proj.MatMulNBits.bias.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 896 + ], + "size_in_bytes": 3584, + "op_tensor_size": 3584, + "offset": 372933056, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_765.const", + "file_size": 3584 + }, + "model.layers.23.mlp.down_proj.MatMulNBits.scales.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "float", + "shape": [ + 34048 + ], + "size_in_bytes": 136192, + "op_tensor_size": 136192, + "offset": 372936640, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_766.const", + "file_size": 136192 + }, + "model.layers.23.mlp.down_proj.MatMulNBits.qzeros.preformat": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "int8", + "shape": [ + 34048 + ], + "size_in_bytes": 34048, + "op_tensor_size": 34048, + "offset": 373072832, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_767.const", + "file_size": 34048 + }, + "model.layers.24.final_norm_layernorm.weight": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 896 + ], + "size_in_bytes": 1792, + "op_tensor_size": 1792, + "offset": 373106880, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_768.const", + "file_size": 1792 + }, + "eps_19_15_47": { + "packed_buffer_label": "const", + "xrt_arg_id": 3, + "dtype": "bfloat16", + "shape": [ + 1 + ], + "size_in_bytes": 2, + "op_tensor_size": 2, + "offset": 373108672, + "file_name": "cache/modellayers.0input_layernormMLADFRMSNORM_769.const", + "file_size": 2 + }, + "present.0.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 0 + }, + "present.0.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 1048576 + }, + "present.1.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 2097152 + }, + "present.1.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 3145728 + }, + "present.2.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 4194304 + }, + "present.2.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 5242880 + }, + "present.3.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 6291456 + }, + "present.3.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 7340032 + }, + "present.4.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 8388608 + }, + "present.4.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 9437184 + }, + "present.5.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 10485760 + }, + "present.5.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 11534336 + }, + "present.6.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 12582912 + }, + "present.6.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 13631488 + }, + "present.7.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 14680064 + }, + "present.7.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 15728640 + }, + "present.8.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 16777216 + }, + "present.8.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 17825792 + }, + "present.9.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 18874368 + }, + "present.9.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 19922944 + }, + "present.10.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 20971520 + }, + "present.10.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 22020096 + }, + "present.11.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 23068672 + }, + "present.11.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 24117248 + }, + "present.12.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 25165824 + }, + "present.12.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 26214400 + }, + "present.13.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 27262976 + }, + "present.13.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 28311552 + }, + "present.14.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 29360128 + }, + "present.14.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 30408704 + }, + "present.15.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 31457280 + }, + "present.15.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 32505856 + }, + "present.16.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 33554432 + }, + "present.16.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 34603008 + }, + "present.17.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 35651584 + }, + "present.17.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 36700160 + }, + "present.18.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 37748736 + }, + "present.18.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 38797312 + }, + "present.19.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 39845888 + }, + "present.19.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 40894464 + }, + "present.20.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 41943040 + }, + "present.20.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 42991616 + }, + "present.21.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 44040192 + }, + "present.21.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 45088768 + }, + "present.22.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 46137344 + }, + "present.22.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 47185920 + }, + "present.23.key": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 48234496 + }, + "present.23.value": { + "packed_buffer_label": "ext_buf_0", + "xrt_arg_id": 5, + "dtype": "bfloat16", + "shape": [ + 1, + 2, + 4096, + 64 + ], + "size_in_bytes": 1048576, + "op_tensor_size": 1048576, + "offset": 49283072 + } + }, + "dynamic_shape_subgraph": true, + "dynamic_shape_list": [ + { + "sequence_length_padded": 128 + }, + { + "sequence_length_padded": 256 + }, + { + "sequence_length_padded": 512 + }, + { + "sequence_length_padded": 768 + }, + { + "sequence_length_padded": 1024 + }, + { + "sequence_length_padded": 1536 + }, + { + "sequence_length_padded": 2048 + }, + { + "sequence_length_padded": 2304 + }, + { + "sequence_length_padded": 3072 + }, + { + "sequence_length_padded": 4096 + } + ], + "aux_info": {} +} \ No newline at end of file