diff --git a/lm-quant-toolkit/scripts/experiment-llama-sensi-7b-cfg-wise.sh b/lm-quant-toolkit/scripts/experiment-llama-sensi-7b-cfg-wise.sh
new file mode 100644
index 0000000000000000000000000000000000000000..2fecbda9da97d63e92ea09697620f08037e1b72a
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama-sensi-7b-cfg-wise.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+ATTEMPT="sensi-cfg-wise-7b"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-milp
+
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+MODELS="0"
+EXP_NAME="${ATTEMPT}_7B"
+echo "=========Run perplexity evaluation on Llama-2-7b========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-llama-sensi-7b-fnorm-avg.sh b/lm-quant-toolkit/scripts/experiment-llama-sensi-7b-fnorm-avg.sh
new file mode 100644
index 0000000000000000000000000000000000000000..7cadad509dee2f0ac77daca1972f9262e832278a
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama-sensi-7b-fnorm-avg.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+ATTEMPT="fnorm-sensi-avg-7b"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-milp
+
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+MODELS="0"
+EXP_NAME="${ATTEMPT}_7B"
+echo "=========Run perplexity evaluation on Llama-2-7b========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-llama-sensi-7b-protect-mlp.sh b/lm-quant-toolkit/scripts/experiment-llama-sensi-7b-protect-mlp.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f9bce5c343017b2c3d3dbe77cf8097028fd85e7c
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama-sensi-7b-protect-mlp.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+ATTEMPT="sensi-protect-mlp-7b"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-milp
+
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+MODELS="0"
+EXP_NAME="${ATTEMPT}"
+echo "=========Run perplexity evaluation on Llama-2-7b========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-llama-sensi-boost.sh b/lm-quant-toolkit/scripts/experiment-llama-sensi-boost.sh
new file mode 100644
index 0000000000000000000000000000000000000000..b4374934578516b03a10449ba6d7a5e8994e4a86
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama-sensi-boost.sh
@@ -0,0 +1,133 @@
+#!/bin/bash
+
+BUDGETS="3.13 3.25 3.51 4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-boost
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+# BOOST_STOPS="2 3"
+BOOST_STOPS="3"
+BOOST_TOP_MS="0"
+
+for BOOST_STOP in $BOOST_STOPS; do
+ for BOOST_TOP_M in $BOOST_TOP_MS; do
+ if [[ $BOOST_STOP -eq 2 && $BOOST_TOP_M -eq 1 ]]; then
+ continue
+ fi
+ ATTEMPT="sensi-boost-${BOOST_STOP}-${BOOST_TOP_M}"
+ EXP_BASE_NAME=$ATTEMPT
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+ log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+ mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+ mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+ # MODELS="0"
+ EXP_NAME="${ATTEMPT}"
+ echo "=========Run perplexity evaluation========="
+ python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --boost-stop $BOOST_STOP \
+ --top-m-layer $BOOST_TOP_M \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ EXIT_CODE=$?
+ if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+ fi
+ echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+ find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+ echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+ python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+ echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+ algo=mxq
+ model_ids=$MODELS
+ for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+ done
+ echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+ find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+ # echo "=========Delete quantized models of batch ${batch_name}========="
+ # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+ OLD_DIR=$(pwd)
+ cd $RESULT_DIR/$EXP_BASE_NAME
+ if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+ fi
+ $OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+ $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+ $OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+ $OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+ $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+ pdflatex table.tex
+
+ # plot configuration allocations for 3 * 12 MXQ combinations
+ for model in $MODEL_NAMES; do
+ for budget in $BUDGETS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+ done
+ cd $OLD_DIR
+ done
+done
+
diff --git a/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-7b.sh b/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-7b.sh
new file mode 100644
index 0000000000000000000000000000000000000000..6b7d87eb5c0f4e18430f217ef4a000bca3ceef90
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-7b.sh
@@ -0,0 +1,119 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+ATTEMPT="llama-sensi-milp-2x"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-milp
+
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+MODELS="0 2"
+EXP_NAME="${ATTEMPT}"
+echo "=========Run perplexity evaluation on Llama-2-7b========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf Meta-Llama-3-8B"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-ablation.sh b/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-ablation.sh
new file mode 100644
index 0000000000000000000000000000000000000000..633119aa50095b4ef62e9d82c32be4db613b5a92
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama-sensi-milp-ablation.sh
@@ -0,0 +1,109 @@
+#!/bin/bash
+
+BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-milp
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+# for SensiMiLP ablation test, all topm values are equivalent
+
+ATTEMPT="sensi-milp-abl"
+EXP_BASE_NAME=$ATTEMPT
+EXP_NAME="${ATTEMPT}"
+
+log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Run perplexity evaluation========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --ablation \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs ]; then
+ mkdir pdfs
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+cd pdfs
+pdflatex table.tex
+cd ..
+
+cd $OLD_DIR
diff --git a/lm-quant-toolkit/scripts/experiment-llama-sensi-milp.sh b/lm-quant-toolkit/scripts/experiment-llama-sensi-milp.sh
new file mode 100644
index 0000000000000000000000000000000000000000..b6bbaac99fe00c5ffe4b6d70bb4bf72f23208a3e
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama-sensi-milp.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+
+BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-milp
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+BOOST_TOP_MS="3"
+
+for BOOST_TOP_M in $BOOST_TOP_MS; do
+ ATTEMPT="sensi-milp-${BOOST_TOP_M}"
+ EXP_BASE_NAME=$ATTEMPT
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+ log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+ mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+ mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+ # MODELS="0"
+ EXP_NAME="${ATTEMPT}"
+ echo "=========Run perplexity evaluation========="
+ python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --top-m-layer $BOOST_TOP_M \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ EXIT_CODE=$?
+ if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+ fi
+ echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+ find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+ echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+ python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+ echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+ algo=mxq
+ model_ids=$MODELS
+ for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+ done
+ echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+ find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+ # echo "=========Delete quantized models of batch ${batch_name}========="
+ # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+ OLD_DIR=$(pwd)
+ cd $RESULT_DIR/$EXP_BASE_NAME
+ if [ ! -d pdfs ]; then
+ mkdir pdfs
+ fi
+ $OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+ $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+ $OLD_DIR/../data-vis/plot-mem-consumption.R -d data/combined.csv
+ $OLD_DIR/../data-vis/plot-quant-speed.R -d data/combined.csv
+ $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+ cd pdfs
+ pdflatex table.tex
+ cd ..
+
+ # plot configuration allocations for 3 * 12 MXQ combinations
+ MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+ BUDGETS=${MXQ_BATCH[@]:1}
+ for model in $MODELS; do
+ for budget in $BUDGETS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --attempt1 mxq1 \
+ --attempt2 $ATTEMPT \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+ done
+ cd $OLD_DIR
+
+done
+
diff --git a/lm-quant-toolkit/scripts/experiment-llama-sensi.sh b/lm-quant-toolkit/scripts/experiment-llama-sensi.sh
new file mode 100644
index 0000000000000000000000000000000000000000..197ceaa629285ecc4d920c821f5402cb0607aa25
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama-sensi.sh
@@ -0,0 +1,190 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+ATTEMPT="llama-sensi"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-directive
+
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+MODELS="1"
+boost_layers="3 39"
+EXP_NAME="${ATTEMPT}_13b"
+echo "=========Run perplexity evaluation on Llama-2-13b========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --boost-layer $boost_layers \
+ --boost-stop 2 \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+MODELS="0 2"
+boost_layers="1 31"
+EXP_NAME="${ATTEMPT}_78"
+echo "=========Run perplexity evaluation on Llama-2-7b & Llama-3-8B========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --boost-layer $boost_layers \
+ --boost-stop 1 \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf Llama-2-13b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-llama2-13b-balanced.sh b/lm-quant-toolkit/scripts/experiment-llama2-13b-balanced.sh
new file mode 100644
index 0000000000000000000000000000000000000000..3aeb3234f9e149043686a8eed3f2e92c7f2d5459
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama2-13b-balanced.sh
@@ -0,0 +1,122 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+MODELS="1"
+ATTEMPT="Llama2-13B-balanced"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-directive
+boost_layers="3 39"
+decline_layers="0"
+EXP_NAME="${ATTEMPT}"
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Run perplexity evaluation on batch ${EXP_NAME}========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --boost-layer $boost_layers \
+ --decline-layer $decline_layers \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-13b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-llama2-13b-boost.sh b/lm-quant-toolkit/scripts/experiment-llama2-13b-boost.sh
new file mode 100644
index 0000000000000000000000000000000000000000..9ed1242ada4bd171a623caa82e881aca111516dd
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama2-13b-boost.sh
@@ -0,0 +1,121 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+MODELS="1"
+ATTEMPT="Llama2-13B-boost"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-directive
+boost_layers="3 39"
+EXP_NAME="${ATTEMPT}"
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Run perplexity evaluation on batch ${EXP_NAME}========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --boost-layer $boost_layers \
+ --boost-stop 2 \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-13b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-llama3-8b-aggressive-reduction.sh b/lm-quant-toolkit/scripts/experiment-llama3-8b-aggressive-reduction.sh
new file mode 100644
index 0000000000000000000000000000000000000000..77f87a6fdee945d097301d3ab198d959f6a2bc0c
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-llama3-8b-aggressive-reduction.sh
@@ -0,0 +1,122 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+MODELS="2"
+ATTEMPT="Llama3-8B-aggressive"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-directive
+boost_layers="31 30"
+decline_layers="0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29"
+EXP_NAME="${ATTEMPT}"
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Run perplexity evaluation on batch ${EXP_NAME}========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --boost-layer $boost_layers \
+ --decline-layer $decline_layers \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Meta-Llama-3-8B"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-sensi-milp-mini-batch.sh b/lm-quant-toolkit/scripts/experiment-sensi-milp-mini-batch.sh
new file mode 100644
index 0000000000000000000000000000000000000000..0808bd6a9a306656b1382b4ec13502559678fd76
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-sensi-milp-mini-batch.sh
@@ -0,0 +1,128 @@
+#!/bin/bash
+
+
+# 100%, 99%, 98%, 97%, 96%, 95% of [3.51, 4.25]
+# MXQ2=(minibatch 3.51 3.47 3.44 3.40 3.37 3.33 4.25 4.21 4.17 4.12 4.08 4.04)
+MXQ2=(minibatch 6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07)
+MXQ_BATCHES=(MXQ2)
+declare -n MXQ_BATCH
+
+ATTEMPT="sensi-milp-mini2"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+EXP_BASE_NAME="${ATTEMPT}"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+# Setup data files directories for reporting
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+
+weight_algo=sensi-milp
+
+for MXQ_BATCH in "${MXQ_BATCHES[@]}"; do
+ batch_name=${MXQ_BATCH[@]:0:1}
+ EXP_NAME="${EXP_BASE_NAME}-${batch_name}"
+ mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+ mkdir -p $RESULT_DIR/${EXP_NAME}_stor
+ log_file="logs/bench-${EXP_NAME}-$(date +%Y%m%d%H%M%S).log"
+
+ echo "=========Run perplexity evaluation on batch ${batch_name}========="
+ python ../src/cli.py llm \
+ --task eval_ppl \
+ --model 0 1 2 \
+ --algo mxq \
+ --weight-algo ${weight_algo} \
+ --config ${MXQ_BATCH[@]:1} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ EXIT_CODE=$?
+ if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+ fi
+ echo "=========Collect perplexity evaluation result on batch ${batch_name}========="
+ find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+ echo "=========Dump quantization configs on batch ${batch_name}========="
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+ python ../src/cli.py dump \
+ --type quant_config \
+ --model 0 1 2 \
+ --budget ${MXQ_BATCH[@]:1} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+ echo "=========Run memory evaluation on batch ${batch_name}========="
+ algo=mxq
+ model_ids="0 1 2"
+ for m in $model_ids; do
+ for cfg in "${MXQ_BATCH[@]:1}"; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+ done
+ echo "=========Collect memory evaluation result on batch ${batch_name}========="
+ find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+
+ # echo "=========Delete quantized models of batch ${batch_name}========="
+ # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+ OLD_DIR=$(pwd)
+ cd $RESULT_DIR/$EXP_BASE_NAME
+ if [ ! -d pdfs ]; then
+ mkdir pdfs
+ fi
+ $OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+ $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+ $OLD_DIR/../data-vis/plot-mem-consumption.R -d data/combined.csv
+ $OLD_DIR/../data-vis/plot-quant-speed.R -d data/combined.csv
+ $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+ cd pdfs
+ pdflatex table.tex
+ cd ..
+
+ # plot configuration allocations for 3 * 12 MXQ combinations
+ MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+ BUDGETS=${MXQ_BATCH[@]:1}
+ for model in $MODELS; do
+ for budget in $BUDGETS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --attempt1 mxq1 \
+ --attempt2 $ATTEMPT \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+ done
+ cd $OLD_DIR
+done
diff --git a/lm-quant-toolkit/scripts/experiment-tail-2-4.sh b/lm-quant-toolkit/scripts/experiment-tail-2-4.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f90a7ed8b71bb01cf90d70fd3fb796150c82a24d
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-tail-2-4.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+
+HEADS=(head-prioritized 1.05 1.10 1.15 1.20)
+# TAILS=(tail-prioritized 1.05 1.10 1.15 1.20)
+TAILS=(tail-prioritized 2.00 4.00)
+# MXQ_BATCHES=(HEADS TAILS)
+MXQ_BATCHES=(TAILS)
+declare -n MXQ_BATCH
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# --config ${MXQ_BATCH[@]:1}
+
+EXP_BASE_NAME="tail-big-factor"
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+for MXQ_BATCH in "${MXQ_BATCHES[@]}"; do
+ weight_algo=${MXQ_BATCH[@]:0:1}
+ for factor in ${MXQ_BATCH[@]:1}; do
+ ATTEMPT="${weight_algo}_${factor/./_}"
+ EXP_NAME="${ATTEMPT}"
+ log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+ mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+ mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+ # echo "=========Run perplexity evaluation on batch ${EXP_NAME}========="
+ # python ../src/cli.py llm \
+ # --task eval_ppl \
+ # --model 0 1 2 \
+ # --algo mxq \
+ # --weight-algo $weight_algo \
+ # --factor $factor \
+ # --config ${BUDGETS} \
+ # --experiment-name "${EXP_NAME}_ppl" \
+ # --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ # --result-dir=$RESULT_DIR \
+ # 2>&1 \
+ # | tee -a $log_file
+ # EXIT_CODE=$?
+ # if [ $EXIT_CODE -ne 0 ]; then
+ # echo "Perplexity evaluation failed!"
+ # exit $EXIT_CODE
+ # fi
+ # echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+ # find $RESULT_DIR/${EXP_NAME}_ppl \
+ # -name "result-*.csv" \
+ # -printf '%T@ %p\n' \
+ # | sort -n \
+ # | tail -1 \
+ # | cut -d' ' -f2 \
+ # | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+ echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+ python ../src/cli.py dump \
+ --type quant_config \
+ --model 0 1 2 \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+ # echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+ # algo=mxq
+ # #model_ids="0 1 2"
+ # model_ids="0"
+ # for m in $model_ids; do
+ # for cfg in ${BUDGETS}; do
+ # python ../src/cli.py llm \
+ # --model $m \
+ # --algo ${algo} \
+ # --config ${cfg} \
+ # --task eval_model_storage \
+ # --experiment-name "${EXP_NAME}_stor" \
+ # --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ # --result-dir=$RESULT_DIR \
+ # 2>&1 \
+ # | tee -a $log_file
+ # done
+ # done
+ # echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+ # find $RESULT_DIR/${EXP_NAME}_stor \
+ # -name "result-*.csv" \
+ # -printf '%T@ %p\n' \
+ # | sort -n \
+ # | tail -1 \
+ # | cut -d' ' -f2 \
+ # | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+ done
+
+
+ # echo "=========Delete quantized models of batch ${batch_name}========="
+ # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+done
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+# MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+MODELS="Llama-2-7b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 tail-prioritized_2_00 \
+ --attempt2 tail-prioritized_4_00 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-tail-boost.sh b/lm-quant-toolkit/scripts/experiment-tail-boost.sh
new file mode 100644
index 0000000000000000000000000000000000000000..6ff717e0f7f8baba9872f3a14d5a48fbf338b4d1
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-tail-boost.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+EXP_BASE_NAME="tail-boost"
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=tail_boost
+ATTEMPT="${weight_algo}"
+EXP_NAME="${ATTEMPT}"
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Run perplexity evaluation on batch ${EXP_NAME}========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model 0 1 2 \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model 0 1 2 \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids="0 1 2"
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 tail_boost \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-tail-reduction.sh b/lm-quant-toolkit/scripts/experiment-tail-reduction.sh
new file mode 100644
index 0000000000000000000000000000000000000000..de284c9e9fb30f4b6b00d31860ac69d62d4d2925
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-tail-reduction.sh
@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+EXP_BASE_NAME="tail-reduction"
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=tail_boost
+ATTEMPT="${EXP_BASE_NAME}"
+EXP_NAME="${ATTEMPT}"
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Run perplexity evaluation on batch ${EXP_NAME}========="
+python ../src/cli.py llm \
+ --task eval_ppl \
+ --model 0 1 2 \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --factor -1 \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+fi
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+# find $RESULT_DIR/${EXP_NAME}_ppl \
+# -name "result-*.csv" \
+# -printf '%T@ %p\n' \
+# | sort -n \
+# | tail -1 \
+# | cut -d' ' -f2 \
+# | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+#
+# echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+# mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+# python ../src/cli.py dump \
+# --type quant_config \
+# --model 0 1 2 \
+# --budget ${BUDGETS} \
+# --attempt $ATTEMPT \
+# --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+# --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+# 2>&1 \
+# | tee -a $log_file
+#
+# echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+# algo=mxq
+# model_ids="0 1 2"
+# for m in $model_ids; do
+# for cfg in ${BUDGETS}; do
+# python ../src/cli.py llm \
+# --model $m \
+# --algo ${algo} \
+# --config ${cfg} \
+# --task eval_model_storage \
+# --experiment-name "${EXP_NAME}_stor" \
+# --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+# --result-dir=$RESULT_DIR \
+# 2>&1 \
+# | tee -a $log_file
+# done
+# done
+# echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+# find $RESULT_DIR/${EXP_NAME}_stor \
+# -name "result-*.csv" \
+# -printf '%T@ %p\n' \
+# | sort -n \
+# | tail -1 \
+# | cut -d' ' -f2 \
+# | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R data/combined.csv
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+BUDGETS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BUDGETS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 tail-reduction \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/experiment-vit-zs-mxq-kurt-boost.sh b/lm-quant-toolkit/scripts/experiment-vit-zs-mxq-kurt-boost.sh
new file mode 100644
index 0000000000000000000000000000000000000000..2fc64436870007de2ca14d039a1543808fa1d039
--- /dev/null
+++ b/lm-quant-toolkit/scripts/experiment-vit-zs-mxq-kurt-boost.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+# python ../src/cli.py vit \
+# --task eval_zeroshot_cls \
+# --model 0 1 \
+# --config 4.51 4.25 4.13 3.51 3.25 3.13\
+# --algo mxq \
+# --weight-algo kurt-boost \
+# --boost-stop 2 \
+# --top-m-layer 1 \
+# --experiment-name eval_zs_BH_mxq_kurt_boost \
+# --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+# --result-dir="/fdata/llm/mxq/results" \
+# 2>&1 \
+# | tee logs/bench-vit-$(date +%Y%m%d%H%M%S).log
+
+python ../src/cli.py vit \
+ --task eval_zeroshot_cls \
+ --model 0 1 \
+ --config 4.51 4.25 4.13 3.51 3.25 3.13\
+ --algo mxq \
+ --weight-algo kurt-boost \
+ --boost-stop 2 \
+ --top-m-layer 2 \
+ --experiment-name eval_zs_BH_mxq_kurt_boost_22 \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-vit-$(date +%Y%m%d%H%M%S).log
+
diff --git a/lm-quant-toolkit/scripts/fix-3.25-llama-7b-sensi-boost.sh b/lm-quant-toolkit/scripts/fix-3.25-llama-7b-sensi-boost.sh
new file mode 100644
index 0000000000000000000000000000000000000000..8720c0ead1a768afdeba65d7d2ad9e0f99932e45
--- /dev/null
+++ b/lm-quant-toolkit/scripts/fix-3.25-llama-7b-sensi-boost.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+
+BUDGETS="3.25"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-boost
+MODELS="0"
+MODEL_NAMES="Llama-2-7b-hf"
+
+
+# BOOST_STOPS="2 3"
+BOOST_STOPS="3"
+BOOST_TOP_MS="3"
+
+for BOOST_STOP in $BOOST_STOPS; do
+ for BOOST_TOP_M in $BOOST_TOP_MS; do
+ if [[ $BOOST_STOP -eq 2 && $BOOST_TOP_M -eq 1 ]]; then
+ continue
+ fi
+ ATTEMPT="sb-3257_2-fix-${BOOST_STOP}-${BOOST_TOP_M}"
+ EXP_BASE_NAME=$ATTEMPT
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+ log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+ mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+ mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+ # MODELS="0"
+ EXP_NAME="${ATTEMPT}"
+ echo "=========Run perplexity evaluation========="
+ python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --boost-stop $BOOST_STOP \
+ --top-m-layer $BOOST_TOP_M \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ EXIT_CODE=$?
+ if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+ fi
+ echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+ find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+ echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+ python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+ echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+ algo=mxq
+ model_ids=$MODELS
+ for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+ done
+ echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+ find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+ # echo "=========Delete quantized models of batch ${batch_name}========="
+ # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+ OLD_DIR=$(pwd)
+ cd $RESULT_DIR/$EXP_BASE_NAME
+ if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+ fi
+ $OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+
+ cd $OLD_DIR
+ done
+done
+
diff --git a/lm-quant-toolkit/scripts/fix-experiment-llama-kurt-milp-ablation.sh b/lm-quant-toolkit/scripts/fix-experiment-llama-kurt-milp-ablation.sh
new file mode 100644
index 0000000000000000000000000000000000000000..6f5ceabba3a040ef0565960c073057726973d2e0
--- /dev/null
+++ b/lm-quant-toolkit/scripts/fix-experiment-llama-kurt-milp-ablation.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=kurt-milp
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+# for SensiMiLP ablation test, all topm values are equivalent
+
+ATTEMPT="kurt-milp-abl"
+EXP_BASE_NAME=$ATTEMPT
+EXP_NAME="${ATTEMPT}"
+
+log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs ]; then
+ mkdir pdfs
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+cd pdfs
+pdflatex table.tex
+cd ..
+
+cd $OLD_DIR
diff --git a/lm-quant-toolkit/scripts/fix-experiment-llama-sensi-milp-ablation.sh b/lm-quant-toolkit/scripts/fix-experiment-llama-sensi-milp-ablation.sh
new file mode 100644
index 0000000000000000000000000000000000000000..94370285fff8692d840fea96e0931d2068eb1fca
--- /dev/null
+++ b/lm-quant-toolkit/scripts/fix-experiment-llama-sensi-milp-ablation.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+
+BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-milp
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+BOOST_TOP_MS="3"
+
+for BOOST_TOP_M in $BOOST_TOP_MS; do
+ ATTEMPT="sensi-milp-abl-${BOOST_TOP_M}"
+ EXP_BASE_NAME=$ATTEMPT
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+ log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+ mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+ mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+ EXP_NAME="${ATTEMPT}"
+
+ echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+ python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+ echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+ algo=mxq
+ model_ids=$MODELS
+ for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+ done
+ echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+ find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+ # echo "=========Delete quantized models of batch ${batch_name}========="
+ # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+ OLD_DIR=$(pwd)
+ cd $RESULT_DIR/$EXP_BASE_NAME
+ if [ ! -d pdfs ]; then
+ mkdir pdfs
+ fi
+ $OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+ $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+ $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+ cd pdfs
+ pdflatex table.tex
+ cd ..
+
+ cd $OLD_DIR
+
+done
+
diff --git a/lm-quant-toolkit/scripts/fix-llama-sensi-milp.sh b/lm-quant-toolkit/scripts/fix-llama-sensi-milp.sh
new file mode 100644
index 0000000000000000000000000000000000000000..b6bbaac99fe00c5ffe4b6d70bb4bf72f23208a3e
--- /dev/null
+++ b/lm-quant-toolkit/scripts/fix-llama-sensi-milp.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+
+BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=sensi-milp
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+BOOST_TOP_MS="3"
+
+for BOOST_TOP_M in $BOOST_TOP_MS; do
+ ATTEMPT="sensi-milp-${BOOST_TOP_M}"
+ EXP_BASE_NAME=$ATTEMPT
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+ log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+ mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+ mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+ # MODELS="0"
+ EXP_NAME="${ATTEMPT}"
+ echo "=========Run perplexity evaluation========="
+ python ../src/cli.py llm \
+ --task eval_ppl \
+ --model $MODELS \
+ --algo mxq \
+ --weight-algo $weight_algo \
+ --top-m-layer $BOOST_TOP_M \
+ --config ${BUDGETS} \
+ --experiment-name "${EXP_NAME}_ppl" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ EXIT_CODE=$?
+ if [ $EXIT_CODE -ne 0 ]; then
+ echo "Perplexity evaluation failed!"
+ exit $EXIT_CODE
+ fi
+ echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+ find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+ echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+ python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+ echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+ algo=mxq
+ model_ids=$MODELS
+ for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+ done
+ echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+ find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+ # echo "=========Delete quantized models of batch ${batch_name}========="
+ # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+ OLD_DIR=$(pwd)
+ cd $RESULT_DIR/$EXP_BASE_NAME
+ if [ ! -d pdfs ]; then
+ mkdir pdfs
+ fi
+ $OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+ $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+ $OLD_DIR/../data-vis/plot-mem-consumption.R -d data/combined.csv
+ $OLD_DIR/../data-vis/plot-quant-speed.R -d data/combined.csv
+ $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+ cd pdfs
+ pdflatex table.tex
+ cd ..
+
+ # plot configuration allocations for 3 * 12 MXQ combinations
+ MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+ BUDGETS=${MXQ_BATCH[@]:1}
+ for model in $MODELS; do
+ for budget in $BUDGETS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --attempt1 mxq1 \
+ --attempt2 $ATTEMPT \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+ done
+ cd $OLD_DIR
+
+done
+
diff --git a/lm-quant-toolkit/scripts/fix_experiment-llama-kurt-milp.sh b/lm-quant-toolkit/scripts/fix_experiment-llama-kurt-milp.sh
new file mode 100644
index 0000000000000000000000000000000000000000..dd2f4738e3a0a9c9af40f2d827e4488c55ffe1a2
--- /dev/null
+++ b/lm-quant-toolkit/scripts/fix_experiment-llama-kurt-milp.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+
+BUDGETS="6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+weight_algo=kurt-milp
+MODELS="0 1 2"
+MODEL_NAMES="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+
+
+BOOST_TOP_MS="1"
+
+for BOOST_TOP_M in $BOOST_TOP_MS; do
+ ATTEMPT="kurt-milp-${BOOST_TOP_M}"
+ EXP_BASE_NAME=$ATTEMPT
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+ log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+ mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+ mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+ # MODELS="0"
+ EXP_NAME="${ATTEMPT}"
+
+ echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+ python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+ echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+ algo=mxq
+ model_ids=$MODELS
+ for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+ done
+ echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+ find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+ # echo "=========Delete quantized models of batch ${batch_name}========="
+ # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+ OLD_DIR=$(pwd)
+ cd $RESULT_DIR/$EXP_BASE_NAME
+ if [ ! -d pdfs ]; then
+ mkdir pdfs
+ fi
+ $OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+ $OLD_DIR/../data-vis/plot-ppl-mem.R -d data/combined.csv
+ $OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+ cd pdfs
+ pdflatex table.tex
+ cd ..
+
+ cd $OLD_DIR
+
+done
+
diff --git a/lm-quant-toolkit/scripts/plot-allot-kurt-scaled-dense.sh b/lm-quant-toolkit/scripts/plot-allot-kurt-scaled-dense.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f0bcb1fadd3e8d4699d72269e210a137c2b941c2
--- /dev/null
+++ b/lm-quant-toolkit/scripts/plot-allot-kurt-scaled-dense.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# MXQ1=(2bit 2.13 2.15 2.17 2.19 2.21 2.23 2.25 2.27 2.29 2.31 2.45 2.47 2.49 2.51 2.53 2.55 2.57)
+# MXQ2=(3bit 3.07 3.09 3.11 3.13 3.15 3.17 3.19 3.21 3.23 3.25 3.27 3.29 3.31 3.33 3.35 3.37 3.39 3.41 3.42 3.43 3.45 3.47 3.49 3.51 3.53 3.55 3.57 3.59 3.61 3.63 3.65 3.67 3.69 3.71 3.73 3.75 3.77 3.79 3.81 3.83 3.85 3.87 3.89 3.91 3.93 3.95 3.97 3.99)
+# Removed 4.99 which is unsolvable with kurt-scaled scheme
+MXQ4=(4bit 4.01 4.03 4.05 4.07 4.09 4.11 4.13 4.15 4.17 4.19 4.21 4.23 4.25 4.27 4.29 4.31 4.33 4.35 4.37 4.39 4.41 4.43 4.45 4.47 4.49 4.51 4.53 4.55 4.57 4.59 4.61 4.63 4.65 4.67 4.69 4.71 4.73 4.75 4.77 4.79 4.81 4.83 4.85 4.87 4.89 4.91 4.93 4.95 4.97)
+# Remove 5.01 5.02, 5.04 5.10 5.32 which is unsolvable with kurt-scaled scheme
+#MXQ5=(5bit 5.00 5.03 5.06 5.08 5.09 5.12 5.14 5.16 5.18 5.20 5.22 5.24 5.26 5.28 5.30 5.33 5.34 5.36 5.38 5.40 5.42 5.44 5.46 5.48 5.50 5.52 5.54 5.56 5.58 5.60 5.62 5.64 5.66 5.68 5.70 5.72 5.74 5.76 5.78 5.80 5.82 5.84 5.86 5.88 5.90 5.92 5.94 5.96 5.98)
+#MXQ6=(6bit 6.00 6.02 6.03 6.04 6.05 6.06 6.07 6.09 6.11 6.13 6.15 6.17 6.19 6.21 6.23 6.25 6.27 6.29 6.31 6.33 6.35 6.37 6.39 6.41 6.43 6.45 6.47 6.49 6.51 6.53 6.55 6.57 6.59 6.61 6.63 6.65 6.68 6.69 6.71 6.72 6.75 6.77 6.78 6.81 6.83 6.86 6.87 6.89 6.92 6.95 6.96 6.98)
+# MXQ7=(78bit 7.01 7.02 7.04 7.06 7.08 7.10 7.13 7.14 7.16 7.19 7.20 7.22 7.24 7.26 7.29 7.30 7.32 7.34 7.36 7.38 7.41 7.42 7.44 7.47 7.48 7.50 7.53 7.54 7.56 7.57 7.60 7.62 7.63 7.66 7.68 7.70 7.72 7.74 7.76 8.13 8.25 8.51)
+MXQ_BATCHES=(MXQ4)
+declare -n MXQ_BATCH
+
+ATTEMPT="kurt-scaled"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+EXP_BASE_NAME="${ATTEMPT}-dense"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+# Setup data files directories for reporting
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs ]; then
+ mkdir pdfs
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+# MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+MODELS="Llama-2-7b-hf"
+BUDGETS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BUDGETS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R -m $model -b $budget \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --attempt1 mxq1 \
+ --attempt2 $ATTEMPT \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/plot-llama-sensi.sh b/lm-quant-toolkit/scripts/plot-llama-sensi.sh
new file mode 100644
index 0000000000000000000000000000000000000000..22b13942f59544ca88b8057498d3b9665218dc53
--- /dev/null
+++ b/lm-quant-toolkit/scripts/plot-llama-sensi.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+ATTEMPT="llama-sensi"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-directive
+
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf"
+BGS="4.13 4.25 4.51"
+for model in $MODELS; do
+ for budget in $BGS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --fnorm FALSE \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/plot-milp-low-bit.sh b/lm-quant-toolkit/scripts/plot-milp-low-bit.sh
new file mode 100644
index 0000000000000000000000000000000000000000..41daba614432618fc21a16d26a0d9e9e24df3039
--- /dev/null
+++ b/lm-quant-toolkit/scripts/plot-milp-low-bit.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+BUDGETS="4.25"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+
+FACTOR=2
+ATTEMPT="sensi-milp-debug-lowbits9"
+EXP_BASE_NAME=$ATTEMPT
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+
+# Use cached dataset to speedup wikitext, c4 ppl evaluation
+export HF_DATASETS_OFFLINE=1
+
+weight_algo=sensi-milp
+
+log_file="logs/bench-${ATTEMPT}-$(date +%Y%m%d%H%M%S).log"
+
+# MODELS="0 1 2"
+MODELS="0"
+EXP_NAME="${ATTEMPT}"
+
+echo "=========Collect perplexity evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_ppl \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/ppl/mxq/$ATTEMPT
+
+echo "=========Dump quantization configs on batch ${EXP_NAME}========="
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+python ../src/cli.py dump \
+ --type quant_config \
+ --model $MODELS \
+ --budget ${BUDGETS} \
+ --attempt $ATTEMPT \
+ --quant-snapshot-dir=$QUANT_SNAPSHOT_DIR \
+ --output-file "$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/quant-allot-${EXP_NAME}.csv" \
+ 2>&1 \
+ | tee -a $log_file
+
+echo "=========Run memory evaluation on batch ${EXP_NAME}========="
+algo=mxq
+model_ids=$MODELS
+for m in $model_ids; do
+ for cfg in ${BUDGETS}; do
+ python ../src/cli.py llm \
+ --model $m \
+ --algo ${algo} \
+ --config ${cfg} \
+ --task eval_model_storage \
+ --experiment-name "${EXP_NAME}_stor" \
+ --quant-snapshot-dir="$QUANT_SNAPSHOT_DIR/$ATTEMPT" \
+ --result-dir=$RESULT_DIR \
+ 2>&1 \
+ | tee -a $log_file
+ done
+done
+echo "=========Collect memory evaluation result on batch ${EXP_NAME}========="
+find $RESULT_DIR/${EXP_NAME}_stor \
+ -name "result-*.csv" \
+ -printf '%T@ %p\n' \
+ | sort -n \
+ | tail -1 \
+ | cut -d' ' -f2 \
+ | xargs -i cp {} $RESULT_DIR/$EXP_BASE_NAME/data/stor/mxq/$ATTEMPT
+
+# echo "=========Delete quantized models of batch ${batch_name}========="
+# find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+$OLD_DIR/../data-vis/plot-mxq-paired.R data/combined.csv
+$OLD_DIR/../data-vis/plot-mem-consumption.R data/combined.csv
+$OLD_DIR/../data-vis/plot-quant-speed.R data/combined.csv
+$OLD_DIR/../data-vis/gen-table-mxq-llm.R --csv_file data/combined.csv --attempt $ATTEMPT
+pdflatex table.tex
+
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf"
+for model in $MODELS; do
+ for budget in $BUDGETS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $ATTEMPT \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/plot-qwen-sensi-metrics.sh b/lm-quant-toolkit/scripts/plot-qwen-sensi-metrics.sh
new file mode 100644
index 0000000000000000000000000000000000000000..78efdf20e3924b0539d4d4e611f1de5631406476
--- /dev/null
+++ b/lm-quant-toolkit/scripts/plot-qwen-sensi-metrics.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+RESULT_BASE_DIR="/fdata/llm/mxq/results"
+CALIB_DATASETS="bos pileval wikitext c4"
+CONFIGS="b2g128 b2g64 b2g32 b3g128 b3g64 b3g32 b4g128 b4g64 b4g32 b8g128 b8g64 b8g32"
+MODELS="Qwen/Qwen2.5-7B Qwen/Qwen2.5-Coder-7B Qwen/Qwen2.5-Coder-7B-Instruct Qwen/Qwen2.5-Math-7B"
+
+EXP_NAME=sensi_qwen25
+RESULT_DIR=$RESULT_BASE_DIR/$EXP_NAME
+mkdir -p $RESULT_DIR/data
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR
+if [ ! -d pdfs ]; then
+ mkdir -p pdfs
+fi
+$OLD_DIR/../data-vis/plot-variant-sensi.R data/
diff --git a/lm-quant-toolkit/scripts/plot-sensi-milp-mini-batch.sh b/lm-quant-toolkit/scripts/plot-sensi-milp-mini-batch.sh
new file mode 100644
index 0000000000000000000000000000000000000000..92d98d1efb0dd0232846f324d435bdf7f2ec9ea0
--- /dev/null
+++ b/lm-quant-toolkit/scripts/plot-sensi-milp-mini-batch.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+
+# 100%, 99%, 98%, 97%, 96%, 95% of [3.51, 4.25]
+# MXQ2=(minibatch 3.51 3.47 3.44 3.40 3.37 3.33 4.25 4.21 4.17 4.12 4.08 4.04)
+MXQ2=(minibatch 6.89 5.72 5.02 4.51 4.25 4.21 4.17 4.13 4.11 4.07 3.95 3.87 3.83 3.65 3.51 3.25 3.19 3.15 3.13 3.11 3.07)
+MXQ_BATCHES=(MXQ2)
+declare -n MXQ_BATCH
+
+ATTEMPT="sensi-milp-mini2"
+RESULT_DIR="/fdata/llm/mxq/results"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+EXP_BASE_NAME="${ATTEMPT}"
+QUANT_SNAPSHOT_DIR="/fdata/llm/mxq/snapshots"
+# Setup data files directories for reporting
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,qnt,stor}
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/{ppl,stor}/mxq/$ATTEMPT
+mkdir -p $QUANT_SNAPSHOT_DIR/$ATTEMPT
+
+weight_algo=sensi-milp
+
+for MXQ_BATCH in "${MXQ_BATCHES[@]}"; do
+ batch_name=${MXQ_BATCH[@]:0:1}
+ EXP_NAME="${EXP_BASE_NAME}-${batch_name}"
+ mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+ mkdir -p $RESULT_DIR/${EXP_NAME}_stor
+ log_file="logs/bench-${EXP_NAME}-$(date +%Y%m%d%H%M%S).log"
+
+ # echo "=========Delete quantized models of batch ${batch_name}========="
+ # find $QUANT_SNAPSHOT_DIR/$ATTEMPT -maxdepth 1 -type d | xargs rm -fr
+
+ OLD_DIR=$(pwd)
+ cd $RESULT_DIR/$EXP_BASE_NAME
+ if [ ! -d pdfs ]; then
+ mkdir pdfs
+ fi
+
+ # plot configuration allocations for 3 * 12 MXQ combinations
+ MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+ BUDGETS=${MXQ_BATCH[@]:1}
+ for model in $MODELS; do
+ for budget in $BUDGETS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --attempt1 mxq1 \
+ --attempt2 $ATTEMPT \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+ done
+ cd $OLD_DIR
+done
diff --git a/lm-quant-toolkit/scripts/quant-llm-13b-awq.sh b/lm-quant-toolkit/scripts/quant-llm-13b-awq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..59f43c68579e863a5fda735612778bb87f206686
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-13b-awq.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 1 \
+ --algo awq \
+ --config b4g32 b4g64 b4g128 \
+ --experiment-name quant_llm_13B-awq2 \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-2bit-dense1-mxq.sh b/lm-quant-toolkit/scripts/quant-llm-2bit-dense1-mxq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..cec2fe1abcddb33f0ac5a195b14a4039056aa05a
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-2bit-dense1-mxq.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 0 1 2 \
+ --algo mxq \
+ --config 2.57 2.55 2.53 2.51 2.49 2.47 2.45 2.31 2.29 2.27 2.25 2.23 2.21 2.19 2.17 2.15 2.13 \
+ --experiment-name quant_llm_2bit_dense1-mxq \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-3bit-dense1-mxq.sh b/lm-quant-toolkit/scripts/quant-llm-3bit-dense1-mxq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..2b6838194b54d95a8f1b99a21e2bc0e06fdaf5f6
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-3bit-dense1-mxq.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 0 1 2 \
+ --algo mxq \
+ --config 3.57 3.55 3.53 3.51 3.49 3.47 3.45 3.31 3.29 3.27 3.25 3.23 3.21 3.19 3.17 3.15 3.13 3.11 3.09 3.07 \
+ --experiment-name quant_llm_3bit_dense1-mxq \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-4_51-mxq.sh b/lm-quant-toolkit/scripts/quant-llm-4_51-mxq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f2832ed6e1921dfa08fd5c28d641d5ed8bdc9625
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-4_51-mxq.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 0 \
+ --algo mxq \
+ --config 4.51 \
+ --experiment-name quant_llm_4_51-mxq \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-4bit-dense1-mxq.sh b/lm-quant-toolkit/scripts/quant-llm-4bit-dense1-mxq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..fe450dbfde5f7378fba45e63183d9d6986a1e4b9
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-4bit-dense1-mxq.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 0 1 2 \
+ --algo mxq \
+ --config 4.61 4.59 4.57 4.55 4.53 4.51 4.49 4.47 4.45 4.43 4.41 4.35 4.33 4.31 4.29 4.27 4.25 4.23 4.21 4.19 4.17 4.15 4.13 4.11 4.09 4.07 4.05 4.03 \
+ --experiment-name quant_llm_4bit_dense1-mxq \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-8bit-gptq.sh b/lm-quant-toolkit/scripts/quant-llm-8bit-gptq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f32305f7a7b2e50eb0ff0c53c9263a1584c562a6
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-8bit-gptq.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 0 1 2 \
+ --algo gptq \
+ --config b8g32 b8g64 b8g128 \
+ --experiment-name quant_llm-8bit-gptq \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-awq.sh b/lm-quant-toolkit/scripts/quant-llm-awq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..2c378867af209dd6467618c56fb1e22dbdda2204
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-awq.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 1 \
+ --algo awq \
+ --config b4g32 b4g64 b4g128 \
+ --experiment-name quant_llm_13B-awq \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-b4-mxq.sh b/lm-quant-toolkit/scripts/quant-llm-b4-mxq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..dcd7dd749d655914857c6af8179f6d2aa85f3be2
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-b4-mxq.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 0 1 2 \
+ --algo mxq \
+ --config 4.51 4.25 4.13 \
+ --experiment-name quant_llm_b4-mxq2 \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-gptq.sh b/lm-quant-toolkit/scripts/quant-llm-gptq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..7f87069f1e6d8c72f6d4c054e72f713d23f9c7c3
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-gptq.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/lm-quant-toolkit/src/cli.py llm \
+ --task quant \
+ --model 0 1 2 \
+ --algo gptq \
+ --experiment-name quant_llm-gptq \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-hqq.sh b/lm-quant-toolkit/scripts/quant-llm-hqq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..7cca1ed873d34d01908446e11a129a49079aa29e
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-hqq.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 0 1 2 \
+ --algo hqq \
+ --experiment-name quant_llm-hqq \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-kurt-13b-mxq.sh b/lm-quant-toolkit/scripts/quant-llm-kurt-13b-mxq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..bd3b388fefb627d31153a7d41b75e96ca06cb5dd
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-kurt-13b-mxq.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 1 \
+ --algo mxq \
+ --config 4.51 4.25 4.13 \
+ --experiment-name quant_llm-weighted-113b-mxq \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots-kurt" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/quant-llm-mxq.sh b/lm-quant-toolkit/scripts/quant-llm-mxq.sh
new file mode 100644
index 0000000000000000000000000000000000000000..8dc71db02c8bb95d74827dbff6f7e51bb8e685c9
--- /dev/null
+++ b/lm-quant-toolkit/scripts/quant-llm-mxq.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# export HF_HOME=/data/hugginface
+# conda activate quant-eval
+
+if [ ! -d logs ]; then
+ mkdir logs
+fi
+
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+
+python ../src/cli.py llm \
+ --task quant \
+ --model 0 1 2 \
+ --algo mxq \
+ --experiment-name quant_llm-mxq \
+ --quant-snapshot-dir="/fdata/llm/mxq/snapshots" \
+ --result-dir="/fdata/llm/mxq/results" \
+ 2>&1 \
+ | tee logs/bench-$(date +%Y%m%d%H%M%S).log
diff --git a/lm-quant-toolkit/scripts/sim-quant-allot.sh b/lm-quant-toolkit/scripts/sim-quant-allot.sh
new file mode 100644
index 0000000000000000000000000000000000000000..ef109cae40e5c37ee5ff7bdc122f0eb04b457eaf
--- /dev/null
+++ b/lm-quant-toolkit/scripts/sim-quant-allot.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+
+TAILS=(tail-prioritized 2.00 4.00)
+# MXQ_BATCHES=(HEADS TAILS)
+MXQ_BATCHES=(TAILS)
+declare -n MXQ_BATCH
+
+EXP_BASE_NAME="tail-factor-search"
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME
+
+for MXQ_BATCH in "${MXQ_BATCHES[@]}"; do
+ weight_algo=${MXQ_BATCH[@]:0:1}
+ for factor in ${MXQ_BATCH[@]:1}; do
+ ATTEMPT="${weight_algo}_${factor/./_}"
+ EXP_NAME="${ATTEMPT}"
+ log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+ mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+ mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+
+ # python -m pdb ../src/cli.py dump \
+ python ../src/cli.py dump \
+ --type quant_config_sim \
+ --model 0 1 2 \
+ --weight-algo $weight_algo \
+ --factor $factor \
+ --budget ${BUDGETS} \
+ --output-file="$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/${EXP_NAME}.csv"\
+ 2>&1 \
+ | tee -a $log_file
+
+ done
+
+done
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+for model in $MODELS; do
+ for budget in $BUDGETS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 tail-prioritized_2_00 \
+ --attempt2 tail-prioritized_4_00 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/scripts/sim-quant-milp.sh b/lm-quant-toolkit/scripts/sim-quant-milp.sh
new file mode 100644
index 0000000000000000000000000000000000000000..14b11a6ec487ac6599f8d73dcdd73f09b28a7dc5
--- /dev/null
+++ b/lm-quant-toolkit/scripts/sim-quant-milp.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# python ../src/lm_quant_toolkit/eval/common.py > 2-8-quant-bits.txt
+python ../src/lm_quant_toolkit/eval/common.py > 2-8-quant-bits-sensi.txt
diff --git a/lm-quant-toolkit/scripts/sim-tail-boost-quant-allot.sh b/lm-quant-toolkit/scripts/sim-tail-boost-quant-allot.sh
new file mode 100644
index 0000000000000000000000000000000000000000..8edd3705bdddab5e99323d5f82c64549b3deb968
--- /dev/null
+++ b/lm-quant-toolkit/scripts/sim-tail-boost-quant-allot.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# BUDGETS="2.13 2.25 2.51 3.13 3.25 3.51 4.13 4.25 4.51"
+BUDGETS="4.13 4.25 4.51"
+RESULT_DIR="/fdata/llm/mxq/results"
+weight_algo="tail_boost"
+EXP_BASE_NAME="$weight_algo"
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME
+
+ATTEMPT="$weight_algo"
+EXP_NAME="${ATTEMPT}"
+log_file="logs/bench-$(date +%Y%m%d%H%M%S).log"
+
+mkdir -p $RESULT_DIR/${EXP_NAME}_ppl
+mkdir -p $RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT
+
+# python -m pdb ../src/cli.py dump \
+python ../src/cli.py dump \
+ --type quant_config_sim \
+ --model 0 1 2 \
+ --weight-algo $weight_algo \
+ --budget ${BUDGETS} \
+ --output-file="$RESULT_DIR/$EXP_BASE_NAME/data/allot/mxq/$ATTEMPT/${EXP_NAME}.csv"\
+ 2>&1 \
+ | tee -a $log_file
+
+
+OLD_DIR=$(pwd)
+cd $RESULT_DIR/$EXP_BASE_NAME
+if [ ! -d pdfs/allot ]; then
+ mkdir -p pdfs/allot
+fi
+$OLD_DIR/../data-vis/combine.R \
+ --baseline_data_dir $OLD_DIR/../data-vis/data \
+ --mxq_data_dir data
+# plot configuration allocations for 3 * 12 MXQ combinations
+MODELS="Llama-2-7b-hf Llama-2-13b-hf Meta-Llama-3-8B"
+for model in $MODELS; do
+ for budget in $BUDGETS; do
+ $OLD_DIR/../data-vis/plot-mxq-allocation.R \
+ -m $model \
+ -b $budget \
+ --attempt1 $weight_algo \
+ --attempt2 mxq1 \
+ --fnorm_data_dir $OLD_DIR/../src/data \
+ --quant_cfg_allot_file data/quant-cfg-allocation.csv
+ done
+done
diff --git a/lm-quant-toolkit/src/data/fnorm-Llama-2-70b-hf.csv b/lm-quant-toolkit/src/data/fnorm-Llama-2-70b-hf.csv
new file mode 100644
index 0000000000000000000000000000000000000000..f8f588393b83c8cabc8bd2d93273fa07b896c51c
--- /dev/null
+++ b/lm-quant-toolkit/src/data/fnorm-Llama-2-70b-hf.csv
@@ -0,0 +1,6721 @@
+layer,module,nbit1,gsize1,nbit2,gsize2,fnorm,memmb,params
+0,self_attn.q_proj,2,32,8,128,33.875,20.0625,67108864
+0,self_attn.q_proj,2,64,8,128,40.5625,18.03125,67108864
+0,self_attn.q_proj,2,128,8,128,46.625,17.015625,67108864
+0,self_attn.q_proj,3,32,8,128,14.78125,28.0625,67108864
+0,self_attn.q_proj,3,64,8,128,18.09375,26.03125,67108864
+0,self_attn.q_proj,3,128,8,128,21.5,25.015625,67108864
+0,self_attn.q_proj,4,32,8,128,6.90234375,36.0625,67108864
+0,self_attn.q_proj,4,64,8,128,8.453125,34.03125,67108864
+0,self_attn.q_proj,4,128,8,128,10.09375,33.015625,67108864
+0,self_attn.q_proj,8,32,8,128,0.41796875,68.0625,67108864
+0,self_attn.q_proj,8,64,8,128,0.51171875,66.03125,67108864
+0,self_attn.q_proj,8,128,8,128,0.609375,65.015625,67108864
+0,self_attn.v_proj,2,32,8,128,9.8515625,2.5078125,8388608
+0,self_attn.v_proj,2,64,8,128,11.765625,2.25390625,8388608
+0,self_attn.v_proj,2,128,8,128,13.578125,2.126953125,8388608
+0,self_attn.v_proj,3,32,8,128,4.21484375,3.5078125,8388608
+0,self_attn.v_proj,3,64,8,128,5.05078125,3.25390625,8388608
+0,self_attn.v_proj,3,128,8,128,5.8515625,3.126953125,8388608
+0,self_attn.v_proj,4,32,8,128,1.9677734375,4.5078125,8388608
+0,self_attn.v_proj,4,64,8,128,2.357421875,4.25390625,8388608
+0,self_attn.v_proj,4,128,8,128,2.73046875,4.126953125,8388608
+0,self_attn.v_proj,8,32,8,128,0.119140625,8.5078125,8388608
+0,self_attn.v_proj,8,64,8,128,0.142578125,8.25390625,8388608
+0,self_attn.v_proj,8,128,8,128,0.16455078125,8.126953125,8388608
+0,self_attn.k_proj,2,32,8,128,27.875,2.5078125,8388608
+0,self_attn.k_proj,2,64,8,128,33.96875,2.25390625,8388608
+0,self_attn.k_proj,2,128,8,128,38.625,2.126953125,8388608
+0,self_attn.k_proj,3,32,8,128,12.40625,3.5078125,8388608
+0,self_attn.k_proj,3,64,8,128,16.21875,3.25390625,8388608
+0,self_attn.k_proj,3,128,8,128,19.703125,3.126953125,8388608
+0,self_attn.k_proj,4,32,8,128,5.79296875,4.5078125,8388608
+0,self_attn.k_proj,4,64,8,128,7.6015625,4.25390625,8388608
+0,self_attn.k_proj,4,128,8,128,9.3125,4.126953125,8388608
+0,self_attn.k_proj,8,32,8,128,0.350830078125,8.5078125,8388608
+0,self_attn.k_proj,8,64,8,128,0.459228515625,8.25390625,8388608
+0,self_attn.k_proj,8,128,8,128,0.56103515625,8.126953125,8388608
+0,self_attn.o_proj,2,32,8,128,22.90625,20.0625,67108864
+0,self_attn.o_proj,2,64,8,128,26.890625,18.03125,67108864
+0,self_attn.o_proj,2,128,8,128,30.609375,17.015625,67108864
+0,self_attn.o_proj,3,32,8,128,9.8671875,28.0625,67108864
+0,self_attn.o_proj,3,64,8,128,11.6484375,26.03125,67108864
+0,self_attn.o_proj,3,128,8,128,13.40625,25.015625,67108864
+0,self_attn.o_proj,4,32,8,128,4.62109375,36.0625,67108864
+0,self_attn.o_proj,4,64,8,128,5.47265625,34.03125,67108864
+0,self_attn.o_proj,4,128,8,128,6.328125,33.015625,67108864
+0,self_attn.o_proj,8,32,8,128,0.2802734375,68.0625,67108864
+0,self_attn.o_proj,8,64,8,128,0.33154296875,66.03125,67108864
+0,self_attn.o_proj,8,128,8,128,0.3828125,65.015625,67108864
+0,mlp.gate_proj,2,32,8,128,56.25,70.21875,234881024
+0,mlp.gate_proj,2,64,8,128,66.6875,63.109375,234881024
+0,mlp.gate_proj,2,128,8,128,76.875,59.5546875,234881024
+0,mlp.gate_proj,3,32,8,128,24.09375,98.21875,234881024
+0,mlp.gate_proj,3,64,8,128,28.734375,91.109375,234881024
+0,mlp.gate_proj,3,128,8,128,33.28125,87.5546875,234881024
+0,mlp.gate_proj,4,32,8,128,11.2421875,126.21875,234881024
+0,mlp.gate_proj,4,64,8,128,13.421875,119.109375,234881024
+0,mlp.gate_proj,4,128,8,128,15.5390625,115.5546875,234881024
+0,mlp.gate_proj,8,32,8,128,0.68408203125,238.21875,234881024
+0,mlp.gate_proj,8,64,8,128,0.8154296875,231.109375,234881024
+0,mlp.gate_proj,8,128,8,128,0.94091796875,227.5546875,234881024
+0,mlp.up_proj,2,32,8,128,54.375,70.21875,234881024
+0,mlp.up_proj,2,64,8,128,64.0625,63.109375,234881024
+0,mlp.up_proj,2,128,8,128,73.25,59.5546875,234881024
+0,mlp.up_proj,3,32,8,128,23.234375,98.21875,234881024
+0,mlp.up_proj,3,64,8,128,27.46875,91.109375,234881024
+0,mlp.up_proj,3,128,8,128,31.375,87.5546875,234881024
+0,mlp.up_proj,4,32,8,128,10.8359375,126.21875,234881024
+0,mlp.up_proj,4,64,8,128,12.8125,119.109375,234881024
+0,mlp.up_proj,4,128,8,128,14.6484375,115.5546875,234881024
+0,mlp.up_proj,8,32,8,128,0.65966796875,238.21875,234881024
+0,mlp.up_proj,8,64,8,128,0.77880859375,231.109375,234881024
+0,mlp.up_proj,8,128,8,128,0.88720703125,227.5546875,234881024
+0,mlp.down_proj,2,32,8,128,65.375,70.21875,234881024
+0,mlp.down_proj,2,64,8,128,76.5625,63.109375,234881024
+0,mlp.down_proj,2,128,8,128,86.6875,59.5546875,234881024
+0,mlp.down_proj,3,32,8,128,28.0625,98.21875,234881024
+0,mlp.down_proj,3,64,8,128,32.875,91.109375,234881024
+0,mlp.down_proj,3,128,8,128,37.46875,87.5546875,234881024
+0,mlp.down_proj,4,32,8,128,13.1015625,126.21875,234881024
+0,mlp.down_proj,4,64,8,128,15.3671875,119.109375,234881024
+0,mlp.down_proj,4,128,8,128,17.515625,115.5546875,234881024
+0,mlp.down_proj,8,32,8,128,0.7958984375,238.21875,234881024
+0,mlp.down_proj,8,64,8,128,0.93359375,231.109375,234881024
+0,mlp.down_proj,8,128,8,128,1.0615234375,227.5546875,234881024
+1,self_attn.q_proj,2,32,8,128,47.625,20.0625,67108864
+1,self_attn.q_proj,2,64,8,128,56.15625,18.03125,67108864
+1,self_attn.q_proj,2,128,8,128,64.0625,17.015625,67108864
+1,self_attn.q_proj,3,32,8,128,21.5,28.0625,67108864
+1,self_attn.q_proj,3,64,8,128,26.359375,26.03125,67108864
+1,self_attn.q_proj,3,128,8,128,31.578125,25.015625,67108864
+1,self_attn.q_proj,4,32,8,128,10.0546875,36.0625,67108864
+1,self_attn.q_proj,4,64,8,128,12.359375,34.03125,67108864
+1,self_attn.q_proj,4,128,8,128,14.8828125,33.015625,67108864
+1,self_attn.q_proj,8,32,8,128,0.60888671875,68.0625,67108864
+1,self_attn.q_proj,8,64,8,128,0.74755859375,66.03125,67108864
+1,self_attn.q_proj,8,128,8,128,0.8974609375,65.015625,67108864
+1,self_attn.v_proj,2,32,8,128,9.65625,2.5078125,8388608
+1,self_attn.v_proj,2,64,8,128,11.4765625,2.25390625,8388608
+1,self_attn.v_proj,2,128,8,128,13.1328125,2.126953125,8388608
+1,self_attn.v_proj,3,32,8,128,4.1328125,3.5078125,8388608
+1,self_attn.v_proj,3,64,8,128,4.921875,3.25390625,8388608
+1,self_attn.v_proj,3,128,8,128,5.65625,3.126953125,8388608
+1,self_attn.v_proj,4,32,8,128,1.927734375,4.5078125,8388608
+1,self_attn.v_proj,4,64,8,128,2.296875,4.25390625,8388608
+1,self_attn.v_proj,4,128,8,128,2.640625,4.126953125,8388608
+1,self_attn.v_proj,8,32,8,128,0.11676025390625,8.5078125,8388608
+1,self_attn.v_proj,8,64,8,128,0.138916015625,8.25390625,8388608
+1,self_attn.v_proj,8,128,8,128,0.1591796875,8.126953125,8388608
+1,self_attn.k_proj,2,32,8,128,32.03125,2.5078125,8388608
+1,self_attn.k_proj,2,64,8,128,38.46875,2.25390625,8388608
+1,self_attn.k_proj,2,128,8,128,43.8125,2.126953125,8388608
+1,self_attn.k_proj,3,32,8,128,14.2734375,3.5078125,8388608
+1,self_attn.k_proj,3,64,8,128,17.796875,3.25390625,8388608
+1,self_attn.k_proj,3,128,8,128,21.3125,3.126953125,8388608
+1,self_attn.k_proj,4,32,8,128,6.671875,4.5078125,8388608
+1,self_attn.k_proj,4,64,8,128,8.34375,4.25390625,8388608
+1,self_attn.k_proj,4,128,8,128,10.0234375,4.126953125,8388608
+1,self_attn.k_proj,8,32,8,128,0.403076171875,8.5078125,8388608
+1,self_attn.k_proj,8,64,8,128,0.50390625,8.25390625,8388608
+1,self_attn.k_proj,8,128,8,128,0.603515625,8.126953125,8388608
+1,self_attn.o_proj,2,32,8,128,27.6875,20.0625,67108864
+1,self_attn.o_proj,2,64,8,128,32.53125,18.03125,67108864
+1,self_attn.o_proj,2,128,8,128,37.15625,17.015625,67108864
+1,self_attn.o_proj,3,32,8,128,11.90625,28.0625,67108864
+1,self_attn.o_proj,3,64,8,128,14.0859375,26.03125,67108864
+1,self_attn.o_proj,3,128,8,128,16.234375,25.015625,67108864
+1,self_attn.o_proj,4,32,8,128,5.5625,36.0625,67108864
+1,self_attn.o_proj,4,64,8,128,6.58984375,34.03125,67108864
+1,self_attn.o_proj,4,128,8,128,7.62109375,33.015625,67108864
+1,self_attn.o_proj,8,32,8,128,0.3369140625,68.0625,67108864
+1,self_attn.o_proj,8,64,8,128,0.398681640625,66.03125,67108864
+1,self_attn.o_proj,8,128,8,128,0.459716796875,65.015625,67108864
+1,mlp.gate_proj,2,32,8,128,73.3125,70.21875,234881024
+1,mlp.gate_proj,2,64,8,128,85.875,63.109375,234881024
+1,mlp.gate_proj,2,128,8,128,97.5625,59.5546875,234881024
+1,mlp.gate_proj,3,32,8,128,31.359375,98.21875,234881024
+1,mlp.gate_proj,3,64,8,128,36.84375,91.109375,234881024
+1,mlp.gate_proj,3,128,8,128,41.96875,87.5546875,234881024
+1,mlp.gate_proj,4,32,8,128,14.6328125,126.21875,234881024
+1,mlp.gate_proj,4,64,8,128,17.1875,119.109375,234881024
+1,mlp.gate_proj,4,128,8,128,19.625,115.5546875,234881024
+1,mlp.gate_proj,8,32,8,128,0.88916015625,238.21875,234881024
+1,mlp.gate_proj,8,64,8,128,1.044921875,231.109375,234881024
+1,mlp.gate_proj,8,128,8,128,1.189453125,227.5546875,234881024
+1,mlp.up_proj,2,32,8,128,71.3125,70.21875,234881024
+1,mlp.up_proj,2,64,8,128,83.1875,63.109375,234881024
+1,mlp.up_proj,2,128,8,128,93.8125,59.5546875,234881024
+1,mlp.up_proj,3,32,8,128,30.484375,98.21875,234881024
+1,mlp.up_proj,3,64,8,128,35.625,91.109375,234881024
+1,mlp.up_proj,3,128,8,128,40.25,87.5546875,234881024
+1,mlp.up_proj,4,32,8,128,14.234375,126.21875,234881024
+1,mlp.up_proj,4,64,8,128,16.609375,119.109375,234881024
+1,mlp.up_proj,4,128,8,128,18.78125,115.5546875,234881024
+1,mlp.up_proj,8,32,8,128,0.86474609375,238.21875,234881024
+1,mlp.up_proj,8,64,8,128,1.009765625,231.109375,234881024
+1,mlp.up_proj,8,128,8,128,1.138671875,227.5546875,234881024
+1,mlp.down_proj,2,32,8,128,76.75,70.21875,234881024
+1,mlp.down_proj,2,64,8,128,89.375,63.109375,234881024
+1,mlp.down_proj,2,128,8,128,100.9375,59.5546875,234881024
+1,mlp.down_proj,3,32,8,128,32.8125,98.21875,234881024
+1,mlp.down_proj,3,64,8,128,38.34375,91.109375,234881024
+1,mlp.down_proj,3,128,8,128,43.3125,87.5546875,234881024
+1,mlp.down_proj,4,32,8,128,15.3046875,126.21875,234881024
+1,mlp.down_proj,4,64,8,128,17.890625,119.109375,234881024
+1,mlp.down_proj,4,128,8,128,20.234375,115.5546875,234881024
+1,mlp.down_proj,8,32,8,128,0.93017578125,238.21875,234881024
+1,mlp.down_proj,8,64,8,128,1.0869140625,231.109375,234881024
+1,mlp.down_proj,8,128,8,128,1.2275390625,227.5546875,234881024
+2,self_attn.q_proj,2,32,8,128,57.03125,20.0625,67108864
+2,self_attn.q_proj,2,64,8,128,67.4375,18.03125,67108864
+2,self_attn.q_proj,2,128,8,128,78.5625,17.015625,67108864
+2,self_attn.q_proj,3,32,8,128,24.5,28.0625,67108864
+2,self_attn.q_proj,3,64,8,128,29.15625,26.03125,67108864
+2,self_attn.q_proj,3,128,8,128,34.3125,25.015625,67108864
+2,self_attn.q_proj,4,32,8,128,11.4296875,36.0625,67108864
+2,self_attn.q_proj,4,64,8,128,13.609375,34.03125,67108864
+2,self_attn.q_proj,4,128,8,128,16.015625,33.015625,67108864
+2,self_attn.q_proj,8,32,8,128,0.69189453125,68.0625,67108864
+2,self_attn.q_proj,8,64,8,128,0.82275390625,66.03125,67108864
+2,self_attn.q_proj,8,128,8,128,0.96630859375,65.015625,67108864
+2,self_attn.v_proj,2,32,8,128,10.03125,2.5078125,8388608
+2,self_attn.v_proj,2,64,8,128,11.75,2.25390625,8388608
+2,self_attn.v_proj,2,128,8,128,13.3125,2.126953125,8388608
+2,self_attn.v_proj,3,32,8,128,4.2890625,3.5078125,8388608
+2,self_attn.v_proj,3,64,8,128,5.02734375,3.25390625,8388608
+2,self_attn.v_proj,3,128,8,128,5.6953125,3.126953125,8388608
+2,self_attn.v_proj,4,32,8,128,2.0,4.5078125,8388608
+2,self_attn.v_proj,4,64,8,128,2.345703125,4.25390625,8388608
+2,self_attn.v_proj,4,128,8,128,2.65625,4.126953125,8388608
+2,self_attn.v_proj,8,32,8,128,0.1212158203125,8.5078125,8388608
+2,self_attn.v_proj,8,64,8,128,0.141845703125,8.25390625,8388608
+2,self_attn.v_proj,8,128,8,128,0.1602783203125,8.126953125,8388608
+2,self_attn.k_proj,2,32,8,128,36.46875,2.5078125,8388608
+2,self_attn.k_proj,2,64,8,128,43.71875,2.25390625,8388608
+2,self_attn.k_proj,2,128,8,128,49.75,2.126953125,8388608
+2,self_attn.k_proj,3,32,8,128,15.796875,3.5078125,8388608
+2,self_attn.k_proj,3,64,8,128,19.1875,3.25390625,8388608
+2,self_attn.k_proj,3,128,8,128,22.40625,3.126953125,8388608
+2,self_attn.k_proj,4,32,8,128,7.37109375,4.5078125,8388608
+2,self_attn.k_proj,4,64,8,128,8.953125,4.25390625,8388608
+2,self_attn.k_proj,4,128,8,128,10.4609375,4.126953125,8388608
+2,self_attn.k_proj,8,32,8,128,0.44580078125,8.5078125,8388608
+2,self_attn.k_proj,8,64,8,128,0.54150390625,8.25390625,8388608
+2,self_attn.k_proj,8,128,8,128,0.63037109375,8.126953125,8388608
+2,self_attn.o_proj,2,32,8,128,34.6875,20.0625,67108864
+2,self_attn.o_proj,2,64,8,128,40.65625,18.03125,67108864
+2,self_attn.o_proj,2,128,8,128,46.28125,17.015625,67108864
+2,self_attn.o_proj,3,32,8,128,14.8828125,28.0625,67108864
+2,self_attn.o_proj,3,64,8,128,17.515625,26.03125,67108864
+2,self_attn.o_proj,3,128,8,128,20.09375,25.015625,67108864
+2,self_attn.o_proj,4,32,8,128,6.94921875,36.0625,67108864
+2,self_attn.o_proj,4,64,8,128,8.1796875,34.03125,67108864
+2,self_attn.o_proj,4,128,8,128,9.3984375,33.015625,67108864
+2,self_attn.o_proj,8,32,8,128,0.420166015625,68.0625,67108864
+2,self_attn.o_proj,8,64,8,128,0.4951171875,66.03125,67108864
+2,self_attn.o_proj,8,128,8,128,0.56640625,65.015625,67108864
+2,mlp.gate_proj,2,32,8,128,80.1875,70.21875,234881024
+2,mlp.gate_proj,2,64,8,128,93.625,63.109375,234881024
+2,mlp.gate_proj,2,128,8,128,106.0625,59.5546875,234881024
+2,mlp.gate_proj,3,32,8,128,34.28125,98.21875,234881024
+2,mlp.gate_proj,3,64,8,128,40.15625,91.109375,234881024
+2,mlp.gate_proj,3,128,8,128,45.5,87.5546875,234881024
+2,mlp.gate_proj,4,32,8,128,15.9921875,126.21875,234881024
+2,mlp.gate_proj,4,64,8,128,18.734375,119.109375,234881024
+2,mlp.gate_proj,4,128,8,128,21.25,115.5546875,234881024
+2,mlp.gate_proj,8,32,8,128,0.97265625,238.21875,234881024
+2,mlp.gate_proj,8,64,8,128,1.1396484375,231.109375,234881024
+2,mlp.gate_proj,8,128,8,128,1.2890625,227.5546875,234881024
+2,mlp.up_proj,2,32,8,128,76.375,70.21875,234881024
+2,mlp.up_proj,2,64,8,128,88.875,63.109375,234881024
+2,mlp.up_proj,2,128,8,128,100.25,59.5546875,234881024
+2,mlp.up_proj,3,32,8,128,32.625,98.21875,234881024
+2,mlp.up_proj,3,64,8,128,38.09375,91.109375,234881024
+2,mlp.up_proj,3,128,8,128,42.9375,87.5546875,234881024
+2,mlp.up_proj,4,32,8,128,15.2265625,126.21875,234881024
+2,mlp.up_proj,4,64,8,128,17.765625,119.109375,234881024
+2,mlp.up_proj,4,128,8,128,20.046875,115.5546875,234881024
+2,mlp.up_proj,8,32,8,128,0.92578125,238.21875,234881024
+2,mlp.up_proj,8,64,8,128,1.080078125,231.109375,234881024
+2,mlp.up_proj,8,128,8,128,1.216796875,227.5546875,234881024
+2,mlp.down_proj,2,32,8,128,79.5,70.21875,234881024
+2,mlp.down_proj,2,64,8,128,92.5,63.109375,234881024
+2,mlp.down_proj,2,128,8,128,104.3125,59.5546875,234881024
+2,mlp.down_proj,3,32,8,128,33.96875,98.21875,234881024
+2,mlp.down_proj,3,64,8,128,39.625,91.109375,234881024
+2,mlp.down_proj,3,128,8,128,44.6875,87.5546875,234881024
+2,mlp.down_proj,4,32,8,128,15.84375,126.21875,234881024
+2,mlp.down_proj,4,64,8,128,18.484375,119.109375,234881024
+2,mlp.down_proj,4,128,8,128,20.859375,115.5546875,234881024
+2,mlp.down_proj,8,32,8,128,0.9638671875,238.21875,234881024
+2,mlp.down_proj,8,64,8,128,1.125,231.109375,234881024
+2,mlp.down_proj,8,128,8,128,1.2666015625,227.5546875,234881024
+3,self_attn.q_proj,2,32,8,128,51.6875,20.0625,67108864
+3,self_attn.q_proj,2,64,8,128,61.3125,18.03125,67108864
+3,self_attn.q_proj,2,128,8,128,70.9375,17.015625,67108864
+3,self_attn.q_proj,3,32,8,128,22.140625,28.0625,67108864
+3,self_attn.q_proj,3,64,8,128,26.328125,26.03125,67108864
+3,self_attn.q_proj,3,128,8,128,30.609375,25.015625,67108864
+3,self_attn.q_proj,4,32,8,128,10.328125,36.0625,67108864
+3,self_attn.q_proj,4,64,8,128,12.28125,34.03125,67108864
+3,self_attn.q_proj,4,128,8,128,14.28125,33.015625,67108864
+3,self_attn.q_proj,8,32,8,128,0.62548828125,68.0625,67108864
+3,self_attn.q_proj,8,64,8,128,0.74267578125,66.03125,67108864
+3,self_attn.q_proj,8,128,8,128,0.86083984375,65.015625,67108864
+3,self_attn.v_proj,2,32,8,128,11.6484375,2.5078125,8388608
+3,self_attn.v_proj,2,64,8,128,13.6328125,2.25390625,8388608
+3,self_attn.v_proj,2,128,8,128,15.4140625,2.126953125,8388608
+3,self_attn.v_proj,3,32,8,128,4.98046875,3.5078125,8388608
+3,self_attn.v_proj,3,64,8,128,5.828125,3.25390625,8388608
+3,self_attn.v_proj,3,128,8,128,6.58984375,3.126953125,8388608
+3,self_attn.v_proj,4,32,8,128,2.32421875,4.5078125,8388608
+3,self_attn.v_proj,4,64,8,128,2.71875,4.25390625,8388608
+3,self_attn.v_proj,4,128,8,128,3.076171875,4.126953125,8388608
+3,self_attn.v_proj,8,32,8,128,0.140869140625,8.5078125,8388608
+3,self_attn.v_proj,8,64,8,128,0.1644287109375,8.25390625,8388608
+3,self_attn.v_proj,8,128,8,128,0.1854248046875,8.126953125,8388608
+3,self_attn.k_proj,2,32,8,128,31.90625,2.5078125,8388608
+3,self_attn.k_proj,2,64,8,128,38.375,2.25390625,8388608
+3,self_attn.k_proj,2,128,8,128,43.78125,2.126953125,8388608
+3,self_attn.k_proj,3,32,8,128,13.875,3.5078125,8388608
+3,self_attn.k_proj,3,64,8,128,16.84375,3.25390625,8388608
+3,self_attn.k_proj,3,128,8,128,19.703125,3.126953125,8388608
+3,self_attn.k_proj,4,32,8,128,6.47265625,4.5078125,8388608
+3,self_attn.k_proj,4,64,8,128,7.86328125,4.25390625,8388608
+3,self_attn.k_proj,4,128,8,128,9.203125,4.126953125,8388608
+3,self_attn.k_proj,8,32,8,128,0.3916015625,8.5078125,8388608
+3,self_attn.k_proj,8,64,8,128,0.4755859375,8.25390625,8388608
+3,self_attn.k_proj,8,128,8,128,0.5546875,8.126953125,8388608
+3,self_attn.o_proj,2,32,8,128,38.125,20.0625,67108864
+3,self_attn.o_proj,2,64,8,128,44.53125,18.03125,67108864
+3,self_attn.o_proj,2,128,8,128,50.40625,17.015625,67108864
+3,self_attn.o_proj,3,32,8,128,16.328125,28.0625,67108864
+3,self_attn.o_proj,3,64,8,128,19.109375,26.03125,67108864
+3,self_attn.o_proj,3,128,8,128,21.703125,25.015625,67108864
+3,self_attn.o_proj,4,32,8,128,7.6171875,36.0625,67108864
+3,self_attn.o_proj,4,64,8,128,8.921875,34.03125,67108864
+3,self_attn.o_proj,4,128,8,128,10.1328125,33.015625,67108864
+3,self_attn.o_proj,8,32,8,128,0.461181640625,68.0625,67108864
+3,self_attn.o_proj,8,64,8,128,0.53955078125,66.03125,67108864
+3,self_attn.o_proj,8,128,8,128,0.61083984375,65.015625,67108864
+3,mlp.gate_proj,2,32,8,128,82.0625,70.21875,234881024
+3,mlp.gate_proj,2,64,8,128,95.875,63.109375,234881024
+3,mlp.gate_proj,2,128,8,128,108.625,59.5546875,234881024
+3,mlp.gate_proj,3,32,8,128,35.09375,98.21875,234881024
+3,mlp.gate_proj,3,64,8,128,41.0625,91.109375,234881024
+3,mlp.gate_proj,3,128,8,128,46.53125,87.5546875,234881024
+3,mlp.gate_proj,4,32,8,128,16.359375,126.21875,234881024
+3,mlp.gate_proj,4,64,8,128,19.171875,119.109375,234881024
+3,mlp.gate_proj,4,128,8,128,21.71875,115.5546875,234881024
+3,mlp.gate_proj,8,32,8,128,0.9951171875,238.21875,234881024
+3,mlp.gate_proj,8,64,8,128,1.166015625,231.109375,234881024
+3,mlp.gate_proj,8,128,8,128,1.3173828125,227.5546875,234881024
+3,mlp.up_proj,2,32,8,128,78.4375,70.21875,234881024
+3,mlp.up_proj,2,64,8,128,91.25,63.109375,234881024
+3,mlp.up_proj,2,128,8,128,103.0,59.5546875,234881024
+3,mlp.up_proj,3,32,8,128,33.5,98.21875,234881024
+3,mlp.up_proj,3,64,8,128,39.09375,91.109375,234881024
+3,mlp.up_proj,3,128,8,128,44.09375,87.5546875,234881024
+3,mlp.up_proj,4,32,8,128,15.6171875,126.21875,234881024
+3,mlp.up_proj,4,64,8,128,18.234375,119.109375,234881024
+3,mlp.up_proj,4,128,8,128,20.59375,115.5546875,234881024
+3,mlp.up_proj,8,32,8,128,0.94970703125,238.21875,234881024
+3,mlp.up_proj,8,64,8,128,1.109375,231.109375,234881024
+3,mlp.up_proj,8,128,8,128,1.2490234375,227.5546875,234881024
+3,mlp.down_proj,2,32,8,128,80.0,70.21875,234881024
+3,mlp.down_proj,2,64,8,128,93.125,63.109375,234881024
+3,mlp.down_proj,2,128,8,128,105.0625,59.5546875,234881024
+3,mlp.down_proj,3,32,8,128,34.1875,98.21875,234881024
+3,mlp.down_proj,3,64,8,128,39.875,91.109375,234881024
+3,mlp.down_proj,3,128,8,128,44.96875,87.5546875,234881024
+3,mlp.down_proj,4,32,8,128,15.9453125,126.21875,234881024
+3,mlp.down_proj,4,64,8,128,18.609375,119.109375,234881024
+3,mlp.down_proj,4,128,8,128,21.0,115.5546875,234881024
+3,mlp.down_proj,8,32,8,128,0.9697265625,238.21875,234881024
+3,mlp.down_proj,8,64,8,128,1.1318359375,231.109375,234881024
+3,mlp.down_proj,8,128,8,128,1.2734375,227.5546875,234881024
+4,self_attn.q_proj,2,32,8,128,48.125,20.0625,67108864
+4,self_attn.q_proj,2,64,8,128,56.875,18.03125,67108864
+4,self_attn.q_proj,2,128,8,128,65.5625,17.015625,67108864
+4,self_attn.q_proj,3,32,8,128,20.59375,28.0625,67108864
+4,self_attn.q_proj,3,64,8,128,24.40625,26.03125,67108864
+4,self_attn.q_proj,3,128,8,128,28.21875,25.015625,67108864
+4,self_attn.q_proj,4,32,8,128,9.609375,36.0625,67108864
+4,self_attn.q_proj,4,64,8,128,11.390625,34.03125,67108864
+4,self_attn.q_proj,4,128,8,128,13.171875,33.015625,67108864
+4,self_attn.q_proj,8,32,8,128,0.58203125,68.0625,67108864
+4,self_attn.q_proj,8,64,8,128,0.68896484375,66.03125,67108864
+4,self_attn.q_proj,8,128,8,128,0.7939453125,65.015625,67108864
+4,self_attn.v_proj,2,32,8,128,11.40625,2.5078125,8388608
+4,self_attn.v_proj,2,64,8,128,13.40625,2.25390625,8388608
+4,self_attn.v_proj,2,128,8,128,15.21875,2.126953125,8388608
+4,self_attn.v_proj,3,32,8,128,4.87890625,3.5078125,8388608
+4,self_attn.v_proj,3,64,8,128,5.73046875,3.25390625,8388608
+4,self_attn.v_proj,3,128,8,128,6.51171875,3.126953125,8388608
+4,self_attn.v_proj,4,32,8,128,2.275390625,4.5078125,8388608
+4,self_attn.v_proj,4,64,8,128,2.67578125,4.25390625,8388608
+4,self_attn.v_proj,4,128,8,128,3.037109375,4.126953125,8388608
+4,self_attn.v_proj,8,32,8,128,0.137939453125,8.5078125,8388608
+4,self_attn.v_proj,8,64,8,128,0.161865234375,8.25390625,8388608
+4,self_attn.v_proj,8,128,8,128,0.18310546875,8.126953125,8388608
+4,self_attn.k_proj,2,32,8,128,27.859375,2.5078125,8388608
+4,self_attn.k_proj,2,64,8,128,33.59375,2.25390625,8388608
+4,self_attn.k_proj,2,128,8,128,38.3125,2.126953125,8388608
+4,self_attn.k_proj,3,32,8,128,12.1484375,3.5078125,8388608
+4,self_attn.k_proj,3,64,8,128,14.90625,3.25390625,8388608
+4,self_attn.k_proj,3,128,8,128,17.546875,3.126953125,8388608
+4,self_attn.k_proj,4,32,8,128,5.66796875,4.5078125,8388608
+4,self_attn.k_proj,4,64,8,128,6.9609375,4.25390625,8388608
+4,self_attn.k_proj,4,128,8,128,8.2109375,4.126953125,8388608
+4,self_attn.k_proj,8,32,8,128,0.343017578125,8.5078125,8388608
+4,self_attn.k_proj,8,64,8,128,0.4208984375,8.25390625,8388608
+4,self_attn.k_proj,8,128,8,128,0.494384765625,8.126953125,8388608
+4,self_attn.o_proj,2,32,8,128,36.09375,20.0625,67108864
+4,self_attn.o_proj,2,64,8,128,42.125,18.03125,67108864
+4,self_attn.o_proj,2,128,8,128,47.6875,17.015625,67108864
+4,self_attn.o_proj,3,32,8,128,15.4453125,28.0625,67108864
+4,self_attn.o_proj,3,64,8,128,18.0625,26.03125,67108864
+4,self_attn.o_proj,3,128,8,128,20.5,25.015625,67108864
+4,self_attn.o_proj,4,32,8,128,7.2109375,36.0625,67108864
+4,self_attn.o_proj,4,64,8,128,8.4375,34.03125,67108864
+4,self_attn.o_proj,4,128,8,128,9.578125,33.015625,67108864
+4,self_attn.o_proj,8,32,8,128,0.436279296875,68.0625,67108864
+4,self_attn.o_proj,8,64,8,128,0.51025390625,66.03125,67108864
+4,self_attn.o_proj,8,128,8,128,0.5771484375,65.015625,67108864
+4,mlp.gate_proj,2,32,8,128,82.8125,70.21875,234881024
+4,mlp.gate_proj,2,64,8,128,96.8125,63.109375,234881024
+4,mlp.gate_proj,2,128,8,128,109.875,59.5546875,234881024
+4,mlp.gate_proj,3,32,8,128,35.40625,98.21875,234881024
+4,mlp.gate_proj,3,64,8,128,41.46875,91.109375,234881024
+4,mlp.gate_proj,3,128,8,128,47.03125,87.5546875,234881024
+4,mlp.gate_proj,4,32,8,128,16.515625,126.21875,234881024
+4,mlp.gate_proj,4,64,8,128,19.359375,119.109375,234881024
+4,mlp.gate_proj,4,128,8,128,21.953125,115.5546875,234881024
+4,mlp.gate_proj,8,32,8,128,1.0048828125,238.21875,234881024
+4,mlp.gate_proj,8,64,8,128,1.177734375,231.109375,234881024
+4,mlp.gate_proj,8,128,8,128,1.33203125,227.5546875,234881024
+4,mlp.up_proj,2,32,8,128,78.75,70.21875,234881024
+4,mlp.up_proj,2,64,8,128,91.625,63.109375,234881024
+4,mlp.up_proj,2,128,8,128,103.375,59.5546875,234881024
+4,mlp.up_proj,3,32,8,128,33.625,98.21875,234881024
+4,mlp.up_proj,3,64,8,128,39.25,91.109375,234881024
+4,mlp.up_proj,3,128,8,128,44.28125,87.5546875,234881024
+4,mlp.up_proj,4,32,8,128,15.6875,126.21875,234881024
+4,mlp.up_proj,4,64,8,128,18.3125,119.109375,234881024
+4,mlp.up_proj,4,128,8,128,20.671875,115.5546875,234881024
+4,mlp.up_proj,8,32,8,128,0.95361328125,238.21875,234881024
+4,mlp.up_proj,8,64,8,128,1.11328125,231.109375,234881024
+4,mlp.up_proj,8,128,8,128,1.25390625,227.5546875,234881024
+4,mlp.down_proj,2,32,8,128,80.0,70.21875,234881024
+4,mlp.down_proj,2,64,8,128,93.0625,63.109375,234881024
+4,mlp.down_proj,2,128,8,128,104.9375,59.5546875,234881024
+4,mlp.down_proj,3,32,8,128,34.1875,98.21875,234881024
+4,mlp.down_proj,3,64,8,128,39.84375,91.109375,234881024
+4,mlp.down_proj,3,128,8,128,44.875,87.5546875,234881024
+4,mlp.down_proj,4,32,8,128,15.9375,126.21875,234881024
+4,mlp.down_proj,4,64,8,128,18.59375,119.109375,234881024
+4,mlp.down_proj,4,128,8,128,20.953125,115.5546875,234881024
+4,mlp.down_proj,8,32,8,128,0.96923828125,238.21875,234881024
+4,mlp.down_proj,8,64,8,128,1.130859375,231.109375,234881024
+4,mlp.down_proj,8,128,8,128,1.271484375,227.5546875,234881024
+5,self_attn.q_proj,2,32,8,128,50.375,20.0625,67108864
+5,self_attn.q_proj,2,64,8,128,59.25,18.03125,67108864
+5,self_attn.q_proj,2,128,8,128,68.1875,17.015625,67108864
+5,self_attn.q_proj,3,32,8,128,21.5625,28.0625,67108864
+5,self_attn.q_proj,3,64,8,128,25.390625,26.03125,67108864
+5,self_attn.q_proj,3,128,8,128,29.28125,25.015625,67108864
+5,self_attn.q_proj,4,32,8,128,10.0625,36.0625,67108864
+5,self_attn.q_proj,4,64,8,128,11.8515625,34.03125,67108864
+5,self_attn.q_proj,4,128,8,128,13.671875,33.015625,67108864
+5,self_attn.q_proj,8,32,8,128,0.609375,68.0625,67108864
+5,self_attn.q_proj,8,64,8,128,0.716796875,66.03125,67108864
+5,self_attn.q_proj,8,128,8,128,0.82421875,65.015625,67108864
+5,self_attn.v_proj,2,32,8,128,11.46875,2.5078125,8388608
+5,self_attn.v_proj,2,64,8,128,13.40625,2.25390625,8388608
+5,self_attn.v_proj,2,128,8,128,15.1640625,2.126953125,8388608
+5,self_attn.v_proj,3,32,8,128,4.90234375,3.5078125,8388608
+5,self_attn.v_proj,3,64,8,128,5.73828125,3.25390625,8388608
+5,self_attn.v_proj,3,128,8,128,6.484375,3.126953125,8388608
+5,self_attn.v_proj,4,32,8,128,2.287109375,4.5078125,8388608
+5,self_attn.v_proj,4,64,8,128,2.67578125,4.25390625,8388608
+5,self_attn.v_proj,4,128,8,128,3.025390625,4.126953125,8388608
+5,self_attn.v_proj,8,32,8,128,0.1385498046875,8.5078125,8388608
+5,self_attn.v_proj,8,64,8,128,0.161865234375,8.25390625,8388608
+5,self_attn.v_proj,8,128,8,128,0.1824951171875,8.126953125,8388608
+5,self_attn.k_proj,2,32,8,128,28.84375,2.5078125,8388608
+5,self_attn.k_proj,2,64,8,128,34.8125,2.25390625,8388608
+5,self_attn.k_proj,2,128,8,128,39.9375,2.126953125,8388608
+5,self_attn.k_proj,3,32,8,128,12.5078125,3.5078125,8388608
+5,self_attn.k_proj,3,64,8,128,15.2890625,3.25390625,8388608
+5,self_attn.k_proj,3,128,8,128,18.0,3.126953125,8388608
+5,self_attn.k_proj,4,32,8,128,5.8359375,4.5078125,8388608
+5,self_attn.k_proj,4,64,8,128,7.140625,4.25390625,8388608
+5,self_attn.k_proj,4,128,8,128,8.40625,4.126953125,8388608
+5,self_attn.k_proj,8,32,8,128,0.353271484375,8.5078125,8388608
+5,self_attn.k_proj,8,64,8,128,0.431640625,8.25390625,8388608
+5,self_attn.k_proj,8,128,8,128,0.5068359375,8.126953125,8388608
+5,self_attn.o_proj,2,32,8,128,37.5,20.0625,67108864
+5,self_attn.o_proj,2,64,8,128,43.71875,18.03125,67108864
+5,self_attn.o_proj,2,128,8,128,49.40625,17.015625,67108864
+5,self_attn.o_proj,3,32,8,128,16.03125,28.0625,67108864
+5,self_attn.o_proj,3,64,8,128,18.71875,26.03125,67108864
+5,self_attn.o_proj,3,128,8,128,21.1875,25.015625,67108864
+5,self_attn.o_proj,4,32,8,128,7.48046875,36.0625,67108864
+5,self_attn.o_proj,4,64,8,128,8.734375,34.03125,67108864
+5,self_attn.o_proj,4,128,8,128,9.890625,33.015625,67108864
+5,self_attn.o_proj,8,32,8,128,0.452880859375,68.0625,67108864
+5,self_attn.o_proj,8,64,8,128,0.5283203125,66.03125,67108864
+5,self_attn.o_proj,8,128,8,128,0.59619140625,65.015625,67108864
+5,mlp.gate_proj,2,32,8,128,83.1875,70.21875,234881024
+5,mlp.gate_proj,2,64,8,128,97.3125,63.109375,234881024
+5,mlp.gate_proj,2,128,8,128,110.3125,59.5546875,234881024
+5,mlp.gate_proj,3,32,8,128,35.59375,98.21875,234881024
+5,mlp.gate_proj,3,64,8,128,41.65625,91.109375,234881024
+5,mlp.gate_proj,3,128,8,128,47.25,87.5546875,234881024
+5,mlp.gate_proj,4,32,8,128,16.59375,126.21875,234881024
+5,mlp.gate_proj,4,64,8,128,19.453125,119.109375,234881024
+5,mlp.gate_proj,4,128,8,128,22.046875,115.5546875,234881024
+5,mlp.gate_proj,8,32,8,128,1.009765625,238.21875,234881024
+5,mlp.gate_proj,8,64,8,128,1.18359375,231.109375,234881024
+5,mlp.gate_proj,8,128,8,128,1.337890625,227.5546875,234881024
+5,mlp.up_proj,2,32,8,128,79.1875,70.21875,234881024
+5,mlp.up_proj,2,64,8,128,92.1875,63.109375,234881024
+5,mlp.up_proj,2,128,8,128,104.0625,59.5546875,234881024
+5,mlp.up_proj,3,32,8,128,33.8125,98.21875,234881024
+5,mlp.up_proj,3,64,8,128,39.5,91.109375,234881024
+5,mlp.up_proj,3,128,8,128,44.53125,87.5546875,234881024
+5,mlp.up_proj,4,32,8,128,15.7734375,126.21875,234881024
+5,mlp.up_proj,4,64,8,128,18.421875,119.109375,234881024
+5,mlp.up_proj,4,128,8,128,20.796875,115.5546875,234881024
+5,mlp.up_proj,8,32,8,128,0.958984375,238.21875,234881024
+5,mlp.up_proj,8,64,8,128,1.1201171875,231.109375,234881024
+5,mlp.up_proj,8,128,8,128,1.26171875,227.5546875,234881024
+5,mlp.down_proj,2,32,8,128,80.0,70.21875,234881024
+5,mlp.down_proj,2,64,8,128,93.0625,63.109375,234881024
+5,mlp.down_proj,2,128,8,128,104.9375,59.5546875,234881024
+5,mlp.down_proj,3,32,8,128,34.1875,98.21875,234881024
+5,mlp.down_proj,3,64,8,128,39.875,91.109375,234881024
+5,mlp.down_proj,3,128,8,128,44.9375,87.5546875,234881024
+5,mlp.down_proj,4,32,8,128,15.9453125,126.21875,234881024
+5,mlp.down_proj,4,64,8,128,18.609375,119.109375,234881024
+5,mlp.down_proj,4,128,8,128,20.96875,115.5546875,234881024
+5,mlp.down_proj,8,32,8,128,0.9697265625,238.21875,234881024
+5,mlp.down_proj,8,64,8,128,1.130859375,231.109375,234881024
+5,mlp.down_proj,8,128,8,128,1.2724609375,227.5546875,234881024
+6,self_attn.q_proj,2,32,8,128,52.15625,20.0625,67108864
+6,self_attn.q_proj,2,64,8,128,61.40625,18.03125,67108864
+6,self_attn.q_proj,2,128,8,128,71.0,17.015625,67108864
+6,self_attn.q_proj,3,32,8,128,22.34375,28.0625,67108864
+6,self_attn.q_proj,3,64,8,128,26.375,26.03125,67108864
+6,self_attn.q_proj,3,128,8,128,30.625,25.015625,67108864
+6,self_attn.q_proj,4,32,8,128,10.4296875,36.0625,67108864
+6,self_attn.q_proj,4,64,8,128,12.3046875,34.03125,67108864
+6,self_attn.q_proj,4,128,8,128,14.3046875,33.015625,67108864
+6,self_attn.q_proj,8,32,8,128,0.63134765625,68.0625,67108864
+6,self_attn.q_proj,8,64,8,128,0.74462890625,66.03125,67108864
+6,self_attn.q_proj,8,128,8,128,0.86181640625,65.015625,67108864
+6,self_attn.v_proj,2,32,8,128,11.5546875,2.5078125,8388608
+6,self_attn.v_proj,2,64,8,128,13.484375,2.25390625,8388608
+6,self_attn.v_proj,2,128,8,128,15.2421875,2.126953125,8388608
+6,self_attn.v_proj,3,32,8,128,4.9375,3.5078125,8388608
+6,self_attn.v_proj,3,64,8,128,5.765625,3.25390625,8388608
+6,self_attn.v_proj,3,128,8,128,6.51953125,3.126953125,8388608
+6,self_attn.v_proj,4,32,8,128,2.3046875,4.5078125,8388608
+6,self_attn.v_proj,4,64,8,128,2.69140625,4.25390625,8388608
+6,self_attn.v_proj,4,128,8,128,3.041015625,4.126953125,8388608
+6,self_attn.v_proj,8,32,8,128,0.1396484375,8.5078125,8388608
+6,self_attn.v_proj,8,64,8,128,0.1627197265625,8.25390625,8388608
+6,self_attn.v_proj,8,128,8,128,0.183349609375,8.126953125,8388608
+6,self_attn.k_proj,2,32,8,128,28.609375,2.5078125,8388608
+6,self_attn.k_proj,2,64,8,128,34.5,2.25390625,8388608
+6,self_attn.k_proj,2,128,8,128,39.5,2.126953125,8388608
+6,self_attn.k_proj,3,32,8,128,12.3671875,3.5078125,8388608
+6,self_attn.k_proj,3,64,8,128,15.1015625,3.25390625,8388608
+6,self_attn.k_proj,3,128,8,128,17.65625,3.126953125,8388608
+6,self_attn.k_proj,4,32,8,128,5.76953125,4.5078125,8388608
+6,self_attn.k_proj,4,64,8,128,7.046875,4.25390625,8388608
+6,self_attn.k_proj,4,128,8,128,8.25,4.126953125,8388608
+6,self_attn.k_proj,8,32,8,128,0.349365234375,8.5078125,8388608
+6,self_attn.k_proj,8,64,8,128,0.42626953125,8.25390625,8388608
+6,self_attn.k_proj,8,128,8,128,0.4970703125,8.126953125,8388608
+6,self_attn.o_proj,2,32,8,128,38.1875,20.0625,67108864
+6,self_attn.o_proj,2,64,8,128,44.5,18.03125,67108864
+6,self_attn.o_proj,2,128,8,128,50.25,17.015625,67108864
+6,self_attn.o_proj,3,32,8,128,16.34375,28.0625,67108864
+6,self_attn.o_proj,3,64,8,128,19.0625,26.03125,67108864
+6,self_attn.o_proj,3,128,8,128,21.546875,25.015625,67108864
+6,self_attn.o_proj,4,32,8,128,7.625,36.0625,67108864
+6,self_attn.o_proj,4,64,8,128,8.8984375,34.03125,67108864
+6,self_attn.o_proj,4,128,8,128,10.0625,33.015625,67108864
+6,self_attn.o_proj,8,32,8,128,0.46142578125,68.0625,67108864
+6,self_attn.o_proj,8,64,8,128,0.53857421875,66.03125,67108864
+6,self_attn.o_proj,8,128,8,128,0.60693359375,65.015625,67108864
+6,mlp.gate_proj,2,32,8,128,83.625,70.21875,234881024
+6,mlp.gate_proj,2,64,8,128,97.75,63.109375,234881024
+6,mlp.gate_proj,2,128,8,128,110.75,59.5546875,234881024
+6,mlp.gate_proj,3,32,8,128,35.78125,98.21875,234881024
+6,mlp.gate_proj,3,64,8,128,41.84375,91.109375,234881024
+6,mlp.gate_proj,3,128,8,128,47.40625,87.5546875,234881024
+6,mlp.gate_proj,4,32,8,128,16.6875,126.21875,234881024
+6,mlp.gate_proj,4,64,8,128,19.546875,119.109375,234881024
+6,mlp.gate_proj,4,128,8,128,22.125,115.5546875,234881024
+6,mlp.gate_proj,8,32,8,128,1.0146484375,238.21875,234881024
+6,mlp.gate_proj,8,64,8,128,1.1884765625,231.109375,234881024
+6,mlp.gate_proj,8,128,8,128,1.3427734375,227.5546875,234881024
+6,mlp.up_proj,2,32,8,128,79.5625,70.21875,234881024
+6,mlp.up_proj,2,64,8,128,92.6875,63.109375,234881024
+6,mlp.up_proj,2,128,8,128,104.5625,59.5546875,234881024
+6,mlp.up_proj,3,32,8,128,34.0,98.21875,234881024
+6,mlp.up_proj,3,64,8,128,39.6875,91.109375,234881024
+6,mlp.up_proj,3,128,8,128,44.75,87.5546875,234881024
+6,mlp.up_proj,4,32,8,128,15.859375,126.21875,234881024
+6,mlp.up_proj,4,64,8,128,18.515625,119.109375,234881024
+6,mlp.up_proj,4,128,8,128,20.890625,115.5546875,234881024
+6,mlp.up_proj,8,32,8,128,0.96435546875,238.21875,234881024
+6,mlp.up_proj,8,64,8,128,1.1259765625,231.109375,234881024
+6,mlp.up_proj,8,128,8,128,1.267578125,227.5546875,234881024
+6,mlp.down_proj,2,32,8,128,80.375,70.21875,234881024
+6,mlp.down_proj,2,64,8,128,93.5625,63.109375,234881024
+6,mlp.down_proj,2,128,8,128,105.5,59.5546875,234881024
+6,mlp.down_proj,3,32,8,128,34.34375,98.21875,234881024
+6,mlp.down_proj,3,64,8,128,40.0625,91.109375,234881024
+6,mlp.down_proj,3,128,8,128,45.15625,87.5546875,234881024
+6,mlp.down_proj,4,32,8,128,16.015625,126.21875,234881024
+6,mlp.down_proj,4,64,8,128,18.6875,119.109375,234881024
+6,mlp.down_proj,4,128,8,128,21.078125,115.5546875,234881024
+6,mlp.down_proj,8,32,8,128,0.97412109375,238.21875,234881024
+6,mlp.down_proj,8,64,8,128,1.13671875,231.109375,234881024
+6,mlp.down_proj,8,128,8,128,1.279296875,227.5546875,234881024
+7,self_attn.q_proj,2,32,8,128,51.46875,20.0625,67108864
+7,self_attn.q_proj,2,64,8,128,60.59375,18.03125,67108864
+7,self_attn.q_proj,2,128,8,128,70.0,17.015625,67108864
+7,self_attn.q_proj,3,32,8,128,22.03125,28.0625,67108864
+7,self_attn.q_proj,3,64,8,128,26.0,26.03125,67108864
+7,self_attn.q_proj,3,128,8,128,30.109375,25.015625,67108864
+7,self_attn.q_proj,4,32,8,128,10.28125,36.0625,67108864
+7,self_attn.q_proj,4,64,8,128,12.1328125,34.03125,67108864
+7,self_attn.q_proj,4,128,8,128,14.0546875,33.015625,67108864
+7,self_attn.q_proj,8,32,8,128,0.62255859375,68.0625,67108864
+7,self_attn.q_proj,8,64,8,128,0.7333984375,66.03125,67108864
+7,self_attn.q_proj,8,128,8,128,0.84716796875,65.015625,67108864
+7,self_attn.v_proj,2,32,8,128,11.3046875,2.5078125,8388608
+7,self_attn.v_proj,2,64,8,128,13.25,2.25390625,8388608
+7,self_attn.v_proj,2,128,8,128,15.0390625,2.126953125,8388608
+7,self_attn.v_proj,3,32,8,128,4.8359375,3.5078125,8388608
+7,self_attn.v_proj,3,64,8,128,5.66796875,3.25390625,8388608
+7,self_attn.v_proj,3,128,8,128,6.4296875,3.126953125,8388608
+7,self_attn.v_proj,4,32,8,128,2.255859375,4.5078125,8388608
+7,self_attn.v_proj,4,64,8,128,2.64453125,4.25390625,8388608
+7,self_attn.v_proj,4,128,8,128,3.001953125,4.126953125,8388608
+7,self_attn.v_proj,8,32,8,128,0.13671875,8.5078125,8388608
+7,self_attn.v_proj,8,64,8,128,0.1600341796875,8.25390625,8388608
+7,self_attn.v_proj,8,128,8,128,0.1810302734375,8.126953125,8388608
+7,self_attn.k_proj,2,32,8,128,28.65625,2.5078125,8388608
+7,self_attn.k_proj,2,64,8,128,34.6875,2.25390625,8388608
+7,self_attn.k_proj,2,128,8,128,39.75,2.126953125,8388608
+7,self_attn.k_proj,3,32,8,128,12.390625,3.5078125,8388608
+7,self_attn.k_proj,3,64,8,128,15.2421875,3.25390625,8388608
+7,self_attn.k_proj,3,128,8,128,17.953125,3.126953125,8388608
+7,self_attn.k_proj,4,32,8,128,5.78125,4.5078125,8388608
+7,self_attn.k_proj,4,64,8,128,7.11328125,4.25390625,8388608
+7,self_attn.k_proj,4,128,8,128,8.3828125,4.126953125,8388608
+7,self_attn.k_proj,8,32,8,128,0.349853515625,8.5078125,8388608
+7,self_attn.k_proj,8,64,8,128,0.43017578125,8.25390625,8388608
+7,self_attn.k_proj,8,128,8,128,0.5048828125,8.126953125,8388608
+7,self_attn.o_proj,2,32,8,128,38.15625,20.0625,67108864
+7,self_attn.o_proj,2,64,8,128,44.46875,18.03125,67108864
+7,self_attn.o_proj,2,128,8,128,50.1875,17.015625,67108864
+7,self_attn.o_proj,3,32,8,128,16.3125,28.0625,67108864
+7,self_attn.o_proj,3,64,8,128,19.03125,26.03125,67108864
+7,self_attn.o_proj,3,128,8,128,21.5,25.015625,67108864
+7,self_attn.o_proj,4,32,8,128,7.61328125,36.0625,67108864
+7,self_attn.o_proj,4,64,8,128,8.8828125,34.03125,67108864
+7,self_attn.o_proj,4,128,8,128,10.0390625,33.015625,67108864
+7,self_attn.o_proj,8,32,8,128,0.4609375,68.0625,67108864
+7,self_attn.o_proj,8,64,8,128,0.537109375,66.03125,67108864
+7,self_attn.o_proj,8,128,8,128,0.60498046875,65.015625,67108864
+7,mlp.gate_proj,2,32,8,128,84.0,70.21875,234881024
+7,mlp.gate_proj,2,64,8,128,98.1875,63.109375,234881024
+7,mlp.gate_proj,2,128,8,128,111.1875,59.5546875,234881024
+7,mlp.gate_proj,3,32,8,128,35.9375,98.21875,234881024
+7,mlp.gate_proj,3,64,8,128,42.03125,91.109375,234881024
+7,mlp.gate_proj,3,128,8,128,47.625,87.5546875,234881024
+7,mlp.gate_proj,4,32,8,128,16.765625,126.21875,234881024
+7,mlp.gate_proj,4,64,8,128,19.625,119.109375,234881024
+7,mlp.gate_proj,4,128,8,128,22.234375,115.5546875,234881024
+7,mlp.gate_proj,8,32,8,128,1.01953125,238.21875,234881024
+7,mlp.gate_proj,8,64,8,128,1.1943359375,231.109375,234881024
+7,mlp.gate_proj,8,128,8,128,1.3486328125,227.5546875,234881024
+7,mlp.up_proj,2,32,8,128,79.9375,70.21875,234881024
+7,mlp.up_proj,2,64,8,128,93.125,63.109375,234881024
+7,mlp.up_proj,2,128,8,128,105.0,59.5546875,234881024
+7,mlp.up_proj,3,32,8,128,34.15625,98.21875,234881024
+7,mlp.up_proj,3,64,8,128,39.875,91.109375,234881024
+7,mlp.up_proj,3,128,8,128,44.9375,87.5546875,234881024
+7,mlp.up_proj,4,32,8,128,15.9375,126.21875,234881024
+7,mlp.up_proj,4,64,8,128,18.59375,119.109375,234881024
+7,mlp.up_proj,4,128,8,128,20.984375,115.5546875,234881024
+7,mlp.up_proj,8,32,8,128,0.96923828125,238.21875,234881024
+7,mlp.up_proj,8,64,8,128,1.130859375,231.109375,234881024
+7,mlp.up_proj,8,128,8,128,1.2724609375,227.5546875,234881024
+7,mlp.down_proj,2,32,8,128,80.75,70.21875,234881024
+7,mlp.down_proj,2,64,8,128,94.0,63.109375,234881024
+7,mlp.down_proj,2,128,8,128,106.0,59.5546875,234881024
+7,mlp.down_proj,3,32,8,128,34.53125,98.21875,234881024
+7,mlp.down_proj,3,64,8,128,40.25,91.109375,234881024
+7,mlp.down_proj,3,128,8,128,45.34375,87.5546875,234881024
+7,mlp.down_proj,4,32,8,128,16.09375,126.21875,234881024
+7,mlp.down_proj,4,64,8,128,18.78125,119.109375,234881024
+7,mlp.down_proj,4,128,8,128,21.1875,115.5546875,234881024
+7,mlp.down_proj,8,32,8,128,0.97900390625,238.21875,234881024
+7,mlp.down_proj,8,64,8,128,1.142578125,231.109375,234881024
+7,mlp.down_proj,8,128,8,128,1.28515625,227.5546875,234881024
+8,self_attn.q_proj,2,32,8,128,52.15625,20.0625,67108864
+8,self_attn.q_proj,2,64,8,128,61.28125,18.03125,67108864
+8,self_attn.q_proj,2,128,8,128,70.625,17.015625,67108864
+8,self_attn.q_proj,3,32,8,128,22.328125,28.0625,67108864
+8,self_attn.q_proj,3,64,8,128,26.28125,26.03125,67108864
+8,self_attn.q_proj,3,128,8,128,30.390625,25.015625,67108864
+8,self_attn.q_proj,4,32,8,128,10.421875,36.0625,67108864
+8,self_attn.q_proj,4,64,8,128,12.2578125,34.03125,67108864
+8,self_attn.q_proj,4,128,8,128,14.1796875,33.015625,67108864
+8,self_attn.q_proj,8,32,8,128,0.630859375,68.0625,67108864
+8,self_attn.q_proj,8,64,8,128,0.74169921875,66.03125,67108864
+8,self_attn.q_proj,8,128,8,128,0.85498046875,65.015625,67108864
+8,self_attn.v_proj,2,32,8,128,11.4921875,2.5078125,8388608
+8,self_attn.v_proj,2,64,8,128,13.453125,2.25390625,8388608
+8,self_attn.v_proj,2,128,8,128,15.25,2.126953125,8388608
+8,self_attn.v_proj,3,32,8,128,4.9140625,3.5078125,8388608
+8,self_attn.v_proj,3,64,8,128,5.7578125,3.25390625,8388608
+8,self_attn.v_proj,3,128,8,128,6.52734375,3.126953125,8388608
+8,self_attn.v_proj,4,32,8,128,2.29296875,4.5078125,8388608
+8,self_attn.v_proj,4,64,8,128,2.685546875,4.25390625,8388608
+8,self_attn.v_proj,4,128,8,128,3.046875,4.126953125,8388608
+8,self_attn.v_proj,8,32,8,128,0.1387939453125,8.5078125,8388608
+8,self_attn.v_proj,8,64,8,128,0.1624755859375,8.25390625,8388608
+8,self_attn.v_proj,8,128,8,128,0.18359375,8.126953125,8388608
+8,self_attn.k_proj,2,32,8,128,30.90625,2.5078125,8388608
+8,self_attn.k_proj,2,64,8,128,37.3125,2.25390625,8388608
+8,self_attn.k_proj,2,128,8,128,42.625,2.126953125,8388608
+8,self_attn.k_proj,3,32,8,128,13.3671875,3.5078125,8388608
+8,self_attn.k_proj,3,64,8,128,16.359375,3.25390625,8388608
+8,self_attn.k_proj,3,128,8,128,19.21875,3.126953125,8388608
+8,self_attn.k_proj,4,32,8,128,6.2421875,4.5078125,8388608
+8,self_attn.k_proj,4,64,8,128,7.63671875,4.25390625,8388608
+8,self_attn.k_proj,4,128,8,128,8.96875,4.126953125,8388608
+8,self_attn.k_proj,8,32,8,128,0.377685546875,8.5078125,8388608
+8,self_attn.k_proj,8,64,8,128,0.4619140625,8.25390625,8388608
+8,self_attn.k_proj,8,128,8,128,0.54052734375,8.126953125,8388608
+8,self_attn.o_proj,2,32,8,128,38.6875,20.0625,67108864
+8,self_attn.o_proj,2,64,8,128,45.1875,18.03125,67108864
+8,self_attn.o_proj,2,128,8,128,51.125,17.015625,67108864
+8,self_attn.o_proj,3,32,8,128,16.578125,28.0625,67108864
+8,self_attn.o_proj,3,64,8,128,19.40625,26.03125,67108864
+8,self_attn.o_proj,3,128,8,128,22.078125,25.015625,67108864
+8,self_attn.o_proj,4,32,8,128,7.75,36.0625,67108864
+8,self_attn.o_proj,4,64,8,128,9.0859375,34.03125,67108864
+8,self_attn.o_proj,4,128,8,128,10.3515625,33.015625,67108864
+8,self_attn.o_proj,8,32,8,128,0.46923828125,68.0625,67108864
+8,self_attn.o_proj,8,64,8,128,0.55029296875,66.03125,67108864
+8,self_attn.o_proj,8,128,8,128,0.62548828125,65.015625,67108864
+8,mlp.gate_proj,2,32,8,128,85.9375,70.21875,234881024
+8,mlp.gate_proj,2,64,8,128,100.8125,63.109375,234881024
+8,mlp.gate_proj,2,128,8,128,114.625,59.5546875,234881024
+8,mlp.gate_proj,3,32,8,128,36.84375,98.21875,234881024
+8,mlp.gate_proj,3,64,8,128,43.3125,91.109375,234881024
+8,mlp.gate_proj,3,128,8,128,49.46875,87.5546875,234881024
+8,mlp.gate_proj,4,32,8,128,17.171875,126.21875,234881024
+8,mlp.gate_proj,4,64,8,128,20.234375,119.109375,234881024
+8,mlp.gate_proj,4,128,8,128,23.125,115.5546875,234881024
+8,mlp.gate_proj,8,32,8,128,1.044921875,238.21875,234881024
+8,mlp.gate_proj,8,64,8,128,1.2314453125,231.109375,234881024
+8,mlp.gate_proj,8,128,8,128,1.40234375,227.5546875,234881024
+8,mlp.up_proj,2,32,8,128,80.1875,70.21875,234881024
+8,mlp.up_proj,2,64,8,128,93.4375,63.109375,234881024
+8,mlp.up_proj,2,128,8,128,105.625,59.5546875,234881024
+8,mlp.up_proj,3,32,8,128,34.28125,98.21875,234881024
+8,mlp.up_proj,3,64,8,128,40.0625,91.109375,234881024
+8,mlp.up_proj,3,128,8,128,45.3125,87.5546875,234881024
+8,mlp.up_proj,4,32,8,128,15.984375,126.21875,234881024
+8,mlp.up_proj,4,64,8,128,18.6875,119.109375,234881024
+8,mlp.up_proj,4,128,8,128,21.140625,115.5546875,234881024
+8,mlp.up_proj,8,32,8,128,0.97216796875,238.21875,234881024
+8,mlp.up_proj,8,64,8,128,1.13671875,231.109375,234881024
+8,mlp.up_proj,8,128,8,128,1.2822265625,227.5546875,234881024
+8,mlp.down_proj,2,32,8,128,80.9375,70.21875,234881024
+8,mlp.down_proj,2,64,8,128,94.1875,63.109375,234881024
+8,mlp.down_proj,2,128,8,128,106.1875,59.5546875,234881024
+8,mlp.down_proj,3,32,8,128,34.59375,98.21875,234881024
+8,mlp.down_proj,3,64,8,128,40.34375,91.109375,234881024
+8,mlp.down_proj,3,128,8,128,45.46875,87.5546875,234881024
+8,mlp.down_proj,4,32,8,128,16.140625,126.21875,234881024
+8,mlp.down_proj,4,64,8,128,18.84375,119.109375,234881024
+8,mlp.down_proj,4,128,8,128,21.25,115.5546875,234881024
+8,mlp.down_proj,8,32,8,128,0.982421875,238.21875,234881024
+8,mlp.down_proj,8,64,8,128,1.1474609375,231.109375,234881024
+8,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024
+9,self_attn.q_proj,2,32,8,128,47.28125,20.0625,67108864
+9,self_attn.q_proj,2,64,8,128,55.5625,18.03125,67108864
+9,self_attn.q_proj,2,128,8,128,63.9375,17.015625,67108864
+9,self_attn.q_proj,3,32,8,128,20.234375,28.0625,67108864
+9,self_attn.q_proj,3,64,8,128,23.8125,26.03125,67108864
+9,self_attn.q_proj,3,128,8,128,27.46875,25.015625,67108864
+9,self_attn.q_proj,4,32,8,128,9.4453125,36.0625,67108864
+9,self_attn.q_proj,4,64,8,128,11.109375,34.03125,67108864
+9,self_attn.q_proj,4,128,8,128,12.8125,33.015625,67108864
+9,self_attn.q_proj,8,32,8,128,0.57177734375,68.0625,67108864
+9,self_attn.q_proj,8,64,8,128,0.67236328125,66.03125,67108864
+9,self_attn.q_proj,8,128,8,128,0.7724609375,65.015625,67108864
+9,self_attn.v_proj,2,32,8,128,12.7734375,2.5078125,8388608
+9,self_attn.v_proj,2,64,8,128,14.9453125,2.25390625,8388608
+9,self_attn.v_proj,2,128,8,128,16.921875,2.126953125,8388608
+9,self_attn.v_proj,3,32,8,128,5.4609375,3.5078125,8388608
+9,self_attn.v_proj,3,64,8,128,6.390625,3.25390625,8388608
+9,self_attn.v_proj,3,128,8,128,7.23828125,3.126953125,8388608
+9,self_attn.v_proj,4,32,8,128,2.546875,4.5078125,8388608
+9,self_attn.v_proj,4,64,8,128,2.982421875,4.25390625,8388608
+9,self_attn.v_proj,4,128,8,128,3.37890625,4.126953125,8388608
+9,self_attn.v_proj,8,32,8,128,0.154296875,8.5078125,8388608
+9,self_attn.v_proj,8,64,8,128,0.180419921875,8.25390625,8388608
+9,self_attn.v_proj,8,128,8,128,0.20361328125,8.126953125,8388608
+9,self_attn.k_proj,2,32,8,128,28.15625,2.5078125,8388608
+9,self_attn.k_proj,2,64,8,128,34.0625,2.25390625,8388608
+9,self_attn.k_proj,2,128,8,128,39.03125,2.126953125,8388608
+9,self_attn.k_proj,3,32,8,128,12.1640625,3.5078125,8388608
+9,self_attn.k_proj,3,64,8,128,14.953125,3.25390625,8388608
+9,self_attn.k_proj,3,128,8,128,17.640625,3.126953125,8388608
+9,self_attn.k_proj,4,32,8,128,5.6796875,4.5078125,8388608
+9,self_attn.k_proj,4,64,8,128,6.98046875,4.25390625,8388608
+9,self_attn.k_proj,4,128,8,128,8.234375,4.126953125,8388608
+9,self_attn.k_proj,8,32,8,128,0.34375,8.5078125,8388608
+9,self_attn.k_proj,8,64,8,128,0.421875,8.25390625,8388608
+9,self_attn.k_proj,8,128,8,128,0.495849609375,8.126953125,8388608
+9,self_attn.o_proj,2,32,8,128,39.5,20.0625,67108864
+9,self_attn.o_proj,2,64,8,128,46.0,18.03125,67108864
+9,self_attn.o_proj,2,128,8,128,51.875,17.015625,67108864
+9,self_attn.o_proj,3,32,8,128,16.890625,28.0625,67108864
+9,self_attn.o_proj,3,64,8,128,19.703125,26.03125,67108864
+9,self_attn.o_proj,3,128,8,128,22.265625,25.015625,67108864
+9,self_attn.o_proj,4,32,8,128,7.8828125,36.0625,67108864
+9,self_attn.o_proj,4,64,8,128,9.1953125,34.03125,67108864
+9,self_attn.o_proj,4,128,8,128,10.390625,33.015625,67108864
+9,self_attn.o_proj,8,32,8,128,0.477294921875,68.0625,67108864
+9,self_attn.o_proj,8,64,8,128,0.55615234375,66.03125,67108864
+9,self_attn.o_proj,8,128,8,128,0.62646484375,65.015625,67108864
+9,mlp.gate_proj,2,32,8,128,86.0625,70.21875,234881024
+9,mlp.gate_proj,2,64,8,128,100.75,63.109375,234881024
+9,mlp.gate_proj,2,128,8,128,114.125,59.5546875,234881024
+9,mlp.gate_proj,3,32,8,128,36.875,98.21875,234881024
+9,mlp.gate_proj,3,64,8,128,43.15625,91.109375,234881024
+9,mlp.gate_proj,3,128,8,128,48.96875,87.5546875,234881024
+9,mlp.gate_proj,4,32,8,128,17.1875,126.21875,234881024
+9,mlp.gate_proj,4,64,8,128,20.15625,119.109375,234881024
+9,mlp.gate_proj,4,128,8,128,22.890625,115.5546875,234881024
+9,mlp.gate_proj,8,32,8,128,1.0458984375,238.21875,234881024
+9,mlp.gate_proj,8,64,8,128,1.2265625,231.109375,234881024
+9,mlp.gate_proj,8,128,8,128,1.3876953125,227.5546875,234881024
+9,mlp.up_proj,2,32,8,128,80.4375,70.21875,234881024
+9,mlp.up_proj,2,64,8,128,93.5625,63.109375,234881024
+9,mlp.up_proj,2,128,8,128,105.5625,59.5546875,234881024
+9,mlp.up_proj,3,32,8,128,34.375,98.21875,234881024
+9,mlp.up_proj,3,64,8,128,40.0625,91.109375,234881024
+9,mlp.up_proj,3,128,8,128,45.125,87.5546875,234881024
+9,mlp.up_proj,4,32,8,128,16.03125,126.21875,234881024
+9,mlp.up_proj,4,64,8,128,18.703125,119.109375,234881024
+9,mlp.up_proj,4,128,8,128,21.0625,115.5546875,234881024
+9,mlp.up_proj,8,32,8,128,0.974609375,238.21875,234881024
+9,mlp.up_proj,8,64,8,128,1.13671875,231.109375,234881024
+9,mlp.up_proj,8,128,8,128,1.2783203125,227.5546875,234881024
+9,mlp.down_proj,2,32,8,128,81.25,70.21875,234881024
+9,mlp.down_proj,2,64,8,128,94.5,63.109375,234881024
+9,mlp.down_proj,2,128,8,128,106.5,59.5546875,234881024
+9,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024
+9,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024
+9,mlp.down_proj,3,128,8,128,45.5625,87.5546875,234881024
+9,mlp.down_proj,4,32,8,128,16.1875,126.21875,234881024
+9,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024
+9,mlp.down_proj,4,128,8,128,21.265625,115.5546875,234881024
+9,mlp.down_proj,8,32,8,128,0.98486328125,238.21875,234881024
+9,mlp.down_proj,8,64,8,128,1.1484375,231.109375,234881024
+9,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024
+10,self_attn.q_proj,2,32,8,128,45.96875,20.0625,67108864
+10,self_attn.q_proj,2,64,8,128,54.625,18.03125,67108864
+10,self_attn.q_proj,2,128,8,128,62.8125,17.015625,67108864
+10,self_attn.q_proj,3,32,8,128,19.71875,28.0625,67108864
+10,self_attn.q_proj,3,64,8,128,23.546875,26.03125,67108864
+10,self_attn.q_proj,3,128,8,128,27.34375,25.015625,67108864
+10,self_attn.q_proj,4,32,8,128,9.203125,36.0625,67108864
+10,self_attn.q_proj,4,64,8,128,10.984375,34.03125,67108864
+10,self_attn.q_proj,4,128,8,128,12.7578125,33.015625,67108864
+10,self_attn.q_proj,8,32,8,128,0.55712890625,68.0625,67108864
+10,self_attn.q_proj,8,64,8,128,0.6650390625,66.03125,67108864
+10,self_attn.q_proj,8,128,8,128,0.76904296875,65.015625,67108864
+10,self_attn.v_proj,2,32,8,128,13.546875,2.5078125,8388608
+10,self_attn.v_proj,2,64,8,128,15.7734375,2.25390625,8388608
+10,self_attn.v_proj,2,128,8,128,17.78125,2.126953125,8388608
+10,self_attn.v_proj,3,32,8,128,5.79296875,3.5078125,8388608
+10,self_attn.v_proj,3,64,8,128,6.75,3.25390625,8388608
+10,self_attn.v_proj,3,128,8,128,7.60546875,3.126953125,8388608
+10,self_attn.v_proj,4,32,8,128,2.703125,4.5078125,8388608
+10,self_attn.v_proj,4,64,8,128,3.1484375,4.25390625,8388608
+10,self_attn.v_proj,4,128,8,128,3.55078125,4.126953125,8388608
+10,self_attn.v_proj,8,32,8,128,0.1636962890625,8.5078125,8388608
+10,self_attn.v_proj,8,64,8,128,0.1905517578125,8.25390625,8388608
+10,self_attn.v_proj,8,128,8,128,0.2139892578125,8.126953125,8388608
+10,self_attn.k_proj,2,32,8,128,26.1875,2.5078125,8388608
+10,self_attn.k_proj,2,64,8,128,31.78125,2.25390625,8388608
+10,self_attn.k_proj,2,128,8,128,36.625,2.126953125,8388608
+10,self_attn.k_proj,3,32,8,128,11.3515625,3.5078125,8388608
+10,self_attn.k_proj,3,64,8,128,14.0390625,3.25390625,8388608
+10,self_attn.k_proj,3,128,8,128,16.703125,3.126953125,8388608
+10,self_attn.k_proj,4,32,8,128,5.296875,4.5078125,8388608
+10,self_attn.k_proj,4,64,8,128,6.5546875,4.25390625,8388608
+10,self_attn.k_proj,4,128,8,128,7.80078125,4.126953125,8388608
+10,self_attn.k_proj,8,32,8,128,0.320556640625,8.5078125,8388608
+10,self_attn.k_proj,8,64,8,128,0.396240234375,8.25390625,8388608
+10,self_attn.k_proj,8,128,8,128,0.47021484375,8.126953125,8388608
+10,self_attn.o_proj,2,32,8,128,38.625,20.0625,67108864
+10,self_attn.o_proj,2,64,8,128,45.03125,18.03125,67108864
+10,self_attn.o_proj,2,128,8,128,50.90625,17.015625,67108864
+10,self_attn.o_proj,3,32,8,128,16.546875,28.0625,67108864
+10,self_attn.o_proj,3,64,8,128,19.359375,26.03125,67108864
+10,self_attn.o_proj,3,128,8,128,21.96875,25.015625,67108864
+10,self_attn.o_proj,4,32,8,128,7.73046875,36.0625,67108864
+10,self_attn.o_proj,4,64,8,128,9.046875,34.03125,67108864
+10,self_attn.o_proj,4,128,8,128,10.2890625,33.015625,67108864
+10,self_attn.o_proj,8,32,8,128,0.468017578125,68.0625,67108864
+10,self_attn.o_proj,8,64,8,128,0.5478515625,66.03125,67108864
+10,self_attn.o_proj,8,128,8,128,0.62060546875,65.015625,67108864
+10,mlp.gate_proj,2,32,8,128,86.5,70.21875,234881024
+10,mlp.gate_proj,2,64,8,128,101.1875,63.109375,234881024
+10,mlp.gate_proj,2,128,8,128,114.5625,59.5546875,234881024
+10,mlp.gate_proj,3,32,8,128,37.03125,98.21875,234881024
+10,mlp.gate_proj,3,64,8,128,43.34375,91.109375,234881024
+10,mlp.gate_proj,3,128,8,128,49.15625,87.5546875,234881024
+10,mlp.gate_proj,4,32,8,128,17.28125,126.21875,234881024
+10,mlp.gate_proj,4,64,8,128,20.234375,119.109375,234881024
+10,mlp.gate_proj,4,128,8,128,22.953125,115.5546875,234881024
+10,mlp.gate_proj,8,32,8,128,1.05078125,238.21875,234881024
+10,mlp.gate_proj,8,64,8,128,1.232421875,231.109375,234881024
+10,mlp.gate_proj,8,128,8,128,1.392578125,227.5546875,234881024
+10,mlp.up_proj,2,32,8,128,80.8125,70.21875,234881024
+10,mlp.up_proj,2,64,8,128,94.0,63.109375,234881024
+10,mlp.up_proj,2,128,8,128,106.0,59.5546875,234881024
+10,mlp.up_proj,3,32,8,128,34.53125,98.21875,234881024
+10,mlp.up_proj,3,64,8,128,40.28125,91.109375,234881024
+10,mlp.up_proj,3,128,8,128,45.375,87.5546875,234881024
+10,mlp.up_proj,4,32,8,128,16.109375,126.21875,234881024
+10,mlp.up_proj,4,64,8,128,18.796875,119.109375,234881024
+10,mlp.up_proj,4,128,8,128,21.171875,115.5546875,234881024
+10,mlp.up_proj,8,32,8,128,0.97900390625,238.21875,234881024
+10,mlp.up_proj,8,64,8,128,1.142578125,231.109375,234881024
+10,mlp.up_proj,8,128,8,128,1.2841796875,227.5546875,234881024
+10,mlp.down_proj,2,32,8,128,81.25,70.21875,234881024
+10,mlp.down_proj,2,64,8,128,94.5625,63.109375,234881024
+10,mlp.down_proj,2,128,8,128,106.5625,59.5546875,234881024
+10,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024
+10,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024
+10,mlp.down_proj,3,128,8,128,45.5625,87.5546875,234881024
+10,mlp.down_proj,4,32,8,128,16.203125,126.21875,234881024
+10,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024
+10,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024
+10,mlp.down_proj,8,32,8,128,0.9853515625,238.21875,234881024
+10,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024
+10,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024
+11,self_attn.q_proj,2,32,8,128,47.5,20.0625,67108864
+11,self_attn.q_proj,2,64,8,128,56.25,18.03125,67108864
+11,self_attn.q_proj,2,128,8,128,64.4375,17.015625,67108864
+11,self_attn.q_proj,3,32,8,128,20.359375,28.0625,67108864
+11,self_attn.q_proj,3,64,8,128,24.1875,26.03125,67108864
+11,self_attn.q_proj,3,128,8,128,27.9375,25.015625,67108864
+11,self_attn.q_proj,4,32,8,128,9.5078125,36.0625,67108864
+11,self_attn.q_proj,4,64,8,128,11.28125,34.03125,67108864
+11,self_attn.q_proj,4,128,8,128,13.03125,33.015625,67108864
+11,self_attn.q_proj,8,32,8,128,0.57568359375,68.0625,67108864
+11,self_attn.q_proj,8,64,8,128,0.6826171875,66.03125,67108864
+11,self_attn.q_proj,8,128,8,128,0.78564453125,65.015625,67108864
+11,self_attn.v_proj,2,32,8,128,12.765625,2.5078125,8388608
+11,self_attn.v_proj,2,64,8,128,14.8828125,2.25390625,8388608
+11,self_attn.v_proj,2,128,8,128,16.796875,2.126953125,8388608
+11,self_attn.v_proj,3,32,8,128,5.4609375,3.5078125,8388608
+11,self_attn.v_proj,3,64,8,128,6.3671875,3.25390625,8388608
+11,self_attn.v_proj,3,128,8,128,7.1875,3.126953125,8388608
+11,self_attn.v_proj,4,32,8,128,2.546875,4.5078125,8388608
+11,self_attn.v_proj,4,64,8,128,2.970703125,4.25390625,8388608
+11,self_attn.v_proj,4,128,8,128,3.353515625,4.126953125,8388608
+11,self_attn.v_proj,8,32,8,128,0.154296875,8.5078125,8388608
+11,self_attn.v_proj,8,64,8,128,0.1796875,8.25390625,8388608
+11,self_attn.v_proj,8,128,8,128,0.2021484375,8.126953125,8388608
+11,self_attn.k_proj,2,32,8,128,27.3125,2.5078125,8388608
+11,self_attn.k_proj,2,64,8,128,32.78125,2.25390625,8388608
+11,self_attn.k_proj,2,128,8,128,37.375,2.126953125,8388608
+11,self_attn.k_proj,3,32,8,128,11.765625,3.5078125,8388608
+11,self_attn.k_proj,3,64,8,128,14.2578125,3.25390625,8388608
+11,self_attn.k_proj,3,128,8,128,16.609375,3.126953125,8388608
+11,self_attn.k_proj,4,32,8,128,5.4921875,4.5078125,8388608
+11,self_attn.k_proj,4,64,8,128,6.65625,4.25390625,8388608
+11,self_attn.k_proj,4,128,8,128,7.75390625,4.126953125,8388608
+11,self_attn.k_proj,8,32,8,128,0.332275390625,8.5078125,8388608
+11,self_attn.k_proj,8,64,8,128,0.402587890625,8.25390625,8388608
+11,self_attn.k_proj,8,128,8,128,0.467041015625,8.126953125,8388608
+11,self_attn.o_proj,2,32,8,128,38.1875,20.0625,67108864
+11,self_attn.o_proj,2,64,8,128,44.53125,18.03125,67108864
+11,self_attn.o_proj,2,128,8,128,50.28125,17.015625,67108864
+11,self_attn.o_proj,3,32,8,128,16.34375,28.0625,67108864
+11,self_attn.o_proj,3,64,8,128,19.109375,26.03125,67108864
+11,self_attn.o_proj,3,128,8,128,21.625,25.015625,67108864
+11,self_attn.o_proj,4,32,8,128,7.6328125,36.0625,67108864
+11,self_attn.o_proj,4,64,8,128,8.921875,34.03125,67108864
+11,self_attn.o_proj,4,128,8,128,10.109375,33.015625,67108864
+11,self_attn.o_proj,8,32,8,128,0.4619140625,68.0625,67108864
+11,self_attn.o_proj,8,64,8,128,0.53955078125,66.03125,67108864
+11,self_attn.o_proj,8,128,8,128,0.60986328125,65.015625,67108864
+11,mlp.gate_proj,2,32,8,128,86.375,70.21875,234881024
+11,mlp.gate_proj,2,64,8,128,101.0,63.109375,234881024
+11,mlp.gate_proj,2,128,8,128,114.3125,59.5546875,234881024
+11,mlp.gate_proj,3,32,8,128,36.96875,98.21875,234881024
+11,mlp.gate_proj,3,64,8,128,43.28125,91.109375,234881024
+11,mlp.gate_proj,3,128,8,128,49.0625,87.5546875,234881024
+11,mlp.gate_proj,4,32,8,128,17.25,126.21875,234881024
+11,mlp.gate_proj,4,64,8,128,20.203125,119.109375,234881024
+11,mlp.gate_proj,4,128,8,128,22.90625,115.5546875,234881024
+11,mlp.gate_proj,8,32,8,128,1.048828125,238.21875,234881024
+11,mlp.gate_proj,8,64,8,128,1.2294921875,231.109375,234881024
+11,mlp.gate_proj,8,128,8,128,1.3896484375,227.5546875,234881024
+11,mlp.up_proj,2,32,8,128,81.0625,70.21875,234881024
+11,mlp.up_proj,2,64,8,128,94.3125,63.109375,234881024
+11,mlp.up_proj,2,128,8,128,106.375,59.5546875,234881024
+11,mlp.up_proj,3,32,8,128,34.625,98.21875,234881024
+11,mlp.up_proj,3,64,8,128,40.40625,91.109375,234881024
+11,mlp.up_proj,3,128,8,128,45.53125,87.5546875,234881024
+11,mlp.up_proj,4,32,8,128,16.15625,126.21875,234881024
+11,mlp.up_proj,4,64,8,128,18.84375,119.109375,234881024
+11,mlp.up_proj,4,128,8,128,21.25,115.5546875,234881024
+11,mlp.up_proj,8,32,8,128,0.982421875,238.21875,234881024
+11,mlp.up_proj,8,64,8,128,1.146484375,231.109375,234881024
+11,mlp.up_proj,8,128,8,128,1.2890625,227.5546875,234881024
+11,mlp.down_proj,2,32,8,128,81.1875,70.21875,234881024
+11,mlp.down_proj,2,64,8,128,94.4375,63.109375,234881024
+11,mlp.down_proj,2,128,8,128,106.5,59.5546875,234881024
+11,mlp.down_proj,3,32,8,128,34.6875,98.21875,234881024
+11,mlp.down_proj,3,64,8,128,40.4375,91.109375,234881024
+11,mlp.down_proj,3,128,8,128,45.53125,87.5546875,234881024
+11,mlp.down_proj,4,32,8,128,16.171875,126.21875,234881024
+11,mlp.down_proj,4,64,8,128,18.859375,119.109375,234881024
+11,mlp.down_proj,4,128,8,128,21.25,115.5546875,234881024
+11,mlp.down_proj,8,32,8,128,0.98388671875,238.21875,234881024
+11,mlp.down_proj,8,64,8,128,1.1474609375,231.109375,234881024
+11,mlp.down_proj,8,128,8,128,1.2900390625,227.5546875,234881024
+12,self_attn.q_proj,2,32,8,128,49.1875,20.0625,67108864
+12,self_attn.q_proj,2,64,8,128,58.25,18.03125,67108864
+12,self_attn.q_proj,2,128,8,128,67.0,17.015625,67108864
+12,self_attn.q_proj,3,32,8,128,21.046875,28.0625,67108864
+12,self_attn.q_proj,3,64,8,128,24.984375,26.03125,67108864
+12,self_attn.q_proj,3,128,8,128,28.828125,25.015625,67108864
+12,self_attn.q_proj,4,32,8,128,9.8203125,36.0625,67108864
+12,self_attn.q_proj,4,64,8,128,11.65625,34.03125,67108864
+12,self_attn.q_proj,4,128,8,128,13.4609375,33.015625,67108864
+12,self_attn.q_proj,8,32,8,128,0.5947265625,68.0625,67108864
+12,self_attn.q_proj,8,64,8,128,0.705078125,66.03125,67108864
+12,self_attn.q_proj,8,128,8,128,0.81103515625,65.015625,67108864
+12,self_attn.v_proj,2,32,8,128,13.3203125,2.5078125,8388608
+12,self_attn.v_proj,2,64,8,128,15.5390625,2.25390625,8388608
+12,self_attn.v_proj,2,128,8,128,17.53125,2.126953125,8388608
+12,self_attn.v_proj,3,32,8,128,5.69140625,3.5078125,8388608
+12,self_attn.v_proj,3,64,8,128,6.640625,3.25390625,8388608
+12,self_attn.v_proj,3,128,8,128,7.5,3.126953125,8388608
+12,self_attn.v_proj,4,32,8,128,2.65625,4.5078125,8388608
+12,self_attn.v_proj,4,64,8,128,3.099609375,4.25390625,8388608
+12,self_attn.v_proj,4,128,8,128,3.5,4.126953125,8388608
+12,self_attn.v_proj,8,32,8,128,0.160888671875,8.5078125,8388608
+12,self_attn.v_proj,8,64,8,128,0.1875,8.25390625,8388608
+12,self_attn.v_proj,8,128,8,128,0.2109375,8.126953125,8388608
+12,self_attn.k_proj,2,32,8,128,28.5625,2.5078125,8388608
+12,self_attn.k_proj,2,64,8,128,34.3125,2.25390625,8388608
+12,self_attn.k_proj,2,128,8,128,39.375,2.126953125,8388608
+12,self_attn.k_proj,3,32,8,128,12.28125,3.5078125,8388608
+12,self_attn.k_proj,3,64,8,128,14.828125,3.25390625,8388608
+12,self_attn.k_proj,3,128,8,128,17.28125,3.126953125,8388608
+12,self_attn.k_proj,4,32,8,128,5.734375,4.5078125,8388608
+12,self_attn.k_proj,4,64,8,128,6.921875,4.25390625,8388608
+12,self_attn.k_proj,4,128,8,128,8.0625,4.126953125,8388608
+12,self_attn.k_proj,8,32,8,128,0.346923828125,8.5078125,8388608
+12,self_attn.k_proj,8,64,8,128,0.418701171875,8.25390625,8388608
+12,self_attn.k_proj,8,128,8,128,0.48583984375,8.126953125,8388608
+12,self_attn.o_proj,2,32,8,128,39.21875,20.0625,67108864
+12,self_attn.o_proj,2,64,8,128,45.78125,18.03125,67108864
+12,self_attn.o_proj,2,128,8,128,51.8125,17.015625,67108864
+12,self_attn.o_proj,3,32,8,128,16.78125,28.0625,67108864
+12,self_attn.o_proj,3,64,8,128,19.625,26.03125,67108864
+12,self_attn.o_proj,3,128,8,128,22.28125,25.015625,67108864
+12,self_attn.o_proj,4,32,8,128,7.828125,36.0625,67108864
+12,self_attn.o_proj,4,64,8,128,9.15625,34.03125,67108864
+12,self_attn.o_proj,4,128,8,128,10.40625,33.015625,67108864
+12,self_attn.o_proj,8,32,8,128,0.473876953125,68.0625,67108864
+12,self_attn.o_proj,8,64,8,128,0.55419921875,66.03125,67108864
+12,self_attn.o_proj,8,128,8,128,0.626953125,65.015625,67108864
+12,mlp.gate_proj,2,32,8,128,85.625,70.21875,234881024
+12,mlp.gate_proj,2,64,8,128,100.1875,63.109375,234881024
+12,mlp.gate_proj,2,128,8,128,113.5,59.5546875,234881024
+12,mlp.gate_proj,3,32,8,128,36.65625,98.21875,234881024
+12,mlp.gate_proj,3,64,8,128,42.90625,91.109375,234881024
+12,mlp.gate_proj,3,128,8,128,48.6875,87.5546875,234881024
+12,mlp.gate_proj,4,32,8,128,17.09375,126.21875,234881024
+12,mlp.gate_proj,4,64,8,128,20.03125,119.109375,234881024
+12,mlp.gate_proj,4,128,8,128,22.703125,115.5546875,234881024
+12,mlp.gate_proj,8,32,8,128,1.0400390625,238.21875,234881024
+12,mlp.gate_proj,8,64,8,128,1.21875,231.109375,234881024
+12,mlp.gate_proj,8,128,8,128,1.3779296875,227.5546875,234881024
+12,mlp.up_proj,2,32,8,128,81.125,70.21875,234881024
+12,mlp.up_proj,2,64,8,128,94.5,63.109375,234881024
+12,mlp.up_proj,2,128,8,128,106.5625,59.5546875,234881024
+12,mlp.up_proj,3,32,8,128,34.6875,98.21875,234881024
+12,mlp.up_proj,3,64,8,128,40.4375,91.109375,234881024
+12,mlp.up_proj,3,128,8,128,45.625,87.5546875,234881024
+12,mlp.up_proj,4,32,8,128,16.171875,126.21875,234881024
+12,mlp.up_proj,4,64,8,128,18.875,119.109375,234881024
+12,mlp.up_proj,4,128,8,128,21.296875,115.5546875,234881024
+12,mlp.up_proj,8,32,8,128,0.9833984375,238.21875,234881024
+12,mlp.up_proj,8,64,8,128,1.1484375,231.109375,234881024
+12,mlp.up_proj,8,128,8,128,1.2919921875,227.5546875,234881024
+12,mlp.down_proj,2,32,8,128,81.125,70.21875,234881024
+12,mlp.down_proj,2,64,8,128,94.375,63.109375,234881024
+12,mlp.down_proj,2,128,8,128,106.5,59.5546875,234881024
+12,mlp.down_proj,3,32,8,128,34.6875,98.21875,234881024
+12,mlp.down_proj,3,64,8,128,40.4375,91.109375,234881024
+12,mlp.down_proj,3,128,8,128,45.5625,87.5546875,234881024
+12,mlp.down_proj,4,32,8,128,16.171875,126.21875,234881024
+12,mlp.down_proj,4,64,8,128,18.875,119.109375,234881024
+12,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024
+12,mlp.down_proj,8,32,8,128,0.9833984375,238.21875,234881024
+12,mlp.down_proj,8,64,8,128,1.1474609375,231.109375,234881024
+12,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024
+13,self_attn.q_proj,2,32,8,128,48.9375,20.0625,67108864
+13,self_attn.q_proj,2,64,8,128,58.0,18.03125,67108864
+13,self_attn.q_proj,2,128,8,128,67.125,17.015625,67108864
+13,self_attn.q_proj,3,32,8,128,20.984375,28.0625,67108864
+13,self_attn.q_proj,3,64,8,128,24.953125,26.03125,67108864
+13,self_attn.q_proj,3,128,8,128,29.0625,25.015625,67108864
+13,self_attn.q_proj,4,32,8,128,9.7890625,36.0625,67108864
+13,self_attn.q_proj,4,64,8,128,11.640625,34.03125,67108864
+13,self_attn.q_proj,4,128,8,128,13.5625,33.015625,67108864
+13,self_attn.q_proj,8,32,8,128,0.5927734375,68.0625,67108864
+13,self_attn.q_proj,8,64,8,128,0.7041015625,66.03125,67108864
+13,self_attn.q_proj,8,128,8,128,0.8173828125,65.015625,67108864
+13,self_attn.v_proj,2,32,8,128,12.734375,2.5078125,8388608
+13,self_attn.v_proj,2,64,8,128,14.890625,2.25390625,8388608
+13,self_attn.v_proj,2,128,8,128,16.84375,2.126953125,8388608
+13,self_attn.v_proj,3,32,8,128,5.4453125,3.5078125,8388608
+13,self_attn.v_proj,3,64,8,128,6.3671875,3.25390625,8388608
+13,self_attn.v_proj,3,128,8,128,7.203125,3.126953125,8388608
+13,self_attn.v_proj,4,32,8,128,2.541015625,4.5078125,8388608
+13,self_attn.v_proj,4,64,8,128,2.970703125,4.25390625,8388608
+13,self_attn.v_proj,4,128,8,128,3.36328125,4.126953125,8388608
+13,self_attn.v_proj,8,32,8,128,0.15380859375,8.5078125,8388608
+13,self_attn.v_proj,8,64,8,128,0.1796875,8.25390625,8388608
+13,self_attn.v_proj,8,128,8,128,0.20263671875,8.126953125,8388608
+13,self_attn.k_proj,2,32,8,128,28.515625,2.5078125,8388608
+13,self_attn.k_proj,2,64,8,128,34.3125,2.25390625,8388608
+13,self_attn.k_proj,2,128,8,128,39.0625,2.126953125,8388608
+13,self_attn.k_proj,3,32,8,128,12.3671875,3.5078125,8388608
+13,self_attn.k_proj,3,64,8,128,15.1640625,3.25390625,8388608
+13,self_attn.k_proj,3,128,8,128,17.828125,3.126953125,8388608
+13,self_attn.k_proj,4,32,8,128,5.76953125,4.5078125,8388608
+13,self_attn.k_proj,4,64,8,128,7.078125,4.25390625,8388608
+13,self_attn.k_proj,4,128,8,128,8.3203125,4.126953125,8388608
+13,self_attn.k_proj,8,32,8,128,0.349365234375,8.5078125,8388608
+13,self_attn.k_proj,8,64,8,128,0.427734375,8.25390625,8388608
+13,self_attn.k_proj,8,128,8,128,0.50146484375,8.126953125,8388608
+13,self_attn.o_proj,2,32,8,128,39.71875,20.0625,67108864
+13,self_attn.o_proj,2,64,8,128,46.3125,18.03125,67108864
+13,self_attn.o_proj,2,128,8,128,52.3125,17.015625,67108864
+13,self_attn.o_proj,3,32,8,128,17.0,28.0625,67108864
+13,self_attn.o_proj,3,64,8,128,19.859375,26.03125,67108864
+13,self_attn.o_proj,3,128,8,128,22.484375,25.015625,67108864
+13,self_attn.o_proj,4,32,8,128,7.9375,36.0625,67108864
+13,self_attn.o_proj,4,64,8,128,9.2734375,34.03125,67108864
+13,self_attn.o_proj,4,128,8,128,10.5078125,33.015625,67108864
+13,self_attn.o_proj,8,32,8,128,0.48046875,68.0625,67108864
+13,self_attn.o_proj,8,64,8,128,0.56103515625,66.03125,67108864
+13,self_attn.o_proj,8,128,8,128,0.6337890625,65.015625,67108864
+13,mlp.gate_proj,2,32,8,128,85.0,70.21875,234881024
+13,mlp.gate_proj,2,64,8,128,99.4375,63.109375,234881024
+13,mlp.gate_proj,2,128,8,128,112.625,59.5546875,234881024
+13,mlp.gate_proj,3,32,8,128,36.375,98.21875,234881024
+13,mlp.gate_proj,3,64,8,128,42.5625,91.109375,234881024
+13,mlp.gate_proj,3,128,8,128,48.25,87.5546875,234881024
+13,mlp.gate_proj,4,32,8,128,16.953125,126.21875,234881024
+13,mlp.gate_proj,4,64,8,128,19.875,119.109375,234881024
+13,mlp.gate_proj,4,128,8,128,22.515625,115.5546875,234881024
+13,mlp.gate_proj,8,32,8,128,1.03125,238.21875,234881024
+13,mlp.gate_proj,8,64,8,128,1.208984375,231.109375,234881024
+13,mlp.gate_proj,8,128,8,128,1.3662109375,227.5546875,234881024
+13,mlp.up_proj,2,32,8,128,81.0,70.21875,234881024
+13,mlp.up_proj,2,64,8,128,94.3125,63.109375,234881024
+13,mlp.up_proj,2,128,8,128,106.375,59.5546875,234881024
+13,mlp.up_proj,3,32,8,128,34.625,98.21875,234881024
+13,mlp.up_proj,3,64,8,128,40.375,91.109375,234881024
+13,mlp.up_proj,3,128,8,128,45.5,87.5546875,234881024
+13,mlp.up_proj,4,32,8,128,16.140625,126.21875,234881024
+13,mlp.up_proj,4,64,8,128,18.84375,119.109375,234881024
+13,mlp.up_proj,4,128,8,128,21.25,115.5546875,234881024
+13,mlp.up_proj,8,32,8,128,0.98193359375,238.21875,234881024
+13,mlp.up_proj,8,64,8,128,1.146484375,231.109375,234881024
+13,mlp.up_proj,8,128,8,128,1.2890625,227.5546875,234881024
+13,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024
+13,mlp.down_proj,2,64,8,128,94.6875,63.109375,234881024
+13,mlp.down_proj,2,128,8,128,106.75,59.5546875,234881024
+13,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024
+13,mlp.down_proj,3,64,8,128,40.53125,91.109375,234881024
+13,mlp.down_proj,3,128,8,128,45.65625,87.5546875,234881024
+13,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024
+13,mlp.down_proj,4,64,8,128,18.921875,119.109375,234881024
+13,mlp.down_proj,4,128,8,128,21.3125,115.5546875,234881024
+13,mlp.down_proj,8,32,8,128,0.986328125,238.21875,234881024
+13,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024
+13,mlp.down_proj,8,128,8,128,1.29296875,227.5546875,234881024
+14,self_attn.q_proj,2,32,8,128,49.21875,20.0625,67108864
+14,self_attn.q_proj,2,64,8,128,58.09375,18.03125,67108864
+14,self_attn.q_proj,2,128,8,128,66.8125,17.015625,67108864
+14,self_attn.q_proj,3,32,8,128,21.0625,28.0625,67108864
+14,self_attn.q_proj,3,64,8,128,24.90625,26.03125,67108864
+14,self_attn.q_proj,3,128,8,128,28.734375,25.015625,67108864
+14,self_attn.q_proj,4,32,8,128,9.828125,36.0625,67108864
+14,self_attn.q_proj,4,64,8,128,11.625,34.03125,67108864
+14,self_attn.q_proj,4,128,8,128,13.4140625,33.015625,67108864
+14,self_attn.q_proj,8,32,8,128,0.59521484375,68.0625,67108864
+14,self_attn.q_proj,8,64,8,128,0.703125,66.03125,67108864
+14,self_attn.q_proj,8,128,8,128,0.80859375,65.015625,67108864
+14,self_attn.v_proj,2,32,8,128,12.6875,2.5078125,8388608
+14,self_attn.v_proj,2,64,8,128,14.8125,2.25390625,8388608
+14,self_attn.v_proj,2,128,8,128,16.734375,2.126953125,8388608
+14,self_attn.v_proj,3,32,8,128,5.42578125,3.5078125,8388608
+14,self_attn.v_proj,3,64,8,128,6.3359375,3.25390625,8388608
+14,self_attn.v_proj,3,128,8,128,7.16015625,3.126953125,8388608
+14,self_attn.v_proj,4,32,8,128,2.529296875,4.5078125,8388608
+14,self_attn.v_proj,4,64,8,128,2.955078125,4.25390625,8388608
+14,self_attn.v_proj,4,128,8,128,3.341796875,4.126953125,8388608
+14,self_attn.v_proj,8,32,8,128,0.1533203125,8.5078125,8388608
+14,self_attn.v_proj,8,64,8,128,0.1788330078125,8.25390625,8388608
+14,self_attn.v_proj,8,128,8,128,0.201416015625,8.126953125,8388608
+14,self_attn.k_proj,2,32,8,128,26.953125,2.5078125,8388608
+14,self_attn.k_proj,2,64,8,128,32.59375,2.25390625,8388608
+14,self_attn.k_proj,2,128,8,128,37.4375,2.126953125,8388608
+14,self_attn.k_proj,3,32,8,128,11.5859375,3.5078125,8388608
+14,self_attn.k_proj,3,64,8,128,14.15625,3.25390625,8388608
+14,self_attn.k_proj,3,128,8,128,16.5625,3.126953125,8388608
+14,self_attn.k_proj,4,32,8,128,5.40625,4.5078125,8388608
+14,self_attn.k_proj,4,64,8,128,6.6015625,4.25390625,8388608
+14,self_attn.k_proj,4,128,8,128,7.73046875,4.126953125,8388608
+14,self_attn.k_proj,8,32,8,128,0.327392578125,8.5078125,8388608
+14,self_attn.k_proj,8,64,8,128,0.3994140625,8.25390625,8388608
+14,self_attn.k_proj,8,128,8,128,0.4658203125,8.126953125,8388608
+14,self_attn.o_proj,2,32,8,128,40.0625,20.0625,67108864
+14,self_attn.o_proj,2,64,8,128,46.65625,18.03125,67108864
+14,self_attn.o_proj,2,128,8,128,52.6875,17.015625,67108864
+14,self_attn.o_proj,3,32,8,128,17.125,28.0625,67108864
+14,self_attn.o_proj,3,64,8,128,20.0,26.03125,67108864
+14,self_attn.o_proj,3,128,8,128,22.609375,25.015625,67108864
+14,self_attn.o_proj,4,32,8,128,7.9921875,36.0625,67108864
+14,self_attn.o_proj,4,64,8,128,9.328125,34.03125,67108864
+14,self_attn.o_proj,4,128,8,128,10.5546875,33.015625,67108864
+14,self_attn.o_proj,8,32,8,128,0.48388671875,68.0625,67108864
+14,self_attn.o_proj,8,64,8,128,0.564453125,66.03125,67108864
+14,self_attn.o_proj,8,128,8,128,0.63623046875,65.015625,67108864
+14,mlp.gate_proj,2,32,8,128,85.25,70.21875,234881024
+14,mlp.gate_proj,2,64,8,128,99.6875,63.109375,234881024
+14,mlp.gate_proj,2,128,8,128,112.875,59.5546875,234881024
+14,mlp.gate_proj,3,32,8,128,36.46875,98.21875,234881024
+14,mlp.gate_proj,3,64,8,128,42.65625,91.109375,234881024
+14,mlp.gate_proj,3,128,8,128,48.375,87.5546875,234881024
+14,mlp.gate_proj,4,32,8,128,17.015625,126.21875,234881024
+14,mlp.gate_proj,4,64,8,128,19.921875,119.109375,234881024
+14,mlp.gate_proj,4,128,8,128,22.578125,115.5546875,234881024
+14,mlp.gate_proj,8,32,8,128,1.0341796875,238.21875,234881024
+14,mlp.gate_proj,8,64,8,128,1.212890625,231.109375,234881024
+14,mlp.gate_proj,8,128,8,128,1.3701171875,227.5546875,234881024
+14,mlp.up_proj,2,32,8,128,81.1875,70.21875,234881024
+14,mlp.up_proj,2,64,8,128,94.5,63.109375,234881024
+14,mlp.up_proj,2,128,8,128,106.625,59.5546875,234881024
+14,mlp.up_proj,3,32,8,128,34.6875,98.21875,234881024
+14,mlp.up_proj,3,64,8,128,40.46875,91.109375,234881024
+14,mlp.up_proj,3,128,8,128,45.625,87.5546875,234881024
+14,mlp.up_proj,4,32,8,128,16.1875,126.21875,234881024
+14,mlp.up_proj,4,64,8,128,18.890625,119.109375,234881024
+14,mlp.up_proj,4,128,8,128,21.296875,115.5546875,234881024
+14,mlp.up_proj,8,32,8,128,0.98388671875,238.21875,234881024
+14,mlp.up_proj,8,64,8,128,1.1484375,231.109375,234881024
+14,mlp.up_proj,8,128,8,128,1.2919921875,227.5546875,234881024
+14,mlp.down_proj,2,32,8,128,81.25,70.21875,234881024
+14,mlp.down_proj,2,64,8,128,94.5625,63.109375,234881024
+14,mlp.down_proj,2,128,8,128,106.625,59.5546875,234881024
+14,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024
+14,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024
+14,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024
+14,mlp.down_proj,4,32,8,128,16.203125,126.21875,234881024
+14,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024
+14,mlp.down_proj,4,128,8,128,21.296875,115.5546875,234881024
+14,mlp.down_proj,8,32,8,128,0.9853515625,238.21875,234881024
+14,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024
+14,mlp.down_proj,8,128,8,128,1.2919921875,227.5546875,234881024
+15,self_attn.q_proj,2,32,8,128,45.375,20.0625,67108864
+15,self_attn.q_proj,2,64,8,128,53.59375,18.03125,67108864
+15,self_attn.q_proj,2,128,8,128,62.0625,17.015625,67108864
+15,self_attn.q_proj,3,32,8,128,19.4375,28.0625,67108864
+15,self_attn.q_proj,3,64,8,128,23.03125,26.03125,67108864
+15,self_attn.q_proj,3,128,8,128,26.828125,25.015625,67108864
+15,self_attn.q_proj,4,32,8,128,9.0703125,36.0625,67108864
+15,self_attn.q_proj,4,64,8,128,10.7421875,34.03125,67108864
+15,self_attn.q_proj,4,128,8,128,12.5234375,33.015625,67108864
+15,self_attn.q_proj,8,32,8,128,0.54931640625,68.0625,67108864
+15,self_attn.q_proj,8,64,8,128,0.650390625,66.03125,67108864
+15,self_attn.q_proj,8,128,8,128,0.7548828125,65.015625,67108864
+15,self_attn.v_proj,2,32,8,128,13.75,2.5078125,8388608
+15,self_attn.v_proj,2,64,8,128,16.078125,2.25390625,8388608
+15,self_attn.v_proj,2,128,8,128,18.171875,2.126953125,8388608
+15,self_attn.v_proj,3,32,8,128,5.87890625,3.5078125,8388608
+15,self_attn.v_proj,3,64,8,128,6.875,3.25390625,8388608
+15,self_attn.v_proj,3,128,8,128,7.77734375,3.126953125,8388608
+15,self_attn.v_proj,4,32,8,128,2.740234375,4.5078125,8388608
+15,self_attn.v_proj,4,64,8,128,3.20703125,4.25390625,8388608
+15,self_attn.v_proj,4,128,8,128,3.630859375,4.126953125,8388608
+15,self_attn.v_proj,8,32,8,128,0.166015625,8.5078125,8388608
+15,self_attn.v_proj,8,64,8,128,0.194091796875,8.25390625,8388608
+15,self_attn.v_proj,8,128,8,128,0.21875,8.126953125,8388608
+15,self_attn.k_proj,2,32,8,128,25.953125,2.5078125,8388608
+15,self_attn.k_proj,2,64,8,128,31.40625,2.25390625,8388608
+15,self_attn.k_proj,2,128,8,128,35.96875,2.126953125,8388608
+15,self_attn.k_proj,3,32,8,128,11.25,3.5078125,8388608
+15,self_attn.k_proj,3,64,8,128,13.890625,3.25390625,8388608
+15,self_attn.k_proj,3,128,8,128,16.4375,3.126953125,8388608
+15,self_attn.k_proj,4,32,8,128,5.24609375,4.5078125,8388608
+15,self_attn.k_proj,4,64,8,128,6.484375,4.25390625,8388608
+15,self_attn.k_proj,4,128,8,128,7.671875,4.126953125,8388608
+15,self_attn.k_proj,8,32,8,128,0.317626953125,8.5078125,8388608
+15,self_attn.k_proj,8,64,8,128,0.391845703125,8.25390625,8388608
+15,self_attn.k_proj,8,128,8,128,0.46240234375,8.126953125,8388608
+15,self_attn.o_proj,2,32,8,128,40.40625,20.0625,67108864
+15,self_attn.o_proj,2,64,8,128,47.0625,18.03125,67108864
+15,self_attn.o_proj,2,128,8,128,53.09375,17.015625,67108864
+15,self_attn.o_proj,3,32,8,128,17.296875,28.0625,67108864
+15,self_attn.o_proj,3,64,8,128,20.171875,26.03125,67108864
+15,self_attn.o_proj,3,128,8,128,22.796875,25.015625,67108864
+15,self_attn.o_proj,4,32,8,128,8.078125,36.0625,67108864
+15,self_attn.o_proj,4,64,8,128,9.421875,34.03125,67108864
+15,self_attn.o_proj,4,128,8,128,10.6640625,33.015625,67108864
+15,self_attn.o_proj,8,32,8,128,0.48876953125,68.0625,67108864
+15,self_attn.o_proj,8,64,8,128,0.5703125,66.03125,67108864
+15,self_attn.o_proj,8,128,8,128,0.64306640625,65.015625,67108864
+15,mlp.gate_proj,2,32,8,128,85.0625,70.21875,234881024
+15,mlp.gate_proj,2,64,8,128,99.5,63.109375,234881024
+15,mlp.gate_proj,2,128,8,128,112.6875,59.5546875,234881024
+15,mlp.gate_proj,3,32,8,128,36.40625,98.21875,234881024
+15,mlp.gate_proj,3,64,8,128,42.59375,91.109375,234881024
+15,mlp.gate_proj,3,128,8,128,48.28125,87.5546875,234881024
+15,mlp.gate_proj,4,32,8,128,16.96875,126.21875,234881024
+15,mlp.gate_proj,4,64,8,128,19.890625,119.109375,234881024
+15,mlp.gate_proj,4,128,8,128,22.53125,115.5546875,234881024
+15,mlp.gate_proj,8,32,8,128,1.033203125,238.21875,234881024
+15,mlp.gate_proj,8,64,8,128,1.2099609375,231.109375,234881024
+15,mlp.gate_proj,8,128,8,128,1.3671875,227.5546875,234881024
+15,mlp.up_proj,2,32,8,128,81.125,70.21875,234881024
+15,mlp.up_proj,2,64,8,128,94.4375,63.109375,234881024
+15,mlp.up_proj,2,128,8,128,106.5,59.5546875,234881024
+15,mlp.up_proj,3,32,8,128,34.6875,98.21875,234881024
+15,mlp.up_proj,3,64,8,128,40.4375,91.109375,234881024
+15,mlp.up_proj,3,128,8,128,45.59375,87.5546875,234881024
+15,mlp.up_proj,4,32,8,128,16.171875,126.21875,234881024
+15,mlp.up_proj,4,64,8,128,18.875,119.109375,234881024
+15,mlp.up_proj,4,128,8,128,21.28125,115.5546875,234881024
+15,mlp.up_proj,8,32,8,128,0.9833984375,238.21875,234881024
+15,mlp.up_proj,8,64,8,128,1.1474609375,231.109375,234881024
+15,mlp.up_proj,8,128,8,128,1.291015625,227.5546875,234881024
+15,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024
+15,mlp.down_proj,2,64,8,128,94.6875,63.109375,234881024
+15,mlp.down_proj,2,128,8,128,106.75,59.5546875,234881024
+15,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024
+15,mlp.down_proj,3,64,8,128,40.53125,91.109375,234881024
+15,mlp.down_proj,3,128,8,128,45.65625,87.5546875,234881024
+15,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024
+15,mlp.down_proj,4,64,8,128,18.921875,119.109375,234881024
+15,mlp.down_proj,4,128,8,128,21.3125,115.5546875,234881024
+15,mlp.down_proj,8,32,8,128,0.986328125,238.21875,234881024
+15,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024
+15,mlp.down_proj,8,128,8,128,1.29296875,227.5546875,234881024
+16,self_attn.q_proj,2,32,8,128,46.375,20.0625,67108864
+16,self_attn.q_proj,2,64,8,128,55.0,18.03125,67108864
+16,self_attn.q_proj,2,128,8,128,63.6875,17.015625,67108864
+16,self_attn.q_proj,3,32,8,128,19.890625,28.0625,67108864
+16,self_attn.q_proj,3,64,8,128,23.671875,26.03125,67108864
+16,self_attn.q_proj,3,128,8,128,27.609375,25.015625,67108864
+16,self_attn.q_proj,4,32,8,128,9.28125,36.0625,67108864
+16,self_attn.q_proj,4,64,8,128,11.046875,34.03125,67108864
+16,self_attn.q_proj,4,128,8,128,12.8828125,33.015625,67108864
+16,self_attn.q_proj,8,32,8,128,0.56201171875,68.0625,67108864
+16,self_attn.q_proj,8,64,8,128,0.66845703125,66.03125,67108864
+16,self_attn.q_proj,8,128,8,128,0.7763671875,65.015625,67108864
+16,self_attn.v_proj,2,32,8,128,13.5859375,2.5078125,8388608
+16,self_attn.v_proj,2,64,8,128,15.828125,2.25390625,8388608
+16,self_attn.v_proj,2,128,8,128,17.859375,2.126953125,8388608
+16,self_attn.v_proj,3,32,8,128,5.8046875,3.5078125,8388608
+16,self_attn.v_proj,3,64,8,128,6.7734375,3.25390625,8388608
+16,self_attn.v_proj,3,128,8,128,7.64453125,3.126953125,8388608
+16,self_attn.v_proj,4,32,8,128,2.708984375,4.5078125,8388608
+16,self_attn.v_proj,4,64,8,128,3.16015625,4.25390625,8388608
+16,self_attn.v_proj,4,128,8,128,3.56640625,4.126953125,8388608
+16,self_attn.v_proj,8,32,8,128,0.1640625,8.5078125,8388608
+16,self_attn.v_proj,8,64,8,128,0.191162109375,8.25390625,8388608
+16,self_attn.v_proj,8,128,8,128,0.2149658203125,8.126953125,8388608
+16,self_attn.k_proj,2,32,8,128,26.9375,2.5078125,8388608
+16,self_attn.k_proj,2,64,8,128,32.5,2.25390625,8388608
+16,self_attn.k_proj,2,128,8,128,37.21875,2.126953125,8388608
+16,self_attn.k_proj,3,32,8,128,11.640625,3.5078125,8388608
+16,self_attn.k_proj,3,64,8,128,14.265625,3.25390625,8388608
+16,self_attn.k_proj,3,128,8,128,16.765625,3.126953125,8388608
+16,self_attn.k_proj,4,32,8,128,5.43359375,4.5078125,8388608
+16,self_attn.k_proj,4,64,8,128,6.65625,4.25390625,8388608
+16,self_attn.k_proj,4,128,8,128,7.83203125,4.126953125,8388608
+16,self_attn.k_proj,8,32,8,128,0.328857421875,8.5078125,8388608
+16,self_attn.k_proj,8,64,8,128,0.402587890625,8.25390625,8388608
+16,self_attn.k_proj,8,128,8,128,0.471923828125,8.126953125,8388608
+16,self_attn.o_proj,2,32,8,128,40.40625,20.0625,67108864
+16,self_attn.o_proj,2,64,8,128,47.125,18.03125,67108864
+16,self_attn.o_proj,2,128,8,128,53.25,17.015625,67108864
+16,self_attn.o_proj,3,32,8,128,17.296875,28.0625,67108864
+16,self_attn.o_proj,3,64,8,128,20.21875,26.03125,67108864
+16,self_attn.o_proj,3,128,8,128,22.921875,25.015625,67108864
+16,self_attn.o_proj,4,32,8,128,8.0703125,36.0625,67108864
+16,self_attn.o_proj,4,64,8,128,9.4375,34.03125,67108864
+16,self_attn.o_proj,4,128,8,128,10.7109375,33.015625,67108864
+16,self_attn.o_proj,8,32,8,128,0.488525390625,68.0625,67108864
+16,self_attn.o_proj,8,64,8,128,0.57080078125,66.03125,67108864
+16,self_attn.o_proj,8,128,8,128,0.64501953125,65.015625,67108864
+16,mlp.gate_proj,2,32,8,128,84.9375,70.21875,234881024
+16,mlp.gate_proj,2,64,8,128,99.375,63.109375,234881024
+16,mlp.gate_proj,2,128,8,128,112.625,59.5546875,234881024
+16,mlp.gate_proj,3,32,8,128,36.375,98.21875,234881024
+16,mlp.gate_proj,3,64,8,128,42.5625,91.109375,234881024
+16,mlp.gate_proj,3,128,8,128,48.28125,87.5546875,234881024
+16,mlp.gate_proj,4,32,8,128,16.953125,126.21875,234881024
+16,mlp.gate_proj,4,64,8,128,19.875,119.109375,234881024
+16,mlp.gate_proj,4,128,8,128,22.53125,115.5546875,234881024
+16,mlp.gate_proj,8,32,8,128,1.03125,238.21875,234881024
+16,mlp.gate_proj,8,64,8,128,1.208984375,231.109375,234881024
+16,mlp.gate_proj,8,128,8,128,1.3671875,227.5546875,234881024
+16,mlp.up_proj,2,32,8,128,81.3125,70.21875,234881024
+16,mlp.up_proj,2,64,8,128,94.6875,63.109375,234881024
+16,mlp.up_proj,2,128,8,128,106.75,59.5546875,234881024
+16,mlp.up_proj,3,32,8,128,34.75,98.21875,234881024
+16,mlp.up_proj,3,64,8,128,40.53125,91.109375,234881024
+16,mlp.up_proj,3,128,8,128,45.71875,87.5546875,234881024
+16,mlp.up_proj,4,32,8,128,16.203125,126.21875,234881024
+16,mlp.up_proj,4,64,8,128,18.921875,119.109375,234881024
+16,mlp.up_proj,4,128,8,128,21.34375,115.5546875,234881024
+16,mlp.up_proj,8,32,8,128,0.98583984375,238.21875,234881024
+16,mlp.up_proj,8,64,8,128,1.150390625,231.109375,234881024
+16,mlp.up_proj,8,128,8,128,1.294921875,227.5546875,234881024
+16,mlp.down_proj,2,32,8,128,81.25,70.21875,234881024
+16,mlp.down_proj,2,64,8,128,94.5625,63.109375,234881024
+16,mlp.down_proj,2,128,8,128,106.625,59.5546875,234881024
+16,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024
+16,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024
+16,mlp.down_proj,3,128,8,128,45.625,87.5546875,234881024
+16,mlp.down_proj,4,32,8,128,16.203125,126.21875,234881024
+16,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024
+16,mlp.down_proj,4,128,8,128,21.296875,115.5546875,234881024
+16,mlp.down_proj,8,32,8,128,0.9853515625,238.21875,234881024
+16,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024
+16,mlp.down_proj,8,128,8,128,1.2919921875,227.5546875,234881024
+17,self_attn.q_proj,2,32,8,128,49.25,20.0625,67108864
+17,self_attn.q_proj,2,64,8,128,58.15625,18.03125,67108864
+17,self_attn.q_proj,2,128,8,128,67.3125,17.015625,67108864
+17,self_attn.q_proj,3,32,8,128,21.09375,28.0625,67108864
+17,self_attn.q_proj,3,64,8,128,24.953125,26.03125,67108864
+17,self_attn.q_proj,3,128,8,128,28.953125,25.015625,67108864
+17,self_attn.q_proj,4,32,8,128,9.84375,36.0625,67108864
+17,self_attn.q_proj,4,64,8,128,11.6484375,34.03125,67108864
+17,self_attn.q_proj,4,128,8,128,13.515625,33.015625,67108864
+17,self_attn.q_proj,8,32,8,128,0.59619140625,68.0625,67108864
+17,self_attn.q_proj,8,64,8,128,0.70458984375,66.03125,67108864
+17,self_attn.q_proj,8,128,8,128,0.814453125,65.015625,67108864
+17,self_attn.v_proj,2,32,8,128,13.4453125,2.5078125,8388608
+17,self_attn.v_proj,2,64,8,128,15.7734375,2.25390625,8388608
+17,self_attn.v_proj,2,128,8,128,17.90625,2.126953125,8388608
+17,self_attn.v_proj,3,32,8,128,5.74609375,3.5078125,8388608
+17,self_attn.v_proj,3,64,8,128,6.75,3.25390625,8388608
+17,self_attn.v_proj,3,128,8,128,7.6640625,3.126953125,8388608
+17,self_attn.v_proj,4,32,8,128,2.681640625,4.5078125,8388608
+17,self_attn.v_proj,4,64,8,128,3.1484375,4.25390625,8388608
+17,self_attn.v_proj,4,128,8,128,3.580078125,4.126953125,8388608
+17,self_attn.v_proj,8,32,8,128,0.1624755859375,8.5078125,8388608
+17,self_attn.v_proj,8,64,8,128,0.1904296875,8.25390625,8388608
+17,self_attn.v_proj,8,128,8,128,0.215576171875,8.126953125,8388608
+17,self_attn.k_proj,2,32,8,128,29.140625,2.5078125,8388608
+17,self_attn.k_proj,2,64,8,128,35.15625,2.25390625,8388608
+17,self_attn.k_proj,2,128,8,128,40.28125,2.126953125,8388608
+17,self_attn.k_proj,3,32,8,128,12.6015625,3.5078125,8388608
+17,self_attn.k_proj,3,64,8,128,15.4140625,3.25390625,8388608
+17,self_attn.k_proj,3,128,8,128,18.125,3.126953125,8388608
+17,self_attn.k_proj,4,32,8,128,5.87890625,4.5078125,8388608
+17,self_attn.k_proj,4,64,8,128,7.19140625,4.25390625,8388608
+17,self_attn.k_proj,4,128,8,128,8.4609375,4.126953125,8388608
+17,self_attn.k_proj,8,32,8,128,0.355712890625,8.5078125,8388608
+17,self_attn.k_proj,8,64,8,128,0.434814453125,8.25390625,8388608
+17,self_attn.k_proj,8,128,8,128,0.51025390625,8.126953125,8388608
+17,self_attn.o_proj,2,32,8,128,40.5625,20.0625,67108864
+17,self_attn.o_proj,2,64,8,128,47.25,18.03125,67108864
+17,self_attn.o_proj,2,128,8,128,53.34375,17.015625,67108864
+17,self_attn.o_proj,3,32,8,128,17.34375,28.0625,67108864
+17,self_attn.o_proj,3,64,8,128,20.234375,26.03125,67108864
+17,self_attn.o_proj,3,128,8,128,22.859375,25.015625,67108864
+17,self_attn.o_proj,4,32,8,128,8.09375,36.0625,67108864
+17,self_attn.o_proj,4,64,8,128,9.4453125,34.03125,67108864
+17,self_attn.o_proj,4,128,8,128,10.6796875,33.015625,67108864
+17,self_attn.o_proj,8,32,8,128,0.490234375,68.0625,67108864
+17,self_attn.o_proj,8,64,8,128,0.57177734375,66.03125,67108864
+17,self_attn.o_proj,8,128,8,128,0.6435546875,65.015625,67108864
+17,mlp.gate_proj,2,32,8,128,85.0,70.21875,234881024
+17,mlp.gate_proj,2,64,8,128,99.5,63.109375,234881024
+17,mlp.gate_proj,2,128,8,128,112.8125,59.5546875,234881024
+17,mlp.gate_proj,3,32,8,128,36.375,98.21875,234881024
+17,mlp.gate_proj,3,64,8,128,42.59375,91.109375,234881024
+17,mlp.gate_proj,3,128,8,128,48.34375,87.5546875,234881024
+17,mlp.gate_proj,4,32,8,128,16.96875,126.21875,234881024
+17,mlp.gate_proj,4,64,8,128,19.890625,119.109375,234881024
+17,mlp.gate_proj,4,128,8,128,22.5625,115.5546875,234881024
+17,mlp.gate_proj,8,32,8,128,1.0322265625,238.21875,234881024
+17,mlp.gate_proj,8,64,8,128,1.2099609375,231.109375,234881024
+17,mlp.gate_proj,8,128,8,128,1.369140625,227.5546875,234881024
+17,mlp.up_proj,2,32,8,128,81.0625,70.21875,234881024
+17,mlp.up_proj,2,64,8,128,94.375,63.109375,234881024
+17,mlp.up_proj,2,128,8,128,106.4375,59.5546875,234881024
+17,mlp.up_proj,3,32,8,128,34.625,98.21875,234881024
+17,mlp.up_proj,3,64,8,128,40.40625,91.109375,234881024
+17,mlp.up_proj,3,128,8,128,45.5625,87.5546875,234881024
+17,mlp.up_proj,4,32,8,128,16.15625,126.21875,234881024
+17,mlp.up_proj,4,64,8,128,18.859375,119.109375,234881024
+17,mlp.up_proj,4,128,8,128,21.265625,115.5546875,234881024
+17,mlp.up_proj,8,32,8,128,0.982421875,238.21875,234881024
+17,mlp.up_proj,8,64,8,128,1.146484375,231.109375,234881024
+17,mlp.up_proj,8,128,8,128,1.2900390625,227.5546875,234881024
+17,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024
+17,mlp.down_proj,2,64,8,128,94.875,63.109375,234881024
+17,mlp.down_proj,2,128,8,128,106.9375,59.5546875,234881024
+17,mlp.down_proj,3,32,8,128,34.84375,98.21875,234881024
+17,mlp.down_proj,3,64,8,128,40.625,91.109375,234881024
+17,mlp.down_proj,3,128,8,128,45.75,87.5546875,234881024
+17,mlp.down_proj,4,32,8,128,16.25,126.21875,234881024
+17,mlp.down_proj,4,64,8,128,18.953125,119.109375,234881024
+17,mlp.down_proj,4,128,8,128,21.359375,115.5546875,234881024
+17,mlp.down_proj,8,32,8,128,0.98876953125,238.21875,234881024
+17,mlp.down_proj,8,64,8,128,1.1533203125,231.109375,234881024
+17,mlp.down_proj,8,128,8,128,1.296875,227.5546875,234881024
+18,self_attn.q_proj,2,32,8,128,46.3125,20.0625,67108864
+18,self_attn.q_proj,2,64,8,128,54.71875,18.03125,67108864
+18,self_attn.q_proj,2,128,8,128,63.125,17.015625,67108864
+18,self_attn.q_proj,3,32,8,128,19.859375,28.0625,67108864
+18,self_attn.q_proj,3,64,8,128,23.53125,26.03125,67108864
+18,self_attn.q_proj,3,128,8,128,27.296875,25.015625,67108864
+18,self_attn.q_proj,4,32,8,128,9.265625,36.0625,67108864
+18,self_attn.q_proj,4,64,8,128,10.9765625,34.03125,67108864
+18,self_attn.q_proj,4,128,8,128,12.7421875,33.015625,67108864
+18,self_attn.q_proj,8,32,8,128,0.56103515625,68.0625,67108864
+18,self_attn.q_proj,8,64,8,128,0.6640625,66.03125,67108864
+18,self_attn.q_proj,8,128,8,128,0.76806640625,65.015625,67108864
+18,self_attn.v_proj,2,32,8,128,14.40625,2.5078125,8388608
+18,self_attn.v_proj,2,64,8,128,16.828125,2.25390625,8388608
+18,self_attn.v_proj,2,128,8,128,19.015625,2.126953125,8388608
+18,self_attn.v_proj,3,32,8,128,6.16015625,3.5078125,8388608
+18,self_attn.v_proj,3,64,8,128,7.1953125,3.25390625,8388608
+18,self_attn.v_proj,3,128,8,128,8.1328125,3.126953125,8388608
+18,self_attn.v_proj,4,32,8,128,2.873046875,4.5078125,8388608
+18,self_attn.v_proj,4,64,8,128,3.357421875,4.25390625,8388608
+18,self_attn.v_proj,4,128,8,128,3.798828125,4.126953125,8388608
+18,self_attn.v_proj,8,32,8,128,0.174072265625,8.5078125,8388608
+18,self_attn.v_proj,8,64,8,128,0.203125,8.25390625,8388608
+18,self_attn.v_proj,8,128,8,128,0.2288818359375,8.126953125,8388608
+18,self_attn.k_proj,2,32,8,128,26.359375,2.5078125,8388608
+18,self_attn.k_proj,2,64,8,128,31.640625,2.25390625,8388608
+18,self_attn.k_proj,2,128,8,128,36.03125,2.126953125,8388608
+18,self_attn.k_proj,3,32,8,128,11.3828125,3.5078125,8388608
+18,self_attn.k_proj,3,64,8,128,13.8515625,3.25390625,8388608
+18,self_attn.k_proj,3,128,8,128,16.203125,3.126953125,8388608
+18,self_attn.k_proj,4,32,8,128,5.3125,4.5078125,8388608
+18,self_attn.k_proj,4,64,8,128,6.46875,4.25390625,8388608
+18,self_attn.k_proj,4,128,8,128,7.56640625,4.126953125,8388608
+18,self_attn.k_proj,8,32,8,128,0.321533203125,8.5078125,8388608
+18,self_attn.k_proj,8,64,8,128,0.39111328125,8.25390625,8388608
+18,self_attn.k_proj,8,128,8,128,0.455810546875,8.126953125,8388608
+18,self_attn.o_proj,2,32,8,128,40.625,20.0625,67108864
+18,self_attn.o_proj,2,64,8,128,47.3125,18.03125,67108864
+18,self_attn.o_proj,2,128,8,128,53.375,17.015625,67108864
+18,self_attn.o_proj,3,32,8,128,17.375,28.0625,67108864
+18,self_attn.o_proj,3,64,8,128,20.265625,26.03125,67108864
+18,self_attn.o_proj,3,128,8,128,22.890625,25.015625,67108864
+18,self_attn.o_proj,4,32,8,128,8.109375,36.0625,67108864
+18,self_attn.o_proj,4,64,8,128,9.4609375,34.03125,67108864
+18,self_attn.o_proj,4,128,8,128,10.6953125,33.015625,67108864
+18,self_attn.o_proj,8,32,8,128,0.4912109375,68.0625,67108864
+18,self_attn.o_proj,8,64,8,128,0.57275390625,66.03125,67108864
+18,self_attn.o_proj,8,128,8,128,0.6455078125,65.015625,67108864
+18,mlp.gate_proj,2,32,8,128,85.25,70.21875,234881024
+18,mlp.gate_proj,2,64,8,128,99.75,63.109375,234881024
+18,mlp.gate_proj,2,128,8,128,113.125,59.5546875,234881024
+18,mlp.gate_proj,3,32,8,128,36.46875,98.21875,234881024
+18,mlp.gate_proj,3,64,8,128,42.71875,91.109375,234881024
+18,mlp.gate_proj,3,128,8,128,48.46875,87.5546875,234881024
+18,mlp.gate_proj,4,32,8,128,17.015625,126.21875,234881024
+18,mlp.gate_proj,4,64,8,128,19.9375,119.109375,234881024
+18,mlp.gate_proj,4,128,8,128,22.625,115.5546875,234881024
+18,mlp.gate_proj,8,32,8,128,1.03515625,238.21875,234881024
+18,mlp.gate_proj,8,64,8,128,1.212890625,231.109375,234881024
+18,mlp.gate_proj,8,128,8,128,1.3720703125,227.5546875,234881024
+18,mlp.up_proj,2,32,8,128,80.9375,70.21875,234881024
+18,mlp.up_proj,2,64,8,128,94.1875,63.109375,234881024
+18,mlp.up_proj,2,128,8,128,106.25,59.5546875,234881024
+18,mlp.up_proj,3,32,8,128,34.59375,98.21875,234881024
+18,mlp.up_proj,3,64,8,128,40.34375,91.109375,234881024
+18,mlp.up_proj,3,128,8,128,45.46875,87.5546875,234881024
+18,mlp.up_proj,4,32,8,128,16.125,126.21875,234881024
+18,mlp.up_proj,4,64,8,128,18.828125,119.109375,234881024
+18,mlp.up_proj,4,128,8,128,21.21875,115.5546875,234881024
+18,mlp.up_proj,8,32,8,128,0.98095703125,238.21875,234881024
+18,mlp.up_proj,8,64,8,128,1.14453125,231.109375,234881024
+18,mlp.up_proj,8,128,8,128,1.2880859375,227.5546875,234881024
+18,mlp.down_proj,2,32,8,128,81.625,70.21875,234881024
+18,mlp.down_proj,2,64,8,128,94.9375,63.109375,234881024
+18,mlp.down_proj,2,128,8,128,107.0625,59.5546875,234881024
+18,mlp.down_proj,3,32,8,128,34.875,98.21875,234881024
+18,mlp.down_proj,3,64,8,128,40.65625,91.109375,234881024
+18,mlp.down_proj,3,128,8,128,45.78125,87.5546875,234881024
+18,mlp.down_proj,4,32,8,128,16.265625,126.21875,234881024
+18,mlp.down_proj,4,64,8,128,18.96875,119.109375,234881024
+18,mlp.down_proj,4,128,8,128,21.375,115.5546875,234881024
+18,mlp.down_proj,8,32,8,128,0.98974609375,238.21875,234881024
+18,mlp.down_proj,8,64,8,128,1.154296875,231.109375,234881024
+18,mlp.down_proj,8,128,8,128,1.2978515625,227.5546875,234881024
+19,self_attn.q_proj,2,32,8,128,45.21875,20.0625,67108864
+19,self_attn.q_proj,2,64,8,128,53.4375,18.03125,67108864
+19,self_attn.q_proj,2,128,8,128,61.78125,17.015625,67108864
+19,self_attn.q_proj,3,32,8,128,19.375,28.0625,67108864
+19,self_attn.q_proj,3,64,8,128,22.953125,26.03125,67108864
+19,self_attn.q_proj,3,128,8,128,26.65625,25.015625,67108864
+19,self_attn.q_proj,4,32,8,128,9.0390625,36.0625,67108864
+19,self_attn.q_proj,4,64,8,128,10.7109375,34.03125,67108864
+19,self_attn.q_proj,4,128,8,128,12.4375,33.015625,67108864
+19,self_attn.q_proj,8,32,8,128,0.54736328125,68.0625,67108864
+19,self_attn.q_proj,8,64,8,128,0.64794921875,66.03125,67108864
+19,self_attn.q_proj,8,128,8,128,0.75,65.015625,67108864
+19,self_attn.v_proj,2,32,8,128,14.84375,2.5078125,8388608
+19,self_attn.v_proj,2,64,8,128,17.34375,2.25390625,8388608
+19,self_attn.v_proj,2,128,8,128,19.59375,2.126953125,8388608
+19,self_attn.v_proj,3,32,8,128,6.34375,3.5078125,8388608
+19,self_attn.v_proj,3,64,8,128,7.41796875,3.25390625,8388608
+19,self_attn.v_proj,3,128,8,128,8.3828125,3.126953125,8388608
+19,self_attn.v_proj,4,32,8,128,2.958984375,4.5078125,8388608
+19,self_attn.v_proj,4,64,8,128,3.4609375,4.25390625,8388608
+19,self_attn.v_proj,4,128,8,128,3.912109375,4.126953125,8388608
+19,self_attn.v_proj,8,32,8,128,0.1793212890625,8.5078125,8388608
+19,self_attn.v_proj,8,64,8,128,0.209228515625,8.25390625,8388608
+19,self_attn.v_proj,8,128,8,128,0.23583984375,8.126953125,8388608
+19,self_attn.k_proj,2,32,8,128,26.90625,2.5078125,8388608
+19,self_attn.k_proj,2,64,8,128,32.4375,2.25390625,8388608
+19,self_attn.k_proj,2,128,8,128,36.9375,2.126953125,8388608
+19,self_attn.k_proj,3,32,8,128,11.6796875,3.5078125,8388608
+19,self_attn.k_proj,3,64,8,128,14.34375,3.25390625,8388608
+19,self_attn.k_proj,3,128,8,128,16.875,3.126953125,8388608
+19,self_attn.k_proj,4,32,8,128,5.44921875,4.5078125,8388608
+19,self_attn.k_proj,4,64,8,128,6.69140625,4.25390625,8388608
+19,self_attn.k_proj,4,128,8,128,7.875,4.126953125,8388608
+19,self_attn.k_proj,8,32,8,128,0.329833984375,8.5078125,8388608
+19,self_attn.k_proj,8,64,8,128,0.40478515625,8.25390625,8388608
+19,self_attn.k_proj,8,128,8,128,0.474609375,8.126953125,8388608
+19,self_attn.o_proj,2,32,8,128,41.875,20.0625,67108864
+19,self_attn.o_proj,2,64,8,128,48.78125,18.03125,67108864
+19,self_attn.o_proj,2,128,8,128,55.03125,17.015625,67108864
+19,self_attn.o_proj,3,32,8,128,17.921875,28.0625,67108864
+19,self_attn.o_proj,3,64,8,128,20.90625,26.03125,67108864
+19,self_attn.o_proj,3,128,8,128,23.625,25.015625,67108864
+19,self_attn.o_proj,4,32,8,128,8.3671875,36.0625,67108864
+19,self_attn.o_proj,4,64,8,128,9.765625,34.03125,67108864
+19,self_attn.o_proj,4,128,8,128,11.0390625,33.015625,67108864
+19,self_attn.o_proj,8,32,8,128,0.50634765625,68.0625,67108864
+19,self_attn.o_proj,8,64,8,128,0.5908203125,66.03125,67108864
+19,self_attn.o_proj,8,128,8,128,0.66552734375,65.015625,67108864
+19,mlp.gate_proj,2,32,8,128,85.0625,70.21875,234881024
+19,mlp.gate_proj,2,64,8,128,99.375,63.109375,234881024
+19,mlp.gate_proj,2,128,8,128,112.5625,59.5546875,234881024
+19,mlp.gate_proj,3,32,8,128,36.375,98.21875,234881024
+19,mlp.gate_proj,3,64,8,128,42.5625,91.109375,234881024
+19,mlp.gate_proj,3,128,8,128,48.21875,87.5546875,234881024
+19,mlp.gate_proj,4,32,8,128,16.96875,126.21875,234881024
+19,mlp.gate_proj,4,64,8,128,19.859375,119.109375,234881024
+19,mlp.gate_proj,4,128,8,128,22.5,115.5546875,234881024
+19,mlp.gate_proj,8,32,8,128,1.0322265625,238.21875,234881024
+19,mlp.gate_proj,8,64,8,128,1.208984375,231.109375,234881024
+19,mlp.gate_proj,8,128,8,128,1.3642578125,227.5546875,234881024
+19,mlp.up_proj,2,32,8,128,81.25,70.21875,234881024
+19,mlp.up_proj,2,64,8,128,94.5625,63.109375,234881024
+19,mlp.up_proj,2,128,8,128,106.625,59.5546875,234881024
+19,mlp.up_proj,3,32,8,128,34.71875,98.21875,234881024
+19,mlp.up_proj,3,64,8,128,40.5,91.109375,234881024
+19,mlp.up_proj,3,128,8,128,45.625,87.5546875,234881024
+19,mlp.up_proj,4,32,8,128,16.1875,126.21875,234881024
+19,mlp.up_proj,4,64,8,128,18.890625,119.109375,234881024
+19,mlp.up_proj,4,128,8,128,21.296875,115.5546875,234881024
+19,mlp.up_proj,8,32,8,128,0.98486328125,238.21875,234881024
+19,mlp.up_proj,8,64,8,128,1.1494140625,231.109375,234881024
+19,mlp.up_proj,8,128,8,128,1.2919921875,227.5546875,234881024
+19,mlp.down_proj,2,32,8,128,81.9375,70.21875,234881024
+19,mlp.down_proj,2,64,8,128,95.3125,63.109375,234881024
+19,mlp.down_proj,2,128,8,128,107.4375,59.5546875,234881024
+19,mlp.down_proj,3,32,8,128,35.03125,98.21875,234881024
+19,mlp.down_proj,3,64,8,128,40.8125,91.109375,234881024
+19,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024
+19,mlp.down_proj,4,32,8,128,16.328125,126.21875,234881024
+19,mlp.down_proj,4,64,8,128,19.046875,119.109375,234881024
+19,mlp.down_proj,4,128,8,128,21.453125,115.5546875,234881024
+19,mlp.down_proj,8,32,8,128,0.9931640625,238.21875,234881024
+19,mlp.down_proj,8,64,8,128,1.158203125,231.109375,234881024
+19,mlp.down_proj,8,128,8,128,1.3017578125,227.5546875,234881024
+20,self_attn.q_proj,2,32,8,128,45.46875,20.0625,67108864
+20,self_attn.q_proj,2,64,8,128,53.75,18.03125,67108864
+20,self_attn.q_proj,2,128,8,128,61.65625,17.015625,67108864
+20,self_attn.q_proj,3,32,8,128,19.5,28.0625,67108864
+20,self_attn.q_proj,3,64,8,128,23.140625,26.03125,67108864
+20,self_attn.q_proj,3,128,8,128,26.75,25.015625,67108864
+20,self_attn.q_proj,4,32,8,128,9.1015625,36.0625,67108864
+20,self_attn.q_proj,4,64,8,128,10.796875,34.03125,67108864
+20,self_attn.q_proj,4,128,8,128,12.484375,33.015625,67108864
+20,self_attn.q_proj,8,32,8,128,0.55126953125,68.0625,67108864
+20,self_attn.q_proj,8,64,8,128,0.6533203125,66.03125,67108864
+20,self_attn.q_proj,8,128,8,128,0.7529296875,65.015625,67108864
+20,self_attn.v_proj,2,32,8,128,13.9765625,2.5078125,8388608
+20,self_attn.v_proj,2,64,8,128,16.375,2.25390625,8388608
+20,self_attn.v_proj,2,128,8,128,18.53125,2.126953125,8388608
+20,self_attn.v_proj,3,32,8,128,5.9765625,3.5078125,8388608
+20,self_attn.v_proj,3,64,8,128,7.00390625,3.25390625,8388608
+20,self_attn.v_proj,3,128,8,128,7.9375,3.126953125,8388608
+20,self_attn.v_proj,4,32,8,128,2.7890625,4.5078125,8388608
+20,self_attn.v_proj,4,64,8,128,3.26953125,4.25390625,8388608
+20,self_attn.v_proj,4,128,8,128,3.703125,4.126953125,8388608
+20,self_attn.v_proj,8,32,8,128,0.1688232421875,8.5078125,8388608
+20,self_attn.v_proj,8,64,8,128,0.19775390625,8.25390625,8388608
+20,self_attn.v_proj,8,128,8,128,0.22314453125,8.126953125,8388608
+20,self_attn.k_proj,2,32,8,128,27.171875,2.5078125,8388608
+20,self_attn.k_proj,2,64,8,128,32.375,2.25390625,8388608
+20,self_attn.k_proj,2,128,8,128,36.53125,2.126953125,8388608
+20,self_attn.k_proj,3,32,8,128,11.84375,3.5078125,8388608
+20,self_attn.k_proj,3,64,8,128,14.4375,3.25390625,8388608
+20,self_attn.k_proj,3,128,8,128,16.875,3.126953125,8388608
+20,self_attn.k_proj,4,32,8,128,5.52734375,4.5078125,8388608
+20,self_attn.k_proj,4,64,8,128,6.7421875,4.25390625,8388608
+20,self_attn.k_proj,4,128,8,128,7.89453125,4.126953125,8388608
+20,self_attn.k_proj,8,32,8,128,0.33447265625,8.5078125,8388608
+20,self_attn.k_proj,8,64,8,128,0.40771484375,8.25390625,8388608
+20,self_attn.k_proj,8,128,8,128,0.475341796875,8.126953125,8388608
+20,self_attn.o_proj,2,32,8,128,39.84375,20.0625,67108864
+20,self_attn.o_proj,2,64,8,128,46.4375,18.03125,67108864
+20,self_attn.o_proj,2,128,8,128,52.40625,17.015625,67108864
+20,self_attn.o_proj,3,32,8,128,17.0625,28.0625,67108864
+20,self_attn.o_proj,3,64,8,128,19.921875,26.03125,67108864
+20,self_attn.o_proj,3,128,8,128,22.53125,25.015625,67108864
+20,self_attn.o_proj,4,32,8,128,7.96875,36.0625,67108864
+20,self_attn.o_proj,4,64,8,128,9.3125,34.03125,67108864
+20,self_attn.o_proj,4,128,8,128,10.5546875,33.015625,67108864
+20,self_attn.o_proj,8,32,8,128,0.4833984375,68.0625,67108864
+20,self_attn.o_proj,8,64,8,128,0.56494140625,66.03125,67108864
+20,self_attn.o_proj,8,128,8,128,0.6396484375,65.015625,67108864
+20,mlp.gate_proj,2,32,8,128,85.1875,70.21875,234881024
+20,mlp.gate_proj,2,64,8,128,99.625,63.109375,234881024
+20,mlp.gate_proj,2,128,8,128,112.8125,59.5546875,234881024
+20,mlp.gate_proj,3,32,8,128,36.46875,98.21875,234881024
+20,mlp.gate_proj,3,64,8,128,42.625,91.109375,234881024
+20,mlp.gate_proj,3,128,8,128,48.3125,87.5546875,234881024
+20,mlp.gate_proj,4,32,8,128,17.0,126.21875,234881024
+20,mlp.gate_proj,4,64,8,128,19.90625,119.109375,234881024
+20,mlp.gate_proj,4,128,8,128,22.546875,115.5546875,234881024
+20,mlp.gate_proj,8,32,8,128,1.0341796875,238.21875,234881024
+20,mlp.gate_proj,8,64,8,128,1.2109375,231.109375,234881024
+20,mlp.gate_proj,8,128,8,128,1.3681640625,227.5546875,234881024
+20,mlp.up_proj,2,32,8,128,81.4375,70.21875,234881024
+20,mlp.up_proj,2,64,8,128,94.8125,63.109375,234881024
+20,mlp.up_proj,2,128,8,128,106.9375,59.5546875,234881024
+20,mlp.up_proj,3,32,8,128,34.8125,98.21875,234881024
+20,mlp.up_proj,3,64,8,128,40.59375,91.109375,234881024
+20,mlp.up_proj,3,128,8,128,45.75,87.5546875,234881024
+20,mlp.up_proj,4,32,8,128,16.234375,126.21875,234881024
+20,mlp.up_proj,4,64,8,128,18.9375,119.109375,234881024
+20,mlp.up_proj,4,128,8,128,21.359375,115.5546875,234881024
+20,mlp.up_proj,8,32,8,128,0.9873046875,238.21875,234881024
+20,mlp.up_proj,8,64,8,128,1.15234375,231.109375,234881024
+20,mlp.up_proj,8,128,8,128,1.2958984375,227.5546875,234881024
+20,mlp.down_proj,2,32,8,128,81.875,70.21875,234881024
+20,mlp.down_proj,2,64,8,128,95.25,63.109375,234881024
+20,mlp.down_proj,2,128,8,128,107.375,59.5546875,234881024
+20,mlp.down_proj,3,32,8,128,35.0,98.21875,234881024
+20,mlp.down_proj,3,64,8,128,40.78125,91.109375,234881024
+20,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024
+20,mlp.down_proj,4,32,8,128,16.328125,126.21875,234881024
+20,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024
+20,mlp.down_proj,4,128,8,128,21.4375,115.5546875,234881024
+20,mlp.down_proj,8,32,8,128,0.99267578125,238.21875,234881024
+20,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024
+20,mlp.down_proj,8,128,8,128,1.30078125,227.5546875,234881024
+21,self_attn.q_proj,2,32,8,128,44.0,20.0625,67108864
+21,self_attn.q_proj,2,64,8,128,52.125,18.03125,67108864
+21,self_attn.q_proj,2,128,8,128,59.9375,17.015625,67108864
+21,self_attn.q_proj,3,32,8,128,19.0,28.0625,67108864
+21,self_attn.q_proj,3,64,8,128,22.75,26.03125,67108864
+21,self_attn.q_proj,3,128,8,128,26.6875,25.015625,67108864
+21,self_attn.q_proj,4,32,8,128,8.859375,36.0625,67108864
+21,self_attn.q_proj,4,64,8,128,10.625,34.03125,67108864
+21,self_attn.q_proj,4,128,8,128,12.4609375,33.015625,67108864
+21,self_attn.q_proj,8,32,8,128,0.53662109375,68.0625,67108864
+21,self_attn.q_proj,8,64,8,128,0.64306640625,66.03125,67108864
+21,self_attn.q_proj,8,128,8,128,0.7509765625,65.015625,67108864
+21,self_attn.v_proj,2,32,8,128,13.7890625,2.5078125,8388608
+21,self_attn.v_proj,2,64,8,128,16.109375,2.25390625,8388608
+21,self_attn.v_proj,2,128,8,128,18.21875,2.126953125,8388608
+21,self_attn.v_proj,3,32,8,128,5.89453125,3.5078125,8388608
+21,self_attn.v_proj,3,64,8,128,6.8984375,3.25390625,8388608
+21,self_attn.v_proj,3,128,8,128,7.80078125,3.126953125,8388608
+21,self_attn.v_proj,4,32,8,128,2.75,4.5078125,8388608
+21,self_attn.v_proj,4,64,8,128,3.216796875,4.25390625,8388608
+21,self_attn.v_proj,4,128,8,128,3.642578125,4.126953125,8388608
+21,self_attn.v_proj,8,32,8,128,0.16650390625,8.5078125,8388608
+21,self_attn.v_proj,8,64,8,128,0.194580078125,8.25390625,8388608
+21,self_attn.v_proj,8,128,8,128,0.219482421875,8.126953125,8388608
+21,self_attn.k_proj,2,32,8,128,26.78125,2.5078125,8388608
+21,self_attn.k_proj,2,64,8,128,31.96875,2.25390625,8388608
+21,self_attn.k_proj,2,128,8,128,36.09375,2.126953125,8388608
+21,self_attn.k_proj,3,32,8,128,11.921875,3.5078125,8388608
+21,self_attn.k_proj,3,64,8,128,14.7734375,3.25390625,8388608
+21,self_attn.k_proj,3,128,8,128,17.484375,3.126953125,8388608
+21,self_attn.k_proj,4,32,8,128,5.56640625,4.5078125,8388608
+21,self_attn.k_proj,4,64,8,128,6.91015625,4.25390625,8388608
+21,self_attn.k_proj,4,128,8,128,8.2109375,4.126953125,8388608
+21,self_attn.k_proj,8,32,8,128,0.3369140625,8.5078125,8388608
+21,self_attn.k_proj,8,64,8,128,0.417724609375,8.25390625,8388608
+21,self_attn.k_proj,8,128,8,128,0.4951171875,8.126953125,8388608
+21,self_attn.o_proj,2,32,8,128,39.4375,20.0625,67108864
+21,self_attn.o_proj,2,64,8,128,45.96875,18.03125,67108864
+21,self_attn.o_proj,2,128,8,128,51.875,17.015625,67108864
+21,self_attn.o_proj,3,32,8,128,16.875,28.0625,67108864
+21,self_attn.o_proj,3,64,8,128,19.703125,26.03125,67108864
+21,self_attn.o_proj,3,128,8,128,22.28125,25.015625,67108864
+21,self_attn.o_proj,4,32,8,128,7.875,36.0625,67108864
+21,self_attn.o_proj,4,64,8,128,9.1953125,34.03125,67108864
+21,self_attn.o_proj,4,128,8,128,10.40625,33.015625,67108864
+21,self_attn.o_proj,8,32,8,128,0.476806640625,68.0625,67108864
+21,self_attn.o_proj,8,64,8,128,0.55615234375,66.03125,67108864
+21,self_attn.o_proj,8,128,8,128,0.62744140625,65.015625,67108864
+21,mlp.gate_proj,2,32,8,128,84.875,70.21875,234881024
+21,mlp.gate_proj,2,64,8,128,99.1875,63.109375,234881024
+21,mlp.gate_proj,2,128,8,128,112.25,59.5546875,234881024
+21,mlp.gate_proj,3,32,8,128,36.3125,98.21875,234881024
+21,mlp.gate_proj,3,64,8,128,42.4375,91.109375,234881024
+21,mlp.gate_proj,3,128,8,128,48.0625,87.5546875,234881024
+21,mlp.gate_proj,4,32,8,128,16.9375,126.21875,234881024
+21,mlp.gate_proj,4,64,8,128,19.828125,119.109375,234881024
+21,mlp.gate_proj,4,128,8,128,22.421875,115.5546875,234881024
+21,mlp.gate_proj,8,32,8,128,1.0302734375,238.21875,234881024
+21,mlp.gate_proj,8,64,8,128,1.2060546875,231.109375,234881024
+21,mlp.gate_proj,8,128,8,128,1.3603515625,227.5546875,234881024
+21,mlp.up_proj,2,32,8,128,81.6875,70.21875,234881024
+21,mlp.up_proj,2,64,8,128,95.0625,63.109375,234881024
+21,mlp.up_proj,2,128,8,128,107.25,59.5546875,234881024
+21,mlp.up_proj,3,32,8,128,34.9375,98.21875,234881024
+21,mlp.up_proj,3,64,8,128,40.71875,91.109375,234881024
+21,mlp.up_proj,3,128,8,128,45.90625,87.5546875,234881024
+21,mlp.up_proj,4,32,8,128,16.28125,126.21875,234881024
+21,mlp.up_proj,4,64,8,128,19.0,119.109375,234881024
+21,mlp.up_proj,4,128,8,128,21.421875,115.5546875,234881024
+21,mlp.up_proj,8,32,8,128,0.99072265625,238.21875,234881024
+21,mlp.up_proj,8,64,8,128,1.1552734375,231.109375,234881024
+21,mlp.up_proj,8,128,8,128,1.298828125,227.5546875,234881024
+21,mlp.down_proj,2,32,8,128,82.0,70.21875,234881024
+21,mlp.down_proj,2,64,8,128,95.375,63.109375,234881024
+21,mlp.down_proj,2,128,8,128,107.5,59.5546875,234881024
+21,mlp.down_proj,3,32,8,128,35.03125,98.21875,234881024
+21,mlp.down_proj,3,64,8,128,40.8125,91.109375,234881024
+21,mlp.down_proj,3,128,8,128,45.96875,87.5546875,234881024
+21,mlp.down_proj,4,32,8,128,16.34375,126.21875,234881024
+21,mlp.down_proj,4,64,8,128,19.0625,119.109375,234881024
+21,mlp.down_proj,4,128,8,128,21.46875,115.5546875,234881024
+21,mlp.down_proj,8,32,8,128,0.994140625,238.21875,234881024
+21,mlp.down_proj,8,64,8,128,1.1591796875,231.109375,234881024
+21,mlp.down_proj,8,128,8,128,1.3017578125,227.5546875,234881024
+22,self_attn.q_proj,2,32,8,128,42.4375,20.0625,67108864
+22,self_attn.q_proj,2,64,8,128,50.625,18.03125,67108864
+22,self_attn.q_proj,2,128,8,128,58.65625,17.015625,67108864
+22,self_attn.q_proj,3,32,8,128,18.21875,28.0625,67108864
+22,self_attn.q_proj,3,64,8,128,21.859375,26.03125,67108864
+22,self_attn.q_proj,3,128,8,128,25.625,25.015625,67108864
+22,self_attn.q_proj,4,32,8,128,8.5,36.0625,67108864
+22,self_attn.q_proj,4,64,8,128,10.203125,34.03125,67108864
+22,self_attn.q_proj,4,128,8,128,11.9609375,33.015625,67108864
+22,self_attn.q_proj,8,32,8,128,0.5146484375,68.0625,67108864
+22,self_attn.q_proj,8,64,8,128,0.61767578125,66.03125,67108864
+22,self_attn.q_proj,8,128,8,128,0.72119140625,65.015625,67108864
+22,self_attn.v_proj,2,32,8,128,15.328125,2.5078125,8388608
+22,self_attn.v_proj,2,64,8,128,17.921875,2.25390625,8388608
+22,self_attn.v_proj,2,128,8,128,20.328125,2.126953125,8388608
+22,self_attn.v_proj,3,32,8,128,6.55078125,3.5078125,8388608
+22,self_attn.v_proj,3,64,8,128,7.671875,3.25390625,8388608
+22,self_attn.v_proj,3,128,8,128,8.703125,3.126953125,8388608
+22,self_attn.v_proj,4,32,8,128,3.056640625,4.5078125,8388608
+22,self_attn.v_proj,4,64,8,128,3.58203125,4.25390625,8388608
+22,self_attn.v_proj,4,128,8,128,4.0625,4.126953125,8388608
+22,self_attn.v_proj,8,32,8,128,0.1851806640625,8.5078125,8388608
+22,self_attn.v_proj,8,64,8,128,0.2166748046875,8.25390625,8388608
+22,self_attn.v_proj,8,128,8,128,0.2447509765625,8.126953125,8388608
+22,self_attn.k_proj,2,32,8,128,27.0,2.5078125,8388608
+22,self_attn.k_proj,2,64,8,128,32.5,2.25390625,8388608
+22,self_attn.k_proj,2,128,8,128,36.9375,2.126953125,8388608
+22,self_attn.k_proj,3,32,8,128,11.7734375,3.5078125,8388608
+22,self_attn.k_proj,3,64,8,128,14.4765625,3.25390625,8388608
+22,self_attn.k_proj,3,128,8,128,17.0625,3.126953125,8388608
+22,self_attn.k_proj,4,32,8,128,5.4921875,4.5078125,8388608
+22,self_attn.k_proj,4,64,8,128,6.75,4.25390625,8388608
+22,self_attn.k_proj,4,128,8,128,7.97265625,4.126953125,8388608
+22,self_attn.k_proj,8,32,8,128,0.33251953125,8.5078125,8388608
+22,self_attn.k_proj,8,64,8,128,0.408203125,8.25390625,8388608
+22,self_attn.k_proj,8,128,8,128,0.480224609375,8.126953125,8388608
+22,self_attn.o_proj,2,32,8,128,40.78125,20.0625,67108864
+22,self_attn.o_proj,2,64,8,128,47.5,18.03125,67108864
+22,self_attn.o_proj,2,128,8,128,53.53125,17.015625,67108864
+22,self_attn.o_proj,3,32,8,128,17.4375,28.0625,67108864
+22,self_attn.o_proj,3,64,8,128,20.3125,26.03125,67108864
+22,self_attn.o_proj,3,128,8,128,22.921875,25.015625,67108864
+22,self_attn.o_proj,4,32,8,128,8.1328125,36.0625,67108864
+22,self_attn.o_proj,4,64,8,128,9.484375,34.03125,67108864
+22,self_attn.o_proj,4,128,8,128,10.703125,33.015625,67108864
+22,self_attn.o_proj,8,32,8,128,0.49267578125,68.0625,67108864
+22,self_attn.o_proj,8,64,8,128,0.57373046875,66.03125,67108864
+22,self_attn.o_proj,8,128,8,128,0.64501953125,65.015625,67108864
+22,mlp.gate_proj,2,32,8,128,84.8125,70.21875,234881024
+22,mlp.gate_proj,2,64,8,128,99.0625,63.109375,234881024
+22,mlp.gate_proj,2,128,8,128,112.0625,59.5546875,234881024
+22,mlp.gate_proj,3,32,8,128,36.28125,98.21875,234881024
+22,mlp.gate_proj,3,64,8,128,42.40625,91.109375,234881024
+22,mlp.gate_proj,3,128,8,128,47.96875,87.5546875,234881024
+22,mlp.gate_proj,4,32,8,128,16.921875,126.21875,234881024
+22,mlp.gate_proj,4,64,8,128,19.796875,119.109375,234881024
+22,mlp.gate_proj,4,128,8,128,22.375,115.5546875,234881024
+22,mlp.gate_proj,8,32,8,128,1.029296875,238.21875,234881024
+22,mlp.gate_proj,8,64,8,128,1.2041015625,231.109375,234881024
+22,mlp.gate_proj,8,128,8,128,1.3583984375,227.5546875,234881024
+22,mlp.up_proj,2,32,8,128,81.8125,70.21875,234881024
+22,mlp.up_proj,2,64,8,128,95.25,63.109375,234881024
+22,mlp.up_proj,2,128,8,128,107.375,59.5546875,234881024
+22,mlp.up_proj,3,32,8,128,34.96875,98.21875,234881024
+22,mlp.up_proj,3,64,8,128,40.75,91.109375,234881024
+22,mlp.up_proj,3,128,8,128,45.9375,87.5546875,234881024
+22,mlp.up_proj,4,32,8,128,16.3125,126.21875,234881024
+22,mlp.up_proj,4,64,8,128,19.03125,119.109375,234881024
+22,mlp.up_proj,4,128,8,128,21.4375,115.5546875,234881024
+22,mlp.up_proj,8,32,8,128,0.9921875,238.21875,234881024
+22,mlp.up_proj,8,64,8,128,1.1572265625,231.109375,234881024
+22,mlp.up_proj,8,128,8,128,1.30078125,227.5546875,234881024
+22,mlp.down_proj,2,32,8,128,81.9375,70.21875,234881024
+22,mlp.down_proj,2,64,8,128,95.3125,63.109375,234881024
+22,mlp.down_proj,2,128,8,128,107.4375,59.5546875,234881024
+22,mlp.down_proj,3,32,8,128,35.03125,98.21875,234881024
+22,mlp.down_proj,3,64,8,128,40.8125,91.109375,234881024
+22,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024
+22,mlp.down_proj,4,32,8,128,16.328125,126.21875,234881024
+22,mlp.down_proj,4,64,8,128,19.046875,119.109375,234881024
+22,mlp.down_proj,4,128,8,128,21.453125,115.5546875,234881024
+22,mlp.down_proj,8,32,8,128,0.99365234375,238.21875,234881024
+22,mlp.down_proj,8,64,8,128,1.158203125,231.109375,234881024
+22,mlp.down_proj,8,128,8,128,1.30078125,227.5546875,234881024
+23,self_attn.q_proj,2,32,8,128,44.6875,20.0625,67108864
+23,self_attn.q_proj,2,64,8,128,52.875,18.03125,67108864
+23,self_attn.q_proj,2,128,8,128,60.71875,17.015625,67108864
+23,self_attn.q_proj,3,32,8,128,19.140625,28.0625,67108864
+23,self_attn.q_proj,3,64,8,128,22.71875,26.03125,67108864
+23,self_attn.q_proj,3,128,8,128,26.234375,25.015625,67108864
+23,self_attn.q_proj,4,32,8,128,8.9296875,36.0625,67108864
+23,self_attn.q_proj,4,64,8,128,10.6015625,34.03125,67108864
+23,self_attn.q_proj,4,128,8,128,12.2421875,33.015625,67108864
+23,self_attn.q_proj,8,32,8,128,0.541015625,68.0625,67108864
+23,self_attn.q_proj,8,64,8,128,0.6416015625,66.03125,67108864
+23,self_attn.q_proj,8,128,8,128,0.73828125,65.015625,67108864
+23,self_attn.v_proj,2,32,8,128,14.8671875,2.5078125,8388608
+23,self_attn.v_proj,2,64,8,128,17.390625,2.25390625,8388608
+23,self_attn.v_proj,2,128,8,128,19.671875,2.126953125,8388608
+23,self_attn.v_proj,3,32,8,128,6.36328125,3.5078125,8388608
+23,self_attn.v_proj,3,64,8,128,7.44140625,3.25390625,8388608
+23,self_attn.v_proj,3,128,8,128,8.421875,3.126953125,8388608
+23,self_attn.v_proj,4,32,8,128,2.966796875,4.5078125,8388608
+23,self_attn.v_proj,4,64,8,128,3.470703125,4.25390625,8388608
+23,self_attn.v_proj,4,128,8,128,3.931640625,4.126953125,8388608
+23,self_attn.v_proj,8,32,8,128,0.1796875,8.5078125,8388608
+23,self_attn.v_proj,8,64,8,128,0.2100830078125,8.25390625,8388608
+23,self_attn.v_proj,8,128,8,128,0.2369384765625,8.126953125,8388608
+23,self_attn.k_proj,2,32,8,128,27.078125,2.5078125,8388608
+23,self_attn.k_proj,2,64,8,128,32.46875,2.25390625,8388608
+23,self_attn.k_proj,2,128,8,128,37.0625,2.126953125,8388608
+23,self_attn.k_proj,3,32,8,128,11.6953125,3.5078125,8388608
+23,self_attn.k_proj,3,64,8,128,14.203125,3.25390625,8388608
+23,self_attn.k_proj,3,128,8,128,16.59375,3.126953125,8388608
+23,self_attn.k_proj,4,32,8,128,5.45703125,4.5078125,8388608
+23,self_attn.k_proj,4,64,8,128,6.625,4.25390625,8388608
+23,self_attn.k_proj,4,128,8,128,7.75,4.126953125,8388608
+23,self_attn.k_proj,8,32,8,128,0.330322265625,8.5078125,8388608
+23,self_attn.k_proj,8,64,8,128,0.400634765625,8.25390625,8388608
+23,self_attn.k_proj,8,128,8,128,0.466552734375,8.126953125,8388608
+23,self_attn.o_proj,2,32,8,128,40.375,20.0625,67108864
+23,self_attn.o_proj,2,64,8,128,47.03125,18.03125,67108864
+23,self_attn.o_proj,2,128,8,128,53.03125,17.015625,67108864
+23,self_attn.o_proj,3,32,8,128,17.265625,28.0625,67108864
+23,self_attn.o_proj,3,64,8,128,20.15625,26.03125,67108864
+23,self_attn.o_proj,3,128,8,128,22.78125,25.015625,67108864
+23,self_attn.o_proj,4,32,8,128,8.0703125,36.0625,67108864
+23,self_attn.o_proj,4,64,8,128,9.4296875,34.03125,67108864
+23,self_attn.o_proj,4,128,8,128,10.6640625,33.015625,67108864
+23,self_attn.o_proj,8,32,8,128,0.4892578125,68.0625,67108864
+23,self_attn.o_proj,8,64,8,128,0.5712890625,66.03125,67108864
+23,self_attn.o_proj,8,128,8,128,0.6455078125,65.015625,67108864
+23,mlp.gate_proj,2,32,8,128,84.8125,70.21875,234881024
+23,mlp.gate_proj,2,64,8,128,99.0625,63.109375,234881024
+23,mlp.gate_proj,2,128,8,128,112.125,59.5546875,234881024
+23,mlp.gate_proj,3,32,8,128,36.28125,98.21875,234881024
+23,mlp.gate_proj,3,64,8,128,42.40625,91.109375,234881024
+23,mlp.gate_proj,3,128,8,128,48.0,87.5546875,234881024
+23,mlp.gate_proj,4,32,8,128,16.921875,126.21875,234881024
+23,mlp.gate_proj,4,64,8,128,19.796875,119.109375,234881024
+23,mlp.gate_proj,4,128,8,128,22.40625,115.5546875,234881024
+23,mlp.gate_proj,8,32,8,128,1.029296875,238.21875,234881024
+23,mlp.gate_proj,8,64,8,128,1.205078125,231.109375,234881024
+23,mlp.gate_proj,8,128,8,128,1.359375,227.5546875,234881024
+23,mlp.up_proj,2,32,8,128,81.875,70.21875,234881024
+23,mlp.up_proj,2,64,8,128,95.3125,63.109375,234881024
+23,mlp.up_proj,2,128,8,128,107.4375,59.5546875,234881024
+23,mlp.up_proj,3,32,8,128,35.0,98.21875,234881024
+23,mlp.up_proj,3,64,8,128,40.78125,91.109375,234881024
+23,mlp.up_proj,3,128,8,128,45.96875,87.5546875,234881024
+23,mlp.up_proj,4,32,8,128,16.3125,126.21875,234881024
+23,mlp.up_proj,4,64,8,128,19.046875,119.109375,234881024
+23,mlp.up_proj,4,128,8,128,21.453125,115.5546875,234881024
+23,mlp.up_proj,8,32,8,128,0.99267578125,238.21875,234881024
+23,mlp.up_proj,8,64,8,128,1.158203125,231.109375,234881024
+23,mlp.up_proj,8,128,8,128,1.3017578125,227.5546875,234881024
+23,mlp.down_proj,2,32,8,128,81.8125,70.21875,234881024
+23,mlp.down_proj,2,64,8,128,95.1875,63.109375,234881024
+23,mlp.down_proj,2,128,8,128,107.3125,59.5546875,234881024
+23,mlp.down_proj,3,32,8,128,34.96875,98.21875,234881024
+23,mlp.down_proj,3,64,8,128,40.75,91.109375,234881024
+23,mlp.down_proj,3,128,8,128,45.875,87.5546875,234881024
+23,mlp.down_proj,4,32,8,128,16.3125,126.21875,234881024
+23,mlp.down_proj,4,64,8,128,19.015625,119.109375,234881024
+23,mlp.down_proj,4,128,8,128,21.40625,115.5546875,234881024
+23,mlp.down_proj,8,32,8,128,0.99169921875,238.21875,234881024
+23,mlp.down_proj,8,64,8,128,1.15625,231.109375,234881024
+23,mlp.down_proj,8,128,8,128,1.2998046875,227.5546875,234881024
+24,self_attn.q_proj,2,32,8,128,45.46875,20.0625,67108864
+24,self_attn.q_proj,2,64,8,128,54.125,18.03125,67108864
+24,self_attn.q_proj,2,128,8,128,62.25,17.015625,67108864
+24,self_attn.q_proj,3,32,8,128,19.5,28.0625,67108864
+24,self_attn.q_proj,3,64,8,128,23.265625,26.03125,67108864
+24,self_attn.q_proj,3,128,8,128,26.984375,25.015625,67108864
+24,self_attn.q_proj,4,32,8,128,9.09375,36.0625,67108864
+24,self_attn.q_proj,4,64,8,128,10.859375,34.03125,67108864
+24,self_attn.q_proj,4,128,8,128,12.5859375,33.015625,67108864
+24,self_attn.q_proj,8,32,8,128,0.55078125,68.0625,67108864
+24,self_attn.q_proj,8,64,8,128,0.6572265625,66.03125,67108864
+24,self_attn.q_proj,8,128,8,128,0.75927734375,65.015625,67108864
+24,self_attn.v_proj,2,32,8,128,13.6640625,2.5078125,8388608
+24,self_attn.v_proj,2,64,8,128,15.953125,2.25390625,8388608
+24,self_attn.v_proj,2,128,8,128,18.015625,2.126953125,8388608
+24,self_attn.v_proj,3,32,8,128,5.84375,3.5078125,8388608
+24,self_attn.v_proj,3,64,8,128,6.8203125,3.25390625,8388608
+24,self_attn.v_proj,3,128,8,128,7.71484375,3.126953125,8388608
+24,self_attn.v_proj,4,32,8,128,2.7265625,4.5078125,8388608
+24,self_attn.v_proj,4,64,8,128,3.18359375,4.25390625,8388608
+24,self_attn.v_proj,4,128,8,128,3.599609375,4.126953125,8388608
+24,self_attn.v_proj,8,32,8,128,0.1651611328125,8.5078125,8388608
+24,self_attn.v_proj,8,64,8,128,0.192626953125,8.25390625,8388608
+24,self_attn.v_proj,8,128,8,128,0.217041015625,8.126953125,8388608
+24,self_attn.k_proj,2,32,8,128,26.734375,2.5078125,8388608
+24,self_attn.k_proj,2,64,8,128,32.0,2.25390625,8388608
+24,self_attn.k_proj,2,128,8,128,36.40625,2.126953125,8388608
+24,self_attn.k_proj,3,32,8,128,11.546875,3.5078125,8388608
+24,self_attn.k_proj,3,64,8,128,13.984375,3.25390625,8388608
+24,self_attn.k_proj,3,128,8,128,16.28125,3.126953125,8388608
+24,self_attn.k_proj,4,32,8,128,5.390625,4.5078125,8388608
+24,self_attn.k_proj,4,64,8,128,6.52734375,4.25390625,8388608
+24,self_attn.k_proj,4,128,8,128,7.60546875,4.126953125,8388608
+24,self_attn.k_proj,8,32,8,128,0.326171875,8.5078125,8388608
+24,self_attn.k_proj,8,64,8,128,0.394775390625,8.25390625,8388608
+24,self_attn.k_proj,8,128,8,128,0.45849609375,8.126953125,8388608
+24,self_attn.o_proj,2,32,8,128,39.0625,20.0625,67108864
+24,self_attn.o_proj,2,64,8,128,45.625,18.03125,67108864
+24,self_attn.o_proj,2,128,8,128,51.65625,17.015625,67108864
+24,self_attn.o_proj,3,32,8,128,16.71875,28.0625,67108864
+24,self_attn.o_proj,3,64,8,128,19.59375,26.03125,67108864
+24,self_attn.o_proj,3,128,8,128,22.296875,25.015625,67108864
+24,self_attn.o_proj,4,32,8,128,7.8046875,36.0625,67108864
+24,self_attn.o_proj,4,64,8,128,9.1484375,34.03125,67108864
+24,self_attn.o_proj,4,128,8,128,10.421875,33.015625,67108864
+24,self_attn.o_proj,8,32,8,128,0.472412109375,68.0625,67108864
+24,self_attn.o_proj,8,64,8,128,0.55322265625,66.03125,67108864
+24,self_attn.o_proj,8,128,8,128,0.6279296875,65.015625,67108864
+24,mlp.gate_proj,2,32,8,128,84.375,70.21875,234881024
+24,mlp.gate_proj,2,64,8,128,98.625,63.109375,234881024
+24,mlp.gate_proj,2,128,8,128,111.6875,59.5546875,234881024
+24,mlp.gate_proj,3,32,8,128,36.09375,98.21875,234881024
+24,mlp.gate_proj,3,64,8,128,42.21875,91.109375,234881024
+24,mlp.gate_proj,3,128,8,128,47.84375,87.5546875,234881024
+24,mlp.gate_proj,4,32,8,128,16.84375,126.21875,234881024
+24,mlp.gate_proj,4,64,8,128,19.71875,119.109375,234881024
+24,mlp.gate_proj,4,128,8,128,22.328125,115.5546875,234881024
+24,mlp.gate_proj,8,32,8,128,1.0244140625,238.21875,234881024
+24,mlp.gate_proj,8,64,8,128,1.19921875,231.109375,234881024
+24,mlp.gate_proj,8,128,8,128,1.3544921875,227.5546875,234881024
+24,mlp.up_proj,2,32,8,128,82.0,70.21875,234881024
+24,mlp.up_proj,2,64,8,128,95.5,63.109375,234881024
+24,mlp.up_proj,2,128,8,128,107.75,59.5546875,234881024
+24,mlp.up_proj,3,32,8,128,35.0625,98.21875,234881024
+24,mlp.up_proj,3,64,8,128,40.90625,91.109375,234881024
+24,mlp.up_proj,3,128,8,128,46.125,87.5546875,234881024
+24,mlp.up_proj,4,32,8,128,16.359375,126.21875,234881024
+24,mlp.up_proj,4,64,8,128,19.09375,119.109375,234881024
+24,mlp.up_proj,4,128,8,128,21.53125,115.5546875,234881024
+24,mlp.up_proj,8,32,8,128,0.99462890625,238.21875,234881024
+24,mlp.up_proj,8,64,8,128,1.1611328125,231.109375,234881024
+24,mlp.up_proj,8,128,8,128,1.3056640625,227.5546875,234881024
+24,mlp.down_proj,2,32,8,128,81.8125,70.21875,234881024
+24,mlp.down_proj,2,64,8,128,95.1875,63.109375,234881024
+24,mlp.down_proj,2,128,8,128,107.4375,59.5546875,234881024
+24,mlp.down_proj,3,32,8,128,34.96875,98.21875,234881024
+24,mlp.down_proj,3,64,8,128,40.78125,91.109375,234881024
+24,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024
+24,mlp.down_proj,4,32,8,128,16.3125,126.21875,234881024
+24,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024
+24,mlp.down_proj,4,128,8,128,21.453125,115.5546875,234881024
+24,mlp.down_proj,8,32,8,128,0.9921875,238.21875,234881024
+24,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024
+24,mlp.down_proj,8,128,8,128,1.3017578125,227.5546875,234881024
+25,self_attn.q_proj,2,32,8,128,48.09375,20.0625,67108864
+25,self_attn.q_proj,2,64,8,128,56.78125,18.03125,67108864
+25,self_attn.q_proj,2,128,8,128,65.9375,17.015625,67108864
+25,self_attn.q_proj,3,32,8,128,20.640625,28.0625,67108864
+25,self_attn.q_proj,3,64,8,128,24.515625,26.03125,67108864
+25,self_attn.q_proj,3,128,8,128,28.765625,25.015625,67108864
+25,self_attn.q_proj,4,32,8,128,9.6328125,36.0625,67108864
+25,self_attn.q_proj,4,64,8,128,11.4375,34.03125,67108864
+25,self_attn.q_proj,4,128,8,128,13.4296875,33.015625,67108864
+25,self_attn.q_proj,8,32,8,128,0.58349609375,68.0625,67108864
+25,self_attn.q_proj,8,64,8,128,0.69189453125,66.03125,67108864
+25,self_attn.q_proj,8,128,8,128,0.8095703125,65.015625,67108864
+25,self_attn.v_proj,2,32,8,128,13.2421875,2.5078125,8388608
+25,self_attn.v_proj,2,64,8,128,15.46875,2.25390625,8388608
+25,self_attn.v_proj,2,128,8,128,17.484375,2.126953125,8388608
+25,self_attn.v_proj,3,32,8,128,5.66015625,3.5078125,8388608
+25,self_attn.v_proj,3,64,8,128,6.6171875,3.25390625,8388608
+25,self_attn.v_proj,3,128,8,128,7.48046875,3.126953125,8388608
+25,self_attn.v_proj,4,32,8,128,2.640625,4.5078125,8388608
+25,self_attn.v_proj,4,64,8,128,3.0859375,4.25390625,8388608
+25,self_attn.v_proj,4,128,8,128,3.4921875,4.126953125,8388608
+25,self_attn.v_proj,8,32,8,128,0.159912109375,8.5078125,8388608
+25,self_attn.v_proj,8,64,8,128,0.186767578125,8.25390625,8388608
+25,self_attn.v_proj,8,128,8,128,0.2105712890625,8.126953125,8388608
+25,self_attn.k_proj,2,32,8,128,30.390625,2.5078125,8388608
+25,self_attn.k_proj,2,64,8,128,36.5625,2.25390625,8388608
+25,self_attn.k_proj,2,128,8,128,41.625,2.126953125,8388608
+25,self_attn.k_proj,3,32,8,128,13.296875,3.5078125,8388608
+25,self_attn.k_proj,3,64,8,128,16.4375,3.25390625,8388608
+25,self_attn.k_proj,3,128,8,128,19.5,3.126953125,8388608
+25,self_attn.k_proj,4,32,8,128,6.20703125,4.5078125,8388608
+25,self_attn.k_proj,4,64,8,128,7.6796875,4.25390625,8388608
+25,self_attn.k_proj,4,128,8,128,9.1171875,4.126953125,8388608
+25,self_attn.k_proj,8,32,8,128,0.37548828125,8.5078125,8388608
+25,self_attn.k_proj,8,64,8,128,0.464111328125,8.25390625,8388608
+25,self_attn.k_proj,8,128,8,128,0.54931640625,8.126953125,8388608
+25,self_attn.o_proj,2,32,8,128,39.8125,20.0625,67108864
+25,self_attn.o_proj,2,64,8,128,46.375,18.03125,67108864
+25,self_attn.o_proj,2,128,8,128,52.34375,17.015625,67108864
+25,self_attn.o_proj,3,32,8,128,17.03125,28.0625,67108864
+25,self_attn.o_proj,3,64,8,128,19.890625,26.03125,67108864
+25,self_attn.o_proj,3,128,8,128,22.484375,25.015625,67108864
+25,self_attn.o_proj,4,32,8,128,7.953125,36.0625,67108864
+25,self_attn.o_proj,4,64,8,128,9.2890625,34.03125,67108864
+25,self_attn.o_proj,4,128,8,128,10.5234375,33.015625,67108864
+25,self_attn.o_proj,8,32,8,128,0.481689453125,68.0625,67108864
+25,self_attn.o_proj,8,64,8,128,0.56201171875,66.03125,67108864
+25,self_attn.o_proj,8,128,8,128,0.634765625,65.015625,67108864
+25,mlp.gate_proj,2,32,8,128,83.875,70.21875,234881024
+25,mlp.gate_proj,2,64,8,128,98.0625,63.109375,234881024
+25,mlp.gate_proj,2,128,8,128,111.1875,59.5546875,234881024
+25,mlp.gate_proj,3,32,8,128,35.875,98.21875,234881024
+25,mlp.gate_proj,3,64,8,128,41.96875,91.109375,234881024
+25,mlp.gate_proj,3,128,8,128,47.625,87.5546875,234881024
+25,mlp.gate_proj,4,32,8,128,16.71875,126.21875,234881024
+25,mlp.gate_proj,4,64,8,128,19.609375,119.109375,234881024
+25,mlp.gate_proj,4,128,8,128,22.21875,115.5546875,234881024
+25,mlp.gate_proj,8,32,8,128,1.017578125,238.21875,234881024
+25,mlp.gate_proj,8,64,8,128,1.1923828125,231.109375,234881024
+25,mlp.gate_proj,8,128,8,128,1.34765625,227.5546875,234881024
+25,mlp.up_proj,2,32,8,128,81.6875,70.21875,234881024
+25,mlp.up_proj,2,64,8,128,95.125,63.109375,234881024
+25,mlp.up_proj,2,128,8,128,107.3125,59.5546875,234881024
+25,mlp.up_proj,3,32,8,128,34.9375,98.21875,234881024
+25,mlp.up_proj,3,64,8,128,40.71875,91.109375,234881024
+25,mlp.up_proj,3,128,8,128,45.9375,87.5546875,234881024
+25,mlp.up_proj,4,32,8,128,16.28125,126.21875,234881024
+25,mlp.up_proj,4,64,8,128,19.015625,119.109375,234881024
+25,mlp.up_proj,4,128,8,128,21.421875,115.5546875,234881024
+25,mlp.up_proj,8,32,8,128,0.99072265625,238.21875,234881024
+25,mlp.up_proj,8,64,8,128,1.15625,231.109375,234881024
+25,mlp.up_proj,8,128,8,128,1.30078125,227.5546875,234881024
+25,mlp.down_proj,2,32,8,128,82.0625,70.21875,234881024
+25,mlp.down_proj,2,64,8,128,95.5,63.109375,234881024
+25,mlp.down_proj,2,128,8,128,107.625,59.5546875,234881024
+25,mlp.down_proj,3,32,8,128,35.09375,98.21875,234881024
+25,mlp.down_proj,3,64,8,128,40.875,91.109375,234881024
+25,mlp.down_proj,3,128,8,128,46.0625,87.5546875,234881024
+25,mlp.down_proj,4,32,8,128,16.359375,126.21875,234881024
+25,mlp.down_proj,4,64,8,128,19.078125,119.109375,234881024
+25,mlp.down_proj,4,128,8,128,21.484375,115.5546875,234881024
+25,mlp.down_proj,8,32,8,128,0.9951171875,238.21875,234881024
+25,mlp.down_proj,8,64,8,128,1.16015625,231.109375,234881024
+25,mlp.down_proj,8,128,8,128,1.3037109375,227.5546875,234881024
+26,self_attn.q_proj,2,32,8,128,45.875,20.0625,67108864
+26,self_attn.q_proj,2,64,8,128,54.21875,18.03125,67108864
+26,self_attn.q_proj,2,128,8,128,62.71875,17.015625,67108864
+26,self_attn.q_proj,3,32,8,128,19.640625,28.0625,67108864
+26,self_attn.q_proj,3,64,8,128,23.265625,26.03125,67108864
+26,self_attn.q_proj,3,128,8,128,27.015625,25.015625,67108864
+26,self_attn.q_proj,4,32,8,128,9.1640625,36.0625,67108864
+26,self_attn.q_proj,4,64,8,128,10.859375,34.03125,67108864
+26,self_attn.q_proj,4,128,8,128,12.609375,33.015625,67108864
+26,self_attn.q_proj,8,32,8,128,0.55517578125,68.0625,67108864
+26,self_attn.q_proj,8,64,8,128,0.65673828125,66.03125,67108864
+26,self_attn.q_proj,8,128,8,128,0.76025390625,65.015625,67108864
+26,self_attn.v_proj,2,32,8,128,14.203125,2.5078125,8388608
+26,self_attn.v_proj,2,64,8,128,16.546875,2.25390625,8388608
+26,self_attn.v_proj,2,128,8,128,18.671875,2.126953125,8388608
+26,self_attn.v_proj,3,32,8,128,6.06640625,3.5078125,8388608
+26,self_attn.v_proj,3,64,8,128,7.08203125,3.25390625,8388608
+26,self_attn.v_proj,3,128,8,128,7.98828125,3.126953125,8388608
+26,self_attn.v_proj,4,32,8,128,2.830078125,4.5078125,8388608
+26,self_attn.v_proj,4,64,8,128,3.302734375,4.25390625,8388608
+26,self_attn.v_proj,4,128,8,128,3.728515625,4.126953125,8388608
+26,self_attn.v_proj,8,32,8,128,0.1715087890625,8.5078125,8388608
+26,self_attn.v_proj,8,64,8,128,0.1998291015625,8.25390625,8388608
+26,self_attn.v_proj,8,128,8,128,0.2247314453125,8.126953125,8388608
+26,self_attn.k_proj,2,32,8,128,27.265625,2.5078125,8388608
+26,self_attn.k_proj,2,64,8,128,32.8125,2.25390625,8388608
+26,self_attn.k_proj,2,128,8,128,37.5,2.126953125,8388608
+26,self_attn.k_proj,3,32,8,128,11.78125,3.5078125,8388608
+26,self_attn.k_proj,3,64,8,128,14.375,3.25390625,8388608
+26,self_attn.k_proj,3,128,8,128,16.84375,3.126953125,8388608
+26,self_attn.k_proj,4,32,8,128,5.5,4.5078125,8388608
+26,self_attn.k_proj,4,64,8,128,6.70703125,4.25390625,8388608
+26,self_attn.k_proj,4,128,8,128,7.86328125,4.126953125,8388608
+26,self_attn.k_proj,8,32,8,128,0.332763671875,8.5078125,8388608
+26,self_attn.k_proj,8,64,8,128,0.40576171875,8.25390625,8388608
+26,self_attn.k_proj,8,128,8,128,0.473876953125,8.126953125,8388608
+26,self_attn.o_proj,2,32,8,128,41.15625,20.0625,67108864
+26,self_attn.o_proj,2,64,8,128,47.9375,18.03125,67108864
+26,self_attn.o_proj,2,128,8,128,54.03125,17.015625,67108864
+26,self_attn.o_proj,3,32,8,128,17.609375,28.0625,67108864
+26,self_attn.o_proj,3,64,8,128,20.515625,26.03125,67108864
+26,self_attn.o_proj,3,128,8,128,23.15625,25.015625,67108864
+26,self_attn.o_proj,4,32,8,128,8.2109375,36.0625,67108864
+26,self_attn.o_proj,4,64,8,128,9.578125,34.03125,67108864
+26,self_attn.o_proj,4,128,8,128,10.8046875,33.015625,67108864
+26,self_attn.o_proj,8,32,8,128,0.497314453125,68.0625,67108864
+26,self_attn.o_proj,8,64,8,128,0.5791015625,66.03125,67108864
+26,self_attn.o_proj,8,128,8,128,0.6513671875,65.015625,67108864
+26,mlp.gate_proj,2,32,8,128,83.9375,70.21875,234881024
+26,mlp.gate_proj,2,64,8,128,98.125,63.109375,234881024
+26,mlp.gate_proj,2,128,8,128,111.1875,59.5546875,234881024
+26,mlp.gate_proj,3,32,8,128,35.90625,98.21875,234881024
+26,mlp.gate_proj,3,64,8,128,42.0,91.109375,234881024
+26,mlp.gate_proj,3,128,8,128,47.59375,87.5546875,234881024
+26,mlp.gate_proj,4,32,8,128,16.734375,126.21875,234881024
+26,mlp.gate_proj,4,64,8,128,19.609375,119.109375,234881024
+26,mlp.gate_proj,4,128,8,128,22.203125,115.5546875,234881024
+26,mlp.gate_proj,8,32,8,128,1.0185546875,238.21875,234881024
+26,mlp.gate_proj,8,64,8,128,1.193359375,231.109375,234881024
+26,mlp.gate_proj,8,128,8,128,1.34765625,227.5546875,234881024
+26,mlp.up_proj,2,32,8,128,81.625,70.21875,234881024
+26,mlp.up_proj,2,64,8,128,95.0,63.109375,234881024
+26,mlp.up_proj,2,128,8,128,107.1875,59.5546875,234881024
+26,mlp.up_proj,3,32,8,128,34.875,98.21875,234881024
+26,mlp.up_proj,3,64,8,128,40.65625,91.109375,234881024
+26,mlp.up_proj,3,128,8,128,45.84375,87.5546875,234881024
+26,mlp.up_proj,4,32,8,128,16.265625,126.21875,234881024
+26,mlp.up_proj,4,64,8,128,18.984375,119.109375,234881024
+26,mlp.up_proj,4,128,8,128,21.40625,115.5546875,234881024
+26,mlp.up_proj,8,32,8,128,0.9892578125,238.21875,234881024
+26,mlp.up_proj,8,64,8,128,1.154296875,231.109375,234881024
+26,mlp.up_proj,8,128,8,128,1.2978515625,227.5546875,234881024
+26,mlp.down_proj,2,32,8,128,81.875,70.21875,234881024
+26,mlp.down_proj,2,64,8,128,95.25,63.109375,234881024
+26,mlp.down_proj,2,128,8,128,107.375,59.5546875,234881024
+26,mlp.down_proj,3,32,8,128,35.0,98.21875,234881024
+26,mlp.down_proj,3,64,8,128,40.78125,91.109375,234881024
+26,mlp.down_proj,3,128,8,128,45.90625,87.5546875,234881024
+26,mlp.down_proj,4,32,8,128,16.328125,126.21875,234881024
+26,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024
+26,mlp.down_proj,4,128,8,128,21.421875,115.5546875,234881024
+26,mlp.down_proj,8,32,8,128,0.99267578125,238.21875,234881024
+26,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024
+26,mlp.down_proj,8,128,8,128,1.30078125,227.5546875,234881024
+27,self_attn.q_proj,2,32,8,128,42.375,20.0625,67108864
+27,self_attn.q_proj,2,64,8,128,50.21875,18.03125,67108864
+27,self_attn.q_proj,2,128,8,128,57.84375,17.015625,67108864
+27,self_attn.q_proj,3,32,8,128,18.234375,28.0625,67108864
+27,self_attn.q_proj,3,64,8,128,21.765625,26.03125,67108864
+27,self_attn.q_proj,3,128,8,128,25.421875,25.015625,67108864
+27,self_attn.q_proj,4,32,8,128,8.5078125,36.0625,67108864
+27,self_attn.q_proj,4,64,8,128,10.1640625,34.03125,67108864
+27,self_attn.q_proj,4,128,8,128,11.875,33.015625,67108864
+27,self_attn.q_proj,8,32,8,128,0.51513671875,68.0625,67108864
+27,self_attn.q_proj,8,64,8,128,0.615234375,66.03125,67108864
+27,self_attn.q_proj,8,128,8,128,0.7158203125,65.015625,67108864
+27,self_attn.v_proj,2,32,8,128,15.1484375,2.5078125,8388608
+27,self_attn.v_proj,2,64,8,128,17.65625,2.25390625,8388608
+27,self_attn.v_proj,2,128,8,128,19.921875,2.126953125,8388608
+27,self_attn.v_proj,3,32,8,128,6.4765625,3.5078125,8388608
+27,self_attn.v_proj,3,64,8,128,7.5546875,3.25390625,8388608
+27,self_attn.v_proj,3,128,8,128,8.5234375,3.126953125,8388608
+27,self_attn.v_proj,4,32,8,128,3.021484375,4.5078125,8388608
+27,self_attn.v_proj,4,64,8,128,3.525390625,4.25390625,8388608
+27,self_attn.v_proj,4,128,8,128,3.98046875,4.126953125,8388608
+27,self_attn.v_proj,8,32,8,128,0.1829833984375,8.5078125,8388608
+27,self_attn.v_proj,8,64,8,128,0.2132568359375,8.25390625,8388608
+27,self_attn.v_proj,8,128,8,128,0.2398681640625,8.126953125,8388608
+27,self_attn.k_proj,2,32,8,128,26.453125,2.5078125,8388608
+27,self_attn.k_proj,2,64,8,128,31.65625,2.25390625,8388608
+27,self_attn.k_proj,2,128,8,128,35.875,2.126953125,8388608
+27,self_attn.k_proj,3,32,8,128,11.5078125,3.5078125,8388608
+27,self_attn.k_proj,3,64,8,128,14.0703125,3.25390625,8388608
+27,self_attn.k_proj,3,128,8,128,16.484375,3.126953125,8388608
+27,self_attn.k_proj,4,32,8,128,5.37109375,4.5078125,8388608
+27,self_attn.k_proj,4,64,8,128,6.56640625,4.25390625,8388608
+27,self_attn.k_proj,4,128,8,128,7.69921875,4.126953125,8388608
+27,self_attn.k_proj,8,32,8,128,0.324951171875,8.5078125,8388608
+27,self_attn.k_proj,8,64,8,128,0.397216796875,8.25390625,8388608
+27,self_attn.k_proj,8,128,8,128,0.464111328125,8.126953125,8388608
+27,self_attn.o_proj,2,32,8,128,40.15625,20.0625,67108864
+27,self_attn.o_proj,2,64,8,128,46.78125,18.03125,67108864
+27,self_attn.o_proj,2,128,8,128,52.8125,17.015625,67108864
+27,self_attn.o_proj,3,32,8,128,17.1875,28.0625,67108864
+27,self_attn.o_proj,3,64,8,128,20.046875,26.03125,67108864
+27,self_attn.o_proj,3,128,8,128,22.640625,25.015625,67108864
+27,self_attn.o_proj,4,32,8,128,8.03125,36.0625,67108864
+27,self_attn.o_proj,4,64,8,128,9.375,34.03125,67108864
+27,self_attn.o_proj,4,128,8,128,10.609375,33.015625,67108864
+27,self_attn.o_proj,8,32,8,128,0.48828125,68.0625,67108864
+27,self_attn.o_proj,8,64,8,128,0.5712890625,66.03125,67108864
+27,self_attn.o_proj,8,128,8,128,0.6474609375,65.015625,67108864
+27,mlp.gate_proj,2,32,8,128,84.0,70.21875,234881024
+27,mlp.gate_proj,2,64,8,128,98.1875,63.109375,234881024
+27,mlp.gate_proj,2,128,8,128,111.25,59.5546875,234881024
+27,mlp.gate_proj,3,32,8,128,35.9375,98.21875,234881024
+27,mlp.gate_proj,3,64,8,128,42.0,91.109375,234881024
+27,mlp.gate_proj,3,128,8,128,47.59375,87.5546875,234881024
+27,mlp.gate_proj,4,32,8,128,16.75,126.21875,234881024
+27,mlp.gate_proj,4,64,8,128,19.625,119.109375,234881024
+27,mlp.gate_proj,4,128,8,128,22.21875,115.5546875,234881024
+27,mlp.gate_proj,8,32,8,128,1.01953125,238.21875,234881024
+27,mlp.gate_proj,8,64,8,128,1.193359375,231.109375,234881024
+27,mlp.gate_proj,8,128,8,128,1.34765625,227.5546875,234881024
+27,mlp.up_proj,2,32,8,128,81.5,70.21875,234881024
+27,mlp.up_proj,2,64,8,128,94.875,63.109375,234881024
+27,mlp.up_proj,2,128,8,128,107.0625,59.5546875,234881024
+27,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024
+27,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024
+27,mlp.up_proj,3,128,8,128,45.78125,87.5546875,234881024
+27,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024
+27,mlp.up_proj,4,64,8,128,18.96875,119.109375,234881024
+27,mlp.up_proj,4,128,8,128,21.375,115.5546875,234881024
+27,mlp.up_proj,8,32,8,128,0.98828125,238.21875,234881024
+27,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024
+27,mlp.up_proj,8,128,8,128,1.296875,227.5546875,234881024
+27,mlp.down_proj,2,32,8,128,81.75,70.21875,234881024
+27,mlp.down_proj,2,64,8,128,95.125,63.109375,234881024
+27,mlp.down_proj,2,128,8,128,107.25,59.5546875,234881024
+27,mlp.down_proj,3,32,8,128,34.96875,98.21875,234881024
+27,mlp.down_proj,3,64,8,128,40.71875,91.109375,234881024
+27,mlp.down_proj,3,128,8,128,45.875,87.5546875,234881024
+27,mlp.down_proj,4,32,8,128,16.296875,126.21875,234881024
+27,mlp.down_proj,4,64,8,128,19.0,119.109375,234881024
+27,mlp.down_proj,4,128,8,128,21.40625,115.5546875,234881024
+27,mlp.down_proj,8,32,8,128,0.99169921875,238.21875,234881024
+27,mlp.down_proj,8,64,8,128,1.15625,231.109375,234881024
+27,mlp.down_proj,8,128,8,128,1.298828125,227.5546875,234881024
+28,self_attn.q_proj,2,32,8,128,43.3125,20.0625,67108864
+28,self_attn.q_proj,2,64,8,128,51.40625,18.03125,67108864
+28,self_attn.q_proj,2,128,8,128,59.53125,17.015625,67108864
+28,self_attn.q_proj,3,32,8,128,18.578125,28.0625,67108864
+28,self_attn.q_proj,3,64,8,128,22.140625,26.03125,67108864
+28,self_attn.q_proj,3,128,8,128,25.859375,25.015625,67108864
+28,self_attn.q_proj,4,32,8,128,8.671875,36.0625,67108864
+28,self_attn.q_proj,4,64,8,128,10.328125,34.03125,67108864
+28,self_attn.q_proj,4,128,8,128,12.0703125,33.015625,67108864
+28,self_attn.q_proj,8,32,8,128,0.52490234375,68.0625,67108864
+28,self_attn.q_proj,8,64,8,128,0.625,66.03125,67108864
+28,self_attn.q_proj,8,128,8,128,0.7275390625,65.015625,67108864
+28,self_attn.v_proj,2,32,8,128,15.625,2.5078125,8388608
+28,self_attn.v_proj,2,64,8,128,18.203125,2.25390625,8388608
+28,self_attn.v_proj,2,128,8,128,20.53125,2.126953125,8388608
+28,self_attn.v_proj,3,32,8,128,6.67578125,3.5078125,8388608
+28,self_attn.v_proj,3,64,8,128,7.78515625,3.25390625,8388608
+28,self_attn.v_proj,3,128,8,128,8.78125,3.126953125,8388608
+28,self_attn.v_proj,4,32,8,128,3.115234375,4.5078125,8388608
+28,self_attn.v_proj,4,64,8,128,3.6328125,4.25390625,8388608
+28,self_attn.v_proj,4,128,8,128,4.09765625,4.126953125,8388608
+28,self_attn.v_proj,8,32,8,128,0.188720703125,8.5078125,8388608
+28,self_attn.v_proj,8,64,8,128,0.2198486328125,8.25390625,8388608
+28,self_attn.v_proj,8,128,8,128,0.2471923828125,8.126953125,8388608
+28,self_attn.k_proj,2,32,8,128,25.515625,2.5078125,8388608
+28,self_attn.k_proj,2,64,8,128,30.84375,2.25390625,8388608
+28,self_attn.k_proj,2,128,8,128,35.40625,2.126953125,8388608
+28,self_attn.k_proj,3,32,8,128,11.0390625,3.5078125,8388608
+28,self_attn.k_proj,3,64,8,128,13.5,3.25390625,8388608
+28,self_attn.k_proj,3,128,8,128,15.890625,3.126953125,8388608
+28,self_attn.k_proj,4,32,8,128,5.1484375,4.5078125,8388608
+28,self_attn.k_proj,4,64,8,128,6.3046875,4.25390625,8388608
+28,self_attn.k_proj,4,128,8,128,7.421875,4.126953125,8388608
+28,self_attn.k_proj,8,32,8,128,0.3115234375,8.5078125,8388608
+28,self_attn.k_proj,8,64,8,128,0.381103515625,8.25390625,8388608
+28,self_attn.k_proj,8,128,8,128,0.447265625,8.126953125,8388608
+28,self_attn.o_proj,2,32,8,128,41.1875,20.0625,67108864
+28,self_attn.o_proj,2,64,8,128,47.96875,18.03125,67108864
+28,self_attn.o_proj,2,128,8,128,54.0625,17.015625,67108864
+28,self_attn.o_proj,3,32,8,128,17.609375,28.0625,67108864
+28,self_attn.o_proj,3,64,8,128,20.515625,26.03125,67108864
+28,self_attn.o_proj,3,128,8,128,23.140625,25.015625,67108864
+28,self_attn.o_proj,4,32,8,128,8.21875,36.0625,67108864
+28,self_attn.o_proj,4,64,8,128,9.578125,34.03125,67108864
+28,self_attn.o_proj,4,128,8,128,10.8046875,33.015625,67108864
+28,self_attn.o_proj,8,32,8,128,0.49755859375,68.0625,67108864
+28,self_attn.o_proj,8,64,8,128,0.5791015625,66.03125,67108864
+28,self_attn.o_proj,8,128,8,128,0.6513671875,65.015625,67108864
+28,mlp.gate_proj,2,32,8,128,84.0,70.21875,234881024
+28,mlp.gate_proj,2,64,8,128,98.1875,63.109375,234881024
+28,mlp.gate_proj,2,128,8,128,111.3125,59.5546875,234881024
+28,mlp.gate_proj,3,32,8,128,35.9375,98.21875,234881024
+28,mlp.gate_proj,3,64,8,128,42.03125,91.109375,234881024
+28,mlp.gate_proj,3,128,8,128,47.625,87.5546875,234881024
+28,mlp.gate_proj,4,32,8,128,16.765625,126.21875,234881024
+28,mlp.gate_proj,4,64,8,128,19.625,119.109375,234881024
+28,mlp.gate_proj,4,128,8,128,22.234375,115.5546875,234881024
+28,mlp.gate_proj,8,32,8,128,1.01953125,238.21875,234881024
+28,mlp.gate_proj,8,64,8,128,1.1943359375,231.109375,234881024
+28,mlp.gate_proj,8,128,8,128,1.3486328125,227.5546875,234881024
+28,mlp.up_proj,2,32,8,128,81.5,70.21875,234881024
+28,mlp.up_proj,2,64,8,128,94.875,63.109375,234881024
+28,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024
+28,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024
+28,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024
+28,mlp.up_proj,3,128,8,128,45.8125,87.5546875,234881024
+28,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024
+28,mlp.up_proj,4,64,8,128,18.953125,119.109375,234881024
+28,mlp.up_proj,4,128,8,128,21.375,115.5546875,234881024
+28,mlp.up_proj,8,32,8,128,0.98828125,238.21875,234881024
+28,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024
+28,mlp.up_proj,8,128,8,128,1.296875,227.5546875,234881024
+28,mlp.down_proj,2,32,8,128,81.75,70.21875,234881024
+28,mlp.down_proj,2,64,8,128,95.125,63.109375,234881024
+28,mlp.down_proj,2,128,8,128,107.1875,59.5546875,234881024
+28,mlp.down_proj,3,32,8,128,34.96875,98.21875,234881024
+28,mlp.down_proj,3,64,8,128,40.71875,91.109375,234881024
+28,mlp.down_proj,3,128,8,128,45.84375,87.5546875,234881024
+28,mlp.down_proj,4,32,8,128,16.296875,126.21875,234881024
+28,mlp.down_proj,4,64,8,128,19.0,119.109375,234881024
+28,mlp.down_proj,4,128,8,128,21.390625,115.5546875,234881024
+28,mlp.down_proj,8,32,8,128,0.99169921875,238.21875,234881024
+28,mlp.down_proj,8,64,8,128,1.15625,231.109375,234881024
+28,mlp.down_proj,8,128,8,128,1.298828125,227.5546875,234881024
+29,self_attn.q_proj,2,32,8,128,42.90625,20.0625,67108864
+29,self_attn.q_proj,2,64,8,128,51.1875,18.03125,67108864
+29,self_attn.q_proj,2,128,8,128,59.125,17.015625,67108864
+29,self_attn.q_proj,3,32,8,128,18.4375,28.0625,67108864
+29,self_attn.q_proj,3,64,8,128,22.15625,26.03125,67108864
+29,self_attn.q_proj,3,128,8,128,25.953125,25.015625,67108864
+29,self_attn.q_proj,4,32,8,128,8.6015625,36.0625,67108864
+29,self_attn.q_proj,4,64,8,128,10.3359375,34.03125,67108864
+29,self_attn.q_proj,4,128,8,128,12.109375,33.015625,67108864
+29,self_attn.q_proj,8,32,8,128,0.52099609375,68.0625,67108864
+29,self_attn.q_proj,8,64,8,128,0.62548828125,66.03125,67108864
+29,self_attn.q_proj,8,128,8,128,0.72998046875,65.015625,67108864
+29,self_attn.v_proj,2,32,8,128,15.2890625,2.5078125,8388608
+29,self_attn.v_proj,2,64,8,128,17.828125,2.25390625,8388608
+29,self_attn.v_proj,2,128,8,128,20.125,2.126953125,8388608
+29,self_attn.v_proj,3,32,8,128,6.5390625,3.5078125,8388608
+29,self_attn.v_proj,3,64,8,128,7.62890625,3.25390625,8388608
+29,self_attn.v_proj,3,128,8,128,8.609375,3.126953125,8388608
+29,self_attn.v_proj,4,32,8,128,3.05078125,4.5078125,8388608
+29,self_attn.v_proj,4,64,8,128,3.560546875,4.25390625,8388608
+29,self_attn.v_proj,4,128,8,128,4.01953125,4.126953125,8388608
+29,self_attn.v_proj,8,32,8,128,0.1846923828125,8.5078125,8388608
+29,self_attn.v_proj,8,64,8,128,0.21533203125,8.25390625,8388608
+29,self_attn.v_proj,8,128,8,128,0.2423095703125,8.126953125,8388608
+29,self_attn.k_proj,2,32,8,128,25.03125,2.5078125,8388608
+29,self_attn.k_proj,2,64,8,128,30.0,2.25390625,8388608
+29,self_attn.k_proj,2,128,8,128,34.0625,2.126953125,8388608
+29,self_attn.k_proj,3,32,8,128,10.8671875,3.5078125,8388608
+29,self_attn.k_proj,3,64,8,128,13.28125,3.25390625,8388608
+29,self_attn.k_proj,3,128,8,128,15.5625,3.126953125,8388608
+29,self_attn.k_proj,4,32,8,128,5.0703125,4.5078125,8388608
+29,self_attn.k_proj,4,64,8,128,6.1953125,4.25390625,8388608
+29,self_attn.k_proj,4,128,8,128,7.2734375,4.126953125,8388608
+29,self_attn.k_proj,8,32,8,128,0.306884765625,8.5078125,8388608
+29,self_attn.k_proj,8,64,8,128,0.374755859375,8.25390625,8388608
+29,self_attn.k_proj,8,128,8,128,0.438232421875,8.126953125,8388608
+29,self_attn.o_proj,2,32,8,128,40.9375,20.0625,67108864
+29,self_attn.o_proj,2,64,8,128,47.65625,18.03125,67108864
+29,self_attn.o_proj,2,128,8,128,53.71875,17.015625,67108864
+29,self_attn.o_proj,3,32,8,128,17.515625,28.0625,67108864
+29,self_attn.o_proj,3,64,8,128,20.421875,26.03125,67108864
+29,self_attn.o_proj,3,128,8,128,23.046875,25.015625,67108864
+29,self_attn.o_proj,4,32,8,128,8.1796875,36.0625,67108864
+29,self_attn.o_proj,4,64,8,128,9.5390625,34.03125,67108864
+29,self_attn.o_proj,4,128,8,128,10.78125,33.015625,67108864
+29,self_attn.o_proj,8,32,8,128,0.495361328125,68.0625,67108864
+29,self_attn.o_proj,8,64,8,128,0.57763671875,66.03125,67108864
+29,self_attn.o_proj,8,128,8,128,0.650390625,65.015625,67108864
+29,mlp.gate_proj,2,32,8,128,84.125,70.21875,234881024
+29,mlp.gate_proj,2,64,8,128,98.3125,63.109375,234881024
+29,mlp.gate_proj,2,128,8,128,111.3125,59.5546875,234881024
+29,mlp.gate_proj,3,32,8,128,35.96875,98.21875,234881024
+29,mlp.gate_proj,3,64,8,128,42.0625,91.109375,234881024
+29,mlp.gate_proj,3,128,8,128,47.65625,87.5546875,234881024
+29,mlp.gate_proj,4,32,8,128,16.78125,126.21875,234881024
+29,mlp.gate_proj,4,64,8,128,19.640625,119.109375,234881024
+29,mlp.gate_proj,4,128,8,128,22.234375,115.5546875,234881024
+29,mlp.gate_proj,8,32,8,128,1.0205078125,238.21875,234881024
+29,mlp.gate_proj,8,64,8,128,1.1953125,231.109375,234881024
+29,mlp.gate_proj,8,128,8,128,1.3486328125,227.5546875,234881024
+29,mlp.up_proj,2,32,8,128,81.5,70.21875,234881024
+29,mlp.up_proj,2,64,8,128,94.875,63.109375,234881024
+29,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024
+29,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024
+29,mlp.up_proj,3,64,8,128,40.59375,91.109375,234881024
+29,mlp.up_proj,3,128,8,128,45.78125,87.5546875,234881024
+29,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024
+29,mlp.up_proj,4,64,8,128,18.953125,119.109375,234881024
+29,mlp.up_proj,4,128,8,128,21.359375,115.5546875,234881024
+29,mlp.up_proj,8,32,8,128,0.98779296875,238.21875,234881024
+29,mlp.up_proj,8,64,8,128,1.15234375,231.109375,234881024
+29,mlp.up_proj,8,128,8,128,1.2958984375,227.5546875,234881024
+29,mlp.down_proj,2,32,8,128,81.6875,70.21875,234881024
+29,mlp.down_proj,2,64,8,128,95.0625,63.109375,234881024
+29,mlp.down_proj,2,128,8,128,107.125,59.5546875,234881024
+29,mlp.down_proj,3,32,8,128,34.9375,98.21875,234881024
+29,mlp.down_proj,3,64,8,128,40.6875,91.109375,234881024
+29,mlp.down_proj,3,128,8,128,45.8125,87.5546875,234881024
+29,mlp.down_proj,4,32,8,128,16.28125,126.21875,234881024
+29,mlp.down_proj,4,64,8,128,18.984375,119.109375,234881024
+29,mlp.down_proj,4,128,8,128,21.375,115.5546875,234881024
+29,mlp.down_proj,8,32,8,128,0.99072265625,238.21875,234881024
+29,mlp.down_proj,8,64,8,128,1.1552734375,231.109375,234881024
+29,mlp.down_proj,8,128,8,128,1.2978515625,227.5546875,234881024
+30,self_attn.q_proj,2,32,8,128,37.03125,20.0625,67108864
+30,self_attn.q_proj,2,64,8,128,44.40625,18.03125,67108864
+30,self_attn.q_proj,2,128,8,128,51.8125,17.015625,67108864
+30,self_attn.q_proj,3,32,8,128,15.9453125,28.0625,67108864
+30,self_attn.q_proj,3,64,8,128,19.3125,26.03125,67108864
+30,self_attn.q_proj,3,128,8,128,22.953125,25.015625,67108864
+30,self_attn.q_proj,4,32,8,128,7.4375,36.0625,67108864
+30,self_attn.q_proj,4,64,8,128,9.015625,34.03125,67108864
+30,self_attn.q_proj,4,128,8,128,10.71875,33.015625,67108864
+30,self_attn.q_proj,8,32,8,128,0.4501953125,68.0625,67108864
+30,self_attn.q_proj,8,64,8,128,0.54541015625,66.03125,67108864
+30,self_attn.q_proj,8,128,8,128,0.646484375,65.015625,67108864
+30,self_attn.v_proj,2,32,8,128,15.7109375,2.5078125,8388608
+30,self_attn.v_proj,2,64,8,128,18.3125,2.25390625,8388608
+30,self_attn.v_proj,2,128,8,128,20.65625,2.126953125,8388608
+30,self_attn.v_proj,3,32,8,128,6.71484375,3.5078125,8388608
+30,self_attn.v_proj,3,64,8,128,7.8359375,3.25390625,8388608
+30,self_attn.v_proj,3,128,8,128,8.84375,3.126953125,8388608
+30,self_attn.v_proj,4,32,8,128,3.134765625,4.5078125,8388608
+30,self_attn.v_proj,4,64,8,128,3.65625,4.25390625,8388608
+30,self_attn.v_proj,4,128,8,128,4.125,4.126953125,8388608
+30,self_attn.v_proj,8,32,8,128,0.1898193359375,8.5078125,8388608
+30,self_attn.v_proj,8,64,8,128,0.22119140625,8.25390625,8388608
+30,self_attn.v_proj,8,128,8,128,0.2486572265625,8.126953125,8388608
+30,self_attn.k_proj,2,32,8,128,23.078125,2.5078125,8388608
+30,self_attn.k_proj,2,64,8,128,27.921875,2.25390625,8388608
+30,self_attn.k_proj,2,128,8,128,32.03125,2.126953125,8388608
+30,self_attn.k_proj,3,32,8,128,10.2109375,3.5078125,8388608
+30,self_attn.k_proj,3,64,8,128,12.84375,3.25390625,8388608
+30,self_attn.k_proj,3,128,8,128,15.5390625,3.126953125,8388608
+30,self_attn.k_proj,4,32,8,128,4.765625,4.5078125,8388608
+30,self_attn.k_proj,4,64,8,128,5.99609375,4.25390625,8388608
+30,self_attn.k_proj,4,128,8,128,7.26953125,4.126953125,8388608
+30,self_attn.k_proj,8,32,8,128,0.288330078125,8.5078125,8388608
+30,self_attn.k_proj,8,64,8,128,0.362548828125,8.25390625,8388608
+30,self_attn.k_proj,8,128,8,128,0.437744140625,8.126953125,8388608
+30,self_attn.o_proj,2,32,8,128,40.15625,20.0625,67108864
+30,self_attn.o_proj,2,64,8,128,46.78125,18.03125,67108864
+30,self_attn.o_proj,2,128,8,128,52.71875,17.015625,67108864
+30,self_attn.o_proj,3,32,8,128,17.171875,28.0625,67108864
+30,self_attn.o_proj,3,64,8,128,20.015625,26.03125,67108864
+30,self_attn.o_proj,3,128,8,128,22.578125,25.015625,67108864
+30,self_attn.o_proj,4,32,8,128,8.015625,36.0625,67108864
+30,self_attn.o_proj,4,64,8,128,9.34375,34.03125,67108864
+30,self_attn.o_proj,4,128,8,128,10.5390625,33.015625,67108864
+30,self_attn.o_proj,8,32,8,128,0.485107421875,68.0625,67108864
+30,self_attn.o_proj,8,64,8,128,0.56494140625,66.03125,67108864
+30,self_attn.o_proj,8,128,8,128,0.63525390625,65.015625,67108864
+30,mlp.gate_proj,2,32,8,128,84.3125,70.21875,234881024
+30,mlp.gate_proj,2,64,8,128,98.5625,63.109375,234881024
+30,mlp.gate_proj,2,128,8,128,111.625,59.5546875,234881024
+30,mlp.gate_proj,3,32,8,128,36.0625,98.21875,234881024
+30,mlp.gate_proj,3,64,8,128,42.1875,91.109375,234881024
+30,mlp.gate_proj,3,128,8,128,47.78125,87.5546875,234881024
+30,mlp.gate_proj,4,32,8,128,16.8125,126.21875,234881024
+30,mlp.gate_proj,4,64,8,128,19.6875,119.109375,234881024
+30,mlp.gate_proj,4,128,8,128,22.296875,115.5546875,234881024
+30,mlp.gate_proj,8,32,8,128,1.0234375,238.21875,234881024
+30,mlp.gate_proj,8,64,8,128,1.1982421875,231.109375,234881024
+30,mlp.gate_proj,8,128,8,128,1.3525390625,227.5546875,234881024
+30,mlp.up_proj,2,32,8,128,81.5,70.21875,234881024
+30,mlp.up_proj,2,64,8,128,94.875,63.109375,234881024
+30,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024
+30,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024
+30,mlp.up_proj,3,64,8,128,40.59375,91.109375,234881024
+30,mlp.up_proj,3,128,8,128,45.78125,87.5546875,234881024
+30,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024
+30,mlp.up_proj,4,64,8,128,18.953125,119.109375,234881024
+30,mlp.up_proj,4,128,8,128,21.359375,115.5546875,234881024
+30,mlp.up_proj,8,32,8,128,0.98828125,238.21875,234881024
+30,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024
+30,mlp.up_proj,8,128,8,128,1.2958984375,227.5546875,234881024
+30,mlp.down_proj,2,32,8,128,81.625,70.21875,234881024
+30,mlp.down_proj,2,64,8,128,94.9375,63.109375,234881024
+30,mlp.down_proj,2,128,8,128,107.0,59.5546875,234881024
+30,mlp.down_proj,3,32,8,128,34.875,98.21875,234881024
+30,mlp.down_proj,3,64,8,128,40.625,91.109375,234881024
+30,mlp.down_proj,3,128,8,128,45.75,87.5546875,234881024
+30,mlp.down_proj,4,32,8,128,16.265625,126.21875,234881024
+30,mlp.down_proj,4,64,8,128,18.96875,119.109375,234881024
+30,mlp.down_proj,4,128,8,128,21.359375,115.5546875,234881024
+30,mlp.down_proj,8,32,8,128,0.9892578125,238.21875,234881024
+30,mlp.down_proj,8,64,8,128,1.1533203125,231.109375,234881024
+30,mlp.down_proj,8,128,8,128,1.2958984375,227.5546875,234881024
+31,self_attn.q_proj,2,32,8,128,42.375,20.0625,67108864
+31,self_attn.q_proj,2,64,8,128,50.46875,18.03125,67108864
+31,self_attn.q_proj,2,128,8,128,58.25,17.015625,67108864
+31,self_attn.q_proj,3,32,8,128,18.1875,28.0625,67108864
+31,self_attn.q_proj,3,64,8,128,21.75,26.03125,67108864
+31,self_attn.q_proj,3,128,8,128,25.328125,25.015625,67108864
+31,self_attn.q_proj,4,32,8,128,8.484375,36.0625,67108864
+31,self_attn.q_proj,4,64,8,128,10.15625,34.03125,67108864
+31,self_attn.q_proj,4,128,8,128,11.828125,33.015625,67108864
+31,self_attn.q_proj,8,32,8,128,0.513671875,68.0625,67108864
+31,self_attn.q_proj,8,64,8,128,0.6142578125,66.03125,67108864
+31,self_attn.q_proj,8,128,8,128,0.71337890625,65.015625,67108864
+31,self_attn.v_proj,2,32,8,128,15.1875,2.5078125,8388608
+31,self_attn.v_proj,2,64,8,128,17.75,2.25390625,8388608
+31,self_attn.v_proj,2,128,8,128,20.0625,2.126953125,8388608
+31,self_attn.v_proj,3,32,8,128,6.4921875,3.5078125,8388608
+31,self_attn.v_proj,3,64,8,128,7.58984375,3.25390625,8388608
+31,self_attn.v_proj,3,128,8,128,8.5859375,3.126953125,8388608
+31,self_attn.v_proj,4,32,8,128,3.03125,4.5078125,8388608
+31,self_attn.v_proj,4,64,8,128,3.54296875,4.25390625,8388608
+31,self_attn.v_proj,4,128,8,128,4.0078125,4.126953125,8388608
+31,self_attn.v_proj,8,32,8,128,0.18359375,8.5078125,8388608
+31,self_attn.v_proj,8,64,8,128,0.21435546875,8.25390625,8388608
+31,self_attn.v_proj,8,128,8,128,0.2415771484375,8.126953125,8388608
+31,self_attn.k_proj,2,32,8,128,24.921875,2.5078125,8388608
+31,self_attn.k_proj,2,64,8,128,29.90625,2.25390625,8388608
+31,self_attn.k_proj,2,128,8,128,33.96875,2.126953125,8388608
+31,self_attn.k_proj,3,32,8,128,10.7890625,3.5078125,8388608
+31,self_attn.k_proj,3,64,8,128,13.1640625,3.25390625,8388608
+31,self_attn.k_proj,3,128,8,128,15.40625,3.126953125,8388608
+31,self_attn.k_proj,4,32,8,128,5.03515625,4.5078125,8388608
+31,self_attn.k_proj,4,64,8,128,6.140625,4.25390625,8388608
+31,self_attn.k_proj,4,128,8,128,7.19921875,4.126953125,8388608
+31,self_attn.k_proj,8,32,8,128,0.3046875,8.5078125,8388608
+31,self_attn.k_proj,8,64,8,128,0.371337890625,8.25390625,8388608
+31,self_attn.k_proj,8,128,8,128,0.433349609375,8.126953125,8388608
+31,self_attn.o_proj,2,32,8,128,40.1875,20.0625,67108864
+31,self_attn.o_proj,2,64,8,128,46.78125,18.03125,67108864
+31,self_attn.o_proj,2,128,8,128,52.71875,17.015625,67108864
+31,self_attn.o_proj,3,32,8,128,17.1875,28.0625,67108864
+31,self_attn.o_proj,3,64,8,128,20.03125,26.03125,67108864
+31,self_attn.o_proj,3,128,8,128,22.59375,25.015625,67108864
+31,self_attn.o_proj,4,32,8,128,8.03125,36.0625,67108864
+31,self_attn.o_proj,4,64,8,128,9.3671875,34.03125,67108864
+31,self_attn.o_proj,4,128,8,128,10.59375,33.015625,67108864
+31,self_attn.o_proj,8,32,8,128,0.48779296875,68.0625,67108864
+31,self_attn.o_proj,8,64,8,128,0.56982421875,66.03125,67108864
+31,self_attn.o_proj,8,128,8,128,0.64404296875,65.015625,67108864
+31,mlp.gate_proj,2,32,8,128,84.375,70.21875,234881024
+31,mlp.gate_proj,2,64,8,128,98.5625,63.109375,234881024
+31,mlp.gate_proj,2,128,8,128,111.625,59.5546875,234881024
+31,mlp.gate_proj,3,32,8,128,36.09375,98.21875,234881024
+31,mlp.gate_proj,3,64,8,128,42.1875,91.109375,234881024
+31,mlp.gate_proj,3,128,8,128,47.78125,87.5546875,234881024
+31,mlp.gate_proj,4,32,8,128,16.828125,126.21875,234881024
+31,mlp.gate_proj,4,64,8,128,19.703125,119.109375,234881024
+31,mlp.gate_proj,4,128,8,128,22.296875,115.5546875,234881024
+31,mlp.gate_proj,8,32,8,128,1.0234375,238.21875,234881024
+31,mlp.gate_proj,8,64,8,128,1.1982421875,231.109375,234881024
+31,mlp.gate_proj,8,128,8,128,1.3525390625,227.5546875,234881024
+31,mlp.up_proj,2,32,8,128,81.5625,70.21875,234881024
+31,mlp.up_proj,2,64,8,128,94.9375,63.109375,234881024
+31,mlp.up_proj,2,128,8,128,107.0625,59.5546875,234881024
+31,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024
+31,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024
+31,mlp.up_proj,3,128,8,128,45.8125,87.5546875,234881024
+31,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024
+31,mlp.up_proj,4,64,8,128,18.96875,119.109375,234881024
+31,mlp.up_proj,4,128,8,128,21.375,115.5546875,234881024
+31,mlp.up_proj,8,32,8,128,0.98876953125,238.21875,234881024
+31,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024
+31,mlp.up_proj,8,128,8,128,1.296875,227.5546875,234881024
+31,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024
+31,mlp.down_proj,2,64,8,128,94.8125,63.109375,234881024
+31,mlp.down_proj,2,128,8,128,106.8125,59.5546875,234881024
+31,mlp.down_proj,3,32,8,128,34.8125,98.21875,234881024
+31,mlp.down_proj,3,64,8,128,40.5625,91.109375,234881024
+31,mlp.down_proj,3,128,8,128,45.6875,87.5546875,234881024
+31,mlp.down_proj,4,32,8,128,16.234375,126.21875,234881024
+31,mlp.down_proj,4,64,8,128,18.9375,119.109375,234881024
+31,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024
+31,mlp.down_proj,8,32,8,128,0.98779296875,238.21875,234881024
+31,mlp.down_proj,8,64,8,128,1.1513671875,231.109375,234881024
+31,mlp.down_proj,8,128,8,128,1.2939453125,227.5546875,234881024
+32,self_attn.q_proj,2,32,8,128,43.125,20.0625,67108864
+32,self_attn.q_proj,2,64,8,128,51.0625,18.03125,67108864
+32,self_attn.q_proj,2,128,8,128,58.71875,17.015625,67108864
+32,self_attn.q_proj,3,32,8,128,18.46875,28.0625,67108864
+32,self_attn.q_proj,3,64,8,128,21.90625,26.03125,67108864
+32,self_attn.q_proj,3,128,8,128,25.296875,25.015625,67108864
+32,self_attn.q_proj,4,32,8,128,8.6171875,36.0625,67108864
+32,self_attn.q_proj,4,64,8,128,10.2265625,34.03125,67108864
+32,self_attn.q_proj,4,128,8,128,11.8046875,33.015625,67108864
+32,self_attn.q_proj,8,32,8,128,0.52197265625,68.0625,67108864
+32,self_attn.q_proj,8,64,8,128,0.61865234375,66.03125,67108864
+32,self_attn.q_proj,8,128,8,128,0.7119140625,65.015625,67108864
+32,self_attn.v_proj,2,32,8,128,15.078125,2.5078125,8388608
+32,self_attn.v_proj,2,64,8,128,17.578125,2.25390625,8388608
+32,self_attn.v_proj,2,128,8,128,19.84375,2.126953125,8388608
+32,self_attn.v_proj,3,32,8,128,6.4453125,3.5078125,8388608
+32,self_attn.v_proj,3,64,8,128,7.5234375,3.25390625,8388608
+32,self_attn.v_proj,3,128,8,128,8.4921875,3.126953125,8388608
+32,self_attn.v_proj,4,32,8,128,3.0078125,4.5078125,8388608
+32,self_attn.v_proj,4,64,8,128,3.5078125,4.25390625,8388608
+32,self_attn.v_proj,4,128,8,128,3.962890625,4.126953125,8388608
+32,self_attn.v_proj,8,32,8,128,0.18212890625,8.5078125,8388608
+32,self_attn.v_proj,8,64,8,128,0.2122802734375,8.25390625,8388608
+32,self_attn.v_proj,8,128,8,128,0.23876953125,8.126953125,8388608
+32,self_attn.k_proj,2,32,8,128,25.640625,2.5078125,8388608
+32,self_attn.k_proj,2,64,8,128,30.875,2.25390625,8388608
+32,self_attn.k_proj,2,128,8,128,35.3125,2.126953125,8388608
+32,self_attn.k_proj,3,32,8,128,11.0625,3.5078125,8388608
+32,self_attn.k_proj,3,64,8,128,13.484375,3.25390625,8388608
+32,self_attn.k_proj,3,128,8,128,15.8046875,3.126953125,8388608
+32,self_attn.k_proj,4,32,8,128,5.1640625,4.5078125,8388608
+32,self_attn.k_proj,4,64,8,128,6.296875,4.25390625,8388608
+32,self_attn.k_proj,4,128,8,128,7.3828125,4.126953125,8388608
+32,self_attn.k_proj,8,32,8,128,0.312744140625,8.5078125,8388608
+32,self_attn.k_proj,8,64,8,128,0.380615234375,8.25390625,8388608
+32,self_attn.k_proj,8,128,8,128,0.44482421875,8.126953125,8388608
+32,self_attn.o_proj,2,32,8,128,39.875,20.0625,67108864
+32,self_attn.o_proj,2,64,8,128,46.5,18.03125,67108864
+32,self_attn.o_proj,2,128,8,128,52.59375,17.015625,67108864
+32,self_attn.o_proj,3,32,8,128,17.0625,28.0625,67108864
+32,self_attn.o_proj,3,64,8,128,19.953125,26.03125,67108864
+32,self_attn.o_proj,3,128,8,128,22.625,25.015625,67108864
+32,self_attn.o_proj,4,32,8,128,7.9609375,36.0625,67108864
+32,self_attn.o_proj,4,64,8,128,9.3125,34.03125,67108864
+32,self_attn.o_proj,4,128,8,128,10.5703125,33.015625,67108864
+32,self_attn.o_proj,8,32,8,128,0.482177734375,68.0625,67108864
+32,self_attn.o_proj,8,64,8,128,0.5634765625,66.03125,67108864
+32,self_attn.o_proj,8,128,8,128,0.63720703125,65.015625,67108864
+32,mlp.gate_proj,2,32,8,128,84.3125,70.21875,234881024
+32,mlp.gate_proj,2,64,8,128,98.4375,63.109375,234881024
+32,mlp.gate_proj,2,128,8,128,111.4375,59.5546875,234881024
+32,mlp.gate_proj,3,32,8,128,36.0625,98.21875,234881024
+32,mlp.gate_proj,3,64,8,128,42.125,91.109375,234881024
+32,mlp.gate_proj,3,128,8,128,47.71875,87.5546875,234881024
+32,mlp.gate_proj,4,32,8,128,16.8125,126.21875,234881024
+32,mlp.gate_proj,4,64,8,128,19.6875,119.109375,234881024
+32,mlp.gate_proj,4,128,8,128,22.265625,115.5546875,234881024
+32,mlp.gate_proj,8,32,8,128,1.0224609375,238.21875,234881024
+32,mlp.gate_proj,8,64,8,128,1.197265625,231.109375,234881024
+32,mlp.gate_proj,8,128,8,128,1.3505859375,227.5546875,234881024
+32,mlp.up_proj,2,32,8,128,81.625,70.21875,234881024
+32,mlp.up_proj,2,64,8,128,95.0,63.109375,234881024
+32,mlp.up_proj,2,128,8,128,107.1875,59.5546875,234881024
+32,mlp.up_proj,3,32,8,128,34.875,98.21875,234881024
+32,mlp.up_proj,3,64,8,128,40.6875,91.109375,234881024
+32,mlp.up_proj,3,128,8,128,45.84375,87.5546875,234881024
+32,mlp.up_proj,4,32,8,128,16.265625,126.21875,234881024
+32,mlp.up_proj,4,64,8,128,18.984375,119.109375,234881024
+32,mlp.up_proj,4,128,8,128,21.40625,115.5546875,234881024
+32,mlp.up_proj,8,32,8,128,0.98974609375,238.21875,234881024
+32,mlp.up_proj,8,64,8,128,1.154296875,231.109375,234881024
+32,mlp.up_proj,8,128,8,128,1.2978515625,227.5546875,234881024
+32,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024
+32,mlp.down_proj,2,64,8,128,94.8125,63.109375,234881024
+32,mlp.down_proj,2,128,8,128,106.875,59.5546875,234881024
+32,mlp.down_proj,3,32,8,128,34.8125,98.21875,234881024
+32,mlp.down_proj,3,64,8,128,40.5625,91.109375,234881024
+32,mlp.down_proj,3,128,8,128,45.6875,87.5546875,234881024
+32,mlp.down_proj,4,32,8,128,16.234375,126.21875,234881024
+32,mlp.down_proj,4,64,8,128,18.9375,119.109375,234881024
+32,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024
+32,mlp.down_proj,8,32,8,128,0.98779296875,238.21875,234881024
+32,mlp.down_proj,8,64,8,128,1.1513671875,231.109375,234881024
+32,mlp.down_proj,8,128,8,128,1.2939453125,227.5546875,234881024
+33,self_attn.q_proj,2,32,8,128,45.8125,20.0625,67108864
+33,self_attn.q_proj,2,64,8,128,53.9375,18.03125,67108864
+33,self_attn.q_proj,2,128,8,128,62.1875,17.015625,67108864
+33,self_attn.q_proj,3,32,8,128,19.609375,28.0625,67108864
+33,self_attn.q_proj,3,64,8,128,23.125,26.03125,67108864
+33,self_attn.q_proj,3,128,8,128,26.75,25.015625,67108864
+33,self_attn.q_proj,4,32,8,128,9.1484375,36.0625,67108864
+33,self_attn.q_proj,4,64,8,128,10.7890625,34.03125,67108864
+33,self_attn.q_proj,4,128,8,128,12.484375,33.015625,67108864
+33,self_attn.q_proj,8,32,8,128,0.55419921875,68.0625,67108864
+33,self_attn.q_proj,8,64,8,128,0.65283203125,66.03125,67108864
+33,self_attn.q_proj,8,128,8,128,0.7529296875,65.015625,67108864
+33,self_attn.v_proj,2,32,8,128,14.828125,2.5078125,8388608
+33,self_attn.v_proj,2,64,8,128,17.34375,2.25390625,8388608
+33,self_attn.v_proj,2,128,8,128,19.65625,2.126953125,8388608
+33,self_attn.v_proj,3,32,8,128,6.33984375,3.5078125,8388608
+33,self_attn.v_proj,3,64,8,128,7.421875,3.25390625,8388608
+33,self_attn.v_proj,3,128,8,128,8.40625,3.126953125,8388608
+33,self_attn.v_proj,4,32,8,128,2.95703125,4.5078125,8388608
+33,self_attn.v_proj,4,64,8,128,3.46484375,4.25390625,8388608
+33,self_attn.v_proj,4,128,8,128,3.92578125,4.126953125,8388608
+33,self_attn.v_proj,8,32,8,128,0.17919921875,8.5078125,8388608
+33,self_attn.v_proj,8,64,8,128,0.20947265625,8.25390625,8388608
+33,self_attn.v_proj,8,128,8,128,0.236572265625,8.126953125,8388608
+33,self_attn.k_proj,2,32,8,128,26.15625,2.5078125,8388608
+33,self_attn.k_proj,2,64,8,128,31.578125,2.25390625,8388608
+33,self_attn.k_proj,2,128,8,128,36.21875,2.126953125,8388608
+33,self_attn.k_proj,3,32,8,128,11.328125,3.5078125,8388608
+33,self_attn.k_proj,3,64,8,128,13.8671875,3.25390625,8388608
+33,self_attn.k_proj,3,128,8,128,16.34375,3.126953125,8388608
+33,self_attn.k_proj,4,32,8,128,5.28515625,4.5078125,8388608
+33,self_attn.k_proj,4,64,8,128,6.47265625,4.25390625,8388608
+33,self_attn.k_proj,4,128,8,128,7.63671875,4.126953125,8388608
+33,self_attn.k_proj,8,32,8,128,0.31982421875,8.5078125,8388608
+33,self_attn.k_proj,8,64,8,128,0.391357421875,8.25390625,8388608
+33,self_attn.k_proj,8,128,8,128,0.4599609375,8.126953125,8388608
+33,self_attn.o_proj,2,32,8,128,40.46875,20.0625,67108864
+33,self_attn.o_proj,2,64,8,128,47.15625,18.03125,67108864
+33,self_attn.o_proj,2,128,8,128,53.21875,17.015625,67108864
+33,self_attn.o_proj,3,32,8,128,17.3125,28.0625,67108864
+33,self_attn.o_proj,3,64,8,128,20.21875,26.03125,67108864
+33,self_attn.o_proj,3,128,8,128,22.84375,25.015625,67108864
+33,self_attn.o_proj,4,32,8,128,8.0859375,36.0625,67108864
+33,self_attn.o_proj,4,64,8,128,9.4375,34.03125,67108864
+33,self_attn.o_proj,4,128,8,128,10.6796875,33.015625,67108864
+33,self_attn.o_proj,8,32,8,128,0.489501953125,68.0625,67108864
+33,self_attn.o_proj,8,64,8,128,0.5712890625,66.03125,67108864
+33,self_attn.o_proj,8,128,8,128,0.64453125,65.015625,67108864
+33,mlp.gate_proj,2,32,8,128,84.125,70.21875,234881024
+33,mlp.gate_proj,2,64,8,128,98.3125,63.109375,234881024
+33,mlp.gate_proj,2,128,8,128,111.3125,59.5546875,234881024
+33,mlp.gate_proj,3,32,8,128,35.96875,98.21875,234881024
+33,mlp.gate_proj,3,64,8,128,42.0625,91.109375,234881024
+33,mlp.gate_proj,3,128,8,128,47.65625,87.5546875,234881024
+33,mlp.gate_proj,4,32,8,128,16.78125,126.21875,234881024
+33,mlp.gate_proj,4,64,8,128,19.640625,119.109375,234881024
+33,mlp.gate_proj,4,128,8,128,22.25,115.5546875,234881024
+33,mlp.gate_proj,8,32,8,128,1.0205078125,238.21875,234881024
+33,mlp.gate_proj,8,64,8,128,1.1953125,231.109375,234881024
+33,mlp.gate_proj,8,128,8,128,1.349609375,227.5546875,234881024
+33,mlp.up_proj,2,32,8,128,81.5,70.21875,234881024
+33,mlp.up_proj,2,64,8,128,94.9375,63.109375,234881024
+33,mlp.up_proj,2,128,8,128,107.0625,59.5546875,234881024
+33,mlp.up_proj,3,32,8,128,34.84375,98.21875,234881024
+33,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024
+33,mlp.up_proj,3,128,8,128,45.78125,87.5546875,234881024
+33,mlp.up_proj,4,32,8,128,16.25,126.21875,234881024
+33,mlp.up_proj,4,64,8,128,18.96875,119.109375,234881024
+33,mlp.up_proj,4,128,8,128,21.375,115.5546875,234881024
+33,mlp.up_proj,8,32,8,128,0.98828125,238.21875,234881024
+33,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024
+33,mlp.up_proj,8,128,8,128,1.296875,227.5546875,234881024
+33,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024
+33,mlp.down_proj,2,64,8,128,94.875,63.109375,234881024
+33,mlp.down_proj,2,128,8,128,106.875,59.5546875,234881024
+33,mlp.down_proj,3,32,8,128,34.84375,98.21875,234881024
+33,mlp.down_proj,3,64,8,128,40.59375,91.109375,234881024
+33,mlp.down_proj,3,128,8,128,45.71875,87.5546875,234881024
+33,mlp.down_proj,4,32,8,128,16.25,126.21875,234881024
+33,mlp.down_proj,4,64,8,128,18.953125,119.109375,234881024
+33,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024
+33,mlp.down_proj,8,32,8,128,0.98828125,238.21875,234881024
+33,mlp.down_proj,8,64,8,128,1.15234375,231.109375,234881024
+33,mlp.down_proj,8,128,8,128,1.294921875,227.5546875,234881024
+34,self_attn.q_proj,2,32,8,128,44.0625,20.0625,67108864
+34,self_attn.q_proj,2,64,8,128,52.4375,18.03125,67108864
+34,self_attn.q_proj,2,128,8,128,60.53125,17.015625,67108864
+34,self_attn.q_proj,3,32,8,128,18.96875,28.0625,67108864
+34,self_attn.q_proj,3,64,8,128,22.734375,26.03125,67108864
+34,self_attn.q_proj,3,128,8,128,26.625,25.015625,67108864
+34,self_attn.q_proj,4,32,8,128,8.8515625,36.0625,67108864
+34,self_attn.q_proj,4,64,8,128,10.609375,34.03125,67108864
+34,self_attn.q_proj,4,128,8,128,12.4296875,33.015625,67108864
+34,self_attn.q_proj,8,32,8,128,0.53564453125,68.0625,67108864
+34,self_attn.q_proj,8,64,8,128,0.6416015625,66.03125,67108864
+34,self_attn.q_proj,8,128,8,128,0.74951171875,65.015625,67108864
+34,self_attn.v_proj,2,32,8,128,15.0859375,2.5078125,8388608
+34,self_attn.v_proj,2,64,8,128,17.625,2.25390625,8388608
+34,self_attn.v_proj,2,128,8,128,19.921875,2.126953125,8388608
+34,self_attn.v_proj,3,32,8,128,6.44921875,3.5078125,8388608
+34,self_attn.v_proj,3,64,8,128,7.54296875,3.25390625,8388608
+34,self_attn.v_proj,3,128,8,128,8.53125,3.126953125,8388608
+34,self_attn.v_proj,4,32,8,128,3.009765625,4.5078125,8388608
+34,self_attn.v_proj,4,64,8,128,3.51953125,4.25390625,8388608
+34,self_attn.v_proj,4,128,8,128,3.98046875,4.126953125,8388608
+34,self_attn.v_proj,8,32,8,128,0.1822509765625,8.5078125,8388608
+34,self_attn.v_proj,8,64,8,128,0.2127685546875,8.25390625,8388608
+34,self_attn.v_proj,8,128,8,128,0.2398681640625,8.126953125,8388608
+34,self_attn.k_proj,2,32,8,128,27.109375,2.5078125,8388608
+34,self_attn.k_proj,2,64,8,128,32.625,2.25390625,8388608
+34,self_attn.k_proj,2,128,8,128,37.15625,2.126953125,8388608
+34,self_attn.k_proj,3,32,8,128,11.7890625,3.5078125,8388608
+34,self_attn.k_proj,3,64,8,128,14.4453125,3.25390625,8388608
+34,self_attn.k_proj,3,128,8,128,16.984375,3.126953125,8388608
+34,self_attn.k_proj,4,32,8,128,5.50390625,4.5078125,8388608
+34,self_attn.k_proj,4,64,8,128,6.73828125,4.25390625,8388608
+34,self_attn.k_proj,4,128,8,128,7.9296875,4.126953125,8388608
+34,self_attn.k_proj,8,32,8,128,0.3330078125,8.5078125,8388608
+34,self_attn.k_proj,8,64,8,128,0.407470703125,8.25390625,8388608
+34,self_attn.k_proj,8,128,8,128,0.477783203125,8.126953125,8388608
+34,self_attn.o_proj,2,32,8,128,39.875,20.0625,67108864
+34,self_attn.o_proj,2,64,8,128,46.46875,18.03125,67108864
+34,self_attn.o_proj,2,128,8,128,52.40625,17.015625,67108864
+34,self_attn.o_proj,3,32,8,128,17.046875,28.0625,67108864
+34,self_attn.o_proj,3,64,8,128,19.890625,26.03125,67108864
+34,self_attn.o_proj,3,128,8,128,22.46875,25.015625,67108864
+34,self_attn.o_proj,4,32,8,128,7.95703125,36.0625,67108864
+34,self_attn.o_proj,4,64,8,128,9.28125,34.03125,67108864
+34,self_attn.o_proj,4,128,8,128,10.484375,33.015625,67108864
+34,self_attn.o_proj,8,32,8,128,0.481689453125,68.0625,67108864
+34,self_attn.o_proj,8,64,8,128,0.5615234375,66.03125,67108864
+34,self_attn.o_proj,8,128,8,128,0.63232421875,65.015625,67108864
+34,mlp.gate_proj,2,32,8,128,84.0625,70.21875,234881024
+34,mlp.gate_proj,2,64,8,128,98.3125,63.109375,234881024
+34,mlp.gate_proj,2,128,8,128,111.5,59.5546875,234881024
+34,mlp.gate_proj,3,32,8,128,35.96875,98.21875,234881024
+34,mlp.gate_proj,3,64,8,128,42.09375,91.109375,234881024
+34,mlp.gate_proj,3,128,8,128,47.75,87.5546875,234881024
+34,mlp.gate_proj,4,32,8,128,16.765625,126.21875,234881024
+34,mlp.gate_proj,4,64,8,128,19.65625,119.109375,234881024
+34,mlp.gate_proj,4,128,8,128,22.28125,115.5546875,234881024
+34,mlp.gate_proj,8,32,8,128,1.0205078125,238.21875,234881024
+34,mlp.gate_proj,8,64,8,128,1.1953125,231.109375,234881024
+34,mlp.gate_proj,8,128,8,128,1.3515625,227.5546875,234881024
+34,mlp.up_proj,2,32,8,128,81.3125,70.21875,234881024
+34,mlp.up_proj,2,64,8,128,94.625,63.109375,234881024
+34,mlp.up_proj,2,128,8,128,106.75,59.5546875,234881024
+34,mlp.up_proj,3,32,8,128,34.75,98.21875,234881024
+34,mlp.up_proj,3,64,8,128,40.53125,91.109375,234881024
+34,mlp.up_proj,3,128,8,128,45.65625,87.5546875,234881024
+34,mlp.up_proj,4,32,8,128,16.203125,126.21875,234881024
+34,mlp.up_proj,4,64,8,128,18.90625,119.109375,234881024
+34,mlp.up_proj,4,128,8,128,21.3125,115.5546875,234881024
+34,mlp.up_proj,8,32,8,128,0.98583984375,238.21875,234881024
+34,mlp.up_proj,8,64,8,128,1.150390625,231.109375,234881024
+34,mlp.up_proj,8,128,8,128,1.29296875,227.5546875,234881024
+34,mlp.down_proj,2,32,8,128,81.4375,70.21875,234881024
+34,mlp.down_proj,2,64,8,128,94.6875,63.109375,234881024
+34,mlp.down_proj,2,128,8,128,106.75,59.5546875,234881024
+34,mlp.down_proj,3,32,8,128,34.8125,98.21875,234881024
+34,mlp.down_proj,3,64,8,128,40.53125,91.109375,234881024
+34,mlp.down_proj,3,128,8,128,45.65625,87.5546875,234881024
+34,mlp.down_proj,4,32,8,128,16.234375,126.21875,234881024
+34,mlp.down_proj,4,64,8,128,18.921875,119.109375,234881024
+34,mlp.down_proj,4,128,8,128,21.3125,115.5546875,234881024
+34,mlp.down_proj,8,32,8,128,0.98681640625,238.21875,234881024
+34,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024
+34,mlp.down_proj,8,128,8,128,1.29296875,227.5546875,234881024
+35,self_attn.q_proj,2,32,8,128,43.4375,20.0625,67108864
+35,self_attn.q_proj,2,64,8,128,51.75,18.03125,67108864
+35,self_attn.q_proj,2,128,8,128,59.71875,17.015625,67108864
+35,self_attn.q_proj,3,32,8,128,18.65625,28.0625,67108864
+35,self_attn.q_proj,3,64,8,128,22.359375,26.03125,67108864
+35,self_attn.q_proj,3,128,8,128,26.125,25.015625,67108864
+35,self_attn.q_proj,4,32,8,128,8.703125,36.0625,67108864
+35,self_attn.q_proj,4,64,8,128,10.4296875,34.03125,67108864
+35,self_attn.q_proj,4,128,8,128,12.1875,33.015625,67108864
+35,self_attn.q_proj,8,32,8,128,0.52734375,68.0625,67108864
+35,self_attn.q_proj,8,64,8,128,0.63134765625,66.03125,67108864
+35,self_attn.q_proj,8,128,8,128,0.73486328125,65.015625,67108864
+35,self_attn.v_proj,2,32,8,128,15.046875,2.5078125,8388608
+35,self_attn.v_proj,2,64,8,128,17.625,2.25390625,8388608
+35,self_attn.v_proj,2,128,8,128,19.96875,2.126953125,8388608
+35,self_attn.v_proj,3,32,8,128,6.4375,3.5078125,8388608
+35,self_attn.v_proj,3,64,8,128,7.5390625,3.25390625,8388608
+35,self_attn.v_proj,3,128,8,128,8.546875,3.126953125,8388608
+35,self_attn.v_proj,4,32,8,128,3.001953125,4.5078125,8388608
+35,self_attn.v_proj,4,64,8,128,3.515625,4.25390625,8388608
+35,self_attn.v_proj,4,128,8,128,3.986328125,4.126953125,8388608
+35,self_attn.v_proj,8,32,8,128,0.181884765625,8.5078125,8388608
+35,self_attn.v_proj,8,64,8,128,0.2127685546875,8.25390625,8388608
+35,self_attn.v_proj,8,128,8,128,0.2403564453125,8.126953125,8388608
+35,self_attn.k_proj,2,32,8,128,24.84375,2.5078125,8388608
+35,self_attn.k_proj,2,64,8,128,29.875,2.25390625,8388608
+35,self_attn.k_proj,2,128,8,128,34.0,2.126953125,8388608
+35,self_attn.k_proj,3,32,8,128,10.8046875,3.5078125,8388608
+35,self_attn.k_proj,3,64,8,128,13.2578125,3.25390625,8388608
+35,self_attn.k_proj,3,128,8,128,15.6015625,3.126953125,8388608
+35,self_attn.k_proj,4,32,8,128,5.04296875,4.5078125,8388608
+35,self_attn.k_proj,4,64,8,128,6.19140625,4.25390625,8388608
+35,self_attn.k_proj,4,128,8,128,7.29296875,4.126953125,8388608
+35,self_attn.k_proj,8,32,8,128,0.30517578125,8.5078125,8388608
+35,self_attn.k_proj,8,64,8,128,0.37451171875,8.25390625,8388608
+35,self_attn.k_proj,8,128,8,128,0.439453125,8.126953125,8388608
+35,self_attn.o_proj,2,32,8,128,39.4375,20.0625,67108864
+35,self_attn.o_proj,2,64,8,128,46.0,18.03125,67108864
+35,self_attn.o_proj,2,128,8,128,51.96875,17.015625,67108864
+35,self_attn.o_proj,3,32,8,128,16.890625,28.0625,67108864
+35,self_attn.o_proj,3,64,8,128,19.734375,26.03125,67108864
+35,self_attn.o_proj,3,128,8,128,22.359375,25.015625,67108864
+35,self_attn.o_proj,4,32,8,128,7.8828125,36.0625,67108864
+35,self_attn.o_proj,4,64,8,128,9.2109375,34.03125,67108864
+35,self_attn.o_proj,4,128,8,128,10.4453125,33.015625,67108864
+35,self_attn.o_proj,8,32,8,128,0.477294921875,68.0625,67108864
+35,self_attn.o_proj,8,64,8,128,0.5576171875,66.03125,67108864
+35,self_attn.o_proj,8,128,8,128,0.63037109375,65.015625,67108864
+35,mlp.gate_proj,2,32,8,128,84.0625,70.21875,234881024
+35,mlp.gate_proj,2,64,8,128,98.3125,63.109375,234881024
+35,mlp.gate_proj,2,128,8,128,111.375,59.5546875,234881024
+35,mlp.gate_proj,3,32,8,128,35.96875,98.21875,234881024
+35,mlp.gate_proj,3,64,8,128,42.0625,91.109375,234881024
+35,mlp.gate_proj,3,128,8,128,47.6875,87.5546875,234881024
+35,mlp.gate_proj,4,32,8,128,16.765625,126.21875,234881024
+35,mlp.gate_proj,4,64,8,128,19.640625,119.109375,234881024
+35,mlp.gate_proj,4,128,8,128,22.25,115.5546875,234881024
+35,mlp.gate_proj,8,32,8,128,1.01953125,238.21875,234881024
+35,mlp.gate_proj,8,64,8,128,1.1943359375,231.109375,234881024
+35,mlp.gate_proj,8,128,8,128,1.349609375,227.5546875,234881024
+35,mlp.up_proj,2,32,8,128,81.1875,70.21875,234881024
+35,mlp.up_proj,2,64,8,128,94.5625,63.109375,234881024
+35,mlp.up_proj,2,128,8,128,106.625,59.5546875,234881024
+35,mlp.up_proj,3,32,8,128,34.71875,98.21875,234881024
+35,mlp.up_proj,3,64,8,128,40.46875,91.109375,234881024
+35,mlp.up_proj,3,128,8,128,45.59375,87.5546875,234881024
+35,mlp.up_proj,4,32,8,128,16.1875,126.21875,234881024
+35,mlp.up_proj,4,64,8,128,18.890625,119.109375,234881024
+35,mlp.up_proj,4,128,8,128,21.28125,115.5546875,234881024
+35,mlp.up_proj,8,32,8,128,0.984375,238.21875,234881024
+35,mlp.up_proj,8,64,8,128,1.1484375,231.109375,234881024
+35,mlp.up_proj,8,128,8,128,1.2919921875,227.5546875,234881024
+35,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024
+35,mlp.down_proj,2,64,8,128,94.625,63.109375,234881024
+35,mlp.down_proj,2,128,8,128,106.625,59.5546875,234881024
+35,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024
+35,mlp.down_proj,3,64,8,128,40.5,91.109375,234881024
+35,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024
+35,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024
+35,mlp.down_proj,4,64,8,128,18.90625,119.109375,234881024
+35,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024
+35,mlp.down_proj,8,32,8,128,0.98583984375,238.21875,234881024
+35,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024
+35,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024
+36,self_attn.q_proj,2,32,8,128,44.28125,20.0625,67108864
+36,self_attn.q_proj,2,64,8,128,52.46875,18.03125,67108864
+36,self_attn.q_proj,2,128,8,128,60.28125,17.015625,67108864
+36,self_attn.q_proj,3,32,8,128,19.015625,28.0625,67108864
+36,self_attn.q_proj,3,64,8,128,22.640625,26.03125,67108864
+36,self_attn.q_proj,3,128,8,128,26.28125,25.015625,67108864
+36,self_attn.q_proj,4,32,8,128,8.875,36.0625,67108864
+36,self_attn.q_proj,4,64,8,128,10.5703125,34.03125,67108864
+36,self_attn.q_proj,4,128,8,128,12.2734375,33.015625,67108864
+36,self_attn.q_proj,8,32,8,128,0.537109375,68.0625,67108864
+36,self_attn.q_proj,8,64,8,128,0.6396484375,66.03125,67108864
+36,self_attn.q_proj,8,128,8,128,0.73974609375,65.015625,67108864
+36,self_attn.v_proj,2,32,8,128,15.265625,2.5078125,8388608
+36,self_attn.v_proj,2,64,8,128,17.84375,2.25390625,8388608
+36,self_attn.v_proj,2,128,8,128,20.1875,2.126953125,8388608
+36,self_attn.v_proj,3,32,8,128,6.5234375,3.5078125,8388608
+36,self_attn.v_proj,3,64,8,128,7.63671875,3.25390625,8388608
+36,self_attn.v_proj,3,128,8,128,8.6484375,3.126953125,8388608
+36,self_attn.v_proj,4,32,8,128,3.044921875,4.5078125,8388608
+36,self_attn.v_proj,4,64,8,128,3.5625,4.25390625,8388608
+36,self_attn.v_proj,4,128,8,128,4.03125,4.126953125,8388608
+36,self_attn.v_proj,8,32,8,128,0.184326171875,8.5078125,8388608
+36,self_attn.v_proj,8,64,8,128,0.215576171875,8.25390625,8388608
+36,self_attn.v_proj,8,128,8,128,0.242919921875,8.126953125,8388608
+36,self_attn.k_proj,2,32,8,128,25.546875,2.5078125,8388608
+36,self_attn.k_proj,2,64,8,128,30.53125,2.25390625,8388608
+36,self_attn.k_proj,2,128,8,128,34.46875,2.126953125,8388608
+36,self_attn.k_proj,3,32,8,128,11.1328125,3.5078125,8388608
+36,self_attn.k_proj,3,64,8,128,13.6015625,3.25390625,8388608
+36,self_attn.k_proj,3,128,8,128,15.890625,3.126953125,8388608
+36,self_attn.k_proj,4,32,8,128,5.1953125,4.5078125,8388608
+36,self_attn.k_proj,4,64,8,128,6.34765625,4.25390625,8388608
+36,self_attn.k_proj,4,128,8,128,7.4296875,4.126953125,8388608
+36,self_attn.k_proj,8,32,8,128,0.314453125,8.5078125,8388608
+36,self_attn.k_proj,8,64,8,128,0.3837890625,8.25390625,8388608
+36,self_attn.k_proj,8,128,8,128,0.44775390625,8.126953125,8388608
+36,self_attn.o_proj,2,32,8,128,39.375,20.0625,67108864
+36,self_attn.o_proj,2,64,8,128,45.9375,18.03125,67108864
+36,self_attn.o_proj,2,128,8,128,51.9375,17.015625,67108864
+36,self_attn.o_proj,3,32,8,128,16.84375,28.0625,67108864
+36,self_attn.o_proj,3,64,8,128,19.6875,26.03125,67108864
+36,self_attn.o_proj,3,128,8,128,22.3125,25.015625,67108864
+36,self_attn.o_proj,4,32,8,128,7.86328125,36.0625,67108864
+36,self_attn.o_proj,4,64,8,128,9.1953125,34.03125,67108864
+36,self_attn.o_proj,4,128,8,128,10.421875,33.015625,67108864
+36,self_attn.o_proj,8,32,8,128,0.47607421875,68.0625,67108864
+36,self_attn.o_proj,8,64,8,128,0.55615234375,66.03125,67108864
+36,self_attn.o_proj,8,128,8,128,0.62841796875,65.015625,67108864
+36,mlp.gate_proj,2,32,8,128,84.125,70.21875,234881024
+36,mlp.gate_proj,2,64,8,128,98.4375,63.109375,234881024
+36,mlp.gate_proj,2,128,8,128,111.6875,59.5546875,234881024
+36,mlp.gate_proj,3,32,8,128,36.0,98.21875,234881024
+36,mlp.gate_proj,3,64,8,128,42.15625,91.109375,234881024
+36,mlp.gate_proj,3,128,8,128,47.8125,87.5546875,234881024
+36,mlp.gate_proj,4,32,8,128,16.78125,126.21875,234881024
+36,mlp.gate_proj,4,64,8,128,19.6875,119.109375,234881024
+36,mlp.gate_proj,4,128,8,128,22.3125,115.5546875,234881024
+36,mlp.gate_proj,8,32,8,128,1.0205078125,238.21875,234881024
+36,mlp.gate_proj,8,64,8,128,1.197265625,231.109375,234881024
+36,mlp.gate_proj,8,128,8,128,1.3544921875,227.5546875,234881024
+36,mlp.up_proj,2,32,8,128,81.125,70.21875,234881024
+36,mlp.up_proj,2,64,8,128,94.375,63.109375,234881024
+36,mlp.up_proj,2,128,8,128,106.5,59.5546875,234881024
+36,mlp.up_proj,3,32,8,128,34.65625,98.21875,234881024
+36,mlp.up_proj,3,64,8,128,40.4375,91.109375,234881024
+36,mlp.up_proj,3,128,8,128,45.5625,87.5546875,234881024
+36,mlp.up_proj,4,32,8,128,16.171875,126.21875,234881024
+36,mlp.up_proj,4,64,8,128,18.859375,119.109375,234881024
+36,mlp.up_proj,4,128,8,128,21.265625,115.5546875,234881024
+36,mlp.up_proj,8,32,8,128,0.98291015625,238.21875,234881024
+36,mlp.up_proj,8,64,8,128,1.1474609375,231.109375,234881024
+36,mlp.up_proj,8,128,8,128,1.2900390625,227.5546875,234881024
+36,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024
+36,mlp.down_proj,2,64,8,128,94.625,63.109375,234881024
+36,mlp.down_proj,2,128,8,128,106.6875,59.5546875,234881024
+36,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024
+36,mlp.down_proj,3,64,8,128,40.5,91.109375,234881024
+36,mlp.down_proj,3,128,8,128,45.625,87.5546875,234881024
+36,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024
+36,mlp.down_proj,4,64,8,128,18.90625,119.109375,234881024
+36,mlp.down_proj,4,128,8,128,21.296875,115.5546875,234881024
+36,mlp.down_proj,8,32,8,128,0.986328125,238.21875,234881024
+36,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024
+36,mlp.down_proj,8,128,8,128,1.2919921875,227.5546875,234881024
+37,self_attn.q_proj,2,32,8,128,44.90625,20.0625,67108864
+37,self_attn.q_proj,2,64,8,128,52.96875,18.03125,67108864
+37,self_attn.q_proj,2,128,8,128,60.71875,17.015625,67108864
+37,self_attn.q_proj,3,32,8,128,19.234375,28.0625,67108864
+37,self_attn.q_proj,3,64,8,128,22.75,26.03125,67108864
+37,self_attn.q_proj,3,128,8,128,26.203125,25.015625,67108864
+37,self_attn.q_proj,4,32,8,128,8.9765625,36.0625,67108864
+37,self_attn.q_proj,4,64,8,128,10.6171875,34.03125,67108864
+37,self_attn.q_proj,4,128,8,128,12.234375,33.015625,67108864
+37,self_attn.q_proj,8,32,8,128,0.54345703125,68.0625,67108864
+37,self_attn.q_proj,8,64,8,128,0.642578125,66.03125,67108864
+37,self_attn.q_proj,8,128,8,128,0.7373046875,65.015625,67108864
+37,self_attn.v_proj,2,32,8,128,15.421875,2.5078125,8388608
+37,self_attn.v_proj,2,64,8,128,18.03125,2.25390625,8388608
+37,self_attn.v_proj,2,128,8,128,20.390625,2.126953125,8388608
+37,self_attn.v_proj,3,32,8,128,6.59375,3.5078125,8388608
+37,self_attn.v_proj,3,64,8,128,7.7109375,3.25390625,8388608
+37,self_attn.v_proj,3,128,8,128,8.71875,3.126953125,8388608
+37,self_attn.v_proj,4,32,8,128,3.076171875,4.5078125,8388608
+37,self_attn.v_proj,4,64,8,128,3.59765625,4.25390625,8388608
+37,self_attn.v_proj,4,128,8,128,4.06640625,4.126953125,8388608
+37,self_attn.v_proj,8,32,8,128,0.186279296875,8.5078125,8388608
+37,self_attn.v_proj,8,64,8,128,0.2176513671875,8.25390625,8388608
+37,self_attn.v_proj,8,128,8,128,0.2452392578125,8.126953125,8388608
+37,self_attn.k_proj,2,32,8,128,25.671875,2.5078125,8388608
+37,self_attn.k_proj,2,64,8,128,30.65625,2.25390625,8388608
+37,self_attn.k_proj,2,128,8,128,34.75,2.126953125,8388608
+37,self_attn.k_proj,3,32,8,128,11.1171875,3.5078125,8388608
+37,self_attn.k_proj,3,64,8,128,13.4765625,3.25390625,8388608
+37,self_attn.k_proj,3,128,8,128,15.703125,3.126953125,8388608
+37,self_attn.k_proj,4,32,8,128,5.1875,4.5078125,8388608
+37,self_attn.k_proj,4,64,8,128,6.296875,4.25390625,8388608
+37,self_attn.k_proj,4,128,8,128,7.34375,4.126953125,8388608
+37,self_attn.k_proj,8,32,8,128,0.314208984375,8.5078125,8388608
+37,self_attn.k_proj,8,64,8,128,0.380615234375,8.25390625,8388608
+37,self_attn.k_proj,8,128,8,128,0.4423828125,8.126953125,8388608
+37,self_attn.o_proj,2,32,8,128,40.15625,20.0625,67108864
+37,self_attn.o_proj,2,64,8,128,46.84375,18.03125,67108864
+37,self_attn.o_proj,2,128,8,128,52.9375,17.015625,67108864
+37,self_attn.o_proj,3,32,8,128,17.171875,28.0625,67108864
+37,self_attn.o_proj,3,64,8,128,20.0625,26.03125,67108864
+37,self_attn.o_proj,3,128,8,128,22.703125,25.015625,67108864
+37,self_attn.o_proj,4,32,8,128,8.015625,36.0625,67108864
+37,self_attn.o_proj,4,64,8,128,9.359375,34.03125,67108864
+37,self_attn.o_proj,4,128,8,128,10.6015625,33.015625,67108864
+37,self_attn.o_proj,8,32,8,128,0.485107421875,68.0625,67108864
+37,self_attn.o_proj,8,64,8,128,0.56640625,66.03125,67108864
+37,self_attn.o_proj,8,128,8,128,0.63916015625,65.015625,67108864
+37,mlp.gate_proj,2,32,8,128,84.25,70.21875,234881024
+37,mlp.gate_proj,2,64,8,128,98.5625,63.109375,234881024
+37,mlp.gate_proj,2,128,8,128,111.875,59.5546875,234881024
+37,mlp.gate_proj,3,32,8,128,36.03125,98.21875,234881024
+37,mlp.gate_proj,3,64,8,128,42.1875,91.109375,234881024
+37,mlp.gate_proj,3,128,8,128,47.90625,87.5546875,234881024
+37,mlp.gate_proj,4,32,8,128,16.796875,126.21875,234881024
+37,mlp.gate_proj,4,64,8,128,19.703125,119.109375,234881024
+37,mlp.gate_proj,4,128,8,128,22.359375,115.5546875,234881024
+37,mlp.gate_proj,8,32,8,128,1.0224609375,238.21875,234881024
+37,mlp.gate_proj,8,64,8,128,1.19921875,231.109375,234881024
+37,mlp.gate_proj,8,128,8,128,1.3564453125,227.5546875,234881024
+37,mlp.up_proj,2,32,8,128,81.125,70.21875,234881024
+37,mlp.up_proj,2,64,8,128,94.4375,63.109375,234881024
+37,mlp.up_proj,2,128,8,128,106.5625,59.5546875,234881024
+37,mlp.up_proj,3,32,8,128,34.6875,98.21875,234881024
+37,mlp.up_proj,3,64,8,128,40.4375,91.109375,234881024
+37,mlp.up_proj,3,128,8,128,45.59375,87.5546875,234881024
+37,mlp.up_proj,4,32,8,128,16.171875,126.21875,234881024
+37,mlp.up_proj,4,64,8,128,18.875,119.109375,234881024
+37,mlp.up_proj,4,128,8,128,21.28125,115.5546875,234881024
+37,mlp.up_proj,8,32,8,128,0.98388671875,238.21875,234881024
+37,mlp.up_proj,8,64,8,128,1.1484375,231.109375,234881024
+37,mlp.up_proj,8,128,8,128,1.291015625,227.5546875,234881024
+37,mlp.down_proj,2,32,8,128,81.4375,70.21875,234881024
+37,mlp.down_proj,2,64,8,128,94.6875,63.109375,234881024
+37,mlp.down_proj,2,128,8,128,106.8125,59.5546875,234881024
+37,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024
+37,mlp.down_proj,3,64,8,128,40.53125,91.109375,234881024
+37,mlp.down_proj,3,128,8,128,45.65625,87.5546875,234881024
+37,mlp.down_proj,4,32,8,128,16.234375,126.21875,234881024
+37,mlp.down_proj,4,64,8,128,18.921875,119.109375,234881024
+37,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024
+37,mlp.down_proj,8,32,8,128,0.98681640625,238.21875,234881024
+37,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024
+37,mlp.down_proj,8,128,8,128,1.29296875,227.5546875,234881024
+38,self_attn.q_proj,2,32,8,128,46.5625,20.0625,67108864
+38,self_attn.q_proj,2,64,8,128,55.25,18.03125,67108864
+38,self_attn.q_proj,2,128,8,128,63.65625,17.015625,67108864
+38,self_attn.q_proj,3,32,8,128,19.96875,28.0625,67108864
+38,self_attn.q_proj,3,64,8,128,23.765625,26.03125,67108864
+38,self_attn.q_proj,3,128,8,128,27.59375,25.015625,67108864
+38,self_attn.q_proj,4,32,8,128,9.3125,36.0625,67108864
+38,self_attn.q_proj,4,64,8,128,11.0859375,34.03125,67108864
+38,self_attn.q_proj,4,128,8,128,12.8828125,33.015625,67108864
+38,self_attn.q_proj,8,32,8,128,0.56396484375,68.0625,67108864
+38,self_attn.q_proj,8,64,8,128,0.6708984375,66.03125,67108864
+38,self_attn.q_proj,8,128,8,128,0.7763671875,65.015625,67108864
+38,self_attn.v_proj,2,32,8,128,14.828125,2.5078125,8388608
+38,self_attn.v_proj,2,64,8,128,17.34375,2.25390625,8388608
+38,self_attn.v_proj,2,128,8,128,19.625,2.126953125,8388608
+38,self_attn.v_proj,3,32,8,128,6.33984375,3.5078125,8388608
+38,self_attn.v_proj,3,64,8,128,7.4140625,3.25390625,8388608
+38,self_attn.v_proj,3,128,8,128,8.390625,3.126953125,8388608
+38,self_attn.v_proj,4,32,8,128,2.95703125,4.5078125,8388608
+38,self_attn.v_proj,4,64,8,128,3.4609375,4.25390625,8388608
+38,self_attn.v_proj,4,128,8,128,3.916015625,4.126953125,8388608
+38,self_attn.v_proj,8,32,8,128,0.1790771484375,8.5078125,8388608
+38,self_attn.v_proj,8,64,8,128,0.20947265625,8.25390625,8388608
+38,self_attn.v_proj,8,128,8,128,0.236083984375,8.126953125,8388608
+38,self_attn.k_proj,2,32,8,128,25.828125,2.5078125,8388608
+38,self_attn.k_proj,2,64,8,128,30.90625,2.25390625,8388608
+38,self_attn.k_proj,2,128,8,128,35.0,2.126953125,8388608
+38,self_attn.k_proj,3,32,8,128,11.1796875,3.5078125,8388608
+38,self_attn.k_proj,3,64,8,128,13.625,3.25390625,8388608
+38,self_attn.k_proj,3,128,8,128,15.890625,3.126953125,8388608
+38,self_attn.k_proj,4,32,8,128,5.21484375,4.5078125,8388608
+38,self_attn.k_proj,4,64,8,128,6.359375,4.25390625,8388608
+38,self_attn.k_proj,4,128,8,128,7.42578125,4.126953125,8388608
+38,self_attn.k_proj,8,32,8,128,0.315673828125,8.5078125,8388608
+38,self_attn.k_proj,8,64,8,128,0.384521484375,8.25390625,8388608
+38,self_attn.k_proj,8,128,8,128,0.447509765625,8.126953125,8388608
+38,self_attn.o_proj,2,32,8,128,40.15625,20.0625,67108864
+38,self_attn.o_proj,2,64,8,128,46.84375,18.03125,67108864
+38,self_attn.o_proj,2,128,8,128,52.96875,17.015625,67108864
+38,self_attn.o_proj,3,32,8,128,17.171875,28.0625,67108864
+38,self_attn.o_proj,3,64,8,128,20.078125,26.03125,67108864
+38,self_attn.o_proj,3,128,8,128,22.78125,25.015625,67108864
+38,self_attn.o_proj,4,32,8,128,8.015625,36.0625,67108864
+38,self_attn.o_proj,4,64,8,128,9.375,34.03125,67108864
+38,self_attn.o_proj,4,128,8,128,10.6328125,33.015625,67108864
+38,self_attn.o_proj,8,32,8,128,0.4853515625,68.0625,67108864
+38,self_attn.o_proj,8,64,8,128,0.56689453125,66.03125,67108864
+38,self_attn.o_proj,8,128,8,128,0.64111328125,65.015625,67108864
+38,mlp.gate_proj,2,32,8,128,84.375,70.21875,234881024
+38,mlp.gate_proj,2,64,8,128,98.8125,63.109375,234881024
+38,mlp.gate_proj,2,128,8,128,112.1875,59.5546875,234881024
+38,mlp.gate_proj,3,32,8,128,36.09375,98.21875,234881024
+38,mlp.gate_proj,3,64,8,128,42.28125,91.109375,234881024
+38,mlp.gate_proj,3,128,8,128,48.03125,87.5546875,234881024
+38,mlp.gate_proj,4,32,8,128,16.828125,126.21875,234881024
+38,mlp.gate_proj,4,64,8,128,19.75,119.109375,234881024
+38,mlp.gate_proj,4,128,8,128,22.421875,115.5546875,234881024
+38,mlp.gate_proj,8,32,8,128,1.0234375,238.21875,234881024
+38,mlp.gate_proj,8,64,8,128,1.201171875,231.109375,234881024
+38,mlp.gate_proj,8,128,8,128,1.3603515625,227.5546875,234881024
+38,mlp.up_proj,2,32,8,128,81.25,70.21875,234881024
+38,mlp.up_proj,2,64,8,128,94.5625,63.109375,234881024
+38,mlp.up_proj,2,128,8,128,106.6875,59.5546875,234881024
+38,mlp.up_proj,3,32,8,128,34.71875,98.21875,234881024
+38,mlp.up_proj,3,64,8,128,40.46875,91.109375,234881024
+38,mlp.up_proj,3,128,8,128,45.65625,87.5546875,234881024
+38,mlp.up_proj,4,32,8,128,16.1875,126.21875,234881024
+38,mlp.up_proj,4,64,8,128,18.890625,119.109375,234881024
+38,mlp.up_proj,4,128,8,128,21.3125,115.5546875,234881024
+38,mlp.up_proj,8,32,8,128,0.98486328125,238.21875,234881024
+38,mlp.up_proj,8,64,8,128,1.1494140625,231.109375,234881024
+38,mlp.up_proj,8,128,8,128,1.29296875,227.5546875,234881024
+38,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024
+38,mlp.down_proj,2,64,8,128,94.8125,63.109375,234881024
+38,mlp.down_proj,2,128,8,128,106.875,59.5546875,234881024
+38,mlp.down_proj,3,32,8,128,34.84375,98.21875,234881024
+38,mlp.down_proj,3,64,8,128,40.59375,91.109375,234881024
+38,mlp.down_proj,3,128,8,128,45.71875,87.5546875,234881024
+38,mlp.down_proj,4,32,8,128,16.25,126.21875,234881024
+38,mlp.down_proj,4,64,8,128,18.953125,119.109375,234881024
+38,mlp.down_proj,4,128,8,128,21.34375,115.5546875,234881024
+38,mlp.down_proj,8,32,8,128,0.98828125,238.21875,234881024
+38,mlp.down_proj,8,64,8,128,1.15234375,231.109375,234881024
+38,mlp.down_proj,8,128,8,128,1.294921875,227.5546875,234881024
+39,self_attn.q_proj,2,32,8,128,48.0,20.0625,67108864
+39,self_attn.q_proj,2,64,8,128,56.46875,18.03125,67108864
+39,self_attn.q_proj,2,128,8,128,65.0,17.015625,67108864
+39,self_attn.q_proj,3,32,8,128,20.546875,28.0625,67108864
+39,self_attn.q_proj,3,64,8,128,24.203125,26.03125,67108864
+39,self_attn.q_proj,3,128,8,128,27.953125,25.015625,67108864
+39,self_attn.q_proj,4,32,8,128,9.5859375,36.0625,67108864
+39,self_attn.q_proj,4,64,8,128,11.2890625,34.03125,67108864
+39,self_attn.q_proj,4,128,8,128,13.0390625,33.015625,67108864
+39,self_attn.q_proj,8,32,8,128,0.58056640625,68.0625,67108864
+39,self_attn.q_proj,8,64,8,128,0.68310546875,66.03125,67108864
+39,self_attn.q_proj,8,128,8,128,0.7861328125,65.015625,67108864
+39,self_attn.v_proj,2,32,8,128,15.125,2.5078125,8388608
+39,self_attn.v_proj,2,64,8,128,17.6875,2.25390625,8388608
+39,self_attn.v_proj,2,128,8,128,20.0,2.126953125,8388608
+39,self_attn.v_proj,3,32,8,128,6.46875,3.5078125,8388608
+39,self_attn.v_proj,3,64,8,128,7.56640625,3.25390625,8388608
+39,self_attn.v_proj,3,128,8,128,8.5625,3.126953125,8388608
+39,self_attn.v_proj,4,32,8,128,3.017578125,4.5078125,8388608
+39,self_attn.v_proj,4,64,8,128,3.53125,4.25390625,8388608
+39,self_attn.v_proj,4,128,8,128,3.99609375,4.126953125,8388608
+39,self_attn.v_proj,8,32,8,128,0.1827392578125,8.5078125,8388608
+39,self_attn.v_proj,8,64,8,128,0.213623046875,8.25390625,8388608
+39,self_attn.v_proj,8,128,8,128,0.240966796875,8.126953125,8388608
+39,self_attn.k_proj,2,32,8,128,25.8125,2.5078125,8388608
+39,self_attn.k_proj,2,64,8,128,30.96875,2.25390625,8388608
+39,self_attn.k_proj,2,128,8,128,35.375,2.126953125,8388608
+39,self_attn.k_proj,3,32,8,128,11.09375,3.5078125,8388608
+39,self_attn.k_proj,3,64,8,128,13.390625,3.25390625,8388608
+39,self_attn.k_proj,3,128,8,128,15.5234375,3.126953125,8388608
+39,self_attn.k_proj,4,32,8,128,5.17578125,4.5078125,8388608
+39,self_attn.k_proj,4,64,8,128,6.25,4.25390625,8388608
+39,self_attn.k_proj,4,128,8,128,7.25,4.126953125,8388608
+39,self_attn.k_proj,8,32,8,128,0.313232421875,8.5078125,8388608
+39,self_attn.k_proj,8,64,8,128,0.3779296875,8.25390625,8388608
+39,self_attn.k_proj,8,128,8,128,0.436767578125,8.126953125,8388608
+39,self_attn.o_proj,2,32,8,128,41.1875,20.0625,67108864
+39,self_attn.o_proj,2,64,8,128,48.0625,18.03125,67108864
+39,self_attn.o_proj,2,128,8,128,54.34375,17.015625,67108864
+39,self_attn.o_proj,3,32,8,128,17.625,28.0625,67108864
+39,self_attn.o_proj,3,64,8,128,20.59375,26.03125,67108864
+39,self_attn.o_proj,3,128,8,128,23.359375,25.015625,67108864
+39,self_attn.o_proj,4,32,8,128,8.21875,36.0625,67108864
+39,self_attn.o_proj,4,64,8,128,9.6171875,34.03125,67108864
+39,self_attn.o_proj,4,128,8,128,10.90625,33.015625,67108864
+39,self_attn.o_proj,8,32,8,128,0.497802734375,68.0625,67108864
+39,self_attn.o_proj,8,64,8,128,0.58203125,66.03125,67108864
+39,self_attn.o_proj,8,128,8,128,0.6572265625,65.015625,67108864
+39,mlp.gate_proj,2,32,8,128,84.625,70.21875,234881024
+39,mlp.gate_proj,2,64,8,128,99.125,63.109375,234881024
+39,mlp.gate_proj,2,128,8,128,112.6875,59.5546875,234881024
+39,mlp.gate_proj,3,32,8,128,36.1875,98.21875,234881024
+39,mlp.gate_proj,3,64,8,128,42.4375,91.109375,234881024
+39,mlp.gate_proj,3,128,8,128,48.28125,87.5546875,234881024
+39,mlp.gate_proj,4,32,8,128,16.875,126.21875,234881024
+39,mlp.gate_proj,4,64,8,128,19.8125,119.109375,234881024
+39,mlp.gate_proj,4,128,8,128,22.53125,115.5546875,234881024
+39,mlp.gate_proj,8,32,8,128,1.0263671875,238.21875,234881024
+39,mlp.gate_proj,8,64,8,128,1.2060546875,231.109375,234881024
+39,mlp.gate_proj,8,128,8,128,1.3671875,227.5546875,234881024
+39,mlp.up_proj,2,32,8,128,81.4375,70.21875,234881024
+39,mlp.up_proj,2,64,8,128,94.875,63.109375,234881024
+39,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024
+39,mlp.up_proj,3,32,8,128,34.8125,98.21875,234881024
+39,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024
+39,mlp.up_proj,3,128,8,128,45.8125,87.5546875,234881024
+39,mlp.up_proj,4,32,8,128,16.234375,126.21875,234881024
+39,mlp.up_proj,4,64,8,128,18.953125,119.109375,234881024
+39,mlp.up_proj,4,128,8,128,21.375,115.5546875,234881024
+39,mlp.up_proj,8,32,8,128,0.98779296875,238.21875,234881024
+39,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024
+39,mlp.up_proj,8,128,8,128,1.296875,227.5546875,234881024
+39,mlp.down_proj,2,32,8,128,81.8125,70.21875,234881024
+39,mlp.down_proj,2,64,8,128,95.25,63.109375,234881024
+39,mlp.down_proj,2,128,8,128,107.375,59.5546875,234881024
+39,mlp.down_proj,3,32,8,128,34.96875,98.21875,234881024
+39,mlp.down_proj,3,64,8,128,40.75,91.109375,234881024
+39,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024
+39,mlp.down_proj,4,32,8,128,16.3125,126.21875,234881024
+39,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024
+39,mlp.down_proj,4,128,8,128,21.453125,115.5546875,234881024
+39,mlp.down_proj,8,32,8,128,0.9921875,238.21875,234881024
+39,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024
+39,mlp.down_proj,8,128,8,128,1.3017578125,227.5546875,234881024
+40,self_attn.q_proj,2,32,8,128,47.28125,20.0625,67108864
+40,self_attn.q_proj,2,64,8,128,55.96875,18.03125,67108864
+40,self_attn.q_proj,2,128,8,128,64.6875,17.015625,67108864
+40,self_attn.q_proj,3,32,8,128,20.25,28.0625,67108864
+40,self_attn.q_proj,3,64,8,128,24.015625,26.03125,67108864
+40,self_attn.q_proj,3,128,8,128,27.875,25.015625,67108864
+40,self_attn.q_proj,4,32,8,128,9.4453125,36.0625,67108864
+40,self_attn.q_proj,4,64,8,128,11.203125,34.03125,67108864
+40,self_attn.q_proj,4,128,8,128,13.0078125,33.015625,67108864
+40,self_attn.q_proj,8,32,8,128,0.572265625,68.0625,67108864
+40,self_attn.q_proj,8,64,8,128,0.677734375,66.03125,67108864
+40,self_attn.q_proj,8,128,8,128,0.7841796875,65.015625,67108864
+40,self_attn.v_proj,2,32,8,128,16.09375,2.5078125,8388608
+40,self_attn.v_proj,2,64,8,128,18.828125,2.25390625,8388608
+40,self_attn.v_proj,2,128,8,128,21.3125,2.126953125,8388608
+40,self_attn.v_proj,3,32,8,128,6.8828125,3.5078125,8388608
+40,self_attn.v_proj,3,64,8,128,8.0546875,3.25390625,8388608
+40,self_attn.v_proj,3,128,8,128,9.1171875,3.126953125,8388608
+40,self_attn.v_proj,4,32,8,128,3.2109375,4.5078125,8388608
+40,self_attn.v_proj,4,64,8,128,3.7578125,4.25390625,8388608
+40,self_attn.v_proj,4,128,8,128,4.25390625,4.126953125,8388608
+40,self_attn.v_proj,8,32,8,128,0.1944580078125,8.5078125,8388608
+40,self_attn.v_proj,8,64,8,128,0.2274169921875,8.25390625,8388608
+40,self_attn.v_proj,8,128,8,128,0.25634765625,8.126953125,8388608
+40,self_attn.k_proj,2,32,8,128,25.421875,2.5078125,8388608
+40,self_attn.k_proj,2,64,8,128,30.59375,2.25390625,8388608
+40,self_attn.k_proj,2,128,8,128,34.875,2.126953125,8388608
+40,self_attn.k_proj,3,32,8,128,10.9453125,3.5078125,8388608
+40,self_attn.k_proj,3,64,8,128,13.3046875,3.25390625,8388608
+40,self_attn.k_proj,3,128,8,128,15.484375,3.126953125,8388608
+40,self_attn.k_proj,4,32,8,128,5.109375,4.5078125,8388608
+40,self_attn.k_proj,4,64,8,128,6.2109375,4.25390625,8388608
+40,self_attn.k_proj,4,128,8,128,7.23046875,4.126953125,8388608
+40,self_attn.k_proj,8,32,8,128,0.309326171875,8.5078125,8388608
+40,self_attn.k_proj,8,64,8,128,0.375732421875,8.25390625,8388608
+40,self_attn.k_proj,8,128,8,128,0.435546875,8.126953125,8388608
+40,self_attn.o_proj,2,32,8,128,43.09375,20.0625,67108864
+40,self_attn.o_proj,2,64,8,128,50.1875,18.03125,67108864
+40,self_attn.o_proj,2,128,8,128,56.59375,17.015625,67108864
+40,self_attn.o_proj,3,32,8,128,18.421875,28.0625,67108864
+40,self_attn.o_proj,3,64,8,128,21.484375,26.03125,67108864
+40,self_attn.o_proj,3,128,8,128,24.25,25.015625,67108864
+40,self_attn.o_proj,4,32,8,128,8.59375,36.0625,67108864
+40,self_attn.o_proj,4,64,8,128,10.0234375,34.03125,67108864
+40,self_attn.o_proj,4,128,8,128,11.3203125,33.015625,67108864
+40,self_attn.o_proj,8,32,8,128,0.5205078125,68.0625,67108864
+40,self_attn.o_proj,8,64,8,128,0.6064453125,66.03125,67108864
+40,self_attn.o_proj,8,128,8,128,0.68212890625,65.015625,67108864
+40,mlp.gate_proj,2,32,8,128,84.5,70.21875,234881024
+40,mlp.gate_proj,2,64,8,128,98.8125,63.109375,234881024
+40,mlp.gate_proj,2,128,8,128,112.125,59.5546875,234881024
+40,mlp.gate_proj,3,32,8,128,36.125,98.21875,234881024
+40,mlp.gate_proj,3,64,8,128,42.3125,91.109375,234881024
+40,mlp.gate_proj,3,128,8,128,48.03125,87.5546875,234881024
+40,mlp.gate_proj,4,32,8,128,16.84375,126.21875,234881024
+40,mlp.gate_proj,4,64,8,128,19.75,119.109375,234881024
+40,mlp.gate_proj,4,128,8,128,22.421875,115.5546875,234881024
+40,mlp.gate_proj,8,32,8,128,1.025390625,238.21875,234881024
+40,mlp.gate_proj,8,64,8,128,1.2021484375,231.109375,234881024
+40,mlp.gate_proj,8,128,8,128,1.359375,227.5546875,234881024
+40,mlp.up_proj,2,32,8,128,81.4375,70.21875,234881024
+40,mlp.up_proj,2,64,8,128,94.8125,63.109375,234881024
+40,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024
+40,mlp.up_proj,3,32,8,128,34.8125,98.21875,234881024
+40,mlp.up_proj,3,64,8,128,40.59375,91.109375,234881024
+40,mlp.up_proj,3,128,8,128,45.75,87.5546875,234881024
+40,mlp.up_proj,4,32,8,128,16.234375,126.21875,234881024
+40,mlp.up_proj,4,64,8,128,18.953125,119.109375,234881024
+40,mlp.up_proj,4,128,8,128,21.359375,115.5546875,234881024
+40,mlp.up_proj,8,32,8,128,0.9873046875,238.21875,234881024
+40,mlp.up_proj,8,64,8,128,1.15234375,231.109375,234881024
+40,mlp.up_proj,8,128,8,128,1.2958984375,227.5546875,234881024
+40,mlp.down_proj,2,32,8,128,82.0,70.21875,234881024
+40,mlp.down_proj,2,64,8,128,95.4375,63.109375,234881024
+40,mlp.down_proj,2,128,8,128,107.625,59.5546875,234881024
+40,mlp.down_proj,3,32,8,128,35.0625,98.21875,234881024
+40,mlp.down_proj,3,64,8,128,40.84375,91.109375,234881024
+40,mlp.down_proj,3,128,8,128,46.03125,87.5546875,234881024
+40,mlp.down_proj,4,32,8,128,16.34375,126.21875,234881024
+40,mlp.down_proj,4,64,8,128,19.078125,119.109375,234881024
+40,mlp.down_proj,4,128,8,128,21.5,115.5546875,234881024
+40,mlp.down_proj,8,32,8,128,0.99462890625,238.21875,234881024
+40,mlp.down_proj,8,64,8,128,1.16015625,231.109375,234881024
+40,mlp.down_proj,8,128,8,128,1.3037109375,227.5546875,234881024
+41,self_attn.q_proj,2,32,8,128,45.75,20.0625,67108864
+41,self_attn.q_proj,2,64,8,128,54.0,18.03125,67108864
+41,self_attn.q_proj,2,128,8,128,62.34375,17.015625,67108864
+41,self_attn.q_proj,3,32,8,128,19.59375,28.0625,67108864
+41,self_attn.q_proj,3,64,8,128,23.1875,26.03125,67108864
+41,self_attn.q_proj,3,128,8,128,26.9375,25.015625,67108864
+41,self_attn.q_proj,4,32,8,128,9.1484375,36.0625,67108864
+41,self_attn.q_proj,4,64,8,128,10.828125,34.03125,67108864
+41,self_attn.q_proj,4,128,8,128,12.5703125,33.015625,67108864
+41,self_attn.q_proj,8,32,8,128,0.5537109375,68.0625,67108864
+41,self_attn.q_proj,8,64,8,128,0.6552734375,66.03125,67108864
+41,self_attn.q_proj,8,128,8,128,0.7578125,65.015625,67108864
+41,self_attn.v_proj,2,32,8,128,15.5546875,2.5078125,8388608
+41,self_attn.v_proj,2,64,8,128,18.15625,2.25390625,8388608
+41,self_attn.v_proj,2,128,8,128,20.515625,2.126953125,8388608
+41,self_attn.v_proj,3,32,8,128,6.65234375,3.5078125,8388608
+41,self_attn.v_proj,3,64,8,128,7.765625,3.25390625,8388608
+41,self_attn.v_proj,3,128,8,128,8.78125,3.126953125,8388608
+41,self_attn.v_proj,4,32,8,128,3.1015625,4.5078125,8388608
+41,self_attn.v_proj,4,64,8,128,3.625,4.25390625,8388608
+41,self_attn.v_proj,4,128,8,128,4.09765625,4.126953125,8388608
+41,self_attn.v_proj,8,32,8,128,0.1878662109375,8.5078125,8388608
+41,self_attn.v_proj,8,64,8,128,0.2193603515625,8.25390625,8388608
+41,self_attn.v_proj,8,128,8,128,0.2469482421875,8.126953125,8388608
+41,self_attn.k_proj,2,32,8,128,26.21875,2.5078125,8388608
+41,self_attn.k_proj,2,64,8,128,31.71875,2.25390625,8388608
+41,self_attn.k_proj,2,128,8,128,36.34375,2.126953125,8388608
+41,self_attn.k_proj,3,32,8,128,11.34375,3.5078125,8388608
+41,self_attn.k_proj,3,64,8,128,13.953125,3.25390625,8388608
+41,self_attn.k_proj,3,128,8,128,16.453125,3.126953125,8388608
+41,self_attn.k_proj,4,32,8,128,5.2890625,4.5078125,8388608
+41,self_attn.k_proj,4,64,8,128,6.51171875,4.25390625,8388608
+41,self_attn.k_proj,4,128,8,128,7.6875,4.126953125,8388608
+41,self_attn.k_proj,8,32,8,128,0.3203125,8.5078125,8388608
+41,self_attn.k_proj,8,64,8,128,0.393798828125,8.25390625,8388608
+41,self_attn.k_proj,8,128,8,128,0.463134765625,8.126953125,8388608
+41,self_attn.o_proj,2,32,8,128,41.625,20.0625,67108864
+41,self_attn.o_proj,2,64,8,128,48.53125,18.03125,67108864
+41,self_attn.o_proj,2,128,8,128,54.8125,17.015625,67108864
+41,self_attn.o_proj,3,32,8,128,17.8125,28.0625,67108864
+41,self_attn.o_proj,3,64,8,128,20.796875,26.03125,67108864
+41,self_attn.o_proj,3,128,8,128,23.53125,25.015625,67108864
+41,self_attn.o_proj,4,32,8,128,8.3203125,36.0625,67108864
+41,self_attn.o_proj,4,64,8,128,9.734375,34.03125,67108864
+41,self_attn.o_proj,4,128,8,128,11.03125,33.015625,67108864
+41,self_attn.o_proj,8,32,8,128,0.5048828125,68.0625,67108864
+41,self_attn.o_proj,8,64,8,128,0.5908203125,66.03125,67108864
+41,self_attn.o_proj,8,128,8,128,0.66845703125,65.015625,67108864
+41,mlp.gate_proj,2,32,8,128,85.125,70.21875,234881024
+41,mlp.gate_proj,2,64,8,128,99.625,63.109375,234881024
+41,mlp.gate_proj,2,128,8,128,112.9375,59.5546875,234881024
+41,mlp.gate_proj,3,32,8,128,36.4375,98.21875,234881024
+41,mlp.gate_proj,3,64,8,128,42.625,91.109375,234881024
+41,mlp.gate_proj,3,128,8,128,48.375,87.5546875,234881024
+41,mlp.gate_proj,4,32,8,128,16.984375,126.21875,234881024
+41,mlp.gate_proj,4,64,8,128,19.90625,119.109375,234881024
+41,mlp.gate_proj,4,128,8,128,22.578125,115.5546875,234881024
+41,mlp.gate_proj,8,32,8,128,1.033203125,238.21875,234881024
+41,mlp.gate_proj,8,64,8,128,1.2109375,231.109375,234881024
+41,mlp.gate_proj,8,128,8,128,1.369140625,227.5546875,234881024
+41,mlp.up_proj,2,32,8,128,80.875,70.21875,234881024
+41,mlp.up_proj,2,64,8,128,94.1875,63.109375,234881024
+41,mlp.up_proj,2,128,8,128,106.25,59.5546875,234881024
+41,mlp.up_proj,3,32,8,128,34.59375,98.21875,234881024
+41,mlp.up_proj,3,64,8,128,40.3125,91.109375,234881024
+41,mlp.up_proj,3,128,8,128,45.4375,87.5546875,234881024
+41,mlp.up_proj,4,32,8,128,16.125,126.21875,234881024
+41,mlp.up_proj,4,64,8,128,18.8125,119.109375,234881024
+41,mlp.up_proj,4,128,8,128,21.21875,115.5546875,234881024
+41,mlp.up_proj,8,32,8,128,0.98095703125,238.21875,234881024
+41,mlp.up_proj,8,64,8,128,1.14453125,231.109375,234881024
+41,mlp.up_proj,8,128,8,128,1.287109375,227.5546875,234881024
+41,mlp.down_proj,2,32,8,128,81.875,70.21875,234881024
+41,mlp.down_proj,2,64,8,128,95.25,63.109375,234881024
+41,mlp.down_proj,2,128,8,128,107.375,59.5546875,234881024
+41,mlp.down_proj,3,32,8,128,35.0,98.21875,234881024
+41,mlp.down_proj,3,64,8,128,40.75,91.109375,234881024
+41,mlp.down_proj,3,128,8,128,45.9375,87.5546875,234881024
+41,mlp.down_proj,4,32,8,128,16.3125,126.21875,234881024
+41,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024
+41,mlp.down_proj,4,128,8,128,21.4375,115.5546875,234881024
+41,mlp.down_proj,8,32,8,128,0.9921875,238.21875,234881024
+41,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024
+41,mlp.down_proj,8,128,8,128,1.30078125,227.5546875,234881024
+42,self_attn.q_proj,2,32,8,128,44.5,20.0625,67108864
+42,self_attn.q_proj,2,64,8,128,52.4375,18.03125,67108864
+42,self_attn.q_proj,2,128,8,128,60.5625,17.015625,67108864
+42,self_attn.q_proj,3,32,8,128,19.0625,28.0625,67108864
+42,self_attn.q_proj,3,64,8,128,22.484375,26.03125,67108864
+42,self_attn.q_proj,3,128,8,128,26.0625,25.015625,67108864
+42,self_attn.q_proj,4,32,8,128,8.890625,36.0625,67108864
+42,self_attn.q_proj,4,64,8,128,10.5,34.03125,67108864
+42,self_attn.q_proj,4,128,8,128,12.1640625,33.015625,67108864
+42,self_attn.q_proj,8,32,8,128,0.53857421875,68.0625,67108864
+42,self_attn.q_proj,8,64,8,128,0.63525390625,66.03125,67108864
+42,self_attn.q_proj,8,128,8,128,0.7333984375,65.015625,67108864
+42,self_attn.v_proj,2,32,8,128,16.25,2.5078125,8388608
+42,self_attn.v_proj,2,64,8,128,19.0625,2.25390625,8388608
+42,self_attn.v_proj,2,128,8,128,21.625,2.126953125,8388608
+42,self_attn.v_proj,3,32,8,128,6.94921875,3.5078125,8388608
+42,self_attn.v_proj,3,64,8,128,8.15625,3.25390625,8388608
+42,self_attn.v_proj,3,128,8,128,9.2578125,3.126953125,8388608
+42,self_attn.v_proj,4,32,8,128,3.2421875,4.5078125,8388608
+42,self_attn.v_proj,4,64,8,128,3.8046875,4.25390625,8388608
+42,self_attn.v_proj,4,128,8,128,4.3203125,4.126953125,8388608
+42,self_attn.v_proj,8,32,8,128,0.1964111328125,8.5078125,8388608
+42,self_attn.v_proj,8,64,8,128,0.230224609375,8.25390625,8388608
+42,self_attn.v_proj,8,128,8,128,0.26025390625,8.126953125,8388608
+42,self_attn.k_proj,2,32,8,128,25.078125,2.5078125,8388608
+42,self_attn.k_proj,2,64,8,128,30.296875,2.25390625,8388608
+42,self_attn.k_proj,2,128,8,128,34.59375,2.126953125,8388608
+42,self_attn.k_proj,3,32,8,128,10.875,3.5078125,8388608
+42,self_attn.k_proj,3,64,8,128,13.40625,3.25390625,8388608
+42,self_attn.k_proj,3,128,8,128,15.796875,3.126953125,8388608
+42,self_attn.k_proj,4,32,8,128,5.07421875,4.5078125,8388608
+42,self_attn.k_proj,4,64,8,128,6.2578125,4.25390625,8388608
+42,self_attn.k_proj,4,128,8,128,7.37890625,4.126953125,8388608
+42,self_attn.k_proj,8,32,8,128,0.30712890625,8.5078125,8388608
+42,self_attn.k_proj,8,64,8,128,0.37841796875,8.25390625,8388608
+42,self_attn.k_proj,8,128,8,128,0.44482421875,8.126953125,8388608
+42,self_attn.o_proj,2,32,8,128,41.90625,20.0625,67108864
+42,self_attn.o_proj,2,64,8,128,48.875,18.03125,67108864
+42,self_attn.o_proj,2,128,8,128,55.25,17.015625,67108864
+42,self_attn.o_proj,3,32,8,128,17.96875,28.0625,67108864
+42,self_attn.o_proj,3,64,8,128,21.03125,26.03125,67108864
+42,self_attn.o_proj,3,128,8,128,23.859375,25.015625,67108864
+42,self_attn.o_proj,4,32,8,128,8.3984375,36.0625,67108864
+42,self_attn.o_proj,4,64,8,128,9.8515625,34.03125,67108864
+42,self_attn.o_proj,4,128,8,128,11.2109375,33.015625,67108864
+42,self_attn.o_proj,8,32,8,128,0.50927734375,68.0625,67108864
+42,self_attn.o_proj,8,64,8,128,0.59716796875,66.03125,67108864
+42,self_attn.o_proj,8,128,8,128,0.67822265625,65.015625,67108864
+42,mlp.gate_proj,2,32,8,128,86.0,70.21875,234881024
+42,mlp.gate_proj,2,64,8,128,100.5625,63.109375,234881024
+42,mlp.gate_proj,2,128,8,128,113.875,59.5546875,234881024
+42,mlp.gate_proj,3,32,8,128,36.8125,98.21875,234881024
+42,mlp.gate_proj,3,64,8,128,43.03125,91.109375,234881024
+42,mlp.gate_proj,3,128,8,128,48.78125,87.5546875,234881024
+42,mlp.gate_proj,4,32,8,128,17.15625,126.21875,234881024
+42,mlp.gate_proj,4,64,8,128,20.09375,119.109375,234881024
+42,mlp.gate_proj,4,128,8,128,22.75,115.5546875,234881024
+42,mlp.gate_proj,8,32,8,128,1.0439453125,238.21875,234881024
+42,mlp.gate_proj,8,64,8,128,1.22265625,231.109375,234881024
+42,mlp.gate_proj,8,128,8,128,1.380859375,227.5546875,234881024
+42,mlp.up_proj,2,32,8,128,80.5625,70.21875,234881024
+42,mlp.up_proj,2,64,8,128,93.8125,63.109375,234881024
+42,mlp.up_proj,2,128,8,128,105.75,59.5546875,234881024
+42,mlp.up_proj,3,32,8,128,34.4375,98.21875,234881024
+42,mlp.up_proj,3,64,8,128,40.15625,91.109375,234881024
+42,mlp.up_proj,3,128,8,128,45.25,87.5546875,234881024
+42,mlp.up_proj,4,32,8,128,16.0625,126.21875,234881024
+42,mlp.up_proj,4,64,8,128,18.734375,119.109375,234881024
+42,mlp.up_proj,4,128,8,128,21.125,115.5546875,234881024
+42,mlp.up_proj,8,32,8,128,0.9765625,238.21875,234881024
+42,mlp.up_proj,8,64,8,128,1.1396484375,231.109375,234881024
+42,mlp.up_proj,8,128,8,128,1.28125,227.5546875,234881024
+42,mlp.down_proj,2,32,8,128,81.6875,70.21875,234881024
+42,mlp.down_proj,2,64,8,128,95.0625,63.109375,234881024
+42,mlp.down_proj,2,128,8,128,107.125,59.5546875,234881024
+42,mlp.down_proj,3,32,8,128,34.90625,98.21875,234881024
+42,mlp.down_proj,3,64,8,128,40.6875,91.109375,234881024
+42,mlp.down_proj,3,128,8,128,45.84375,87.5546875,234881024
+42,mlp.down_proj,4,32,8,128,16.28125,126.21875,234881024
+42,mlp.down_proj,4,64,8,128,18.984375,119.109375,234881024
+42,mlp.down_proj,4,128,8,128,21.390625,115.5546875,234881024
+42,mlp.down_proj,8,32,8,128,0.990234375,238.21875,234881024
+42,mlp.down_proj,8,64,8,128,1.1552734375,231.109375,234881024
+42,mlp.down_proj,8,128,8,128,1.2978515625,227.5546875,234881024
+43,self_attn.q_proj,2,32,8,128,44.625,20.0625,67108864
+43,self_attn.q_proj,2,64,8,128,53.125,18.03125,67108864
+43,self_attn.q_proj,2,128,8,128,61.53125,17.015625,67108864
+43,self_attn.q_proj,3,32,8,128,19.171875,28.0625,67108864
+43,self_attn.q_proj,3,64,8,128,22.953125,26.03125,67108864
+43,self_attn.q_proj,3,128,8,128,26.90625,25.015625,67108864
+43,self_attn.q_proj,4,32,8,128,8.9453125,36.0625,67108864
+43,self_attn.q_proj,4,64,8,128,10.7109375,34.03125,67108864
+43,self_attn.q_proj,4,128,8,128,12.5625,33.015625,67108864
+43,self_attn.q_proj,8,32,8,128,0.54150390625,68.0625,67108864
+43,self_attn.q_proj,8,64,8,128,0.6484375,66.03125,67108864
+43,self_attn.q_proj,8,128,8,128,0.75732421875,65.015625,67108864
+43,self_attn.v_proj,2,32,8,128,16.046875,2.5078125,8388608
+43,self_attn.v_proj,2,64,8,128,18.765625,2.25390625,8388608
+43,self_attn.v_proj,2,128,8,128,21.25,2.126953125,8388608
+43,self_attn.v_proj,3,32,8,128,6.86328125,3.5078125,8388608
+43,self_attn.v_proj,3,64,8,128,8.03125,3.25390625,8388608
+43,self_attn.v_proj,3,128,8,128,9.09375,3.126953125,8388608
+43,self_attn.v_proj,4,32,8,128,3.201171875,4.5078125,8388608
+43,self_attn.v_proj,4,64,8,128,3.748046875,4.25390625,8388608
+43,self_attn.v_proj,4,128,8,128,4.24609375,4.126953125,8388608
+43,self_attn.v_proj,8,32,8,128,0.1939697265625,8.5078125,8388608
+43,self_attn.v_proj,8,64,8,128,0.2266845703125,8.25390625,8388608
+43,self_attn.v_proj,8,128,8,128,0.256103515625,8.126953125,8388608
+43,self_attn.k_proj,2,32,8,128,24.953125,2.5078125,8388608
+43,self_attn.k_proj,2,64,8,128,30.046875,2.25390625,8388608
+43,self_attn.k_proj,2,128,8,128,34.3125,2.126953125,8388608
+43,self_attn.k_proj,3,32,8,128,10.765625,3.5078125,8388608
+43,self_attn.k_proj,3,64,8,128,13.1328125,3.25390625,8388608
+43,self_attn.k_proj,3,128,8,128,15.3515625,3.126953125,8388608
+43,self_attn.k_proj,4,32,8,128,5.01953125,4.5078125,8388608
+43,self_attn.k_proj,4,64,8,128,6.12890625,4.25390625,8388608
+43,self_attn.k_proj,4,128,8,128,7.17578125,4.126953125,8388608
+43,self_attn.k_proj,8,32,8,128,0.303955078125,8.5078125,8388608
+43,self_attn.k_proj,8,64,8,128,0.370849609375,8.25390625,8388608
+43,self_attn.k_proj,8,128,8,128,0.43212890625,8.126953125,8388608
+43,self_attn.o_proj,2,32,8,128,41.0,20.0625,67108864
+43,self_attn.o_proj,2,64,8,128,47.8125,18.03125,67108864
+43,self_attn.o_proj,2,128,8,128,54.0,17.015625,67108864
+43,self_attn.o_proj,3,32,8,128,17.53125,28.0625,67108864
+43,self_attn.o_proj,3,64,8,128,20.46875,26.03125,67108864
+43,self_attn.o_proj,3,128,8,128,23.171875,25.015625,67108864
+43,self_attn.o_proj,4,32,8,128,8.1796875,36.0625,67108864
+43,self_attn.o_proj,4,64,8,128,9.5546875,34.03125,67108864
+43,self_attn.o_proj,4,128,8,128,10.8203125,33.015625,67108864
+43,self_attn.o_proj,8,32,8,128,0.495361328125,68.0625,67108864
+43,self_attn.o_proj,8,64,8,128,0.578125,66.03125,67108864
+43,self_attn.o_proj,8,128,8,128,0.65234375,65.015625,67108864
+43,mlp.gate_proj,2,32,8,128,86.6875,70.21875,234881024
+43,mlp.gate_proj,2,64,8,128,101.4375,63.109375,234881024
+43,mlp.gate_proj,2,128,8,128,114.875,59.5546875,234881024
+43,mlp.gate_proj,3,32,8,128,37.09375,98.21875,234881024
+43,mlp.gate_proj,3,64,8,128,43.375,91.109375,234881024
+43,mlp.gate_proj,3,128,8,128,49.21875,87.5546875,234881024
+43,mlp.gate_proj,4,32,8,128,17.296875,126.21875,234881024
+43,mlp.gate_proj,4,64,8,128,20.265625,119.109375,234881024
+43,mlp.gate_proj,4,128,8,128,22.953125,115.5546875,234881024
+43,mlp.gate_proj,8,32,8,128,1.052734375,238.21875,234881024
+43,mlp.gate_proj,8,64,8,128,1.2333984375,231.109375,234881024
+43,mlp.gate_proj,8,128,8,128,1.392578125,227.5546875,234881024
+43,mlp.up_proj,2,32,8,128,80.625,70.21875,234881024
+43,mlp.up_proj,2,64,8,128,93.875,63.109375,234881024
+43,mlp.up_proj,2,128,8,128,105.875,59.5546875,234881024
+43,mlp.up_proj,3,32,8,128,34.46875,98.21875,234881024
+43,mlp.up_proj,3,64,8,128,40.1875,91.109375,234881024
+43,mlp.up_proj,3,128,8,128,45.34375,87.5546875,234881024
+43,mlp.up_proj,4,32,8,128,16.0625,126.21875,234881024
+43,mlp.up_proj,4,64,8,128,18.765625,119.109375,234881024
+43,mlp.up_proj,4,128,8,128,21.15625,115.5546875,234881024
+43,mlp.up_proj,8,32,8,128,0.97705078125,238.21875,234881024
+43,mlp.up_proj,8,64,8,128,1.140625,231.109375,234881024
+43,mlp.up_proj,8,128,8,128,1.283203125,227.5546875,234881024
+43,mlp.down_proj,2,32,8,128,81.75,70.21875,234881024
+43,mlp.down_proj,2,64,8,128,95.125,63.109375,234881024
+43,mlp.down_proj,2,128,8,128,107.25,59.5546875,234881024
+43,mlp.down_proj,3,32,8,128,34.9375,98.21875,234881024
+43,mlp.down_proj,3,64,8,128,40.71875,91.109375,234881024
+43,mlp.down_proj,3,128,8,128,45.875,87.5546875,234881024
+43,mlp.down_proj,4,32,8,128,16.296875,126.21875,234881024
+43,mlp.down_proj,4,64,8,128,19.015625,119.109375,234881024
+43,mlp.down_proj,4,128,8,128,21.421875,115.5546875,234881024
+43,mlp.down_proj,8,32,8,128,0.9912109375,238.21875,234881024
+43,mlp.down_proj,8,64,8,128,1.15625,231.109375,234881024
+43,mlp.down_proj,8,128,8,128,1.2998046875,227.5546875,234881024
+44,self_attn.q_proj,2,32,8,128,45.71875,20.0625,67108864
+44,self_attn.q_proj,2,64,8,128,53.8125,18.03125,67108864
+44,self_attn.q_proj,2,128,8,128,62.28125,17.015625,67108864
+44,self_attn.q_proj,3,32,8,128,19.578125,28.0625,67108864
+44,self_attn.q_proj,3,64,8,128,23.0625,26.03125,67108864
+44,self_attn.q_proj,3,128,8,128,26.796875,25.015625,67108864
+44,self_attn.q_proj,4,32,8,128,9.1328125,36.0625,67108864
+44,self_attn.q_proj,4,64,8,128,10.765625,34.03125,67108864
+44,self_attn.q_proj,4,128,8,128,12.5,33.015625,67108864
+44,self_attn.q_proj,8,32,8,128,0.55322265625,68.0625,67108864
+44,self_attn.q_proj,8,64,8,128,0.65185546875,66.03125,67108864
+44,self_attn.q_proj,8,128,8,128,0.75390625,65.015625,67108864
+44,self_attn.v_proj,2,32,8,128,16.65625,2.5078125,8388608
+44,self_attn.v_proj,2,64,8,128,19.484375,2.25390625,8388608
+44,self_attn.v_proj,2,128,8,128,22.046875,2.126953125,8388608
+44,self_attn.v_proj,3,32,8,128,7.125,3.5078125,8388608
+44,self_attn.v_proj,3,64,8,128,8.3359375,3.25390625,8388608
+44,self_attn.v_proj,3,128,8,128,9.4375,3.126953125,8388608
+44,self_attn.v_proj,4,32,8,128,3.32421875,4.5078125,8388608
+44,self_attn.v_proj,4,64,8,128,3.888671875,4.25390625,8388608
+44,self_attn.v_proj,4,128,8,128,4.40234375,4.126953125,8388608
+44,self_attn.v_proj,8,32,8,128,0.201416015625,8.5078125,8388608
+44,self_attn.v_proj,8,64,8,128,0.2353515625,8.25390625,8388608
+44,self_attn.v_proj,8,128,8,128,0.265380859375,8.126953125,8388608
+44,self_attn.k_proj,2,32,8,128,24.78125,2.5078125,8388608
+44,self_attn.k_proj,2,64,8,128,29.78125,2.25390625,8388608
+44,self_attn.k_proj,2,128,8,128,33.59375,2.126953125,8388608
+44,self_attn.k_proj,3,32,8,128,10.765625,3.5078125,8388608
+44,self_attn.k_proj,3,64,8,128,13.28125,3.25390625,8388608
+44,self_attn.k_proj,3,128,8,128,15.53125,3.126953125,8388608
+44,self_attn.k_proj,4,32,8,128,5.0234375,4.5078125,8388608
+44,self_attn.k_proj,4,64,8,128,6.203125,4.25390625,8388608
+44,self_attn.k_proj,4,128,8,128,7.2578125,4.126953125,8388608
+44,self_attn.k_proj,8,32,8,128,0.30419921875,8.5078125,8388608
+44,self_attn.k_proj,8,64,8,128,0.375244140625,8.25390625,8388608
+44,self_attn.k_proj,8,128,8,128,0.4375,8.126953125,8388608
+44,self_attn.o_proj,2,32,8,128,44.03125,20.0625,67108864
+44,self_attn.o_proj,2,64,8,128,51.3125,18.03125,67108864
+44,self_attn.o_proj,2,128,8,128,57.875,17.015625,67108864
+44,self_attn.o_proj,3,32,8,128,18.875,28.0625,67108864
+44,self_attn.o_proj,3,64,8,128,22.046875,26.03125,67108864
+44,self_attn.o_proj,3,128,8,128,24.953125,25.015625,67108864
+44,self_attn.o_proj,4,32,8,128,8.8359375,36.0625,67108864
+44,self_attn.o_proj,4,64,8,128,10.3359375,34.03125,67108864
+44,self_attn.o_proj,4,128,8,128,11.7265625,33.015625,67108864
+44,self_attn.o_proj,8,32,8,128,0.53515625,68.0625,67108864
+44,self_attn.o_proj,8,64,8,128,0.6259765625,66.03125,67108864
+44,self_attn.o_proj,8,128,8,128,0.70849609375,65.015625,67108864
+44,mlp.gate_proj,2,32,8,128,87.8125,70.21875,234881024
+44,mlp.gate_proj,2,64,8,128,102.625,63.109375,234881024
+44,mlp.gate_proj,2,128,8,128,116.0,59.5546875,234881024
+44,mlp.gate_proj,3,32,8,128,37.59375,98.21875,234881024
+44,mlp.gate_proj,3,64,8,128,43.875,91.109375,234881024
+44,mlp.gate_proj,3,128,8,128,49.6875,87.5546875,234881024
+44,mlp.gate_proj,4,32,8,128,17.515625,126.21875,234881024
+44,mlp.gate_proj,4,64,8,128,20.484375,119.109375,234881024
+44,mlp.gate_proj,4,128,8,128,23.171875,115.5546875,234881024
+44,mlp.gate_proj,8,32,8,128,1.06640625,238.21875,234881024
+44,mlp.gate_proj,8,64,8,128,1.2470703125,231.109375,234881024
+44,mlp.gate_proj,8,128,8,128,1.40625,227.5546875,234881024
+44,mlp.up_proj,2,32,8,128,79.8125,70.21875,234881024
+44,mlp.up_proj,2,64,8,128,92.875,63.109375,234881024
+44,mlp.up_proj,2,128,8,128,104.75,59.5546875,234881024
+44,mlp.up_proj,3,32,8,128,34.09375,98.21875,234881024
+44,mlp.up_proj,3,64,8,128,39.78125,91.109375,234881024
+44,mlp.up_proj,3,128,8,128,44.8125,87.5546875,234881024
+44,mlp.up_proj,4,32,8,128,15.90625,126.21875,234881024
+44,mlp.up_proj,4,64,8,128,18.5625,119.109375,234881024
+44,mlp.up_proj,4,128,8,128,20.921875,115.5546875,234881024
+44,mlp.up_proj,8,32,8,128,0.96728515625,238.21875,234881024
+44,mlp.up_proj,8,64,8,128,1.12890625,231.109375,234881024
+44,mlp.up_proj,8,128,8,128,1.26953125,227.5546875,234881024
+44,mlp.down_proj,2,32,8,128,81.3125,70.21875,234881024
+44,mlp.down_proj,2,64,8,128,94.5625,63.109375,234881024
+44,mlp.down_proj,2,128,8,128,106.5625,59.5546875,234881024
+44,mlp.down_proj,3,32,8,128,34.75,98.21875,234881024
+44,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024
+44,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024
+44,mlp.down_proj,4,32,8,128,16.203125,126.21875,234881024
+44,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024
+44,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024
+44,mlp.down_proj,8,32,8,128,0.98583984375,238.21875,234881024
+44,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024
+44,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024
+45,self_attn.q_proj,2,32,8,128,40.46875,20.0625,67108864
+45,self_attn.q_proj,2,64,8,128,47.71875,18.03125,67108864
+45,self_attn.q_proj,2,128,8,128,55.40625,17.015625,67108864
+45,self_attn.q_proj,3,32,8,128,17.328125,28.0625,67108864
+45,self_attn.q_proj,3,64,8,128,20.484375,26.03125,67108864
+45,self_attn.q_proj,3,128,8,128,23.890625,25.015625,67108864
+45,self_attn.q_proj,4,32,8,128,8.078125,36.0625,67108864
+45,self_attn.q_proj,4,64,8,128,9.5546875,34.03125,67108864
+45,self_attn.q_proj,4,128,8,128,11.140625,33.015625,67108864
+45,self_attn.q_proj,8,32,8,128,0.4892578125,68.0625,67108864
+45,self_attn.q_proj,8,64,8,128,0.57861328125,66.03125,67108864
+45,self_attn.q_proj,8,128,8,128,0.671875,65.015625,67108864
+45,self_attn.v_proj,2,32,8,128,17.234375,2.5078125,8388608
+45,self_attn.v_proj,2,64,8,128,20.140625,2.25390625,8388608
+45,self_attn.v_proj,2,128,8,128,22.78125,2.126953125,8388608
+45,self_attn.v_proj,3,32,8,128,7.37109375,3.5078125,8388608
+45,self_attn.v_proj,3,64,8,128,8.6171875,3.25390625,8388608
+45,self_attn.v_proj,3,128,8,128,9.75,3.126953125,8388608
+45,self_attn.v_proj,4,32,8,128,3.4375,4.5078125,8388608
+45,self_attn.v_proj,4,64,8,128,4.01953125,4.25390625,8388608
+45,self_attn.v_proj,4,128,8,128,4.55078125,4.126953125,8388608
+45,self_attn.v_proj,8,32,8,128,0.208251953125,8.5078125,8388608
+45,self_attn.v_proj,8,64,8,128,0.243408203125,8.25390625,8388608
+45,self_attn.v_proj,8,128,8,128,0.2744140625,8.126953125,8388608
+45,self_attn.k_proj,2,32,8,128,23.234375,2.5078125,8388608
+45,self_attn.k_proj,2,64,8,128,28.171875,2.25390625,8388608
+45,self_attn.k_proj,2,128,8,128,32.5,2.126953125,8388608
+45,self_attn.k_proj,3,32,8,128,10.015625,3.5078125,8388608
+45,self_attn.k_proj,3,64,8,128,12.3203125,3.25390625,8388608
+45,self_attn.k_proj,3,128,8,128,14.546875,3.126953125,8388608
+45,self_attn.k_proj,4,32,8,128,4.671875,4.5078125,8388608
+45,self_attn.k_proj,4,64,8,128,5.75390625,4.25390625,8388608
+45,self_attn.k_proj,4,128,8,128,6.796875,4.126953125,8388608
+45,self_attn.k_proj,8,32,8,128,0.282958984375,8.5078125,8388608
+45,self_attn.k_proj,8,64,8,128,0.34814453125,8.25390625,8388608
+45,self_attn.k_proj,8,128,8,128,0.40966796875,8.126953125,8388608
+45,self_attn.o_proj,2,32,8,128,42.21875,20.0625,67108864
+45,self_attn.o_proj,2,64,8,128,49.25,18.03125,67108864
+45,self_attn.o_proj,2,128,8,128,55.65625,17.015625,67108864
+45,self_attn.o_proj,3,32,8,128,18.109375,28.0625,67108864
+45,self_attn.o_proj,3,64,8,128,21.171875,26.03125,67108864
+45,self_attn.o_proj,3,128,8,128,24.03125,25.015625,67108864
+45,self_attn.o_proj,4,32,8,128,8.4609375,36.0625,67108864
+45,self_attn.o_proj,4,64,8,128,9.90625,34.03125,67108864
+45,self_attn.o_proj,4,128,8,128,11.25,33.015625,67108864
+45,self_attn.o_proj,8,32,8,128,0.51220703125,68.0625,67108864
+45,self_attn.o_proj,8,64,8,128,0.59912109375,66.03125,67108864
+45,self_attn.o_proj,8,128,8,128,0.67822265625,65.015625,67108864
+45,mlp.gate_proj,2,32,8,128,88.3125,70.21875,234881024
+45,mlp.gate_proj,2,64,8,128,103.125,63.109375,234881024
+45,mlp.gate_proj,2,128,8,128,116.5,59.5546875,234881024
+45,mlp.gate_proj,3,32,8,128,37.8125,98.21875,234881024
+45,mlp.gate_proj,3,64,8,128,44.09375,91.109375,234881024
+45,mlp.gate_proj,3,128,8,128,49.90625,87.5546875,234881024
+45,mlp.gate_proj,4,32,8,128,17.625,126.21875,234881024
+45,mlp.gate_proj,4,64,8,128,20.59375,119.109375,234881024
+45,mlp.gate_proj,4,128,8,128,23.28125,115.5546875,234881024
+45,mlp.gate_proj,8,32,8,128,1.072265625,238.21875,234881024
+45,mlp.gate_proj,8,64,8,128,1.2529296875,231.109375,234881024
+45,mlp.gate_proj,8,128,8,128,1.412109375,227.5546875,234881024
+45,mlp.up_proj,2,32,8,128,79.75,70.21875,234881024
+45,mlp.up_proj,2,64,8,128,92.8125,63.109375,234881024
+45,mlp.up_proj,2,128,8,128,104.6875,59.5546875,234881024
+45,mlp.up_proj,3,32,8,128,34.09375,98.21875,234881024
+45,mlp.up_proj,3,64,8,128,39.75,91.109375,234881024
+45,mlp.up_proj,3,128,8,128,44.78125,87.5546875,234881024
+45,mlp.up_proj,4,32,8,128,15.8984375,126.21875,234881024
+45,mlp.up_proj,4,64,8,128,18.546875,119.109375,234881024
+45,mlp.up_proj,4,128,8,128,20.890625,115.5546875,234881024
+45,mlp.up_proj,8,32,8,128,0.966796875,238.21875,234881024
+45,mlp.up_proj,8,64,8,128,1.1279296875,231.109375,234881024
+45,mlp.up_proj,8,128,8,128,1.2685546875,227.5546875,234881024
+45,mlp.down_proj,2,32,8,128,81.1875,70.21875,234881024
+45,mlp.down_proj,2,64,8,128,94.5,63.109375,234881024
+45,mlp.down_proj,2,128,8,128,106.5,59.5546875,234881024
+45,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024
+45,mlp.down_proj,3,64,8,128,40.4375,91.109375,234881024
+45,mlp.down_proj,3,128,8,128,45.53125,87.5546875,234881024
+45,mlp.down_proj,4,32,8,128,16.1875,126.21875,234881024
+45,mlp.down_proj,4,64,8,128,18.875,119.109375,234881024
+45,mlp.down_proj,4,128,8,128,21.25,115.5546875,234881024
+45,mlp.down_proj,8,32,8,128,0.984375,238.21875,234881024
+45,mlp.down_proj,8,64,8,128,1.1474609375,231.109375,234881024
+45,mlp.down_proj,8,128,8,128,1.2890625,227.5546875,234881024
+46,self_attn.q_proj,2,32,8,128,39.03125,20.0625,67108864
+46,self_attn.q_proj,2,64,8,128,45.9375,18.03125,67108864
+46,self_attn.q_proj,2,128,8,128,53.15625,17.015625,67108864
+46,self_attn.q_proj,3,32,8,128,16.703125,28.0625,67108864
+46,self_attn.q_proj,3,64,8,128,19.703125,26.03125,67108864
+46,self_attn.q_proj,3,128,8,128,22.859375,25.015625,67108864
+46,self_attn.q_proj,4,32,8,128,7.79296875,36.0625,67108864
+46,self_attn.q_proj,4,64,8,128,9.1953125,34.03125,67108864
+46,self_attn.q_proj,4,128,8,128,10.6640625,33.015625,67108864
+46,self_attn.q_proj,8,32,8,128,0.471923828125,68.0625,67108864
+46,self_attn.q_proj,8,64,8,128,0.55615234375,66.03125,67108864
+46,self_attn.q_proj,8,128,8,128,0.64306640625,65.015625,67108864
+46,self_attn.v_proj,2,32,8,128,17.0,2.5078125,8388608
+46,self_attn.v_proj,2,64,8,128,19.796875,2.25390625,8388608
+46,self_attn.v_proj,2,128,8,128,22.3125,2.126953125,8388608
+46,self_attn.v_proj,3,32,8,128,7.2734375,3.5078125,8388608
+46,self_attn.v_proj,3,64,8,128,8.46875,3.25390625,8388608
+46,self_attn.v_proj,3,128,8,128,9.546875,3.126953125,8388608
+46,self_attn.v_proj,4,32,8,128,3.392578125,4.5078125,8388608
+46,self_attn.v_proj,4,64,8,128,3.951171875,4.25390625,8388608
+46,self_attn.v_proj,4,128,8,128,4.45703125,4.126953125,8388608
+46,self_attn.v_proj,8,32,8,128,0.2054443359375,8.5078125,8388608
+46,self_attn.v_proj,8,64,8,128,0.2391357421875,8.25390625,8388608
+46,self_attn.v_proj,8,128,8,128,0.2685546875,8.126953125,8388608
+46,self_attn.k_proj,2,32,8,128,22.765625,2.5078125,8388608
+46,self_attn.k_proj,2,64,8,128,27.484375,2.25390625,8388608
+46,self_attn.k_proj,2,128,8,128,31.296875,2.126953125,8388608
+46,self_attn.k_proj,3,32,8,128,9.828125,3.5078125,8388608
+46,self_attn.k_proj,3,64,8,128,12.0390625,3.25390625,8388608
+46,self_attn.k_proj,3,128,8,128,14.078125,3.126953125,8388608
+46,self_attn.k_proj,4,32,8,128,4.58203125,4.5078125,8388608
+46,self_attn.k_proj,4,64,8,128,5.6171875,4.25390625,8388608
+46,self_attn.k_proj,4,128,8,128,6.57421875,4.126953125,8388608
+46,self_attn.k_proj,8,32,8,128,0.277587890625,8.5078125,8388608
+46,self_attn.k_proj,8,64,8,128,0.33984375,8.25390625,8388608
+46,self_attn.k_proj,8,128,8,128,0.396240234375,8.126953125,8388608
+46,self_attn.o_proj,2,32,8,128,42.3125,20.0625,67108864
+46,self_attn.o_proj,2,64,8,128,49.28125,18.03125,67108864
+46,self_attn.o_proj,2,128,8,128,55.59375,17.015625,67108864
+46,self_attn.o_proj,3,32,8,128,18.125,28.0625,67108864
+46,self_attn.o_proj,3,64,8,128,21.140625,26.03125,67108864
+46,self_attn.o_proj,3,128,8,128,23.90625,25.015625,67108864
+46,self_attn.o_proj,4,32,8,128,8.4609375,36.0625,67108864
+46,self_attn.o_proj,4,64,8,128,9.8828125,34.03125,67108864
+46,self_attn.o_proj,4,128,8,128,11.1875,33.015625,67108864
+46,self_attn.o_proj,8,32,8,128,0.51220703125,68.0625,67108864
+46,self_attn.o_proj,8,64,8,128,0.59765625,66.03125,67108864
+46,self_attn.o_proj,8,128,8,128,0.6748046875,65.015625,67108864
+46,mlp.gate_proj,2,32,8,128,88.9375,70.21875,234881024
+46,mlp.gate_proj,2,64,8,128,103.875,63.109375,234881024
+46,mlp.gate_proj,2,128,8,128,117.3125,59.5546875,234881024
+46,mlp.gate_proj,3,32,8,128,38.0625,98.21875,234881024
+46,mlp.gate_proj,3,64,8,128,44.40625,91.109375,234881024
+46,mlp.gate_proj,3,128,8,128,50.25,87.5546875,234881024
+46,mlp.gate_proj,4,32,8,128,17.75,126.21875,234881024
+46,mlp.gate_proj,4,64,8,128,20.71875,119.109375,234881024
+46,mlp.gate_proj,4,128,8,128,23.453125,115.5546875,234881024
+46,mlp.gate_proj,8,32,8,128,1.080078125,238.21875,234881024
+46,mlp.gate_proj,8,64,8,128,1.26171875,231.109375,234881024
+46,mlp.gate_proj,8,128,8,128,1.421875,227.5546875,234881024
+46,mlp.up_proj,2,32,8,128,79.6875,70.21875,234881024
+46,mlp.up_proj,2,64,8,128,92.6875,63.109375,234881024
+46,mlp.up_proj,2,128,8,128,104.5,59.5546875,234881024
+46,mlp.up_proj,3,32,8,128,34.03125,98.21875,234881024
+46,mlp.up_proj,3,64,8,128,39.6875,91.109375,234881024
+46,mlp.up_proj,3,128,8,128,44.6875,87.5546875,234881024
+46,mlp.up_proj,4,32,8,128,15.875,126.21875,234881024
+46,mlp.up_proj,4,64,8,128,18.515625,119.109375,234881024
+46,mlp.up_proj,4,128,8,128,20.859375,115.5546875,234881024
+46,mlp.up_proj,8,32,8,128,0.96533203125,238.21875,234881024
+46,mlp.up_proj,8,64,8,128,1.1259765625,231.109375,234881024
+46,mlp.up_proj,8,128,8,128,1.265625,227.5546875,234881024
+46,mlp.down_proj,2,32,8,128,81.0,70.21875,234881024
+46,mlp.down_proj,2,64,8,128,94.25,63.109375,234881024
+46,mlp.down_proj,2,128,8,128,106.1875,59.5546875,234881024
+46,mlp.down_proj,3,32,8,128,34.625,98.21875,234881024
+46,mlp.down_proj,3,64,8,128,40.34375,91.109375,234881024
+46,mlp.down_proj,3,128,8,128,45.4375,87.5546875,234881024
+46,mlp.down_proj,4,32,8,128,16.140625,126.21875,234881024
+46,mlp.down_proj,4,64,8,128,18.8125,119.109375,234881024
+46,mlp.down_proj,4,128,8,128,21.1875,115.5546875,234881024
+46,mlp.down_proj,8,32,8,128,0.98193359375,238.21875,234881024
+46,mlp.down_proj,8,64,8,128,1.14453125,231.109375,234881024
+46,mlp.down_proj,8,128,8,128,1.2861328125,227.5546875,234881024
+47,self_attn.q_proj,2,32,8,128,44.0625,20.0625,67108864
+47,self_attn.q_proj,2,64,8,128,52.0,18.03125,67108864
+47,self_attn.q_proj,2,128,8,128,59.84375,17.015625,67108864
+47,self_attn.q_proj,3,32,8,128,18.875,28.0625,67108864
+47,self_attn.q_proj,3,64,8,128,22.328125,26.03125,67108864
+47,self_attn.q_proj,3,128,8,128,25.828125,25.015625,67108864
+47,self_attn.q_proj,4,32,8,128,8.8046875,36.0625,67108864
+47,self_attn.q_proj,4,64,8,128,10.421875,34.03125,67108864
+47,self_attn.q_proj,4,128,8,128,12.0546875,33.015625,67108864
+47,self_attn.q_proj,8,32,8,128,0.533203125,68.0625,67108864
+47,self_attn.q_proj,8,64,8,128,0.63037109375,66.03125,67108864
+47,self_attn.q_proj,8,128,8,128,0.72705078125,65.015625,67108864
+47,self_attn.v_proj,2,32,8,128,16.75,2.5078125,8388608
+47,self_attn.v_proj,2,64,8,128,19.625,2.25390625,8388608
+47,self_attn.v_proj,2,128,8,128,22.234375,2.126953125,8388608
+47,self_attn.v_proj,3,32,8,128,7.1640625,3.5078125,8388608
+47,self_attn.v_proj,3,64,8,128,8.390625,3.25390625,8388608
+47,self_attn.v_proj,3,128,8,128,9.5078125,3.126953125,8388608
+47,self_attn.v_proj,4,32,8,128,3.341796875,4.5078125,8388608
+47,self_attn.v_proj,4,64,8,128,3.9140625,4.25390625,8388608
+47,self_attn.v_proj,4,128,8,128,4.44140625,4.126953125,8388608
+47,self_attn.v_proj,8,32,8,128,0.202392578125,8.5078125,8388608
+47,self_attn.v_proj,8,64,8,128,0.2369384765625,8.25390625,8388608
+47,self_attn.v_proj,8,128,8,128,0.267578125,8.126953125,8388608
+47,self_attn.k_proj,2,32,8,128,25.765625,2.5078125,8388608
+47,self_attn.k_proj,2,64,8,128,31.03125,2.25390625,8388608
+47,self_attn.k_proj,2,128,8,128,35.4375,2.126953125,8388608
+47,self_attn.k_proj,3,32,8,128,11.109375,3.5078125,8388608
+47,self_attn.k_proj,3,64,8,128,13.546875,3.25390625,8388608
+47,self_attn.k_proj,3,128,8,128,15.8203125,3.126953125,8388608
+47,self_attn.k_proj,4,32,8,128,5.18359375,4.5078125,8388608
+47,self_attn.k_proj,4,64,8,128,6.3203125,4.25390625,8388608
+47,self_attn.k_proj,4,128,8,128,7.390625,4.126953125,8388608
+47,self_attn.k_proj,8,32,8,128,0.313720703125,8.5078125,8388608
+47,self_attn.k_proj,8,64,8,128,0.38232421875,8.25390625,8388608
+47,self_attn.k_proj,8,128,8,128,0.4453125,8.126953125,8388608
+47,self_attn.o_proj,2,32,8,128,42.0,20.0625,67108864
+47,self_attn.o_proj,2,64,8,128,48.96875,18.03125,67108864
+47,self_attn.o_proj,2,128,8,128,55.34375,17.015625,67108864
+47,self_attn.o_proj,3,32,8,128,18.015625,28.0625,67108864
+47,self_attn.o_proj,3,64,8,128,21.078125,26.03125,67108864
+47,self_attn.o_proj,3,128,8,128,23.9375,25.015625,67108864
+47,self_attn.o_proj,4,32,8,128,8.4453125,36.0625,67108864
+47,self_attn.o_proj,4,64,8,128,9.90625,34.03125,67108864
+47,self_attn.o_proj,4,128,8,128,11.296875,33.015625,67108864
+47,self_attn.o_proj,8,32,8,128,0.51416015625,68.0625,67108864
+47,self_attn.o_proj,8,64,8,128,0.60546875,66.03125,67108864
+47,self_attn.o_proj,8,128,8,128,0.69287109375,65.015625,67108864
+47,mlp.gate_proj,2,32,8,128,89.375,70.21875,234881024
+47,mlp.gate_proj,2,64,8,128,104.375,63.109375,234881024
+47,mlp.gate_proj,2,128,8,128,117.875,59.5546875,234881024
+47,mlp.gate_proj,3,32,8,128,38.25,98.21875,234881024
+47,mlp.gate_proj,3,64,8,128,44.65625,91.109375,234881024
+47,mlp.gate_proj,3,128,8,128,50.46875,87.5546875,234881024
+47,mlp.gate_proj,4,32,8,128,17.84375,126.21875,234881024
+47,mlp.gate_proj,4,64,8,128,20.828125,119.109375,234881024
+47,mlp.gate_proj,4,128,8,128,23.5625,115.5546875,234881024
+47,mlp.gate_proj,8,32,8,128,1.0859375,238.21875,234881024
+47,mlp.gate_proj,8,64,8,128,1.2685546875,231.109375,234881024
+47,mlp.gate_proj,8,128,8,128,1.4287109375,227.5546875,234881024
+47,mlp.up_proj,2,32,8,128,79.5,70.21875,234881024
+47,mlp.up_proj,2,64,8,128,92.5,63.109375,234881024
+47,mlp.up_proj,2,128,8,128,104.25,59.5546875,234881024
+47,mlp.up_proj,3,32,8,128,33.96875,98.21875,234881024
+47,mlp.up_proj,3,64,8,128,39.59375,91.109375,234881024
+47,mlp.up_proj,3,128,8,128,44.59375,87.5546875,234881024
+47,mlp.up_proj,4,32,8,128,15.8359375,126.21875,234881024
+47,mlp.up_proj,4,64,8,128,18.46875,119.109375,234881024
+47,mlp.up_proj,4,128,8,128,20.828125,115.5546875,234881024
+47,mlp.up_proj,8,32,8,128,0.962890625,238.21875,234881024
+47,mlp.up_proj,8,64,8,128,1.123046875,231.109375,234881024
+47,mlp.up_proj,8,128,8,128,1.263671875,227.5546875,234881024
+47,mlp.down_proj,2,32,8,128,80.875,70.21875,234881024
+47,mlp.down_proj,2,64,8,128,94.0625,63.109375,234881024
+47,mlp.down_proj,2,128,8,128,106.0,59.5546875,234881024
+47,mlp.down_proj,3,32,8,128,34.5625,98.21875,234881024
+47,mlp.down_proj,3,64,8,128,40.28125,91.109375,234881024
+47,mlp.down_proj,3,128,8,128,45.34375,87.5546875,234881024
+47,mlp.down_proj,4,32,8,128,16.125,126.21875,234881024
+47,mlp.down_proj,4,64,8,128,18.796875,119.109375,234881024
+47,mlp.down_proj,4,128,8,128,21.171875,115.5546875,234881024
+47,mlp.down_proj,8,32,8,128,0.98046875,238.21875,234881024
+47,mlp.down_proj,8,64,8,128,1.1435546875,231.109375,234881024
+47,mlp.down_proj,8,128,8,128,1.2841796875,227.5546875,234881024
+48,self_attn.q_proj,2,32,8,128,37.4375,20.0625,67108864
+48,self_attn.q_proj,2,64,8,128,44.21875,18.03125,67108864
+48,self_attn.q_proj,2,128,8,128,51.0,17.015625,67108864
+48,self_attn.q_proj,3,32,8,128,16.03125,28.0625,67108864
+48,self_attn.q_proj,3,64,8,128,19.0,26.03125,67108864
+48,self_attn.q_proj,3,128,8,128,22.0625,25.015625,67108864
+48,self_attn.q_proj,4,32,8,128,7.484375,36.0625,67108864
+48,self_attn.q_proj,4,64,8,128,8.8671875,34.03125,67108864
+48,self_attn.q_proj,4,128,8,128,10.3046875,33.015625,67108864
+48,self_attn.q_proj,8,32,8,128,0.453125,68.0625,67108864
+48,self_attn.q_proj,8,64,8,128,0.53662109375,66.03125,67108864
+48,self_attn.q_proj,8,128,8,128,0.62109375,65.015625,67108864
+48,self_attn.v_proj,2,32,8,128,17.5,2.5078125,8388608
+48,self_attn.v_proj,2,64,8,128,20.453125,2.25390625,8388608
+48,self_attn.v_proj,2,128,8,128,23.125,2.126953125,8388608
+48,self_attn.v_proj,3,32,8,128,7.484375,3.5078125,8388608
+48,self_attn.v_proj,3,64,8,128,8.75,3.25390625,8388608
+48,self_attn.v_proj,3,128,8,128,9.90625,3.126953125,8388608
+48,self_attn.v_proj,4,32,8,128,3.4921875,4.5078125,8388608
+48,self_attn.v_proj,4,64,8,128,4.08203125,4.25390625,8388608
+48,self_attn.v_proj,4,128,8,128,4.62109375,4.126953125,8388608
+48,self_attn.v_proj,8,32,8,128,0.21142578125,8.5078125,8388608
+48,self_attn.v_proj,8,64,8,128,0.2470703125,8.25390625,8388608
+48,self_attn.v_proj,8,128,8,128,0.278564453125,8.126953125,8388608
+48,self_attn.k_proj,2,32,8,128,22.171875,2.5078125,8388608
+48,self_attn.k_proj,2,64,8,128,26.515625,2.25390625,8388608
+48,self_attn.k_proj,2,128,8,128,29.859375,2.126953125,8388608
+48,self_attn.k_proj,3,32,8,128,9.71875,3.5078125,8388608
+48,self_attn.k_proj,3,64,8,128,12.0,3.25390625,8388608
+48,self_attn.k_proj,3,128,8,128,14.1328125,3.126953125,8388608
+48,self_attn.k_proj,4,32,8,128,4.53515625,4.5078125,8388608
+48,self_attn.k_proj,4,64,8,128,5.60546875,4.25390625,8388608
+48,self_attn.k_proj,4,128,8,128,6.609375,4.126953125,8388608
+48,self_attn.k_proj,8,32,8,128,0.2744140625,8.5078125,8388608
+48,self_attn.k_proj,8,64,8,128,0.3388671875,8.25390625,8388608
+48,self_attn.k_proj,8,128,8,128,0.3984375,8.126953125,8388608
+48,self_attn.o_proj,2,32,8,128,42.09375,20.0625,67108864
+48,self_attn.o_proj,2,64,8,128,49.03125,18.03125,67108864
+48,self_attn.o_proj,2,128,8,128,55.25,17.015625,67108864
+48,self_attn.o_proj,3,32,8,128,18.015625,28.0625,67108864
+48,self_attn.o_proj,3,64,8,128,21.03125,26.03125,67108864
+48,self_attn.o_proj,3,128,8,128,23.75,25.015625,67108864
+48,self_attn.o_proj,4,32,8,128,8.4140625,36.0625,67108864
+48,self_attn.o_proj,4,64,8,128,9.8203125,34.03125,67108864
+48,self_attn.o_proj,4,128,8,128,11.1171875,33.015625,67108864
+48,self_attn.o_proj,8,32,8,128,0.509765625,68.0625,67108864
+48,self_attn.o_proj,8,64,8,128,0.5947265625,66.03125,67108864
+48,self_attn.o_proj,8,128,8,128,0.6708984375,65.015625,67108864
+48,mlp.gate_proj,2,32,8,128,89.5625,70.21875,234881024
+48,mlp.gate_proj,2,64,8,128,104.5625,63.109375,234881024
+48,mlp.gate_proj,2,128,8,128,118.0625,59.5546875,234881024
+48,mlp.gate_proj,3,32,8,128,38.34375,98.21875,234881024
+48,mlp.gate_proj,3,64,8,128,44.71875,91.109375,234881024
+48,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024
+48,mlp.gate_proj,4,32,8,128,17.875,126.21875,234881024
+48,mlp.gate_proj,4,64,8,128,20.875,119.109375,234881024
+48,mlp.gate_proj,4,128,8,128,23.59375,115.5546875,234881024
+48,mlp.gate_proj,8,32,8,128,1.087890625,238.21875,234881024
+48,mlp.gate_proj,8,64,8,128,1.2705078125,231.109375,234881024
+48,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024
+48,mlp.up_proj,2,32,8,128,79.5625,70.21875,234881024
+48,mlp.up_proj,2,64,8,128,92.5625,63.109375,234881024
+48,mlp.up_proj,2,128,8,128,104.3125,59.5546875,234881024
+48,mlp.up_proj,3,32,8,128,34.0,98.21875,234881024
+48,mlp.up_proj,3,64,8,128,39.625,91.109375,234881024
+48,mlp.up_proj,3,128,8,128,44.65625,87.5546875,234881024
+48,mlp.up_proj,4,32,8,128,15.8515625,126.21875,234881024
+48,mlp.up_proj,4,64,8,128,18.484375,119.109375,234881024
+48,mlp.up_proj,4,128,8,128,20.84375,115.5546875,234881024
+48,mlp.up_proj,8,32,8,128,0.9638671875,238.21875,234881024
+48,mlp.up_proj,8,64,8,128,1.1240234375,231.109375,234881024
+48,mlp.up_proj,8,128,8,128,1.2646484375,227.5546875,234881024
+48,mlp.down_proj,2,32,8,128,80.875,70.21875,234881024
+48,mlp.down_proj,2,64,8,128,94.0625,63.109375,234881024
+48,mlp.down_proj,2,128,8,128,106.0,59.5546875,234881024
+48,mlp.down_proj,3,32,8,128,34.5625,98.21875,234881024
+48,mlp.down_proj,3,64,8,128,40.25,91.109375,234881024
+48,mlp.down_proj,3,128,8,128,45.3125,87.5546875,234881024
+48,mlp.down_proj,4,32,8,128,16.109375,126.21875,234881024
+48,mlp.down_proj,4,64,8,128,18.78125,119.109375,234881024
+48,mlp.down_proj,4,128,8,128,21.15625,115.5546875,234881024
+48,mlp.down_proj,8,32,8,128,0.97998046875,238.21875,234881024
+48,mlp.down_proj,8,64,8,128,1.142578125,231.109375,234881024
+48,mlp.down_proj,8,128,8,128,1.283203125,227.5546875,234881024
+49,self_attn.q_proj,2,32,8,128,34.875,20.0625,67108864
+49,self_attn.q_proj,2,64,8,128,41.625,18.03125,67108864
+49,self_attn.q_proj,2,128,8,128,48.09375,17.015625,67108864
+49,self_attn.q_proj,3,32,8,128,15.0078125,28.0625,67108864
+49,self_attn.q_proj,3,64,8,128,18.0625,26.03125,67108864
+49,self_attn.q_proj,3,128,8,128,21.140625,25.015625,67108864
+49,self_attn.q_proj,4,32,8,128,7.00390625,36.0625,67108864
+49,self_attn.q_proj,4,64,8,128,8.4296875,34.03125,67108864
+49,self_attn.q_proj,4,128,8,128,9.875,33.015625,67108864
+49,self_attn.q_proj,8,32,8,128,0.423828125,68.0625,67108864
+49,self_attn.q_proj,8,64,8,128,0.51025390625,66.03125,67108864
+49,self_attn.q_proj,8,128,8,128,0.595703125,65.015625,67108864
+49,self_attn.v_proj,2,32,8,128,17.171875,2.5078125,8388608
+49,self_attn.v_proj,2,64,8,128,20.0625,2.25390625,8388608
+49,self_attn.v_proj,2,128,8,128,22.6875,2.126953125,8388608
+49,self_attn.v_proj,3,32,8,128,7.33984375,3.5078125,8388608
+49,self_attn.v_proj,3,64,8,128,8.578125,3.25390625,8388608
+49,self_attn.v_proj,3,128,8,128,9.703125,3.126953125,8388608
+49,self_attn.v_proj,4,32,8,128,3.42578125,4.5078125,8388608
+49,self_attn.v_proj,4,64,8,128,4.00390625,4.25390625,8388608
+49,self_attn.v_proj,4,128,8,128,4.52734375,4.126953125,8388608
+49,self_attn.v_proj,8,32,8,128,0.20751953125,8.5078125,8388608
+49,self_attn.v_proj,8,64,8,128,0.2421875,8.25390625,8388608
+49,self_attn.v_proj,8,128,8,128,0.27294921875,8.126953125,8388608
+49,self_attn.k_proj,2,32,8,128,21.328125,2.5078125,8388608
+49,self_attn.k_proj,2,64,8,128,25.625,2.25390625,8388608
+49,self_attn.k_proj,2,128,8,128,29.171875,2.126953125,8388608
+49,self_attn.k_proj,3,32,8,128,9.28125,3.5078125,8388608
+49,self_attn.k_proj,3,64,8,128,11.40625,3.25390625,8388608
+49,self_attn.k_proj,3,128,8,128,13.4296875,3.126953125,8388608
+49,self_attn.k_proj,4,32,8,128,4.33203125,4.5078125,8388608
+49,self_attn.k_proj,4,64,8,128,5.32421875,4.25390625,8388608
+49,self_attn.k_proj,4,128,8,128,6.28125,4.126953125,8388608
+49,self_attn.k_proj,8,32,8,128,0.262451171875,8.5078125,8388608
+49,self_attn.k_proj,8,64,8,128,0.322021484375,8.25390625,8388608
+49,self_attn.k_proj,8,128,8,128,0.37841796875,8.126953125,8388608
+49,self_attn.o_proj,2,32,8,128,40.0,20.0625,67108864
+49,self_attn.o_proj,2,64,8,128,46.65625,18.03125,67108864
+49,self_attn.o_proj,2,128,8,128,52.75,17.015625,67108864
+49,self_attn.o_proj,3,32,8,128,17.140625,28.0625,67108864
+49,self_attn.o_proj,3,64,8,128,20.046875,26.03125,67108864
+49,self_attn.o_proj,3,128,8,128,22.734375,25.015625,67108864
+49,self_attn.o_proj,4,32,8,128,8.0,36.0625,67108864
+49,self_attn.o_proj,4,64,8,128,9.359375,34.03125,67108864
+49,self_attn.o_proj,4,128,8,128,10.6328125,33.015625,67108864
+49,self_attn.o_proj,8,32,8,128,0.484375,68.0625,67108864
+49,self_attn.o_proj,8,64,8,128,0.56640625,66.03125,67108864
+49,self_attn.o_proj,8,128,8,128,0.6416015625,65.015625,67108864
+49,mlp.gate_proj,2,32,8,128,89.6875,70.21875,234881024
+49,mlp.gate_proj,2,64,8,128,104.6875,63.109375,234881024
+49,mlp.gate_proj,2,128,8,128,118.125,59.5546875,234881024
+49,mlp.gate_proj,3,32,8,128,38.375,98.21875,234881024
+49,mlp.gate_proj,3,64,8,128,44.78125,91.109375,234881024
+49,mlp.gate_proj,3,128,8,128,50.625,87.5546875,234881024
+49,mlp.gate_proj,4,32,8,128,17.890625,126.21875,234881024
+49,mlp.gate_proj,4,64,8,128,20.890625,119.109375,234881024
+49,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024
+49,mlp.gate_proj,8,32,8,128,1.0888671875,238.21875,234881024
+49,mlp.gate_proj,8,64,8,128,1.271484375,231.109375,234881024
+49,mlp.gate_proj,8,128,8,128,1.4326171875,227.5546875,234881024
+49,mlp.up_proj,2,32,8,128,79.6875,70.21875,234881024
+49,mlp.up_proj,2,64,8,128,92.75,63.109375,234881024
+49,mlp.up_proj,2,128,8,128,104.5625,59.5546875,234881024
+49,mlp.up_proj,3,32,8,128,34.0625,98.21875,234881024
+49,mlp.up_proj,3,64,8,128,39.71875,91.109375,234881024
+49,mlp.up_proj,3,128,8,128,44.75,87.5546875,234881024
+49,mlp.up_proj,4,32,8,128,15.8828125,126.21875,234881024
+49,mlp.up_proj,4,64,8,128,18.515625,119.109375,234881024
+49,mlp.up_proj,4,128,8,128,20.875,115.5546875,234881024
+49,mlp.up_proj,8,32,8,128,0.9658203125,238.21875,234881024
+49,mlp.up_proj,8,64,8,128,1.126953125,231.109375,234881024
+49,mlp.up_proj,8,128,8,128,1.2666015625,227.5546875,234881024
+49,mlp.down_proj,2,32,8,128,80.875,70.21875,234881024
+49,mlp.down_proj,2,64,8,128,94.0625,63.109375,234881024
+49,mlp.down_proj,2,128,8,128,106.0,59.5546875,234881024
+49,mlp.down_proj,3,32,8,128,34.5625,98.21875,234881024
+49,mlp.down_proj,3,64,8,128,40.28125,91.109375,234881024
+49,mlp.down_proj,3,128,8,128,45.34375,87.5546875,234881024
+49,mlp.down_proj,4,32,8,128,16.125,126.21875,234881024
+49,mlp.down_proj,4,64,8,128,18.796875,119.109375,234881024
+49,mlp.down_proj,4,128,8,128,21.15625,115.5546875,234881024
+49,mlp.down_proj,8,32,8,128,0.98046875,238.21875,234881024
+49,mlp.down_proj,8,64,8,128,1.142578125,231.109375,234881024
+49,mlp.down_proj,8,128,8,128,1.2841796875,227.5546875,234881024
+50,self_attn.q_proj,2,32,8,128,34.96875,20.0625,67108864
+50,self_attn.q_proj,2,64,8,128,40.96875,18.03125,67108864
+50,self_attn.q_proj,2,128,8,128,47.53125,17.015625,67108864
+50,self_attn.q_proj,3,32,8,128,14.9921875,28.0625,67108864
+50,self_attn.q_proj,3,64,8,128,17.640625,26.03125,67108864
+50,self_attn.q_proj,3,128,8,128,20.609375,25.015625,67108864
+50,self_attn.q_proj,4,32,8,128,6.99609375,36.0625,67108864
+50,self_attn.q_proj,4,64,8,128,8.234375,34.03125,67108864
+50,self_attn.q_proj,4,128,8,128,9.6171875,33.015625,67108864
+50,self_attn.q_proj,8,32,8,128,0.42333984375,68.0625,67108864
+50,self_attn.q_proj,8,64,8,128,0.498046875,66.03125,67108864
+50,self_attn.q_proj,8,128,8,128,0.580078125,65.015625,67108864
+50,self_attn.v_proj,2,32,8,128,17.59375,2.5078125,8388608
+50,self_attn.v_proj,2,64,8,128,20.5625,2.25390625,8388608
+50,self_attn.v_proj,2,128,8,128,23.234375,2.126953125,8388608
+50,self_attn.v_proj,3,32,8,128,7.51953125,3.5078125,8388608
+50,self_attn.v_proj,3,64,8,128,8.7890625,3.25390625,8388608
+50,self_attn.v_proj,3,128,8,128,9.9375,3.126953125,8388608
+50,self_attn.v_proj,4,32,8,128,3.5078125,4.5078125,8388608
+50,self_attn.v_proj,4,64,8,128,4.1015625,4.25390625,8388608
+50,self_attn.v_proj,4,128,8,128,4.63671875,4.126953125,8388608
+50,self_attn.v_proj,8,32,8,128,0.2125244140625,8.5078125,8388608
+50,self_attn.v_proj,8,64,8,128,0.2481689453125,8.25390625,8388608
+50,self_attn.v_proj,8,128,8,128,0.279541015625,8.126953125,8388608
+50,self_attn.k_proj,2,32,8,128,20.46875,2.5078125,8388608
+50,self_attn.k_proj,2,64,8,128,24.515625,2.25390625,8388608
+50,self_attn.k_proj,2,128,8,128,27.734375,2.126953125,8388608
+50,self_attn.k_proj,3,32,8,128,8.9609375,3.5078125,8388608
+50,self_attn.k_proj,3,64,8,128,11.0625,3.25390625,8388608
+50,self_attn.k_proj,3,128,8,128,13.0078125,3.126953125,8388608
+50,self_attn.k_proj,4,32,8,128,4.18359375,4.5078125,8388608
+50,self_attn.k_proj,4,64,8,128,5.1796875,4.25390625,8388608
+50,self_attn.k_proj,4,128,8,128,6.109375,4.126953125,8388608
+50,self_attn.k_proj,8,32,8,128,0.253173828125,8.5078125,8388608
+50,self_attn.k_proj,8,64,8,128,0.31298828125,8.25390625,8388608
+50,self_attn.k_proj,8,128,8,128,0.368408203125,8.126953125,8388608
+50,self_attn.o_proj,2,32,8,128,41.15625,20.0625,67108864
+50,self_attn.o_proj,2,64,8,128,47.90625,18.03125,67108864
+50,self_attn.o_proj,2,128,8,128,53.96875,17.015625,67108864
+50,self_attn.o_proj,3,32,8,128,17.609375,28.0625,67108864
+50,self_attn.o_proj,3,64,8,128,20.515625,26.03125,67108864
+50,self_attn.o_proj,3,128,8,128,23.125,25.015625,67108864
+50,self_attn.o_proj,4,32,8,128,8.2109375,36.0625,67108864
+50,self_attn.o_proj,4,64,8,128,9.578125,34.03125,67108864
+50,self_attn.o_proj,4,128,8,128,10.796875,33.015625,67108864
+50,self_attn.o_proj,8,32,8,128,0.497314453125,68.0625,67108864
+50,self_attn.o_proj,8,64,8,128,0.5791015625,66.03125,67108864
+50,self_attn.o_proj,8,128,8,128,0.65087890625,65.015625,67108864
+50,mlp.gate_proj,2,32,8,128,89.8125,70.21875,234881024
+50,mlp.gate_proj,2,64,8,128,104.8125,63.109375,234881024
+50,mlp.gate_proj,2,128,8,128,118.3125,59.5546875,234881024
+50,mlp.gate_proj,3,32,8,128,38.4375,98.21875,234881024
+50,mlp.gate_proj,3,64,8,128,44.8125,91.109375,234881024
+50,mlp.gate_proj,3,128,8,128,50.6875,87.5546875,234881024
+50,mlp.gate_proj,4,32,8,128,17.921875,126.21875,234881024
+50,mlp.gate_proj,4,64,8,128,20.921875,119.109375,234881024
+50,mlp.gate_proj,4,128,8,128,23.640625,115.5546875,234881024
+50,mlp.gate_proj,8,32,8,128,1.0908203125,238.21875,234881024
+50,mlp.gate_proj,8,64,8,128,1.2734375,231.109375,234881024
+50,mlp.gate_proj,8,128,8,128,1.4345703125,227.5546875,234881024
+50,mlp.up_proj,2,32,8,128,79.8125,70.21875,234881024
+50,mlp.up_proj,2,64,8,128,92.8125,63.109375,234881024
+50,mlp.up_proj,2,128,8,128,104.625,59.5546875,234881024
+50,mlp.up_proj,3,32,8,128,34.09375,98.21875,234881024
+50,mlp.up_proj,3,64,8,128,39.75,91.109375,234881024
+50,mlp.up_proj,3,128,8,128,44.75,87.5546875,234881024
+50,mlp.up_proj,4,32,8,128,15.8984375,126.21875,234881024
+50,mlp.up_proj,4,64,8,128,18.53125,119.109375,234881024
+50,mlp.up_proj,4,128,8,128,20.890625,115.5546875,234881024
+50,mlp.up_proj,8,32,8,128,0.966796875,238.21875,234881024
+50,mlp.up_proj,8,64,8,128,1.126953125,231.109375,234881024
+50,mlp.up_proj,8,128,8,128,1.267578125,227.5546875,234881024
+50,mlp.down_proj,2,32,8,128,80.875,70.21875,234881024
+50,mlp.down_proj,2,64,8,128,94.0625,63.109375,234881024
+50,mlp.down_proj,2,128,8,128,106.0,59.5546875,234881024
+50,mlp.down_proj,3,32,8,128,34.5625,98.21875,234881024
+50,mlp.down_proj,3,64,8,128,40.25,91.109375,234881024
+50,mlp.down_proj,3,128,8,128,45.3125,87.5546875,234881024
+50,mlp.down_proj,4,32,8,128,16.125,126.21875,234881024
+50,mlp.down_proj,4,64,8,128,18.78125,119.109375,234881024
+50,mlp.down_proj,4,128,8,128,21.15625,115.5546875,234881024
+50,mlp.down_proj,8,32,8,128,0.97998046875,238.21875,234881024
+50,mlp.down_proj,8,64,8,128,1.142578125,231.109375,234881024
+50,mlp.down_proj,8,128,8,128,1.283203125,227.5546875,234881024
+51,self_attn.q_proj,2,32,8,128,37.65625,20.0625,67108864
+51,self_attn.q_proj,2,64,8,128,44.4375,18.03125,67108864
+51,self_attn.q_proj,2,128,8,128,51.3125,17.015625,67108864
+51,self_attn.q_proj,3,32,8,128,16.140625,28.0625,67108864
+51,self_attn.q_proj,3,64,8,128,19.109375,26.03125,67108864
+51,self_attn.q_proj,3,128,8,128,22.21875,25.015625,67108864
+51,self_attn.q_proj,4,32,8,128,7.53125,36.0625,67108864
+51,self_attn.q_proj,4,64,8,128,8.9140625,34.03125,67108864
+51,self_attn.q_proj,4,128,8,128,10.375,33.015625,67108864
+51,self_attn.q_proj,8,32,8,128,0.4560546875,68.0625,67108864
+51,self_attn.q_proj,8,64,8,128,0.53955078125,66.03125,67108864
+51,self_attn.q_proj,8,128,8,128,0.62548828125,65.015625,67108864
+51,self_attn.v_proj,2,32,8,128,18.046875,2.5078125,8388608
+51,self_attn.v_proj,2,64,8,128,21.109375,2.25390625,8388608
+51,self_attn.v_proj,2,128,8,128,23.890625,2.126953125,8388608
+51,self_attn.v_proj,3,32,8,128,7.71484375,3.5078125,8388608
+51,self_attn.v_proj,3,64,8,128,9.0234375,3.25390625,8388608
+51,self_attn.v_proj,3,128,8,128,10.21875,3.126953125,8388608
+51,self_attn.v_proj,4,32,8,128,3.59765625,4.5078125,8388608
+51,self_attn.v_proj,4,64,8,128,4.2109375,4.25390625,8388608
+51,self_attn.v_proj,4,128,8,128,4.76953125,4.126953125,8388608
+51,self_attn.v_proj,8,32,8,128,0.218017578125,8.5078125,8388608
+51,self_attn.v_proj,8,64,8,128,0.2548828125,8.25390625,8388608
+51,self_attn.v_proj,8,128,8,128,0.28759765625,8.126953125,8388608
+51,self_attn.k_proj,2,32,8,128,21.640625,2.5078125,8388608
+51,self_attn.k_proj,2,64,8,128,26.203125,2.25390625,8388608
+51,self_attn.k_proj,2,128,8,128,30.03125,2.126953125,8388608
+51,self_attn.k_proj,3,32,8,128,9.3515625,3.5078125,8388608
+51,self_attn.k_proj,3,64,8,128,11.515625,3.25390625,8388608
+51,self_attn.k_proj,3,128,8,128,13.5625,3.126953125,8388608
+51,self_attn.k_proj,4,32,8,128,4.36328125,4.5078125,8388608
+51,self_attn.k_proj,4,64,8,128,5.375,4.25390625,8388608
+51,self_attn.k_proj,4,128,8,128,6.33203125,4.126953125,8388608
+51,self_attn.k_proj,8,32,8,128,0.26416015625,8.5078125,8388608
+51,self_attn.k_proj,8,64,8,128,0.3251953125,8.25390625,8388608
+51,self_attn.k_proj,8,128,8,128,0.381591796875,8.126953125,8388608
+51,self_attn.o_proj,2,32,8,128,40.8125,20.0625,67108864
+51,self_attn.o_proj,2,64,8,128,47.53125,18.03125,67108864
+51,self_attn.o_proj,2,128,8,128,53.59375,17.015625,67108864
+51,self_attn.o_proj,3,32,8,128,17.46875,28.0625,67108864
+51,self_attn.o_proj,3,64,8,128,20.359375,26.03125,67108864
+51,self_attn.o_proj,3,128,8,128,22.96875,25.015625,67108864
+51,self_attn.o_proj,4,32,8,128,8.1484375,36.0625,67108864
+51,self_attn.o_proj,4,64,8,128,9.5,34.03125,67108864
+51,self_attn.o_proj,4,128,8,128,10.7265625,33.015625,67108864
+51,self_attn.o_proj,8,32,8,128,0.493408203125,68.0625,67108864
+51,self_attn.o_proj,8,64,8,128,0.57470703125,66.03125,67108864
+51,self_attn.o_proj,8,128,8,128,0.64697265625,65.015625,67108864
+51,mlp.gate_proj,2,32,8,128,89.6875,70.21875,234881024
+51,mlp.gate_proj,2,64,8,128,104.6875,63.109375,234881024
+51,mlp.gate_proj,2,128,8,128,118.125,59.5546875,234881024
+51,mlp.gate_proj,3,32,8,128,38.375,98.21875,234881024
+51,mlp.gate_proj,3,64,8,128,44.75,91.109375,234881024
+51,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024
+51,mlp.gate_proj,4,32,8,128,17.90625,126.21875,234881024
+51,mlp.gate_proj,4,64,8,128,20.890625,119.109375,234881024
+51,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024
+51,mlp.gate_proj,8,32,8,128,1.0888671875,238.21875,234881024
+51,mlp.gate_proj,8,64,8,128,1.271484375,231.109375,234881024
+51,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024
+51,mlp.up_proj,2,32,8,128,80.0,70.21875,234881024
+51,mlp.up_proj,2,64,8,128,93.0,63.109375,234881024
+51,mlp.up_proj,2,128,8,128,104.8125,59.5546875,234881024
+51,mlp.up_proj,3,32,8,128,34.15625,98.21875,234881024
+51,mlp.up_proj,3,64,8,128,39.8125,91.109375,234881024
+51,mlp.up_proj,3,128,8,128,44.84375,87.5546875,234881024
+51,mlp.up_proj,4,32,8,128,15.9375,126.21875,234881024
+51,mlp.up_proj,4,64,8,128,18.578125,119.109375,234881024
+51,mlp.up_proj,4,128,8,128,20.9375,115.5546875,234881024
+51,mlp.up_proj,8,32,8,128,0.96923828125,238.21875,234881024
+51,mlp.up_proj,8,64,8,128,1.1298828125,231.109375,234881024
+51,mlp.up_proj,8,128,8,128,1.2705078125,227.5546875,234881024
+51,mlp.down_proj,2,32,8,128,80.9375,70.21875,234881024
+51,mlp.down_proj,2,64,8,128,94.1875,63.109375,234881024
+51,mlp.down_proj,2,128,8,128,106.125,59.5546875,234881024
+51,mlp.down_proj,3,32,8,128,34.59375,98.21875,234881024
+51,mlp.down_proj,3,64,8,128,40.3125,91.109375,234881024
+51,mlp.down_proj,3,128,8,128,45.375,87.5546875,234881024
+51,mlp.down_proj,4,32,8,128,16.140625,126.21875,234881024
+51,mlp.down_proj,4,64,8,128,18.8125,119.109375,234881024
+51,mlp.down_proj,4,128,8,128,21.1875,115.5546875,234881024
+51,mlp.down_proj,8,32,8,128,0.9814453125,238.21875,234881024
+51,mlp.down_proj,8,64,8,128,1.1435546875,231.109375,234881024
+51,mlp.down_proj,8,128,8,128,1.28515625,227.5546875,234881024
+52,self_attn.q_proj,2,32,8,128,40.40625,20.0625,67108864
+52,self_attn.q_proj,2,64,8,128,47.34375,18.03125,67108864
+52,self_attn.q_proj,2,128,8,128,54.6875,17.015625,67108864
+52,self_attn.q_proj,3,32,8,128,17.296875,28.0625,67108864
+52,self_attn.q_proj,3,64,8,128,20.3125,26.03125,67108864
+52,self_attn.q_proj,3,128,8,128,23.546875,25.015625,67108864
+52,self_attn.q_proj,4,32,8,128,8.0703125,36.0625,67108864
+52,self_attn.q_proj,4,64,8,128,9.484375,34.03125,67108864
+52,self_attn.q_proj,4,128,8,128,11.0,33.015625,67108864
+52,self_attn.q_proj,8,32,8,128,0.48876953125,68.0625,67108864
+52,self_attn.q_proj,8,64,8,128,0.57373046875,66.03125,67108864
+52,self_attn.q_proj,8,128,8,128,0.66357421875,65.015625,67108864
+52,self_attn.v_proj,2,32,8,128,17.765625,2.5078125,8388608
+52,self_attn.v_proj,2,64,8,128,20.78125,2.25390625,8388608
+52,self_attn.v_proj,2,128,8,128,23.515625,2.126953125,8388608
+52,self_attn.v_proj,3,32,8,128,7.59765625,3.5078125,8388608
+52,self_attn.v_proj,3,64,8,128,8.890625,3.25390625,8388608
+52,self_attn.v_proj,3,128,8,128,10.0546875,3.126953125,8388608
+52,self_attn.v_proj,4,32,8,128,3.54296875,4.5078125,8388608
+52,self_attn.v_proj,4,64,8,128,4.1484375,4.25390625,8388608
+52,self_attn.v_proj,4,128,8,128,4.69140625,4.126953125,8388608
+52,self_attn.v_proj,8,32,8,128,0.214599609375,8.5078125,8388608
+52,self_attn.v_proj,8,64,8,128,0.2509765625,8.25390625,8388608
+52,self_attn.v_proj,8,128,8,128,0.282958984375,8.126953125,8388608
+52,self_attn.k_proj,2,32,8,128,22.265625,2.5078125,8388608
+52,self_attn.k_proj,2,64,8,128,26.6875,2.25390625,8388608
+52,self_attn.k_proj,2,128,8,128,30.09375,2.126953125,8388608
+52,self_attn.k_proj,3,32,8,128,9.6796875,3.5078125,8388608
+52,self_attn.k_proj,3,64,8,128,11.8984375,3.25390625,8388608
+52,self_attn.k_proj,3,128,8,128,13.90625,3.126953125,8388608
+52,self_attn.k_proj,4,32,8,128,4.515625,4.5078125,8388608
+52,self_attn.k_proj,4,64,8,128,5.55859375,4.25390625,8388608
+52,self_attn.k_proj,4,128,8,128,6.5,4.126953125,8388608
+52,self_attn.k_proj,8,32,8,128,0.2734375,8.5078125,8388608
+52,self_attn.k_proj,8,64,8,128,0.3359375,8.25390625,8388608
+52,self_attn.k_proj,8,128,8,128,0.391845703125,8.126953125,8388608
+52,self_attn.o_proj,2,32,8,128,41.8125,20.0625,67108864
+52,self_attn.o_proj,2,64,8,128,48.71875,18.03125,67108864
+52,self_attn.o_proj,2,128,8,128,55.0,17.015625,67108864
+52,self_attn.o_proj,3,32,8,128,17.9375,28.0625,67108864
+52,self_attn.o_proj,3,64,8,128,20.96875,26.03125,67108864
+52,self_attn.o_proj,3,128,8,128,23.75,25.015625,67108864
+52,self_attn.o_proj,4,32,8,128,8.421875,36.0625,67108864
+52,self_attn.o_proj,4,64,8,128,9.875,34.03125,67108864
+52,self_attn.o_proj,4,128,8,128,11.234375,33.015625,67108864
+52,self_attn.o_proj,8,32,8,128,0.51123046875,68.0625,67108864
+52,self_attn.o_proj,8,64,8,128,0.60009765625,66.03125,67108864
+52,self_attn.o_proj,8,128,8,128,0.68359375,65.015625,67108864
+52,mlp.gate_proj,2,32,8,128,89.75,70.21875,234881024
+52,mlp.gate_proj,2,64,8,128,104.8125,63.109375,234881024
+52,mlp.gate_proj,2,128,8,128,118.1875,59.5546875,234881024
+52,mlp.gate_proj,3,32,8,128,38.40625,98.21875,234881024
+52,mlp.gate_proj,3,64,8,128,44.78125,91.109375,234881024
+52,mlp.gate_proj,3,128,8,128,50.625,87.5546875,234881024
+52,mlp.gate_proj,4,32,8,128,17.921875,126.21875,234881024
+52,mlp.gate_proj,4,64,8,128,20.90625,119.109375,234881024
+52,mlp.gate_proj,4,128,8,128,23.625,115.5546875,234881024
+52,mlp.gate_proj,8,32,8,128,1.08984375,238.21875,234881024
+52,mlp.gate_proj,8,64,8,128,1.2724609375,231.109375,234881024
+52,mlp.gate_proj,8,128,8,128,1.4326171875,227.5546875,234881024
+52,mlp.up_proj,2,32,8,128,80.0625,70.21875,234881024
+52,mlp.up_proj,2,64,8,128,93.125,63.109375,234881024
+52,mlp.up_proj,2,128,8,128,104.9375,59.5546875,234881024
+52,mlp.up_proj,3,32,8,128,34.1875,98.21875,234881024
+52,mlp.up_proj,3,64,8,128,39.875,91.109375,234881024
+52,mlp.up_proj,3,128,8,128,44.90625,87.5546875,234881024
+52,mlp.up_proj,4,32,8,128,15.953125,126.21875,234881024
+52,mlp.up_proj,4,64,8,128,18.59375,119.109375,234881024
+52,mlp.up_proj,4,128,8,128,20.953125,115.5546875,234881024
+52,mlp.up_proj,8,32,8,128,0.9697265625,238.21875,234881024
+52,mlp.up_proj,8,64,8,128,1.130859375,231.109375,234881024
+52,mlp.up_proj,8,128,8,128,1.271484375,227.5546875,234881024
+52,mlp.down_proj,2,32,8,128,80.9375,70.21875,234881024
+52,mlp.down_proj,2,64,8,128,94.1875,63.109375,234881024
+52,mlp.down_proj,2,128,8,128,106.125,59.5546875,234881024
+52,mlp.down_proj,3,32,8,128,34.59375,98.21875,234881024
+52,mlp.down_proj,3,64,8,128,40.3125,91.109375,234881024
+52,mlp.down_proj,3,128,8,128,45.375,87.5546875,234881024
+52,mlp.down_proj,4,32,8,128,16.140625,126.21875,234881024
+52,mlp.down_proj,4,64,8,128,18.8125,119.109375,234881024
+52,mlp.down_proj,4,128,8,128,21.171875,115.5546875,234881024
+52,mlp.down_proj,8,32,8,128,0.9814453125,238.21875,234881024
+52,mlp.down_proj,8,64,8,128,1.1435546875,231.109375,234881024
+52,mlp.down_proj,8,128,8,128,1.28515625,227.5546875,234881024
+53,self_attn.q_proj,2,32,8,128,33.6875,20.0625,67108864
+53,self_attn.q_proj,2,64,8,128,39.71875,18.03125,67108864
+53,self_attn.q_proj,2,128,8,128,46.0,17.015625,67108864
+53,self_attn.q_proj,3,32,8,128,14.4609375,28.0625,67108864
+53,self_attn.q_proj,3,64,8,128,17.125,26.03125,67108864
+53,self_attn.q_proj,3,128,8,128,19.96875,25.015625,67108864
+53,self_attn.q_proj,4,32,8,128,6.75,36.0625,67108864
+53,self_attn.q_proj,4,64,8,128,7.98828125,34.03125,67108864
+53,self_attn.q_proj,4,128,8,128,9.3203125,33.015625,67108864
+53,self_attn.q_proj,8,32,8,128,0.408447265625,68.0625,67108864
+53,self_attn.q_proj,8,64,8,128,0.4833984375,66.03125,67108864
+53,self_attn.q_proj,8,128,8,128,0.56201171875,65.015625,67108864
+53,self_attn.v_proj,2,32,8,128,18.265625,2.5078125,8388608
+53,self_attn.v_proj,2,64,8,128,21.40625,2.25390625,8388608
+53,self_attn.v_proj,2,128,8,128,24.28125,2.126953125,8388608
+53,self_attn.v_proj,3,32,8,128,7.8125,3.5078125,8388608
+53,self_attn.v_proj,3,64,8,128,9.15625,3.25390625,8388608
+53,self_attn.v_proj,3,128,8,128,10.390625,3.126953125,8388608
+53,self_attn.v_proj,4,32,8,128,3.64453125,4.5078125,8388608
+53,self_attn.v_proj,4,64,8,128,4.2734375,4.25390625,8388608
+53,self_attn.v_proj,4,128,8,128,4.84765625,4.126953125,8388608
+53,self_attn.v_proj,8,32,8,128,0.2208251953125,8.5078125,8388608
+53,self_attn.v_proj,8,64,8,128,0.258544921875,8.25390625,8388608
+53,self_attn.v_proj,8,128,8,128,0.2919921875,8.126953125,8388608
+53,self_attn.k_proj,2,32,8,128,20.03125,2.5078125,8388608
+53,self_attn.k_proj,2,64,8,128,24.0625,2.25390625,8388608
+53,self_attn.k_proj,2,128,8,128,27.328125,2.126953125,8388608
+53,self_attn.k_proj,3,32,8,128,8.8125,3.5078125,8388608
+53,self_attn.k_proj,3,64,8,128,10.953125,3.25390625,8388608
+53,self_attn.k_proj,3,128,8,128,13.0078125,3.126953125,8388608
+53,self_attn.k_proj,4,32,8,128,4.1171875,4.5078125,8388608
+53,self_attn.k_proj,4,64,8,128,5.12109375,4.25390625,8388608
+53,self_attn.k_proj,4,128,8,128,6.10546875,4.126953125,8388608
+53,self_attn.k_proj,8,32,8,128,0.2491455078125,8.5078125,8388608
+53,self_attn.k_proj,8,64,8,128,0.309814453125,8.25390625,8388608
+53,self_attn.k_proj,8,128,8,128,0.3681640625,8.126953125,8388608
+53,self_attn.o_proj,2,32,8,128,41.4375,20.0625,67108864
+53,self_attn.o_proj,2,64,8,128,48.25,18.03125,67108864
+53,self_attn.o_proj,2,128,8,128,54.375,17.015625,67108864
+53,self_attn.o_proj,3,32,8,128,17.71875,28.0625,67108864
+53,self_attn.o_proj,3,64,8,128,20.65625,26.03125,67108864
+53,self_attn.o_proj,3,128,8,128,23.3125,25.015625,67108864
+53,self_attn.o_proj,4,32,8,128,8.265625,36.0625,67108864
+53,self_attn.o_proj,4,64,8,128,9.640625,34.03125,67108864
+53,self_attn.o_proj,4,128,8,128,10.8828125,33.015625,67108864
+53,self_attn.o_proj,8,32,8,128,0.50048828125,68.0625,67108864
+53,self_attn.o_proj,8,64,8,128,0.5830078125,66.03125,67108864
+53,self_attn.o_proj,8,128,8,128,0.65625,65.015625,67108864
+53,mlp.gate_proj,2,32,8,128,89.75,70.21875,234881024
+53,mlp.gate_proj,2,64,8,128,104.8125,63.109375,234881024
+53,mlp.gate_proj,2,128,8,128,118.1875,59.5546875,234881024
+53,mlp.gate_proj,3,32,8,128,38.40625,98.21875,234881024
+53,mlp.gate_proj,3,64,8,128,44.8125,91.109375,234881024
+53,mlp.gate_proj,3,128,8,128,50.625,87.5546875,234881024
+53,mlp.gate_proj,4,32,8,128,17.921875,126.21875,234881024
+53,mlp.gate_proj,4,64,8,128,20.90625,119.109375,234881024
+53,mlp.gate_proj,4,128,8,128,23.625,115.5546875,234881024
+53,mlp.gate_proj,8,32,8,128,1.08984375,238.21875,234881024
+53,mlp.gate_proj,8,64,8,128,1.2724609375,231.109375,234881024
+53,mlp.gate_proj,8,128,8,128,1.4326171875,227.5546875,234881024
+53,mlp.up_proj,2,32,8,128,80.1875,70.21875,234881024
+53,mlp.up_proj,2,64,8,128,93.3125,63.109375,234881024
+53,mlp.up_proj,2,128,8,128,105.125,59.5546875,234881024
+53,mlp.up_proj,3,32,8,128,34.25,98.21875,234881024
+53,mlp.up_proj,3,64,8,128,39.9375,91.109375,234881024
+53,mlp.up_proj,3,128,8,128,44.96875,87.5546875,234881024
+53,mlp.up_proj,4,32,8,128,15.9765625,126.21875,234881024
+53,mlp.up_proj,4,64,8,128,18.625,119.109375,234881024
+53,mlp.up_proj,4,128,8,128,20.984375,115.5546875,234881024
+53,mlp.up_proj,8,32,8,128,0.9716796875,238.21875,234881024
+53,mlp.up_proj,8,64,8,128,1.1328125,231.109375,234881024
+53,mlp.up_proj,8,128,8,128,1.2734375,227.5546875,234881024
+53,mlp.down_proj,2,32,8,128,81.0,70.21875,234881024
+53,mlp.down_proj,2,64,8,128,94.25,63.109375,234881024
+53,mlp.down_proj,2,128,8,128,106.1875,59.5546875,234881024
+53,mlp.down_proj,3,32,8,128,34.625,98.21875,234881024
+53,mlp.down_proj,3,64,8,128,40.34375,91.109375,234881024
+53,mlp.down_proj,3,128,8,128,45.40625,87.5546875,234881024
+53,mlp.down_proj,4,32,8,128,16.140625,126.21875,234881024
+53,mlp.down_proj,4,64,8,128,18.8125,119.109375,234881024
+53,mlp.down_proj,4,128,8,128,21.1875,115.5546875,234881024
+53,mlp.down_proj,8,32,8,128,0.98193359375,238.21875,234881024
+53,mlp.down_proj,8,64,8,128,1.14453125,231.109375,234881024
+53,mlp.down_proj,8,128,8,128,1.28515625,227.5546875,234881024
+54,self_attn.q_proj,2,32,8,128,34.46875,20.0625,67108864
+54,self_attn.q_proj,2,64,8,128,40.625,18.03125,67108864
+54,self_attn.q_proj,2,128,8,128,47.125,17.015625,67108864
+54,self_attn.q_proj,3,32,8,128,14.78125,28.0625,67108864
+54,self_attn.q_proj,3,64,8,128,17.5,26.03125,67108864
+54,self_attn.q_proj,3,128,8,128,20.484375,25.015625,67108864
+54,self_attn.q_proj,4,32,8,128,6.8984375,36.0625,67108864
+54,self_attn.q_proj,4,64,8,128,8.171875,34.03125,67108864
+54,self_attn.q_proj,4,128,8,128,9.5625,33.015625,67108864
+54,self_attn.q_proj,8,32,8,128,0.41748046875,68.0625,67108864
+54,self_attn.q_proj,8,64,8,128,0.494384765625,66.03125,67108864
+54,self_attn.q_proj,8,128,8,128,0.5771484375,65.015625,67108864
+54,self_attn.v_proj,2,32,8,128,18.25,2.5078125,8388608
+54,self_attn.v_proj,2,64,8,128,21.421875,2.25390625,8388608
+54,self_attn.v_proj,2,128,8,128,24.296875,2.126953125,8388608
+54,self_attn.v_proj,3,32,8,128,7.8046875,3.5078125,8388608
+54,self_attn.v_proj,3,64,8,128,9.1640625,3.25390625,8388608
+54,self_attn.v_proj,3,128,8,128,10.3984375,3.126953125,8388608
+54,self_attn.v_proj,4,32,8,128,3.640625,4.5078125,8388608
+54,self_attn.v_proj,4,64,8,128,4.2734375,4.25390625,8388608
+54,self_attn.v_proj,4,128,8,128,4.8515625,4.126953125,8388608
+54,self_attn.v_proj,8,32,8,128,0.220458984375,8.5078125,8388608
+54,self_attn.v_proj,8,64,8,128,0.258544921875,8.25390625,8388608
+54,self_attn.v_proj,8,128,8,128,0.29248046875,8.126953125,8388608
+54,self_attn.k_proj,2,32,8,128,19.71875,2.5078125,8388608
+54,self_attn.k_proj,2,64,8,128,23.78125,2.25390625,8388608
+54,self_attn.k_proj,2,128,8,128,27.046875,2.126953125,8388608
+54,self_attn.k_proj,3,32,8,128,8.6484375,3.5078125,8388608
+54,self_attn.k_proj,3,64,8,128,10.765625,3.25390625,8388608
+54,self_attn.k_proj,3,128,8,128,12.7890625,3.126953125,8388608
+54,self_attn.k_proj,4,32,8,128,4.03515625,4.5078125,8388608
+54,self_attn.k_proj,4,64,8,128,5.03125,4.25390625,8388608
+54,self_attn.k_proj,4,128,8,128,5.98828125,4.126953125,8388608
+54,self_attn.k_proj,8,32,8,128,0.2442626953125,8.5078125,8388608
+54,self_attn.k_proj,8,64,8,128,0.30419921875,8.25390625,8388608
+54,self_attn.k_proj,8,128,8,128,0.360595703125,8.126953125,8388608
+54,self_attn.o_proj,2,32,8,128,40.59375,20.0625,67108864
+54,self_attn.o_proj,2,64,8,128,47.25,18.03125,67108864
+54,self_attn.o_proj,2,128,8,128,53.28125,17.015625,67108864
+54,self_attn.o_proj,3,32,8,128,17.359375,28.0625,67108864
+54,self_attn.o_proj,3,64,8,128,20.21875,26.03125,67108864
+54,self_attn.o_proj,3,128,8,128,22.796875,25.015625,67108864
+54,self_attn.o_proj,4,32,8,128,8.1015625,36.0625,67108864
+54,self_attn.o_proj,4,64,8,128,9.4375,34.03125,67108864
+54,self_attn.o_proj,4,128,8,128,10.6484375,33.015625,67108864
+54,self_attn.o_proj,8,32,8,128,0.490478515625,68.0625,67108864
+54,self_attn.o_proj,8,64,8,128,0.5712890625,66.03125,67108864
+54,self_attn.o_proj,8,128,8,128,0.64208984375,65.015625,67108864
+54,mlp.gate_proj,2,32,8,128,89.6875,70.21875,234881024
+54,mlp.gate_proj,2,64,8,128,104.6875,63.109375,234881024
+54,mlp.gate_proj,2,128,8,128,118.125,59.5546875,234881024
+54,mlp.gate_proj,3,32,8,128,38.40625,98.21875,234881024
+54,mlp.gate_proj,3,64,8,128,44.75,91.109375,234881024
+54,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024
+54,mlp.gate_proj,4,32,8,128,17.90625,126.21875,234881024
+54,mlp.gate_proj,4,64,8,128,20.890625,119.109375,234881024
+54,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024
+54,mlp.gate_proj,8,32,8,128,1.0888671875,238.21875,234881024
+54,mlp.gate_proj,8,64,8,128,1.271484375,231.109375,234881024
+54,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024
+54,mlp.up_proj,2,32,8,128,80.375,70.21875,234881024
+54,mlp.up_proj,2,64,8,128,93.5,63.109375,234881024
+54,mlp.up_proj,2,128,8,128,105.375,59.5546875,234881024
+54,mlp.up_proj,3,32,8,128,34.34375,98.21875,234881024
+54,mlp.up_proj,3,64,8,128,40.03125,91.109375,234881024
+54,mlp.up_proj,3,128,8,128,45.0625,87.5546875,234881024
+54,mlp.up_proj,4,32,8,128,16.015625,126.21875,234881024
+54,mlp.up_proj,4,64,8,128,18.671875,119.109375,234881024
+54,mlp.up_proj,4,128,8,128,21.03125,115.5546875,234881024
+54,mlp.up_proj,8,32,8,128,0.9736328125,238.21875,234881024
+54,mlp.up_proj,8,64,8,128,1.1357421875,231.109375,234881024
+54,mlp.up_proj,8,128,8,128,1.2763671875,227.5546875,234881024
+54,mlp.down_proj,2,32,8,128,81.0625,70.21875,234881024
+54,mlp.down_proj,2,64,8,128,94.3125,63.109375,234881024
+54,mlp.down_proj,2,128,8,128,106.3125,59.5546875,234881024
+54,mlp.down_proj,3,32,8,128,34.65625,98.21875,234881024
+54,mlp.down_proj,3,64,8,128,40.34375,91.109375,234881024
+54,mlp.down_proj,3,128,8,128,45.4375,87.5546875,234881024
+54,mlp.down_proj,4,32,8,128,16.15625,126.21875,234881024
+54,mlp.down_proj,4,64,8,128,18.828125,119.109375,234881024
+54,mlp.down_proj,4,128,8,128,21.203125,115.5546875,234881024
+54,mlp.down_proj,8,32,8,128,0.98291015625,238.21875,234881024
+54,mlp.down_proj,8,64,8,128,1.1455078125,231.109375,234881024
+54,mlp.down_proj,8,128,8,128,1.2861328125,227.5546875,234881024
+55,self_attn.q_proj,2,32,8,128,38.9375,20.0625,67108864
+55,self_attn.q_proj,2,64,8,128,45.9375,18.03125,67108864
+55,self_attn.q_proj,2,128,8,128,53.3125,17.015625,67108864
+55,self_attn.q_proj,3,32,8,128,16.734375,28.0625,67108864
+55,self_attn.q_proj,3,64,8,128,19.84375,26.03125,67108864
+55,self_attn.q_proj,3,128,8,128,23.25,25.015625,67108864
+55,self_attn.q_proj,4,32,8,128,7.8046875,36.0625,67108864
+55,self_attn.q_proj,4,64,8,128,9.2578125,34.03125,67108864
+55,self_attn.q_proj,4,128,8,128,10.8515625,33.015625,67108864
+55,self_attn.q_proj,8,32,8,128,0.47265625,68.0625,67108864
+55,self_attn.q_proj,8,64,8,128,0.560546875,66.03125,67108864
+55,self_attn.q_proj,8,128,8,128,0.65478515625,65.015625,67108864
+55,self_attn.v_proj,2,32,8,128,18.609375,2.5078125,8388608
+55,self_attn.v_proj,2,64,8,128,21.71875,2.25390625,8388608
+55,self_attn.v_proj,2,128,8,128,24.5625,2.126953125,8388608
+55,self_attn.v_proj,3,32,8,128,7.95703125,3.5078125,8388608
+55,self_attn.v_proj,3,64,8,128,9.296875,3.25390625,8388608
+55,self_attn.v_proj,3,128,8,128,10.5078125,3.126953125,8388608
+55,self_attn.v_proj,4,32,8,128,3.712890625,4.5078125,8388608
+55,self_attn.v_proj,4,64,8,128,4.3359375,4.25390625,8388608
+55,self_attn.v_proj,4,128,8,128,4.90625,4.126953125,8388608
+55,self_attn.v_proj,8,32,8,128,0.224853515625,8.5078125,8388608
+55,self_attn.v_proj,8,64,8,128,0.262451171875,8.25390625,8388608
+55,self_attn.v_proj,8,128,8,128,0.295654296875,8.126953125,8388608
+55,self_attn.k_proj,2,32,8,128,22.234375,2.5078125,8388608
+55,self_attn.k_proj,2,64,8,128,26.609375,2.25390625,8388608
+55,self_attn.k_proj,2,128,8,128,30.140625,2.126953125,8388608
+55,self_attn.k_proj,3,32,8,128,9.703125,3.5078125,8388608
+55,self_attn.k_proj,3,64,8,128,11.890625,3.25390625,8388608
+55,self_attn.k_proj,3,128,8,128,13.9765625,3.126953125,8388608
+55,self_attn.k_proj,4,32,8,128,4.52734375,4.5078125,8388608
+55,self_attn.k_proj,4,64,8,128,5.55078125,4.25390625,8388608
+55,self_attn.k_proj,4,128,8,128,6.52734375,4.126953125,8388608
+55,self_attn.k_proj,8,32,8,128,0.274169921875,8.5078125,8388608
+55,self_attn.k_proj,8,64,8,128,0.335693359375,8.25390625,8388608
+55,self_attn.k_proj,8,128,8,128,0.3935546875,8.126953125,8388608
+55,self_attn.o_proj,2,32,8,128,43.15625,20.0625,67108864
+55,self_attn.o_proj,2,64,8,128,50.25,18.03125,67108864
+55,self_attn.o_proj,2,128,8,128,56.59375,17.015625,67108864
+55,self_attn.o_proj,3,32,8,128,18.453125,28.0625,67108864
+55,self_attn.o_proj,3,64,8,128,21.5,26.03125,67108864
+55,self_attn.o_proj,3,128,8,128,24.234375,25.015625,67108864
+55,self_attn.o_proj,4,32,8,128,8.609375,36.0625,67108864
+55,self_attn.o_proj,4,64,8,128,10.03125,34.03125,67108864
+55,self_attn.o_proj,4,128,8,128,11.3125,33.015625,67108864
+55,self_attn.o_proj,8,32,8,128,0.521484375,68.0625,67108864
+55,self_attn.o_proj,8,64,8,128,0.60693359375,66.03125,67108864
+55,self_attn.o_proj,8,128,8,128,0.681640625,65.015625,67108864
+55,mlp.gate_proj,2,32,8,128,89.5625,70.21875,234881024
+55,mlp.gate_proj,2,64,8,128,104.5,63.109375,234881024
+55,mlp.gate_proj,2,128,8,128,117.9375,59.5546875,234881024
+55,mlp.gate_proj,3,32,8,128,38.3125,98.21875,234881024
+55,mlp.gate_proj,3,64,8,128,44.6875,91.109375,234881024
+55,mlp.gate_proj,3,128,8,128,50.5,87.5546875,234881024
+55,mlp.gate_proj,4,32,8,128,17.875,126.21875,234881024
+55,mlp.gate_proj,4,64,8,128,20.859375,119.109375,234881024
+55,mlp.gate_proj,4,128,8,128,23.5625,115.5546875,234881024
+55,mlp.gate_proj,8,32,8,128,1.0869140625,238.21875,234881024
+55,mlp.gate_proj,8,64,8,128,1.26953125,231.109375,234881024
+55,mlp.gate_proj,8,128,8,128,1.4287109375,227.5546875,234881024
+55,mlp.up_proj,2,32,8,128,80.625,70.21875,234881024
+55,mlp.up_proj,2,64,8,128,93.8125,63.109375,234881024
+55,mlp.up_proj,2,128,8,128,105.6875,59.5546875,234881024
+55,mlp.up_proj,3,32,8,128,34.46875,98.21875,234881024
+55,mlp.up_proj,3,64,8,128,40.15625,91.109375,234881024
+55,mlp.up_proj,3,128,8,128,45.21875,87.5546875,234881024
+55,mlp.up_proj,4,32,8,128,16.0625,126.21875,234881024
+55,mlp.up_proj,4,64,8,128,18.734375,119.109375,234881024
+55,mlp.up_proj,4,128,8,128,21.109375,115.5546875,234881024
+55,mlp.up_proj,8,32,8,128,0.97705078125,238.21875,234881024
+55,mlp.up_proj,8,64,8,128,1.1396484375,231.109375,234881024
+55,mlp.up_proj,8,128,8,128,1.2802734375,227.5546875,234881024
+55,mlp.down_proj,2,32,8,128,81.25,70.21875,234881024
+55,mlp.down_proj,2,64,8,128,94.5,63.109375,234881024
+55,mlp.down_proj,2,128,8,128,106.4375,59.5546875,234881024
+55,mlp.down_proj,3,32,8,128,34.71875,98.21875,234881024
+55,mlp.down_proj,3,64,8,128,40.4375,91.109375,234881024
+55,mlp.down_proj,3,128,8,128,45.53125,87.5546875,234881024
+55,mlp.down_proj,4,32,8,128,16.1875,126.21875,234881024
+55,mlp.down_proj,4,64,8,128,18.875,119.109375,234881024
+55,mlp.down_proj,4,128,8,128,21.25,115.5546875,234881024
+55,mlp.down_proj,8,32,8,128,0.98486328125,238.21875,234881024
+55,mlp.down_proj,8,64,8,128,1.1484375,231.109375,234881024
+55,mlp.down_proj,8,128,8,128,1.2890625,227.5546875,234881024
+56,self_attn.q_proj,2,32,8,128,36.5,20.0625,67108864
+56,self_attn.q_proj,2,64,8,128,43.28125,18.03125,67108864
+56,self_attn.q_proj,2,128,8,128,50.125,17.015625,67108864
+56,self_attn.q_proj,3,32,8,128,15.65625,28.0625,67108864
+56,self_attn.q_proj,3,64,8,128,18.65625,26.03125,67108864
+56,self_attn.q_proj,3,128,8,128,21.8125,25.015625,67108864
+56,self_attn.q_proj,4,32,8,128,7.30859375,36.0625,67108864
+56,self_attn.q_proj,4,64,8,128,8.7109375,34.03125,67108864
+56,self_attn.q_proj,4,128,8,128,10.1875,33.015625,67108864
+56,self_attn.q_proj,8,32,8,128,0.4423828125,68.0625,67108864
+56,self_attn.q_proj,8,64,8,128,0.52734375,66.03125,67108864
+56,self_attn.q_proj,8,128,8,128,0.6142578125,65.015625,67108864
+56,self_attn.v_proj,2,32,8,128,18.21875,2.5078125,8388608
+56,self_attn.v_proj,2,64,8,128,21.359375,2.25390625,8388608
+56,self_attn.v_proj,2,128,8,128,24.1875,2.126953125,8388608
+56,self_attn.v_proj,3,32,8,128,7.7890625,3.5078125,8388608
+56,self_attn.v_proj,3,64,8,128,9.1328125,3.25390625,8388608
+56,self_attn.v_proj,3,128,8,128,10.34375,3.126953125,8388608
+56,self_attn.v_proj,4,32,8,128,3.6328125,4.5078125,8388608
+56,self_attn.v_proj,4,64,8,128,4.2578125,4.25390625,8388608
+56,self_attn.v_proj,4,128,8,128,4.828125,4.126953125,8388608
+56,self_attn.v_proj,8,32,8,128,0.219970703125,8.5078125,8388608
+56,self_attn.v_proj,8,64,8,128,0.257568359375,8.25390625,8388608
+56,self_attn.v_proj,8,128,8,128,0.291015625,8.126953125,8388608
+56,self_attn.k_proj,2,32,8,128,22.421875,2.5078125,8388608
+56,self_attn.k_proj,2,64,8,128,26.84375,2.25390625,8388608
+56,self_attn.k_proj,2,128,8,128,30.40625,2.126953125,8388608
+56,self_attn.k_proj,3,32,8,128,9.8359375,3.5078125,8388608
+56,self_attn.k_proj,3,64,8,128,12.140625,3.25390625,8388608
+56,self_attn.k_proj,3,128,8,128,14.34375,3.126953125,8388608
+56,self_attn.k_proj,4,32,8,128,4.59375,4.5078125,8388608
+56,self_attn.k_proj,4,64,8,128,5.67578125,4.25390625,8388608
+56,self_attn.k_proj,4,128,8,128,6.7109375,4.126953125,8388608
+56,self_attn.k_proj,8,32,8,128,0.278076171875,8.5078125,8388608
+56,self_attn.k_proj,8,64,8,128,0.343017578125,8.25390625,8388608
+56,self_attn.k_proj,8,128,8,128,0.404541015625,8.126953125,8388608
+56,self_attn.o_proj,2,32,8,128,40.84375,20.0625,67108864
+56,self_attn.o_proj,2,64,8,128,47.65625,18.03125,67108864
+56,self_attn.o_proj,2,128,8,128,53.8125,17.015625,67108864
+56,self_attn.o_proj,3,32,8,128,17.53125,28.0625,67108864
+56,self_attn.o_proj,3,64,8,128,20.5,26.03125,67108864
+56,self_attn.o_proj,3,128,8,128,23.265625,25.015625,67108864
+56,self_attn.o_proj,4,32,8,128,8.203125,36.0625,67108864
+56,self_attn.o_proj,4,64,8,128,9.6171875,34.03125,67108864
+56,self_attn.o_proj,4,128,8,128,10.9609375,33.015625,67108864
+56,self_attn.o_proj,8,32,8,128,0.499267578125,68.0625,67108864
+56,self_attn.o_proj,8,64,8,128,0.58642578125,66.03125,67108864
+56,self_attn.o_proj,8,128,8,128,0.66943359375,65.015625,67108864
+56,mlp.gate_proj,2,32,8,128,89.5,70.21875,234881024
+56,mlp.gate_proj,2,64,8,128,104.5,63.109375,234881024
+56,mlp.gate_proj,2,128,8,128,118.0,59.5546875,234881024
+56,mlp.gate_proj,3,32,8,128,38.3125,98.21875,234881024
+56,mlp.gate_proj,3,64,8,128,44.65625,91.109375,234881024
+56,mlp.gate_proj,3,128,8,128,50.5,87.5546875,234881024
+56,mlp.gate_proj,4,32,8,128,17.859375,126.21875,234881024
+56,mlp.gate_proj,4,64,8,128,20.84375,119.109375,234881024
+56,mlp.gate_proj,4,128,8,128,23.5625,115.5546875,234881024
+56,mlp.gate_proj,8,32,8,128,1.0869140625,238.21875,234881024
+56,mlp.gate_proj,8,64,8,128,1.2685546875,231.109375,234881024
+56,mlp.gate_proj,8,128,8,128,1.4287109375,227.5546875,234881024
+56,mlp.up_proj,2,32,8,128,80.75,70.21875,234881024
+56,mlp.up_proj,2,64,8,128,93.9375,63.109375,234881024
+56,mlp.up_proj,2,128,8,128,105.875,59.5546875,234881024
+56,mlp.up_proj,3,32,8,128,34.5,98.21875,234881024
+56,mlp.up_proj,3,64,8,128,40.21875,91.109375,234881024
+56,mlp.up_proj,3,128,8,128,45.28125,87.5546875,234881024
+56,mlp.up_proj,4,32,8,128,16.09375,126.21875,234881024
+56,mlp.up_proj,4,64,8,128,18.765625,119.109375,234881024
+56,mlp.up_proj,4,128,8,128,21.140625,115.5546875,234881024
+56,mlp.up_proj,8,32,8,128,0.97900390625,238.21875,234881024
+56,mlp.up_proj,8,64,8,128,1.1416015625,231.109375,234881024
+56,mlp.up_proj,8,128,8,128,1.2822265625,227.5546875,234881024
+56,mlp.down_proj,2,32,8,128,81.3125,70.21875,234881024
+56,mlp.down_proj,2,64,8,128,94.5625,63.109375,234881024
+56,mlp.down_proj,2,128,8,128,106.5625,59.5546875,234881024
+56,mlp.down_proj,3,32,8,128,34.75,98.21875,234881024
+56,mlp.down_proj,3,64,8,128,40.46875,91.109375,234881024
+56,mlp.down_proj,3,128,8,128,45.53125,87.5546875,234881024
+56,mlp.down_proj,4,32,8,128,16.203125,126.21875,234881024
+56,mlp.down_proj,4,64,8,128,18.890625,119.109375,234881024
+56,mlp.down_proj,4,128,8,128,21.25,115.5546875,234881024
+56,mlp.down_proj,8,32,8,128,0.9853515625,238.21875,234881024
+56,mlp.down_proj,8,64,8,128,1.1484375,231.109375,234881024
+56,mlp.down_proj,8,128,8,128,1.2900390625,227.5546875,234881024
+57,self_attn.q_proj,2,32,8,128,34.75,20.0625,67108864
+57,self_attn.q_proj,2,64,8,128,40.90625,18.03125,67108864
+57,self_attn.q_proj,2,128,8,128,47.28125,17.015625,67108864
+57,self_attn.q_proj,3,32,8,128,14.8828125,28.0625,67108864
+57,self_attn.q_proj,3,64,8,128,17.546875,26.03125,67108864
+57,self_attn.q_proj,3,128,8,128,20.390625,25.015625,67108864
+57,self_attn.q_proj,4,32,8,128,6.9453125,36.0625,67108864
+57,self_attn.q_proj,4,64,8,128,8.1953125,34.03125,67108864
+57,self_attn.q_proj,4,128,8,128,9.515625,33.015625,67108864
+57,self_attn.q_proj,8,32,8,128,0.42041015625,68.0625,67108864
+57,self_attn.q_proj,8,64,8,128,0.49560546875,66.03125,67108864
+57,self_attn.q_proj,8,128,8,128,0.57421875,65.015625,67108864
+57,self_attn.v_proj,2,32,8,128,18.65625,2.5078125,8388608
+57,self_attn.v_proj,2,64,8,128,21.78125,2.25390625,8388608
+57,self_attn.v_proj,2,128,8,128,24.59375,2.126953125,8388608
+57,self_attn.v_proj,3,32,8,128,7.96484375,3.5078125,8388608
+57,self_attn.v_proj,3,64,8,128,9.3125,3.25390625,8388608
+57,self_attn.v_proj,3,128,8,128,10.53125,3.126953125,8388608
+57,self_attn.v_proj,4,32,8,128,3.71875,4.5078125,8388608
+57,self_attn.v_proj,4,64,8,128,4.34765625,4.25390625,8388608
+57,self_attn.v_proj,4,128,8,128,4.9140625,4.126953125,8388608
+57,self_attn.v_proj,8,32,8,128,0.2252197265625,8.5078125,8388608
+57,self_attn.v_proj,8,64,8,128,0.262939453125,8.25390625,8388608
+57,self_attn.v_proj,8,128,8,128,0.296142578125,8.126953125,8388608
+57,self_attn.k_proj,2,32,8,128,20.46875,2.5078125,8388608
+57,self_attn.k_proj,2,64,8,128,24.796875,2.25390625,8388608
+57,self_attn.k_proj,2,128,8,128,28.28125,2.126953125,8388608
+57,self_attn.k_proj,3,32,8,128,8.921875,3.5078125,8388608
+57,self_attn.k_proj,3,64,8,128,11.1015625,3.25390625,8388608
+57,self_attn.k_proj,3,128,8,128,13.1796875,3.126953125,8388608
+57,self_attn.k_proj,4,32,8,128,4.16015625,4.5078125,8388608
+57,self_attn.k_proj,4,64,8,128,5.1796875,4.25390625,8388608
+57,self_attn.k_proj,4,128,8,128,6.15625,4.126953125,8388608
+57,self_attn.k_proj,8,32,8,128,0.251953125,8.5078125,8388608
+57,self_attn.k_proj,8,64,8,128,0.313232421875,8.25390625,8388608
+57,self_attn.k_proj,8,128,8,128,0.37109375,8.126953125,8388608
+57,self_attn.o_proj,2,32,8,128,42.21875,20.0625,67108864
+57,self_attn.o_proj,2,64,8,128,49.125,18.03125,67108864
+57,self_attn.o_proj,2,128,8,128,55.34375,17.015625,67108864
+57,self_attn.o_proj,3,32,8,128,18.046875,28.0625,67108864
+57,self_attn.o_proj,3,64,8,128,21.015625,26.03125,67108864
+57,self_attn.o_proj,3,128,8,128,23.6875,25.015625,67108864
+57,self_attn.o_proj,4,32,8,128,8.421875,36.0625,67108864
+57,self_attn.o_proj,4,64,8,128,9.8125,34.03125,67108864
+57,self_attn.o_proj,4,128,8,128,11.0546875,33.015625,67108864
+57,self_attn.o_proj,8,32,8,128,0.509765625,68.0625,67108864
+57,self_attn.o_proj,8,64,8,128,0.59326171875,66.03125,67108864
+57,self_attn.o_proj,8,128,8,128,0.66650390625,65.015625,67108864
+57,mlp.gate_proj,2,32,8,128,89.5,70.21875,234881024
+57,mlp.gate_proj,2,64,8,128,104.5,63.109375,234881024
+57,mlp.gate_proj,2,128,8,128,118.0,59.5546875,234881024
+57,mlp.gate_proj,3,32,8,128,38.3125,98.21875,234881024
+57,mlp.gate_proj,3,64,8,128,44.6875,91.109375,234881024
+57,mlp.gate_proj,3,128,8,128,50.5625,87.5546875,234881024
+57,mlp.gate_proj,4,32,8,128,17.859375,126.21875,234881024
+57,mlp.gate_proj,4,64,8,128,20.859375,119.109375,234881024
+57,mlp.gate_proj,4,128,8,128,23.578125,115.5546875,234881024
+57,mlp.gate_proj,8,32,8,128,1.0869140625,238.21875,234881024
+57,mlp.gate_proj,8,64,8,128,1.26953125,231.109375,234881024
+57,mlp.gate_proj,8,128,8,128,1.4306640625,227.5546875,234881024
+57,mlp.up_proj,2,32,8,128,80.9375,70.21875,234881024
+57,mlp.up_proj,2,64,8,128,94.125,63.109375,234881024
+57,mlp.up_proj,2,128,8,128,106.125,59.5546875,234881024
+57,mlp.up_proj,3,32,8,128,34.59375,98.21875,234881024
+57,mlp.up_proj,3,64,8,128,40.3125,91.109375,234881024
+57,mlp.up_proj,3,128,8,128,45.40625,87.5546875,234881024
+57,mlp.up_proj,4,32,8,128,16.125,126.21875,234881024
+57,mlp.up_proj,4,64,8,128,18.8125,119.109375,234881024
+57,mlp.up_proj,4,128,8,128,21.1875,115.5546875,234881024
+57,mlp.up_proj,8,32,8,128,0.98095703125,238.21875,234881024
+57,mlp.up_proj,8,64,8,128,1.1435546875,231.109375,234881024
+57,mlp.up_proj,8,128,8,128,1.28515625,227.5546875,234881024
+57,mlp.down_proj,2,32,8,128,81.3125,70.21875,234881024
+57,mlp.down_proj,2,64,8,128,94.625,63.109375,234881024
+57,mlp.down_proj,2,128,8,128,106.5625,59.5546875,234881024
+57,mlp.down_proj,3,32,8,128,34.75,98.21875,234881024
+57,mlp.down_proj,3,64,8,128,40.5,91.109375,234881024
+57,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024
+57,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024
+57,mlp.down_proj,4,64,8,128,18.90625,119.109375,234881024
+57,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024
+57,mlp.down_proj,8,32,8,128,0.98583984375,238.21875,234881024
+57,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024
+57,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024
+58,self_attn.q_proj,2,32,8,128,34.375,20.0625,67108864
+58,self_attn.q_proj,2,64,8,128,40.875,18.03125,67108864
+58,self_attn.q_proj,2,128,8,128,47.6875,17.015625,67108864
+58,self_attn.q_proj,3,32,8,128,14.8203125,28.0625,67108864
+58,self_attn.q_proj,3,64,8,128,17.765625,26.03125,67108864
+58,self_attn.q_proj,3,128,8,128,21.0625,25.015625,67108864
+58,self_attn.q_proj,4,32,8,128,6.9140625,36.0625,67108864
+58,self_attn.q_proj,4,64,8,128,8.296875,34.03125,67108864
+58,self_attn.q_proj,4,128,8,128,9.8359375,33.015625,67108864
+58,self_attn.q_proj,8,32,8,128,0.41845703125,68.0625,67108864
+58,self_attn.q_proj,8,64,8,128,0.501953125,66.03125,67108864
+58,self_attn.q_proj,8,128,8,128,0.59326171875,65.015625,67108864
+58,self_attn.v_proj,2,32,8,128,17.875,2.5078125,8388608
+58,self_attn.v_proj,2,64,8,128,20.890625,2.25390625,8388608
+58,self_attn.v_proj,2,128,8,128,23.625,2.126953125,8388608
+58,self_attn.v_proj,3,32,8,128,7.64453125,3.5078125,8388608
+58,self_attn.v_proj,3,64,8,128,8.9375,3.25390625,8388608
+58,self_attn.v_proj,3,128,8,128,10.1171875,3.126953125,8388608
+58,self_attn.v_proj,4,32,8,128,3.56640625,4.5078125,8388608
+58,self_attn.v_proj,4,64,8,128,4.171875,4.25390625,8388608
+58,self_attn.v_proj,4,128,8,128,4.72265625,4.126953125,8388608
+58,self_attn.v_proj,8,32,8,128,0.2161865234375,8.5078125,8388608
+58,self_attn.v_proj,8,64,8,128,0.25244140625,8.25390625,8388608
+58,self_attn.v_proj,8,128,8,128,0.28466796875,8.126953125,8388608
+58,self_attn.k_proj,2,32,8,128,20.4375,2.5078125,8388608
+58,self_attn.k_proj,2,64,8,128,24.609375,2.25390625,8388608
+58,self_attn.k_proj,2,128,8,128,28.109375,2.126953125,8388608
+58,self_attn.k_proj,3,32,8,128,9.0546875,3.5078125,8388608
+58,self_attn.k_proj,3,64,8,128,11.328125,3.25390625,8388608
+58,self_attn.k_proj,3,128,8,128,13.5703125,3.126953125,8388608
+58,self_attn.k_proj,4,32,8,128,4.2265625,4.5078125,8388608
+58,self_attn.k_proj,4,64,8,128,5.30078125,4.25390625,8388608
+58,self_attn.k_proj,4,128,8,128,6.36328125,4.126953125,8388608
+58,self_attn.k_proj,8,32,8,128,0.255615234375,8.5078125,8388608
+58,self_attn.k_proj,8,64,8,128,0.3203125,8.25390625,8388608
+58,self_attn.k_proj,8,128,8,128,0.38330078125,8.126953125,8388608
+58,self_attn.o_proj,2,32,8,128,40.78125,20.0625,67108864
+58,self_attn.o_proj,2,64,8,128,47.5,18.03125,67108864
+58,self_attn.o_proj,2,128,8,128,53.5625,17.015625,67108864
+58,self_attn.o_proj,3,32,8,128,17.453125,28.0625,67108864
+58,self_attn.o_proj,3,64,8,128,20.359375,26.03125,67108864
+58,self_attn.o_proj,3,128,8,128,23.0,25.015625,67108864
+58,self_attn.o_proj,4,32,8,128,8.140625,36.0625,67108864
+58,self_attn.o_proj,4,64,8,128,9.5078125,34.03125,67108864
+58,self_attn.o_proj,4,128,8,128,10.75,33.015625,67108864
+58,self_attn.o_proj,8,32,8,128,0.4931640625,68.0625,67108864
+58,self_attn.o_proj,8,64,8,128,0.5751953125,66.03125,67108864
+58,self_attn.o_proj,8,128,8,128,0.64794921875,65.015625,67108864
+58,mlp.gate_proj,2,32,8,128,89.5625,70.21875,234881024
+58,mlp.gate_proj,2,64,8,128,104.5625,63.109375,234881024
+58,mlp.gate_proj,2,128,8,128,118.1875,59.5546875,234881024
+58,mlp.gate_proj,3,32,8,128,38.3125,98.21875,234881024
+58,mlp.gate_proj,3,64,8,128,44.71875,91.109375,234881024
+58,mlp.gate_proj,3,128,8,128,50.625,87.5546875,234881024
+58,mlp.gate_proj,4,32,8,128,17.875,126.21875,234881024
+58,mlp.gate_proj,4,64,8,128,20.875,119.109375,234881024
+58,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024
+58,mlp.gate_proj,8,32,8,128,1.0869140625,238.21875,234881024
+58,mlp.gate_proj,8,64,8,128,1.2705078125,231.109375,234881024
+58,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024
+58,mlp.up_proj,2,32,8,128,81.0625,70.21875,234881024
+58,mlp.up_proj,2,64,8,128,94.3125,63.109375,234881024
+58,mlp.up_proj,2,128,8,128,106.25,59.5546875,234881024
+58,mlp.up_proj,3,32,8,128,34.625,98.21875,234881024
+58,mlp.up_proj,3,64,8,128,40.375,91.109375,234881024
+58,mlp.up_proj,3,128,8,128,45.46875,87.5546875,234881024
+58,mlp.up_proj,4,32,8,128,16.15625,126.21875,234881024
+58,mlp.up_proj,4,64,8,128,18.828125,119.109375,234881024
+58,mlp.up_proj,4,128,8,128,21.21875,115.5546875,234881024
+58,mlp.up_proj,8,32,8,128,0.982421875,238.21875,234881024
+58,mlp.up_proj,8,64,8,128,1.1455078125,231.109375,234881024
+58,mlp.up_proj,8,128,8,128,1.287109375,227.5546875,234881024
+58,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024
+58,mlp.down_proj,2,64,8,128,94.625,63.109375,234881024
+58,mlp.down_proj,2,128,8,128,106.625,59.5546875,234881024
+58,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024
+58,mlp.down_proj,3,64,8,128,40.5,91.109375,234881024
+58,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024
+58,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024
+58,mlp.down_proj,4,64,8,128,18.90625,119.109375,234881024
+58,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024
+58,mlp.down_proj,8,32,8,128,0.986328125,238.21875,234881024
+58,mlp.down_proj,8,64,8,128,1.1494140625,231.109375,234881024
+58,mlp.down_proj,8,128,8,128,1.291015625,227.5546875,234881024
+59,self_attn.q_proj,2,32,8,128,33.15625,20.0625,67108864
+59,self_attn.q_proj,2,64,8,128,39.125,18.03125,67108864
+59,self_attn.q_proj,2,128,8,128,45.59375,17.015625,67108864
+59,self_attn.q_proj,3,32,8,128,14.21875,28.0625,67108864
+59,self_attn.q_proj,3,64,8,128,16.84375,26.03125,67108864
+59,self_attn.q_proj,3,128,8,128,19.796875,25.015625,67108864
+59,self_attn.q_proj,4,32,8,128,6.63671875,36.0625,67108864
+59,self_attn.q_proj,4,64,8,128,7.8671875,34.03125,67108864
+59,self_attn.q_proj,4,128,8,128,9.25,33.015625,67108864
+59,self_attn.q_proj,8,32,8,128,0.401611328125,68.0625,67108864
+59,self_attn.q_proj,8,64,8,128,0.475830078125,66.03125,67108864
+59,self_attn.q_proj,8,128,8,128,0.55810546875,65.015625,67108864
+59,self_attn.v_proj,2,32,8,128,18.171875,2.5078125,8388608
+59,self_attn.v_proj,2,64,8,128,21.234375,2.25390625,8388608
+59,self_attn.v_proj,2,128,8,128,24.0,2.126953125,8388608
+59,self_attn.v_proj,3,32,8,128,7.765625,3.5078125,8388608
+59,self_attn.v_proj,3,64,8,128,9.078125,3.25390625,8388608
+59,self_attn.v_proj,3,128,8,128,10.2734375,3.126953125,8388608
+59,self_attn.v_proj,4,32,8,128,3.623046875,4.5078125,8388608
+59,self_attn.v_proj,4,64,8,128,4.23828125,4.25390625,8388608
+59,self_attn.v_proj,4,128,8,128,4.79296875,4.126953125,8388608
+59,self_attn.v_proj,8,32,8,128,0.219482421875,8.5078125,8388608
+59,self_attn.v_proj,8,64,8,128,0.25634765625,8.25390625,8388608
+59,self_attn.v_proj,8,128,8,128,0.288818359375,8.126953125,8388608
+59,self_attn.k_proj,2,32,8,128,20.1875,2.5078125,8388608
+59,self_attn.k_proj,2,64,8,128,24.40625,2.25390625,8388608
+59,self_attn.k_proj,2,128,8,128,28.0,2.126953125,8388608
+59,self_attn.k_proj,3,32,8,128,8.8515625,3.5078125,8388608
+59,self_attn.k_proj,3,64,8,128,11.03125,3.25390625,8388608
+59,self_attn.k_proj,3,128,8,128,13.1953125,3.126953125,8388608
+59,self_attn.k_proj,4,32,8,128,4.1328125,4.5078125,8388608
+59,self_attn.k_proj,4,64,8,128,5.15234375,4.25390625,8388608
+59,self_attn.k_proj,4,128,8,128,6.171875,4.126953125,8388608
+59,self_attn.k_proj,8,32,8,128,0.25,8.5078125,8388608
+59,self_attn.k_proj,8,64,8,128,0.3115234375,8.25390625,8388608
+59,self_attn.k_proj,8,128,8,128,0.371826171875,8.126953125,8388608
+59,self_attn.o_proj,2,32,8,128,40.5625,20.0625,67108864
+59,self_attn.o_proj,2,64,8,128,47.21875,18.03125,67108864
+59,self_attn.o_proj,2,128,8,128,53.25,17.015625,67108864
+59,self_attn.o_proj,3,32,8,128,17.34375,28.0625,67108864
+59,self_attn.o_proj,3,64,8,128,20.234375,26.03125,67108864
+59,self_attn.o_proj,3,128,8,128,22.84375,25.015625,67108864
+59,self_attn.o_proj,4,32,8,128,8.09375,36.0625,67108864
+59,self_attn.o_proj,4,64,8,128,9.4375,34.03125,67108864
+59,self_attn.o_proj,4,128,8,128,10.671875,33.015625,67108864
+59,self_attn.o_proj,8,32,8,128,0.489990234375,68.0625,67108864
+59,self_attn.o_proj,8,64,8,128,0.5712890625,66.03125,67108864
+59,self_attn.o_proj,8,128,8,128,0.642578125,65.015625,67108864
+59,mlp.gate_proj,2,32,8,128,89.5,70.21875,234881024
+59,mlp.gate_proj,2,64,8,128,104.625,63.109375,234881024
+59,mlp.gate_proj,2,128,8,128,118.1875,59.5546875,234881024
+59,mlp.gate_proj,3,32,8,128,38.3125,98.21875,234881024
+59,mlp.gate_proj,3,64,8,128,44.71875,91.109375,234881024
+59,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024
+59,mlp.gate_proj,4,32,8,128,17.859375,126.21875,234881024
+59,mlp.gate_proj,4,64,8,128,20.875,119.109375,234881024
+59,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024
+59,mlp.gate_proj,8,32,8,128,1.0869140625,238.21875,234881024
+59,mlp.gate_proj,8,64,8,128,1.2705078125,231.109375,234881024
+59,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024
+59,mlp.up_proj,2,32,8,128,81.1875,70.21875,234881024
+59,mlp.up_proj,2,64,8,128,94.5,63.109375,234881024
+59,mlp.up_proj,2,128,8,128,106.5,59.5546875,234881024
+59,mlp.up_proj,3,32,8,128,34.71875,98.21875,234881024
+59,mlp.up_proj,3,64,8,128,40.4375,91.109375,234881024
+59,mlp.up_proj,3,128,8,128,45.53125,87.5546875,234881024
+59,mlp.up_proj,4,32,8,128,16.1875,126.21875,234881024
+59,mlp.up_proj,4,64,8,128,18.875,119.109375,234881024
+59,mlp.up_proj,4,128,8,128,21.265625,115.5546875,234881024
+59,mlp.up_proj,8,32,8,128,0.984375,238.21875,234881024
+59,mlp.up_proj,8,64,8,128,1.1474609375,231.109375,234881024
+59,mlp.up_proj,8,128,8,128,1.2900390625,227.5546875,234881024
+59,mlp.down_proj,2,32,8,128,81.375,70.21875,234881024
+59,mlp.down_proj,2,64,8,128,94.6875,63.109375,234881024
+59,mlp.down_proj,2,128,8,128,106.6875,59.5546875,234881024
+59,mlp.down_proj,3,32,8,128,34.78125,98.21875,234881024
+59,mlp.down_proj,3,64,8,128,40.53125,91.109375,234881024
+59,mlp.down_proj,3,128,8,128,45.59375,87.5546875,234881024
+59,mlp.down_proj,4,32,8,128,16.21875,126.21875,234881024
+59,mlp.down_proj,4,64,8,128,18.90625,119.109375,234881024
+59,mlp.down_proj,4,128,8,128,21.28125,115.5546875,234881024
+59,mlp.down_proj,8,32,8,128,0.98681640625,238.21875,234881024
+59,mlp.down_proj,8,64,8,128,1.150390625,231.109375,234881024
+59,mlp.down_proj,8,128,8,128,1.2919921875,227.5546875,234881024
+60,self_attn.q_proj,2,32,8,128,26.671875,20.0625,67108864
+60,self_attn.q_proj,2,64,8,128,31.390625,18.03125,67108864
+60,self_attn.q_proj,2,128,8,128,35.96875,17.015625,67108864
+60,self_attn.q_proj,3,32,8,128,11.4609375,28.0625,67108864
+60,self_attn.q_proj,3,64,8,128,13.5859375,26.03125,67108864
+60,self_attn.q_proj,3,128,8,128,15.7265625,25.015625,67108864
+60,self_attn.q_proj,4,32,8,128,5.34765625,36.0625,67108864
+60,self_attn.q_proj,4,64,8,128,6.34375,34.03125,67108864
+60,self_attn.q_proj,4,128,8,128,7.34375,33.015625,67108864
+60,self_attn.q_proj,8,32,8,128,0.323974609375,68.0625,67108864
+60,self_attn.q_proj,8,64,8,128,0.383544921875,66.03125,67108864
+60,self_attn.q_proj,8,128,8,128,0.44287109375,65.015625,67108864
+60,self_attn.v_proj,2,32,8,128,18.1875,2.5078125,8388608
+60,self_attn.v_proj,2,64,8,128,21.25,2.25390625,8388608
+60,self_attn.v_proj,2,128,8,128,24.046875,2.126953125,8388608
+60,self_attn.v_proj,3,32,8,128,7.7734375,3.5078125,8388608
+60,self_attn.v_proj,3,64,8,128,9.09375,3.25390625,8388608
+60,self_attn.v_proj,3,128,8,128,10.28125,3.126953125,8388608
+60,self_attn.v_proj,4,32,8,128,3.626953125,4.5078125,8388608
+60,self_attn.v_proj,4,64,8,128,4.2421875,4.25390625,8388608
+60,self_attn.v_proj,4,128,8,128,4.80078125,4.126953125,8388608
+60,self_attn.v_proj,8,32,8,128,0.2197265625,8.5078125,8388608
+60,self_attn.v_proj,8,64,8,128,0.256591796875,8.25390625,8388608
+60,self_attn.v_proj,8,128,8,128,0.289306640625,8.126953125,8388608
+60,self_attn.k_proj,2,32,8,128,16.640625,2.5078125,8388608
+60,self_attn.k_proj,2,64,8,128,19.96875,2.25390625,8388608
+60,self_attn.k_proj,2,128,8,128,22.671875,2.126953125,8388608
+60,self_attn.k_proj,3,32,8,128,7.30078125,3.5078125,8388608
+60,self_attn.k_proj,3,64,8,128,8.9921875,3.25390625,8388608
+60,self_attn.k_proj,3,128,8,128,10.5234375,3.126953125,8388608
+60,self_attn.k_proj,4,32,8,128,3.416015625,4.5078125,8388608
+60,self_attn.k_proj,4,64,8,128,4.22265625,4.25390625,8388608
+60,self_attn.k_proj,4,128,8,128,4.96875,4.126953125,8388608
+60,self_attn.k_proj,8,32,8,128,0.206787109375,8.5078125,8388608
+60,self_attn.k_proj,8,64,8,128,0.25537109375,8.25390625,8388608
+60,self_attn.k_proj,8,128,8,128,0.2998046875,8.126953125,8388608
+60,self_attn.o_proj,2,32,8,128,39.34375,20.0625,67108864
+60,self_attn.o_proj,2,64,8,128,45.78125,18.03125,67108864
+60,self_attn.o_proj,2,128,8,128,51.53125,17.015625,67108864
+60,self_attn.o_proj,3,32,8,128,16.8125,28.0625,67108864
+60,self_attn.o_proj,3,64,8,128,19.59375,26.03125,67108864
+60,self_attn.o_proj,3,128,8,128,22.078125,25.015625,67108864
+60,self_attn.o_proj,4,32,8,128,7.84765625,36.0625,67108864
+60,self_attn.o_proj,4,64,8,128,9.140625,34.03125,67108864
+60,self_attn.o_proj,4,128,8,128,10.3046875,33.015625,67108864
+60,self_attn.o_proj,8,32,8,128,0.47509765625,68.0625,67108864
+60,self_attn.o_proj,8,64,8,128,0.55322265625,66.03125,67108864
+60,self_attn.o_proj,8,128,8,128,0.62109375,65.015625,67108864
+60,mlp.gate_proj,2,32,8,128,89.4375,70.21875,234881024
+60,mlp.gate_proj,2,64,8,128,104.5,63.109375,234881024
+60,mlp.gate_proj,2,128,8,128,118.125,59.5546875,234881024
+60,mlp.gate_proj,3,32,8,128,38.28125,98.21875,234881024
+60,mlp.gate_proj,3,64,8,128,44.6875,91.109375,234881024
+60,mlp.gate_proj,3,128,8,128,50.5625,87.5546875,234881024
+60,mlp.gate_proj,4,32,8,128,17.84375,126.21875,234881024
+60,mlp.gate_proj,4,64,8,128,20.859375,119.109375,234881024
+60,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024
+60,mlp.gate_proj,8,32,8,128,1.0859375,238.21875,234881024
+60,mlp.gate_proj,8,64,8,128,1.26953125,231.109375,234881024
+60,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024
+60,mlp.up_proj,2,32,8,128,81.375,70.21875,234881024
+60,mlp.up_proj,2,64,8,128,94.6875,63.109375,234881024
+60,mlp.up_proj,2,128,8,128,106.75,59.5546875,234881024
+60,mlp.up_proj,3,32,8,128,34.78125,98.21875,234881024
+60,mlp.up_proj,3,64,8,128,40.53125,91.109375,234881024
+60,mlp.up_proj,3,128,8,128,45.65625,87.5546875,234881024
+60,mlp.up_proj,4,32,8,128,16.21875,126.21875,234881024
+60,mlp.up_proj,4,64,8,128,18.921875,119.109375,234881024
+60,mlp.up_proj,4,128,8,128,21.3125,115.5546875,234881024
+60,mlp.up_proj,8,32,8,128,0.98681640625,238.21875,234881024
+60,mlp.up_proj,8,64,8,128,1.150390625,231.109375,234881024
+60,mlp.up_proj,8,128,8,128,1.29296875,227.5546875,234881024
+60,mlp.down_proj,2,32,8,128,81.5,70.21875,234881024
+60,mlp.down_proj,2,64,8,128,94.8125,63.109375,234881024
+60,mlp.down_proj,2,128,8,128,106.8125,59.5546875,234881024
+60,mlp.down_proj,3,32,8,128,34.84375,98.21875,234881024
+60,mlp.down_proj,3,64,8,128,40.5625,91.109375,234881024
+60,mlp.down_proj,3,128,8,128,45.65625,87.5546875,234881024
+60,mlp.down_proj,4,32,8,128,16.25,126.21875,234881024
+60,mlp.down_proj,4,64,8,128,18.9375,119.109375,234881024
+60,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024
+60,mlp.down_proj,8,32,8,128,0.98779296875,238.21875,234881024
+60,mlp.down_proj,8,64,8,128,1.1513671875,231.109375,234881024
+60,mlp.down_proj,8,128,8,128,1.29296875,227.5546875,234881024
+61,self_attn.q_proj,2,32,8,128,31.328125,20.0625,67108864
+61,self_attn.q_proj,2,64,8,128,37.21875,18.03125,67108864
+61,self_attn.q_proj,2,128,8,128,43.0625,17.015625,67108864
+61,self_attn.q_proj,3,32,8,128,13.421875,28.0625,67108864
+61,self_attn.q_proj,3,64,8,128,16.015625,26.03125,67108864
+61,self_attn.q_proj,3,128,8,128,18.640625,25.015625,67108864
+61,self_attn.q_proj,4,32,8,128,6.265625,36.0625,67108864
+61,self_attn.q_proj,4,64,8,128,7.47265625,34.03125,67108864
+61,self_attn.q_proj,4,128,8,128,8.6953125,33.015625,67108864
+61,self_attn.q_proj,8,32,8,128,0.379150390625,68.0625,67108864
+61,self_attn.q_proj,8,64,8,128,0.451904296875,66.03125,67108864
+61,self_attn.q_proj,8,128,8,128,0.52490234375,65.015625,67108864
+61,self_attn.v_proj,2,32,8,128,18.53125,2.5078125,8388608
+61,self_attn.v_proj,2,64,8,128,21.640625,2.25390625,8388608
+61,self_attn.v_proj,2,128,8,128,24.4375,2.126953125,8388608
+61,self_attn.v_proj,3,32,8,128,7.921875,3.5078125,8388608
+61,self_attn.v_proj,3,64,8,128,9.2578125,3.25390625,8388608
+61,self_attn.v_proj,3,128,8,128,10.4609375,3.126953125,8388608
+61,self_attn.v_proj,4,32,8,128,3.697265625,4.5078125,8388608
+61,self_attn.v_proj,4,64,8,128,4.31640625,4.25390625,8388608
+61,self_attn.v_proj,4,128,8,128,4.87890625,4.126953125,8388608
+61,self_attn.v_proj,8,32,8,128,0.223876953125,8.5078125,8388608
+61,self_attn.v_proj,8,64,8,128,0.26123046875,8.25390625,8388608
+61,self_attn.v_proj,8,128,8,128,0.294189453125,8.126953125,8388608
+61,self_attn.k_proj,2,32,8,128,19.453125,2.5078125,8388608
+61,self_attn.k_proj,2,64,8,128,23.359375,2.25390625,8388608
+61,self_attn.k_proj,2,128,8,128,26.578125,2.126953125,8388608
+61,self_attn.k_proj,3,32,8,128,8.546875,3.5078125,8388608
+61,self_attn.k_proj,3,64,8,128,10.6015625,3.25390625,8388608
+61,self_attn.k_proj,3,128,8,128,12.5234375,3.126953125,8388608
+61,self_attn.k_proj,4,32,8,128,3.990234375,4.5078125,8388608
+61,self_attn.k_proj,4,64,8,128,4.953125,4.25390625,8388608
+61,self_attn.k_proj,4,128,8,128,5.8671875,4.126953125,8388608
+61,self_attn.k_proj,8,32,8,128,0.241455078125,8.5078125,8388608
+61,self_attn.k_proj,8,64,8,128,0.299560546875,8.25390625,8388608
+61,self_attn.k_proj,8,128,8,128,0.353759765625,8.126953125,8388608
+61,self_attn.o_proj,2,32,8,128,40.96875,20.0625,67108864
+61,self_attn.o_proj,2,64,8,128,47.6875,18.03125,67108864
+61,self_attn.o_proj,2,128,8,128,53.6875,17.015625,67108864
+61,self_attn.o_proj,3,32,8,128,17.515625,28.0625,67108864
+61,self_attn.o_proj,3,64,8,128,20.390625,26.03125,67108864
+61,self_attn.o_proj,3,128,8,128,22.984375,25.015625,67108864
+61,self_attn.o_proj,4,32,8,128,8.171875,36.0625,67108864
+61,self_attn.o_proj,4,64,8,128,9.5234375,34.03125,67108864
+61,self_attn.o_proj,4,128,8,128,10.7265625,33.015625,67108864
+61,self_attn.o_proj,8,32,8,128,0.494873046875,68.0625,67108864
+61,self_attn.o_proj,8,64,8,128,0.57568359375,66.03125,67108864
+61,self_attn.o_proj,8,128,8,128,0.646484375,65.015625,67108864
+61,mlp.gate_proj,2,32,8,128,89.375,70.21875,234881024
+61,mlp.gate_proj,2,64,8,128,104.4375,63.109375,234881024
+61,mlp.gate_proj,2,128,8,128,118.0625,59.5546875,234881024
+61,mlp.gate_proj,3,32,8,128,38.25,98.21875,234881024
+61,mlp.gate_proj,3,64,8,128,44.65625,91.109375,234881024
+61,mlp.gate_proj,3,128,8,128,50.5625,87.5546875,234881024
+61,mlp.gate_proj,4,32,8,128,17.828125,126.21875,234881024
+61,mlp.gate_proj,4,64,8,128,20.84375,119.109375,234881024
+61,mlp.gate_proj,4,128,8,128,23.578125,115.5546875,234881024
+61,mlp.gate_proj,8,32,8,128,1.0849609375,238.21875,234881024
+61,mlp.gate_proj,8,64,8,128,1.2685546875,231.109375,234881024
+61,mlp.gate_proj,8,128,8,128,1.4306640625,227.5546875,234881024
+61,mlp.up_proj,2,32,8,128,81.5625,70.21875,234881024
+61,mlp.up_proj,2,64,8,128,94.9375,63.109375,234881024
+61,mlp.up_proj,2,128,8,128,107.0,59.5546875,234881024
+61,mlp.up_proj,3,32,8,128,34.875,98.21875,234881024
+61,mlp.up_proj,3,64,8,128,40.625,91.109375,234881024
+61,mlp.up_proj,3,128,8,128,45.78125,87.5546875,234881024
+61,mlp.up_proj,4,32,8,128,16.265625,126.21875,234881024
+61,mlp.up_proj,4,64,8,128,18.96875,119.109375,234881024
+61,mlp.up_proj,4,128,8,128,21.359375,115.5546875,234881024
+61,mlp.up_proj,8,32,8,128,0.9892578125,238.21875,234881024
+61,mlp.up_proj,8,64,8,128,1.1533203125,231.109375,234881024
+61,mlp.up_proj,8,128,8,128,1.2958984375,227.5546875,234881024
+61,mlp.down_proj,2,32,8,128,81.625,70.21875,234881024
+61,mlp.down_proj,2,64,8,128,94.9375,63.109375,234881024
+61,mlp.down_proj,2,128,8,128,107.0,59.5546875,234881024
+61,mlp.down_proj,3,32,8,128,34.875,98.21875,234881024
+61,mlp.down_proj,3,64,8,128,40.625,91.109375,234881024
+61,mlp.down_proj,3,128,8,128,45.71875,87.5546875,234881024
+61,mlp.down_proj,4,32,8,128,16.265625,126.21875,234881024
+61,mlp.down_proj,4,64,8,128,18.953125,119.109375,234881024
+61,mlp.down_proj,4,128,8,128,21.328125,115.5546875,234881024
+61,mlp.down_proj,8,32,8,128,0.9892578125,238.21875,234881024
+61,mlp.down_proj,8,64,8,128,1.1533203125,231.109375,234881024
+61,mlp.down_proj,8,128,8,128,1.294921875,227.5546875,234881024
+62,self_attn.q_proj,2,32,8,128,32.375,20.0625,67108864
+62,self_attn.q_proj,2,64,8,128,38.5,18.03125,67108864
+62,self_attn.q_proj,2,128,8,128,44.875,17.015625,67108864
+62,self_attn.q_proj,3,32,8,128,13.9140625,28.0625,67108864
+62,self_attn.q_proj,3,64,8,128,16.640625,26.03125,67108864
+62,self_attn.q_proj,3,128,8,128,19.640625,25.015625,67108864
+62,self_attn.q_proj,4,32,8,128,6.48828125,36.0625,67108864
+62,self_attn.q_proj,4,64,8,128,7.765625,34.03125,67108864
+62,self_attn.q_proj,4,128,8,128,9.1640625,33.015625,67108864
+62,self_attn.q_proj,8,32,8,128,0.392822265625,68.0625,67108864
+62,self_attn.q_proj,8,64,8,128,0.4697265625,66.03125,67108864
+62,self_attn.q_proj,8,128,8,128,0.552734375,65.015625,67108864
+62,self_attn.v_proj,2,32,8,128,18.25,2.5078125,8388608
+62,self_attn.v_proj,2,64,8,128,21.28125,2.25390625,8388608
+62,self_attn.v_proj,2,128,8,128,24.015625,2.126953125,8388608
+62,self_attn.v_proj,3,32,8,128,7.8046875,3.5078125,8388608
+62,self_attn.v_proj,3,64,8,128,9.1015625,3.25390625,8388608
+62,self_attn.v_proj,3,128,8,128,10.28125,3.126953125,8388608
+62,self_attn.v_proj,4,32,8,128,3.642578125,4.5078125,8388608
+62,self_attn.v_proj,4,64,8,128,4.25,4.25390625,8388608
+62,self_attn.v_proj,4,128,8,128,4.80078125,4.126953125,8388608
+62,self_attn.v_proj,8,32,8,128,0.2205810546875,8.5078125,8388608
+62,self_attn.v_proj,8,64,8,128,0.257080078125,8.25390625,8388608
+62,self_attn.v_proj,8,128,8,128,0.289306640625,8.126953125,8388608
+62,self_attn.k_proj,2,32,8,128,20.046875,2.5078125,8388608
+62,self_attn.k_proj,2,64,8,128,24.140625,2.25390625,8388608
+62,self_attn.k_proj,2,128,8,128,27.5625,2.126953125,8388608
+62,self_attn.k_proj,3,32,8,128,8.8828125,3.5078125,8388608
+62,self_attn.k_proj,3,64,8,128,11.1484375,3.25390625,8388608
+62,self_attn.k_proj,3,128,8,128,13.3515625,3.126953125,8388608
+62,self_attn.k_proj,4,32,8,128,4.14453125,4.5078125,8388608
+62,self_attn.k_proj,4,64,8,128,5.2109375,4.25390625,8388608
+62,self_attn.k_proj,4,128,8,128,6.265625,4.126953125,8388608
+62,self_attn.k_proj,8,32,8,128,0.250732421875,8.5078125,8388608
+62,self_attn.k_proj,8,64,8,128,0.315185546875,8.25390625,8388608
+62,self_attn.k_proj,8,128,8,128,0.377685546875,8.126953125,8388608
+62,self_attn.o_proj,2,32,8,128,40.40625,20.0625,67108864
+62,self_attn.o_proj,2,64,8,128,47.03125,18.03125,67108864
+62,self_attn.o_proj,2,128,8,128,52.96875,17.015625,67108864
+62,self_attn.o_proj,3,32,8,128,17.265625,28.0625,67108864
+62,self_attn.o_proj,3,64,8,128,20.125,26.03125,67108864
+62,self_attn.o_proj,3,128,8,128,22.671875,25.015625,67108864
+62,self_attn.o_proj,4,32,8,128,8.0546875,36.0625,67108864
+62,self_attn.o_proj,4,64,8,128,9.3828125,34.03125,67108864
+62,self_attn.o_proj,4,128,8,128,10.578125,33.015625,67108864
+62,self_attn.o_proj,8,32,8,128,0.48779296875,68.0625,67108864
+62,self_attn.o_proj,8,64,8,128,0.56787109375,66.03125,67108864
+62,self_attn.o_proj,8,128,8,128,0.6376953125,65.015625,67108864
+62,mlp.gate_proj,2,32,8,128,89.3125,70.21875,234881024
+62,mlp.gate_proj,2,64,8,128,104.375,63.109375,234881024
+62,mlp.gate_proj,2,128,8,128,118.0,59.5546875,234881024
+62,mlp.gate_proj,3,32,8,128,38.21875,98.21875,234881024
+62,mlp.gate_proj,3,64,8,128,44.625,91.109375,234881024
+62,mlp.gate_proj,3,128,8,128,50.53125,87.5546875,234881024
+62,mlp.gate_proj,4,32,8,128,17.8125,126.21875,234881024
+62,mlp.gate_proj,4,64,8,128,20.828125,119.109375,234881024
+62,mlp.gate_proj,4,128,8,128,23.5625,115.5546875,234881024
+62,mlp.gate_proj,8,32,8,128,1.083984375,238.21875,234881024
+62,mlp.gate_proj,8,64,8,128,1.267578125,231.109375,234881024
+62,mlp.gate_proj,8,128,8,128,1.4296875,227.5546875,234881024
+62,mlp.up_proj,2,32,8,128,81.75,70.21875,234881024
+62,mlp.up_proj,2,64,8,128,95.125,63.109375,234881024
+62,mlp.up_proj,2,128,8,128,107.25,59.5546875,234881024
+62,mlp.up_proj,3,32,8,128,34.9375,98.21875,234881024
+62,mlp.up_proj,3,64,8,128,40.71875,91.109375,234881024
+62,mlp.up_proj,3,128,8,128,45.875,87.5546875,234881024
+62,mlp.up_proj,4,32,8,128,16.296875,126.21875,234881024
+62,mlp.up_proj,4,64,8,128,19.0,119.109375,234881024
+62,mlp.up_proj,4,128,8,128,21.40625,115.5546875,234881024
+62,mlp.up_proj,8,32,8,128,0.9912109375,238.21875,234881024
+62,mlp.up_proj,8,64,8,128,1.15625,231.109375,234881024
+62,mlp.up_proj,8,128,8,128,1.298828125,227.5546875,234881024
+62,mlp.down_proj,2,32,8,128,81.6875,70.21875,234881024
+62,mlp.down_proj,2,64,8,128,95.0,63.109375,234881024
+62,mlp.down_proj,2,128,8,128,107.0625,59.5546875,234881024
+62,mlp.down_proj,3,32,8,128,34.90625,98.21875,234881024
+62,mlp.down_proj,3,64,8,128,40.65625,91.109375,234881024
+62,mlp.down_proj,3,128,8,128,45.78125,87.5546875,234881024
+62,mlp.down_proj,4,32,8,128,16.28125,126.21875,234881024
+62,mlp.down_proj,4,64,8,128,18.96875,119.109375,234881024
+62,mlp.down_proj,4,128,8,128,21.359375,115.5546875,234881024
+62,mlp.down_proj,8,32,8,128,0.990234375,238.21875,234881024
+62,mlp.down_proj,8,64,8,128,1.154296875,231.109375,234881024
+62,mlp.down_proj,8,128,8,128,1.2958984375,227.5546875,234881024
+63,self_attn.q_proj,2,32,8,128,31.921875,20.0625,67108864
+63,self_attn.q_proj,2,64,8,128,37.90625,18.03125,67108864
+63,self_attn.q_proj,2,128,8,128,43.84375,17.015625,67108864
+63,self_attn.q_proj,3,32,8,128,13.7421875,28.0625,67108864
+63,self_attn.q_proj,3,64,8,128,16.421875,26.03125,67108864
+63,self_attn.q_proj,3,128,8,128,19.25,25.015625,67108864
+63,self_attn.q_proj,4,32,8,128,6.4140625,36.0625,67108864
+63,self_attn.q_proj,4,64,8,128,7.671875,34.03125,67108864
+63,self_attn.q_proj,4,128,8,128,9.0,33.015625,67108864
+63,self_attn.q_proj,8,32,8,128,0.38818359375,68.0625,67108864
+63,self_attn.q_proj,8,64,8,128,0.464111328125,66.03125,67108864
+63,self_attn.q_proj,8,128,8,128,0.54296875,65.015625,67108864
+63,self_attn.v_proj,2,32,8,128,18.125,2.5078125,8388608
+63,self_attn.v_proj,2,64,8,128,21.28125,2.25390625,8388608
+63,self_attn.v_proj,2,128,8,128,24.203125,2.126953125,8388608
+63,self_attn.v_proj,3,32,8,128,7.75,3.5078125,8388608
+63,self_attn.v_proj,3,64,8,128,9.109375,3.25390625,8388608
+63,self_attn.v_proj,3,128,8,128,10.359375,3.126953125,8388608
+63,self_attn.v_proj,4,32,8,128,3.615234375,4.5078125,8388608
+63,self_attn.v_proj,4,64,8,128,4.25,4.25390625,8388608
+63,self_attn.v_proj,4,128,8,128,4.83203125,4.126953125,8388608
+63,self_attn.v_proj,8,32,8,128,0.218994140625,8.5078125,8388608
+63,self_attn.v_proj,8,64,8,128,0.257080078125,8.25390625,8388608
+63,self_attn.v_proj,8,128,8,128,0.291259765625,8.126953125,8388608
+63,self_attn.k_proj,2,32,8,128,19.3125,2.5078125,8388608
+63,self_attn.k_proj,2,64,8,128,23.109375,2.25390625,8388608
+63,self_attn.k_proj,2,128,8,128,26.109375,2.126953125,8388608
+63,self_attn.k_proj,3,32,8,128,8.5859375,3.5078125,8388608
+63,self_attn.k_proj,3,64,8,128,10.640625,3.25390625,8388608
+63,self_attn.k_proj,3,128,8,128,12.5390625,3.126953125,8388608
+63,self_attn.k_proj,4,32,8,128,4.015625,4.5078125,8388608
+63,self_attn.k_proj,4,64,8,128,4.9921875,4.25390625,8388608
+63,self_attn.k_proj,4,128,8,128,5.9140625,4.126953125,8388608
+63,self_attn.k_proj,8,32,8,128,0.2431640625,8.5078125,8388608
+63,self_attn.k_proj,8,64,8,128,0.3017578125,8.25390625,8388608
+63,self_attn.k_proj,8,128,8,128,0.356689453125,8.126953125,8388608
+63,self_attn.o_proj,2,32,8,128,38.96875,20.0625,67108864
+63,self_attn.o_proj,2,64,8,128,45.34375,18.03125,67108864
+63,self_attn.o_proj,2,128,8,128,51.125,17.015625,67108864
+63,self_attn.o_proj,3,32,8,128,16.671875,28.0625,67108864
+63,self_attn.o_proj,3,64,8,128,19.4375,26.03125,67108864
+63,self_attn.o_proj,3,128,8,128,21.9375,25.015625,67108864
+63,self_attn.o_proj,4,32,8,128,7.77734375,36.0625,67108864
+63,self_attn.o_proj,4,64,8,128,9.0703125,34.03125,67108864
+63,self_attn.o_proj,4,128,8,128,10.25,33.015625,67108864
+63,self_attn.o_proj,8,32,8,128,0.47119140625,68.0625,67108864
+63,self_attn.o_proj,8,64,8,128,0.54931640625,66.03125,67108864
+63,self_attn.o_proj,8,128,8,128,0.6181640625,65.015625,67108864
+63,mlp.gate_proj,2,32,8,128,89.3125,70.21875,234881024
+63,mlp.gate_proj,2,64,8,128,104.375,63.109375,234881024
+63,mlp.gate_proj,2,128,8,128,118.0625,59.5546875,234881024
+63,mlp.gate_proj,3,32,8,128,38.21875,98.21875,234881024
+63,mlp.gate_proj,3,64,8,128,44.625,91.109375,234881024
+63,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024
+63,mlp.gate_proj,4,32,8,128,17.8125,126.21875,234881024
+63,mlp.gate_proj,4,64,8,128,20.84375,119.109375,234881024
+63,mlp.gate_proj,4,128,8,128,23.59375,115.5546875,234881024
+63,mlp.gate_proj,8,32,8,128,1.083984375,238.21875,234881024
+63,mlp.gate_proj,8,64,8,128,1.267578125,231.109375,234881024
+63,mlp.gate_proj,8,128,8,128,1.4306640625,227.5546875,234881024
+63,mlp.up_proj,2,32,8,128,81.875,70.21875,234881024
+63,mlp.up_proj,2,64,8,128,95.3125,63.109375,234881024
+63,mlp.up_proj,2,128,8,128,107.4375,59.5546875,234881024
+63,mlp.up_proj,3,32,8,128,35.0,98.21875,234881024
+63,mlp.up_proj,3,64,8,128,40.78125,91.109375,234881024
+63,mlp.up_proj,3,128,8,128,45.9375,87.5546875,234881024
+63,mlp.up_proj,4,32,8,128,16.328125,126.21875,234881024
+63,mlp.up_proj,4,64,8,128,19.03125,119.109375,234881024
+63,mlp.up_proj,4,128,8,128,21.4375,115.5546875,234881024
+63,mlp.up_proj,8,32,8,128,0.99267578125,238.21875,234881024
+63,mlp.up_proj,8,64,8,128,1.158203125,231.109375,234881024
+63,mlp.up_proj,8,128,8,128,1.30078125,227.5546875,234881024
+63,mlp.down_proj,2,32,8,128,81.6875,70.21875,234881024
+63,mlp.down_proj,2,64,8,128,95.0625,63.109375,234881024
+63,mlp.down_proj,2,128,8,128,107.125,59.5546875,234881024
+63,mlp.down_proj,3,32,8,128,34.9375,98.21875,234881024
+63,mlp.down_proj,3,64,8,128,40.6875,91.109375,234881024
+63,mlp.down_proj,3,128,8,128,45.78125,87.5546875,234881024
+63,mlp.down_proj,4,32,8,128,16.28125,126.21875,234881024
+63,mlp.down_proj,4,64,8,128,18.984375,119.109375,234881024
+63,mlp.down_proj,4,128,8,128,21.375,115.5546875,234881024
+63,mlp.down_proj,8,32,8,128,0.99072265625,238.21875,234881024
+63,mlp.down_proj,8,64,8,128,1.1552734375,231.109375,234881024
+63,mlp.down_proj,8,128,8,128,1.296875,227.5546875,234881024
+64,self_attn.q_proj,2,32,8,128,34.875,20.0625,67108864
+64,self_attn.q_proj,2,64,8,128,41.46875,18.03125,67108864
+64,self_attn.q_proj,2,128,8,128,48.0625,17.015625,67108864
+64,self_attn.q_proj,3,32,8,128,14.9765625,28.0625,67108864
+64,self_attn.q_proj,3,64,8,128,17.90625,26.03125,67108864
+64,self_attn.q_proj,3,128,8,128,20.984375,25.015625,67108864
+64,self_attn.q_proj,4,32,8,128,6.98828125,36.0625,67108864
+64,self_attn.q_proj,4,64,8,128,8.359375,34.03125,67108864
+64,self_attn.q_proj,4,128,8,128,9.796875,33.015625,67108864
+64,self_attn.q_proj,8,32,8,128,0.4228515625,68.0625,67108864
+64,self_attn.q_proj,8,64,8,128,0.505859375,66.03125,67108864
+64,self_attn.q_proj,8,128,8,128,0.5908203125,65.015625,67108864
+64,self_attn.v_proj,2,32,8,128,18.640625,2.5078125,8388608
+64,self_attn.v_proj,2,64,8,128,21.71875,2.25390625,8388608
+64,self_attn.v_proj,2,128,8,128,24.484375,2.126953125,8388608
+64,self_attn.v_proj,3,32,8,128,7.96875,3.5078125,8388608
+64,self_attn.v_proj,3,64,8,128,9.2890625,3.25390625,8388608
+64,self_attn.v_proj,3,128,8,128,10.46875,3.126953125,8388608
+64,self_attn.v_proj,4,32,8,128,3.71875,4.5078125,8388608
+64,self_attn.v_proj,4,64,8,128,4.33203125,4.25390625,8388608
+64,self_attn.v_proj,4,128,8,128,4.88671875,4.126953125,8388608
+64,self_attn.v_proj,8,32,8,128,0.225341796875,8.5078125,8388608
+64,self_attn.v_proj,8,64,8,128,0.26220703125,8.25390625,8388608
+64,self_attn.v_proj,8,128,8,128,0.294677734375,8.126953125,8388608
+64,self_attn.k_proj,2,32,8,128,21.859375,2.5078125,8388608
+64,self_attn.k_proj,2,64,8,128,26.1875,2.25390625,8388608
+64,self_attn.k_proj,2,128,8,128,29.6875,2.126953125,8388608
+64,self_attn.k_proj,3,32,8,128,9.6171875,3.5078125,8388608
+64,self_attn.k_proj,3,64,8,128,11.859375,3.25390625,8388608
+64,self_attn.k_proj,3,128,8,128,14.015625,3.126953125,8388608
+64,self_attn.k_proj,4,32,8,128,4.48828125,4.5078125,8388608
+64,self_attn.k_proj,4,64,8,128,5.54296875,4.25390625,8388608
+64,self_attn.k_proj,4,128,8,128,6.55078125,4.126953125,8388608
+64,self_attn.k_proj,8,32,8,128,0.271728515625,8.5078125,8388608
+64,self_attn.k_proj,8,64,8,128,0.3349609375,8.25390625,8388608
+64,self_attn.k_proj,8,128,8,128,0.39501953125,8.126953125,8388608
+64,self_attn.o_proj,2,32,8,128,41.84375,20.0625,67108864
+64,self_attn.o_proj,2,64,8,128,48.6875,18.03125,67108864
+64,self_attn.o_proj,2,128,8,128,54.84375,17.015625,67108864
+64,self_attn.o_proj,3,32,8,128,17.890625,28.0625,67108864
+64,self_attn.o_proj,3,64,8,128,20.828125,26.03125,67108864
+64,self_attn.o_proj,3,128,8,128,23.484375,25.015625,67108864
+64,self_attn.o_proj,4,32,8,128,8.34375,36.0625,67108864
+64,self_attn.o_proj,4,64,8,128,9.71875,34.03125,67108864
+64,self_attn.o_proj,4,128,8,128,10.9609375,33.015625,67108864
+64,self_attn.o_proj,8,32,8,128,0.50537109375,68.0625,67108864
+64,self_attn.o_proj,8,64,8,128,0.58837890625,66.03125,67108864
+64,self_attn.o_proj,8,128,8,128,0.66064453125,65.015625,67108864
+64,mlp.gate_proj,2,32,8,128,89.25,70.21875,234881024
+64,mlp.gate_proj,2,64,8,128,104.4375,63.109375,234881024
+64,mlp.gate_proj,2,128,8,128,118.125,59.5546875,234881024
+64,mlp.gate_proj,3,32,8,128,38.21875,98.21875,234881024
+64,mlp.gate_proj,3,64,8,128,44.625,91.109375,234881024
+64,mlp.gate_proj,3,128,8,128,50.59375,87.5546875,234881024
+64,mlp.gate_proj,4,32,8,128,17.8125,126.21875,234881024
+64,mlp.gate_proj,4,64,8,128,20.828125,119.109375,234881024
+64,mlp.gate_proj,4,128,8,128,23.609375,115.5546875,234881024
+64,mlp.gate_proj,8,32,8,128,1.083984375,238.21875,234881024
+64,mlp.gate_proj,8,64,8,128,1.2685546875,231.109375,234881024
+64,mlp.gate_proj,8,128,8,128,1.431640625,227.5546875,234881024
+64,mlp.up_proj,2,32,8,128,82.0,70.21875,234881024
+64,mlp.up_proj,2,64,8,128,95.4375,63.109375,234881024
+64,mlp.up_proj,2,128,8,128,107.625,59.5546875,234881024
+64,mlp.up_proj,3,32,8,128,35.0625,98.21875,234881024
+64,mlp.up_proj,3,64,8,128,40.875,91.109375,234881024
+64,mlp.up_proj,3,128,8,128,46.03125,87.5546875,234881024
+64,mlp.up_proj,4,32,8,128,16.359375,126.21875,234881024
+64,mlp.up_proj,4,64,8,128,19.078125,119.109375,234881024
+64,mlp.up_proj,4,128,8,128,21.484375,115.5546875,234881024
+64,mlp.up_proj,8,32,8,128,0.99462890625,238.21875,234881024
+64,mlp.up_proj,8,64,8,128,1.16015625,231.109375,234881024
+64,mlp.up_proj,8,128,8,128,1.3037109375,227.5546875,234881024
+64,mlp.down_proj,2,32,8,128,81.75,70.21875,234881024
+64,mlp.down_proj,2,64,8,128,95.0625,63.109375,234881024
+64,mlp.down_proj,2,128,8,128,107.125,59.5546875,234881024
+64,mlp.down_proj,3,32,8,128,34.9375,98.21875,234881024
+64,mlp.down_proj,3,64,8,128,40.6875,91.109375,234881024
+64,mlp.down_proj,3,128,8,128,45.84375,87.5546875,234881024
+64,mlp.down_proj,4,32,8,128,16.296875,126.21875,234881024
+64,mlp.down_proj,4,64,8,128,18.984375,119.109375,234881024
+64,mlp.down_proj,4,128,8,128,21.375,115.5546875,234881024
+64,mlp.down_proj,8,32,8,128,0.9912109375,238.21875,234881024
+64,mlp.down_proj,8,64,8,128,1.1552734375,231.109375,234881024
+64,mlp.down_proj,8,128,8,128,1.296875,227.5546875,234881024
+65,self_attn.q_proj,2,32,8,128,28.375,20.0625,67108864
+65,self_attn.q_proj,2,64,8,128,33.5625,18.03125,67108864
+65,self_attn.q_proj,2,128,8,128,38.375,17.015625,67108864
+65,self_attn.q_proj,3,32,8,128,12.21875,28.0625,67108864
+65,self_attn.q_proj,3,64,8,128,14.5625,26.03125,67108864
+65,self_attn.q_proj,3,128,8,128,16.875,25.015625,67108864
+65,self_attn.q_proj,4,32,8,128,5.69921875,36.0625,67108864
+65,self_attn.q_proj,4,64,8,128,6.796875,34.03125,67108864
+65,self_attn.q_proj,4,128,8,128,7.8828125,33.015625,67108864
+65,self_attn.q_proj,8,32,8,128,0.34521484375,68.0625,67108864
+65,self_attn.q_proj,8,64,8,128,0.4111328125,66.03125,67108864
+65,self_attn.q_proj,8,128,8,128,0.4755859375,65.015625,67108864
+65,self_attn.v_proj,2,32,8,128,17.84375,2.5078125,8388608
+65,self_attn.v_proj,2,64,8,128,20.953125,2.25390625,8388608
+65,self_attn.v_proj,2,128,8,128,23.8125,2.126953125,8388608
+65,self_attn.v_proj,3,32,8,128,7.6328125,3.5078125,8388608
+65,self_attn.v_proj,3,64,8,128,8.9609375,3.25390625,8388608
+65,self_attn.v_proj,3,128,8,128,10.1875,3.126953125,8388608
+65,self_attn.v_proj,4,32,8,128,3.55859375,4.5078125,8388608
+65,self_attn.v_proj,4,64,8,128,4.18359375,4.25390625,8388608
+65,self_attn.v_proj,4,128,8,128,4.75390625,4.126953125,8388608
+65,self_attn.v_proj,8,32,8,128,0.2154541015625,8.5078125,8388608
+65,self_attn.v_proj,8,64,8,128,0.2529296875,8.25390625,8388608
+65,self_attn.v_proj,8,128,8,128,0.28662109375,8.126953125,8388608
+65,self_attn.k_proj,2,32,8,128,18.078125,2.5078125,8388608
+65,self_attn.k_proj,2,64,8,128,21.609375,2.25390625,8388608
+65,self_attn.k_proj,2,128,8,128,24.46875,2.126953125,8388608
+65,self_attn.k_proj,3,32,8,128,8.0859375,3.5078125,8388608
+65,self_attn.k_proj,3,64,8,128,9.9765625,3.25390625,8388608
+65,self_attn.k_proj,3,128,8,128,11.7109375,3.126953125,8388608
+65,self_attn.k_proj,4,32,8,128,3.791015625,4.5078125,8388608
+65,self_attn.k_proj,4,64,8,128,4.70703125,4.25390625,8388608
+65,self_attn.k_proj,4,128,8,128,5.5703125,4.126953125,8388608
+65,self_attn.k_proj,8,32,8,128,0.2293701171875,8.5078125,8388608
+65,self_attn.k_proj,8,64,8,128,0.284912109375,8.25390625,8388608
+65,self_attn.k_proj,8,128,8,128,0.33642578125,8.126953125,8388608
+65,self_attn.o_proj,2,32,8,128,37.8125,20.0625,67108864
+65,self_attn.o_proj,2,64,8,128,44.03125,18.03125,67108864
+65,self_attn.o_proj,2,128,8,128,49.625,17.015625,67108864
+65,self_attn.o_proj,3,32,8,128,16.171875,28.0625,67108864
+65,self_attn.o_proj,3,64,8,128,18.828125,26.03125,67108864
+65,self_attn.o_proj,3,128,8,128,21.25,25.015625,67108864
+65,self_attn.o_proj,4,32,8,128,7.546875,36.0625,67108864
+65,self_attn.o_proj,4,64,8,128,8.7890625,34.03125,67108864
+65,self_attn.o_proj,4,128,8,128,9.9140625,33.015625,67108864
+65,self_attn.o_proj,8,32,8,128,0.456787109375,68.0625,67108864
+65,self_attn.o_proj,8,64,8,128,0.53173828125,66.03125,67108864
+65,self_attn.o_proj,8,128,8,128,0.59765625,65.015625,67108864
+65,mlp.gate_proj,2,32,8,128,89.125,70.21875,234881024
+65,mlp.gate_proj,2,64,8,128,104.25,63.109375,234881024
+65,mlp.gate_proj,2,128,8,128,117.9375,59.5546875,234881024
+65,mlp.gate_proj,3,32,8,128,38.15625,98.21875,234881024
+65,mlp.gate_proj,3,64,8,128,44.5625,91.109375,234881024
+65,mlp.gate_proj,3,128,8,128,50.5,87.5546875,234881024
+65,mlp.gate_proj,4,32,8,128,17.796875,126.21875,234881024
+65,mlp.gate_proj,4,64,8,128,20.796875,119.109375,234881024
+65,mlp.gate_proj,4,128,8,128,23.5625,115.5546875,234881024
+65,mlp.gate_proj,8,32,8,128,1.08203125,238.21875,234881024
+65,mlp.gate_proj,8,64,8,128,1.2666015625,231.109375,234881024
+65,mlp.gate_proj,8,128,8,128,1.4296875,227.5546875,234881024
+65,mlp.up_proj,2,32,8,128,82.25,70.21875,234881024
+65,mlp.up_proj,2,64,8,128,95.6875,63.109375,234881024
+65,mlp.up_proj,2,128,8,128,107.9375,59.5546875,234881024
+65,mlp.up_proj,3,32,8,128,35.15625,98.21875,234881024
+65,mlp.up_proj,3,64,8,128,40.96875,91.109375,234881024
+65,mlp.up_proj,3,128,8,128,46.15625,87.5546875,234881024
+65,mlp.up_proj,4,32,8,128,16.390625,126.21875,234881024
+65,mlp.up_proj,4,64,8,128,19.125,119.109375,234881024
+65,mlp.up_proj,4,128,8,128,21.546875,115.5546875,234881024
+65,mlp.up_proj,8,32,8,128,0.9970703125,238.21875,234881024
+65,mlp.up_proj,8,64,8,128,1.1630859375,231.109375,234881024
+65,mlp.up_proj,8,128,8,128,1.306640625,227.5546875,234881024
+65,mlp.down_proj,2,32,8,128,81.875,70.21875,234881024
+65,mlp.down_proj,2,64,8,128,95.25,63.109375,234881024
+65,mlp.down_proj,2,128,8,128,107.375,59.5546875,234881024
+65,mlp.down_proj,3,32,8,128,35.0,98.21875,234881024
+65,mlp.down_proj,3,64,8,128,40.75,91.109375,234881024
+65,mlp.down_proj,3,128,8,128,45.875,87.5546875,234881024
+65,mlp.down_proj,4,32,8,128,16.328125,126.21875,234881024
+65,mlp.down_proj,4,64,8,128,19.03125,119.109375,234881024
+65,mlp.down_proj,4,128,8,128,21.421875,115.5546875,234881024
+65,mlp.down_proj,8,32,8,128,0.99267578125,238.21875,234881024
+65,mlp.down_proj,8,64,8,128,1.1572265625,231.109375,234881024
+65,mlp.down_proj,8,128,8,128,1.2998046875,227.5546875,234881024
+66,self_attn.q_proj,2,32,8,128,34.4375,20.0625,67108864
+66,self_attn.q_proj,2,64,8,128,41.375,18.03125,67108864
+66,self_attn.q_proj,2,128,8,128,48.28125,17.015625,67108864
+66,self_attn.q_proj,3,32,8,128,14.828125,28.0625,67108864
+66,self_attn.q_proj,3,64,8,128,17.96875,26.03125,67108864
+66,self_attn.q_proj,3,128,8,128,21.3125,25.015625,67108864
+66,self_attn.q_proj,4,32,8,128,6.91796875,36.0625,67108864
+66,self_attn.q_proj,4,64,8,128,8.3828125,34.03125,67108864
+66,self_attn.q_proj,4,128,8,128,9.953125,33.015625,67108864
+66,self_attn.q_proj,8,32,8,128,0.418701171875,68.0625,67108864
+66,self_attn.q_proj,8,64,8,128,0.50732421875,66.03125,67108864
+66,self_attn.q_proj,8,128,8,128,0.60009765625,65.015625,67108864
+66,self_attn.v_proj,2,32,8,128,17.921875,2.5078125,8388608
+66,self_attn.v_proj,2,64,8,128,20.890625,2.25390625,8388608
+66,self_attn.v_proj,2,128,8,128,23.578125,2.126953125,8388608
+66,self_attn.v_proj,3,32,8,128,7.66015625,3.5078125,8388608
+66,self_attn.v_proj,3,64,8,128,8.9453125,3.25390625,8388608
+66,self_attn.v_proj,3,128,8,128,10.0859375,3.126953125,8388608
+66,self_attn.v_proj,4,32,8,128,3.57421875,4.5078125,8388608
+66,self_attn.v_proj,4,64,8,128,4.16796875,4.25390625,8388608
+66,self_attn.v_proj,4,128,8,128,4.70703125,4.126953125,8388608
+66,self_attn.v_proj,8,32,8,128,0.2164306640625,8.5078125,8388608
+66,self_attn.v_proj,8,64,8,128,0.25244140625,8.25390625,8388608
+66,self_attn.v_proj,8,128,8,128,0.28369140625,8.126953125,8388608
+66,self_attn.k_proj,2,32,8,128,20.59375,2.5078125,8388608
+66,self_attn.k_proj,2,64,8,128,24.984375,2.25390625,8388608
+66,self_attn.k_proj,2,128,8,128,28.6875,2.126953125,8388608
+66,self_attn.k_proj,3,32,8,128,9.09375,3.5078125,8388608
+66,self_attn.k_proj,3,64,8,128,11.4140625,3.25390625,8388608
+66,self_attn.k_proj,3,128,8,128,13.734375,3.126953125,8388608
+66,self_attn.k_proj,4,32,8,128,4.2421875,4.5078125,8388608
+66,self_attn.k_proj,4,64,8,128,5.33203125,4.25390625,8388608
+66,self_attn.k_proj,4,128,8,128,6.421875,4.126953125,8388608
+66,self_attn.k_proj,8,32,8,128,0.25634765625,8.5078125,8388608
+66,self_attn.k_proj,8,64,8,128,0.322265625,8.25390625,8388608
+66,self_attn.k_proj,8,128,8,128,0.386962890625,8.126953125,8388608
+66,self_attn.o_proj,2,32,8,128,40.6875,20.0625,67108864
+66,self_attn.o_proj,2,64,8,128,47.375,18.03125,67108864
+66,self_attn.o_proj,2,128,8,128,53.375,17.015625,67108864
+66,self_attn.o_proj,3,32,8,128,17.40625,28.0625,67108864
+66,self_attn.o_proj,3,64,8,128,20.265625,26.03125,67108864
+66,self_attn.o_proj,3,128,8,128,22.859375,25.015625,67108864
+66,self_attn.o_proj,4,32,8,128,8.1171875,36.0625,67108864
+66,self_attn.o_proj,4,64,8,128,9.4609375,34.03125,67108864
+66,self_attn.o_proj,4,128,8,128,10.671875,33.015625,67108864
+66,self_attn.o_proj,8,32,8,128,0.49169921875,68.0625,67108864
+66,self_attn.o_proj,8,64,8,128,0.572265625,66.03125,67108864
+66,self_attn.o_proj,8,128,8,128,0.64306640625,65.015625,67108864
+66,mlp.gate_proj,2,32,8,128,89.0,70.21875,234881024
+66,mlp.gate_proj,2,64,8,128,104.125,63.109375,234881024
+66,mlp.gate_proj,2,128,8,128,117.8125,59.5546875,234881024
+66,mlp.gate_proj,3,32,8,128,38.09375,98.21875,234881024
+66,mlp.gate_proj,3,64,8,128,44.5,91.109375,234881024
+66,mlp.gate_proj,3,128,8,128,50.4375,87.5546875,234881024
+66,mlp.gate_proj,4,32,8,128,17.75,126.21875,234881024
+66,mlp.gate_proj,4,64,8,128,20.765625,119.109375,234881024
+66,mlp.gate_proj,4,128,8,128,23.53125,115.5546875,234881024
+66,mlp.gate_proj,8,32,8,128,1.080078125,238.21875,234881024
+66,mlp.gate_proj,8,64,8,128,1.2646484375,231.109375,234881024
+66,mlp.gate_proj,8,128,8,128,1.427734375,227.5546875,234881024
+66,mlp.up_proj,2,32,8,128,82.4375,70.21875,234881024
+66,mlp.up_proj,2,64,8,128,95.9375,63.109375,234881024
+66,mlp.up_proj,2,128,8,128,108.1875,59.5546875,234881024
+66,mlp.up_proj,3,32,8,128,35.25,98.21875,234881024
+66,mlp.up_proj,3,64,8,128,41.0625,91.109375,234881024
+66,mlp.up_proj,3,128,8,128,46.25,87.5546875,234881024
+66,mlp.up_proj,4,32,8,128,16.4375,126.21875,234881024
+66,mlp.up_proj,4,64,8,128,19.171875,119.109375,234881024
+66,mlp.up_proj,4,128,8,128,21.578125,115.5546875,234881024
+66,mlp.up_proj,8,32,8,128,0.99951171875,238.21875,234881024
+66,mlp.up_proj,8,64,8,128,1.166015625,231.109375,234881024
+66,mlp.up_proj,8,128,8,128,1.3095703125,227.5546875,234881024
+66,mlp.down_proj,2,32,8,128,82.0625,70.21875,234881024
+66,mlp.down_proj,2,64,8,128,95.4375,63.109375,234881024
+66,mlp.down_proj,2,128,8,128,107.5,59.5546875,234881024
+66,mlp.down_proj,3,32,8,128,35.0625,98.21875,234881024
+66,mlp.down_proj,3,64,8,128,40.84375,91.109375,234881024
+66,mlp.down_proj,3,128,8,128,45.96875,87.5546875,234881024
+66,mlp.down_proj,4,32,8,128,16.34375,126.21875,234881024
+66,mlp.down_proj,4,64,8,128,19.0625,119.109375,234881024
+66,mlp.down_proj,4,128,8,128,21.453125,115.5546875,234881024
+66,mlp.down_proj,8,32,8,128,0.99462890625,238.21875,234881024
+66,mlp.down_proj,8,64,8,128,1.1591796875,231.109375,234881024
+66,mlp.down_proj,8,128,8,128,1.3017578125,227.5546875,234881024
+67,self_attn.q_proj,2,32,8,128,29.65625,20.0625,67108864
+67,self_attn.q_proj,2,64,8,128,35.28125,18.03125,67108864
+67,self_attn.q_proj,2,128,8,128,40.8125,17.015625,67108864
+67,self_attn.q_proj,3,32,8,128,12.859375,28.0625,67108864
+67,self_attn.q_proj,3,64,8,128,15.5234375,26.03125,67108864
+67,self_attn.q_proj,3,128,8,128,18.375,25.015625,67108864
+67,self_attn.q_proj,4,32,8,128,6.0,36.0625,67108864
+67,self_attn.q_proj,4,64,8,128,7.24609375,34.03125,67108864
+67,self_attn.q_proj,4,128,8,128,8.5859375,33.015625,67108864
+67,self_attn.q_proj,8,32,8,128,0.36328125,68.0625,67108864
+67,self_attn.q_proj,8,64,8,128,0.4384765625,66.03125,67108864
+67,self_attn.q_proj,8,128,8,128,0.51806640625,65.015625,67108864
+67,self_attn.v_proj,2,32,8,128,18.140625,2.5078125,8388608
+67,self_attn.v_proj,2,64,8,128,21.5,2.25390625,8388608
+67,self_attn.v_proj,2,128,8,128,24.515625,2.126953125,8388608
+67,self_attn.v_proj,3,32,8,128,7.765625,3.5078125,8388608
+67,self_attn.v_proj,3,64,8,128,9.21875,3.25390625,8388608
+67,self_attn.v_proj,3,128,8,128,10.546875,3.126953125,8388608
+67,self_attn.v_proj,4,32,8,128,3.623046875,4.5078125,8388608
+67,self_attn.v_proj,4,64,8,128,4.30078125,4.25390625,8388608
+67,self_attn.v_proj,4,128,8,128,4.92578125,4.126953125,8388608
+67,self_attn.v_proj,8,32,8,128,0.2193603515625,8.5078125,8388608
+67,self_attn.v_proj,8,64,8,128,0.260009765625,8.25390625,8388608
+67,self_attn.v_proj,8,128,8,128,0.296875,8.126953125,8388608
+67,self_attn.k_proj,2,32,8,128,19.78125,2.5078125,8388608
+67,self_attn.k_proj,2,64,8,128,23.4375,2.25390625,8388608
+67,self_attn.k_proj,2,128,8,128,26.453125,2.126953125,8388608
+67,self_attn.k_proj,3,32,8,128,8.9453125,3.5078125,8388608
+67,self_attn.k_proj,3,64,8,128,11.03125,3.25390625,8388608
+67,self_attn.k_proj,3,128,8,128,12.8203125,3.126953125,8388608
+67,self_attn.k_proj,4,32,8,128,4.2265625,4.5078125,8388608
+67,self_attn.k_proj,4,64,8,128,5.2734375,4.25390625,8388608
+67,self_attn.k_proj,4,128,8,128,6.20703125,4.126953125,8388608
+67,self_attn.k_proj,8,32,8,128,0.256591796875,8.5078125,8388608
+67,self_attn.k_proj,8,64,8,128,0.32177734375,8.25390625,8388608
+67,self_attn.k_proj,8,128,8,128,0.3798828125,8.126953125,8388608
+67,self_attn.o_proj,2,32,8,128,36.25,20.0625,67108864
+67,self_attn.o_proj,2,64,8,128,42.3125,18.03125,67108864
+67,self_attn.o_proj,2,128,8,128,47.90625,17.015625,67108864
+67,self_attn.o_proj,3,32,8,128,15.546875,28.0625,67108864
+67,self_attn.o_proj,3,64,8,128,18.21875,26.03125,67108864
+67,self_attn.o_proj,3,128,8,128,20.765625,25.015625,67108864
+67,self_attn.o_proj,4,32,8,128,7.26953125,36.0625,67108864
+67,self_attn.o_proj,4,64,8,128,8.53125,34.03125,67108864
+67,self_attn.o_proj,4,128,8,128,9.734375,33.015625,67108864
+67,self_attn.o_proj,8,32,8,128,0.4404296875,68.0625,67108864
+67,self_attn.o_proj,8,64,8,128,0.51708984375,66.03125,67108864
+67,self_attn.o_proj,8,128,8,128,0.58984375,65.015625,67108864
+67,mlp.gate_proj,2,32,8,128,88.8125,70.21875,234881024
+67,mlp.gate_proj,2,64,8,128,103.8125,63.109375,234881024
+67,mlp.gate_proj,2,128,8,128,117.4375,59.5546875,234881024
+67,mlp.gate_proj,3,32,8,128,38.0,98.21875,234881024
+67,mlp.gate_proj,3,64,8,128,44.40625,91.109375,234881024
+67,mlp.gate_proj,3,128,8,128,50.3125,87.5546875,234881024
+67,mlp.gate_proj,4,32,8,128,17.71875,126.21875,234881024
+67,mlp.gate_proj,4,64,8,128,20.71875,119.109375,234881024
+67,mlp.gate_proj,4,128,8,128,23.46875,115.5546875,234881024
+67,mlp.gate_proj,8,32,8,128,1.078125,238.21875,234881024
+67,mlp.gate_proj,8,64,8,128,1.26171875,231.109375,234881024
+67,mlp.gate_proj,8,128,8,128,1.423828125,227.5546875,234881024
+67,mlp.up_proj,2,32,8,128,82.6875,70.21875,234881024
+67,mlp.up_proj,2,64,8,128,96.25,63.109375,234881024
+67,mlp.up_proj,2,128,8,128,108.5,59.5546875,234881024
+67,mlp.up_proj,3,32,8,128,35.34375,98.21875,234881024
+67,mlp.up_proj,3,64,8,128,41.1875,91.109375,234881024
+67,mlp.up_proj,3,128,8,128,46.40625,87.5546875,234881024
+67,mlp.up_proj,4,32,8,128,16.484375,126.21875,234881024
+67,mlp.up_proj,4,64,8,128,19.21875,119.109375,234881024
+67,mlp.up_proj,4,128,8,128,21.65625,115.5546875,234881024
+67,mlp.up_proj,8,32,8,128,1.0029296875,238.21875,234881024
+67,mlp.up_proj,8,64,8,128,1.1689453125,231.109375,234881024
+67,mlp.up_proj,8,128,8,128,1.3134765625,227.5546875,234881024
+67,mlp.down_proj,2,32,8,128,82.1875,70.21875,234881024
+67,mlp.down_proj,2,64,8,128,95.625,63.109375,234881024
+67,mlp.down_proj,2,128,8,128,107.75,59.5546875,234881024
+67,mlp.down_proj,3,32,8,128,35.125,98.21875,234881024
+67,mlp.down_proj,3,64,8,128,40.90625,91.109375,234881024
+67,mlp.down_proj,3,128,8,128,46.0625,87.5546875,234881024
+67,mlp.down_proj,4,32,8,128,16.390625,126.21875,234881024
+67,mlp.down_proj,4,64,8,128,19.09375,119.109375,234881024
+67,mlp.down_proj,4,128,8,128,21.5,115.5546875,234881024
+67,mlp.down_proj,8,32,8,128,0.99658203125,238.21875,234881024
+67,mlp.down_proj,8,64,8,128,1.162109375,231.109375,234881024
+67,mlp.down_proj,8,128,8,128,1.3046875,227.5546875,234881024
+68,self_attn.q_proj,2,32,8,128,35.21875,20.0625,67108864
+68,self_attn.q_proj,2,64,8,128,41.5,18.03125,67108864
+68,self_attn.q_proj,2,128,8,128,48.0625,17.015625,67108864
+68,self_attn.q_proj,3,32,8,128,15.078125,28.0625,67108864
+68,self_attn.q_proj,3,64,8,128,17.796875,26.03125,67108864
+68,self_attn.q_proj,3,128,8,128,20.6875,25.015625,67108864
+68,self_attn.q_proj,4,32,8,128,7.03515625,36.0625,67108864
+68,self_attn.q_proj,4,64,8,128,8.3046875,34.03125,67108864
+68,self_attn.q_proj,4,128,8,128,9.65625,33.015625,67108864
+68,self_attn.q_proj,8,32,8,128,0.42578125,68.0625,67108864
+68,self_attn.q_proj,8,64,8,128,0.50244140625,66.03125,67108864
+68,self_attn.q_proj,8,128,8,128,0.58251953125,65.015625,67108864
+68,self_attn.v_proj,2,32,8,128,20.0,2.5078125,8388608
+68,self_attn.v_proj,2,64,8,128,23.515625,2.25390625,8388608
+68,self_attn.v_proj,2,128,8,128,26.78125,2.126953125,8388608
+68,self_attn.v_proj,3,32,8,128,8.5546875,3.5078125,8388608
+68,self_attn.v_proj,3,64,8,128,10.0625,3.25390625,8388608
+68,self_attn.v_proj,3,128,8,128,11.4609375,3.126953125,8388608
+68,self_attn.v_proj,4,32,8,128,3.9921875,4.5078125,8388608
+68,self_attn.v_proj,4,64,8,128,4.6953125,4.25390625,8388608
+68,self_attn.v_proj,4,128,8,128,5.34765625,4.126953125,8388608
+68,self_attn.v_proj,8,32,8,128,0.24169921875,8.5078125,8388608
+68,self_attn.v_proj,8,64,8,128,0.2841796875,8.25390625,8388608
+68,self_attn.v_proj,8,128,8,128,0.322265625,8.126953125,8388608
+68,self_attn.k_proj,2,32,8,128,21.234375,2.5078125,8388608
+68,self_attn.k_proj,2,64,8,128,25.640625,2.25390625,8388608
+68,self_attn.k_proj,2,128,8,128,29.390625,2.126953125,8388608
+68,self_attn.k_proj,3,32,8,128,9.1796875,3.5078125,8388608
+68,self_attn.k_proj,3,64,8,128,11.25,3.25390625,8388608
+68,self_attn.k_proj,3,128,8,128,13.2421875,3.126953125,8388608
+68,self_attn.k_proj,4,32,8,128,4.28515625,4.5078125,8388608
+68,self_attn.k_proj,4,64,8,128,5.25390625,4.25390625,8388608
+68,self_attn.k_proj,4,128,8,128,6.1796875,4.126953125,8388608
+68,self_attn.k_proj,8,32,8,128,0.25927734375,8.5078125,8388608
+68,self_attn.k_proj,8,64,8,128,0.317626953125,8.25390625,8388608
+68,self_attn.k_proj,8,128,8,128,0.37255859375,8.126953125,8388608
+68,self_attn.o_proj,2,32,8,128,42.1875,20.0625,67108864
+68,self_attn.o_proj,2,64,8,128,49.09375,18.03125,67108864
+68,self_attn.o_proj,2,128,8,128,55.34375,17.015625,67108864
+68,self_attn.o_proj,3,32,8,128,18.046875,28.0625,67108864
+68,self_attn.o_proj,3,64,8,128,21.015625,26.03125,67108864
+68,self_attn.o_proj,3,128,8,128,23.6875,25.015625,67108864
+68,self_attn.o_proj,4,32,8,128,8.4140625,36.0625,67108864
+68,self_attn.o_proj,4,64,8,128,9.8046875,34.03125,67108864
+68,self_attn.o_proj,4,128,8,128,11.0546875,33.015625,67108864
+68,self_attn.o_proj,8,32,8,128,0.509765625,68.0625,67108864
+68,self_attn.o_proj,8,64,8,128,0.59326171875,66.03125,67108864
+68,self_attn.o_proj,8,128,8,128,0.66650390625,65.015625,67108864
+68,mlp.gate_proj,2,32,8,128,88.6875,70.21875,234881024
+68,mlp.gate_proj,2,64,8,128,103.6875,63.109375,234881024
+68,mlp.gate_proj,2,128,8,128,117.375,59.5546875,234881024
+68,mlp.gate_proj,3,32,8,128,37.9375,98.21875,234881024
+68,mlp.gate_proj,3,64,8,128,44.375,91.109375,234881024
+68,mlp.gate_proj,3,128,8,128,50.28125,87.5546875,234881024
+68,mlp.gate_proj,4,32,8,128,17.6875,126.21875,234881024
+68,mlp.gate_proj,4,64,8,128,20.703125,119.109375,234881024
+68,mlp.gate_proj,4,128,8,128,23.453125,115.5546875,234881024
+68,mlp.gate_proj,8,32,8,128,1.076171875,238.21875,234881024
+68,mlp.gate_proj,8,64,8,128,1.259765625,231.109375,234881024
+68,mlp.gate_proj,8,128,8,128,1.4228515625,227.5546875,234881024
+68,mlp.up_proj,2,32,8,128,82.875,70.21875,234881024
+68,mlp.up_proj,2,64,8,128,96.4375,63.109375,234881024
+68,mlp.up_proj,2,128,8,128,108.75,59.5546875,234881024
+68,mlp.up_proj,3,32,8,128,35.4375,98.21875,234881024
+68,mlp.up_proj,3,64,8,128,41.28125,91.109375,234881024
+68,mlp.up_proj,3,128,8,128,46.53125,87.5546875,234881024
+68,mlp.up_proj,4,32,8,128,16.515625,126.21875,234881024
+68,mlp.up_proj,4,64,8,128,19.265625,119.109375,234881024
+68,mlp.up_proj,4,128,8,128,21.703125,115.5546875,234881024
+68,mlp.up_proj,8,32,8,128,1.0048828125,238.21875,234881024
+68,mlp.up_proj,8,64,8,128,1.171875,231.109375,234881024
+68,mlp.up_proj,8,128,8,128,1.3173828125,227.5546875,234881024
+68,mlp.down_proj,2,32,8,128,82.3125,70.21875,234881024
+68,mlp.down_proj,2,64,8,128,95.75,63.109375,234881024
+68,mlp.down_proj,2,128,8,128,107.9375,59.5546875,234881024
+68,mlp.down_proj,3,32,8,128,35.1875,98.21875,234881024
+68,mlp.down_proj,3,64,8,128,40.96875,91.109375,234881024
+68,mlp.down_proj,3,128,8,128,46.125,87.5546875,234881024
+68,mlp.down_proj,4,32,8,128,16.40625,126.21875,234881024
+68,mlp.down_proj,4,64,8,128,19.125,119.109375,234881024
+68,mlp.down_proj,4,128,8,128,21.53125,115.5546875,234881024
+68,mlp.down_proj,8,32,8,128,0.998046875,238.21875,234881024
+68,mlp.down_proj,8,64,8,128,1.1640625,231.109375,234881024
+68,mlp.down_proj,8,128,8,128,1.306640625,227.5546875,234881024
+69,self_attn.q_proj,2,32,8,128,38.34375,20.0625,67108864
+69,self_attn.q_proj,2,64,8,128,45.1875,18.03125,67108864
+69,self_attn.q_proj,2,128,8,128,52.25,17.015625,67108864
+69,self_attn.q_proj,3,32,8,128,16.421875,28.0625,67108864
+69,self_attn.q_proj,3,64,8,128,19.390625,26.03125,67108864
+69,self_attn.q_proj,3,128,8,128,22.5,25.015625,67108864
+69,self_attn.q_proj,4,32,8,128,7.6640625,36.0625,67108864
+69,self_attn.q_proj,4,64,8,128,9.0546875,34.03125,67108864
+69,self_attn.q_proj,4,128,8,128,10.5078125,33.015625,67108864
+69,self_attn.q_proj,8,32,8,128,0.4638671875,68.0625,67108864
+69,self_attn.q_proj,8,64,8,128,0.5478515625,66.03125,67108864
+69,self_attn.q_proj,8,128,8,128,0.6337890625,65.015625,67108864
+69,self_attn.v_proj,2,32,8,128,20.828125,2.5078125,8388608
+69,self_attn.v_proj,2,64,8,128,24.359375,2.25390625,8388608
+69,self_attn.v_proj,2,128,8,128,27.546875,2.126953125,8388608
+69,self_attn.v_proj,3,32,8,128,8.90625,3.5078125,8388608
+69,self_attn.v_proj,3,64,8,128,10.421875,3.25390625,8388608
+69,self_attn.v_proj,3,128,8,128,11.7890625,3.126953125,8388608
+69,self_attn.v_proj,4,32,8,128,4.15625,4.5078125,8388608
+69,self_attn.v_proj,4,64,8,128,4.859375,4.25390625,8388608
+69,self_attn.v_proj,4,128,8,128,5.5,4.126953125,8388608
+69,self_attn.v_proj,8,32,8,128,0.251708984375,8.5078125,8388608
+69,self_attn.v_proj,8,64,8,128,0.294189453125,8.25390625,8388608
+69,self_attn.v_proj,8,128,8,128,0.33154296875,8.126953125,8388608
+69,self_attn.k_proj,2,32,8,128,22.328125,2.5078125,8388608
+69,self_attn.k_proj,2,64,8,128,26.796875,2.25390625,8388608
+69,self_attn.k_proj,2,128,8,128,30.84375,2.126953125,8388608
+69,self_attn.k_proj,3,32,8,128,9.578125,3.5078125,8388608
+69,self_attn.k_proj,3,64,8,128,11.5625,3.25390625,8388608
+69,self_attn.k_proj,3,128,8,128,13.46875,3.126953125,8388608
+69,self_attn.k_proj,4,32,8,128,4.46875,4.5078125,8388608
+69,self_attn.k_proj,4,64,8,128,5.39453125,4.25390625,8388608
+69,self_attn.k_proj,4,128,8,128,6.28515625,4.126953125,8388608
+69,self_attn.k_proj,8,32,8,128,0.270751953125,8.5078125,8388608
+69,self_attn.k_proj,8,64,8,128,0.326416015625,8.25390625,8388608
+69,self_attn.k_proj,8,128,8,128,0.37890625,8.126953125,8388608
+69,self_attn.o_proj,2,32,8,128,44.28125,20.0625,67108864
+69,self_attn.o_proj,2,64,8,128,51.5625,18.03125,67108864
+69,self_attn.o_proj,2,128,8,128,58.125,17.015625,67108864
+69,self_attn.o_proj,3,32,8,128,18.9375,28.0625,67108864
+69,self_attn.o_proj,3,64,8,128,22.0625,26.03125,67108864
+69,self_attn.o_proj,3,128,8,128,24.890625,25.015625,67108864
+69,self_attn.o_proj,4,32,8,128,8.8359375,36.0625,67108864
+69,self_attn.o_proj,4,64,8,128,10.296875,34.03125,67108864
+69,self_attn.o_proj,4,128,8,128,11.6171875,33.015625,67108864
+69,self_attn.o_proj,8,32,8,128,0.53515625,68.0625,67108864
+69,self_attn.o_proj,8,64,8,128,0.623046875,66.03125,67108864
+69,self_attn.o_proj,8,128,8,128,0.7001953125,65.015625,67108864
+69,mlp.gate_proj,2,32,8,128,88.375,70.21875,234881024
+69,mlp.gate_proj,2,64,8,128,103.3125,63.109375,234881024
+69,mlp.gate_proj,2,128,8,128,116.875,59.5546875,234881024
+69,mlp.gate_proj,3,32,8,128,37.8125,98.21875,234881024
+69,mlp.gate_proj,3,64,8,128,44.1875,91.109375,234881024
+69,mlp.gate_proj,3,128,8,128,50.0625,87.5546875,234881024
+69,mlp.gate_proj,4,32,8,128,17.625,126.21875,234881024
+69,mlp.gate_proj,4,64,8,128,20.625,119.109375,234881024
+69,mlp.gate_proj,4,128,8,128,23.359375,115.5546875,234881024
+69,mlp.gate_proj,8,32,8,128,1.072265625,238.21875,234881024
+69,mlp.gate_proj,8,64,8,128,1.2548828125,231.109375,234881024
+69,mlp.gate_proj,8,128,8,128,1.4169921875,227.5546875,234881024
+69,mlp.up_proj,2,32,8,128,83.1875,70.21875,234881024
+69,mlp.up_proj,2,64,8,128,96.875,63.109375,234881024
+69,mlp.up_proj,2,128,8,128,109.1875,59.5546875,234881024
+69,mlp.up_proj,3,32,8,128,35.5625,98.21875,234881024
+69,mlp.up_proj,3,64,8,128,41.4375,91.109375,234881024
+69,mlp.up_proj,3,128,8,128,46.71875,87.5546875,234881024
+69,mlp.up_proj,4,32,8,128,16.59375,126.21875,234881024
+69,mlp.up_proj,4,64,8,128,19.34375,119.109375,234881024
+69,mlp.up_proj,4,128,8,128,21.796875,115.5546875,234881024
+69,mlp.up_proj,8,32,8,128,1.0087890625,238.21875,234881024
+69,mlp.up_proj,8,64,8,128,1.1767578125,231.109375,234881024
+69,mlp.up_proj,8,128,8,128,1.322265625,227.5546875,234881024
+69,mlp.down_proj,2,32,8,128,82.625,70.21875,234881024
+69,mlp.down_proj,2,64,8,128,96.1875,63.109375,234881024
+69,mlp.down_proj,2,128,8,128,108.375,59.5546875,234881024
+69,mlp.down_proj,3,32,8,128,35.34375,98.21875,234881024
+69,mlp.down_proj,3,64,8,128,41.125,91.109375,234881024
+69,mlp.down_proj,3,128,8,128,46.3125,87.5546875,234881024
+69,mlp.down_proj,4,32,8,128,16.484375,126.21875,234881024
+69,mlp.down_proj,4,64,8,128,19.203125,119.109375,234881024
+69,mlp.down_proj,4,128,8,128,21.625,115.5546875,234881024
+69,mlp.down_proj,8,32,8,128,1.001953125,238.21875,234881024
+69,mlp.down_proj,8,64,8,128,1.16796875,231.109375,234881024
+69,mlp.down_proj,8,128,8,128,1.3115234375,227.5546875,234881024
+70,self_attn.q_proj,2,32,8,128,38.1875,20.0625,67108864
+70,self_attn.q_proj,2,64,8,128,45.15625,18.03125,67108864
+70,self_attn.q_proj,2,128,8,128,52.03125,17.015625,67108864
+70,self_attn.q_proj,3,32,8,128,16.359375,28.0625,67108864
+70,self_attn.q_proj,3,64,8,128,19.421875,26.03125,67108864
+70,self_attn.q_proj,3,128,8,128,22.46875,25.015625,67108864
+70,self_attn.q_proj,4,32,8,128,7.63671875,36.0625,67108864
+70,self_attn.q_proj,4,64,8,128,9.0625,34.03125,67108864
+70,self_attn.q_proj,4,128,8,128,10.4921875,33.015625,67108864
+70,self_attn.q_proj,8,32,8,128,0.46240234375,68.0625,67108864
+70,self_attn.q_proj,8,64,8,128,0.54833984375,66.03125,67108864
+70,self_attn.q_proj,8,128,8,128,0.6328125,65.015625,67108864
+70,self_attn.v_proj,2,32,8,128,19.4375,2.5078125,8388608
+70,self_attn.v_proj,2,64,8,128,22.71875,2.25390625,8388608
+70,self_attn.v_proj,2,128,8,128,25.71875,2.126953125,8388608
+70,self_attn.v_proj,3,32,8,128,8.3125,3.5078125,8388608
+70,self_attn.v_proj,3,64,8,128,9.71875,3.25390625,8388608
+70,self_attn.v_proj,3,128,8,128,11.0078125,3.126953125,8388608
+70,self_attn.v_proj,4,32,8,128,3.876953125,4.5078125,8388608
+70,self_attn.v_proj,4,64,8,128,4.53515625,4.25390625,8388608
+70,self_attn.v_proj,4,128,8,128,5.13671875,4.126953125,8388608
+70,self_attn.v_proj,8,32,8,128,0.23486328125,8.5078125,8388608
+70,self_attn.v_proj,8,64,8,128,0.2744140625,8.25390625,8388608
+70,self_attn.v_proj,8,128,8,128,0.3095703125,8.126953125,8388608
+70,self_attn.k_proj,2,32,8,128,21.90625,2.5078125,8388608
+70,self_attn.k_proj,2,64,8,128,26.3125,2.25390625,8388608
+70,self_attn.k_proj,2,128,8,128,30.1875,2.126953125,8388608
+70,self_attn.k_proj,3,32,8,128,9.4375,3.5078125,8388608
+70,self_attn.k_proj,3,64,8,128,11.4609375,3.25390625,8388608
+70,self_attn.k_proj,3,128,8,128,13.3828125,3.126953125,8388608
+70,self_attn.k_proj,4,32,8,128,4.40234375,4.5078125,8388608
+70,self_attn.k_proj,4,64,8,128,5.34765625,4.25390625,8388608
+70,self_attn.k_proj,4,128,8,128,6.24609375,4.126953125,8388608
+70,self_attn.k_proj,8,32,8,128,0.2666015625,8.5078125,8388608
+70,self_attn.k_proj,8,64,8,128,0.323486328125,8.25390625,8388608
+70,self_attn.k_proj,8,128,8,128,0.376708984375,8.126953125,8388608
+70,self_attn.o_proj,2,32,8,128,41.8125,20.0625,67108864
+70,self_attn.o_proj,2,64,8,128,48.71875,18.03125,67108864
+70,self_attn.o_proj,2,128,8,128,54.9375,17.015625,67108864
+70,self_attn.o_proj,3,32,8,128,17.875,28.0625,67108864
+70,self_attn.o_proj,3,64,8,128,20.859375,26.03125,67108864
+70,self_attn.o_proj,3,128,8,128,23.546875,25.015625,67108864
+70,self_attn.o_proj,4,32,8,128,8.34375,36.0625,67108864
+70,self_attn.o_proj,4,64,8,128,9.734375,34.03125,67108864
+70,self_attn.o_proj,4,128,8,128,10.9921875,33.015625,67108864
+70,self_attn.o_proj,8,32,8,128,0.50537109375,68.0625,67108864
+70,self_attn.o_proj,8,64,8,128,0.5888671875,66.03125,67108864
+70,self_attn.o_proj,8,128,8,128,0.66259765625,65.015625,67108864
+70,mlp.gate_proj,2,32,8,128,88.25,70.21875,234881024
+70,mlp.gate_proj,2,64,8,128,103.125,63.109375,234881024
+70,mlp.gate_proj,2,128,8,128,116.75,59.5546875,234881024
+70,mlp.gate_proj,3,32,8,128,37.75,98.21875,234881024
+70,mlp.gate_proj,3,64,8,128,44.125,91.109375,234881024
+70,mlp.gate_proj,3,128,8,128,50.0,87.5546875,234881024
+70,mlp.gate_proj,4,32,8,128,17.609375,126.21875,234881024
+70,mlp.gate_proj,4,64,8,128,20.59375,119.109375,234881024
+70,mlp.gate_proj,4,128,8,128,23.328125,115.5546875,234881024
+70,mlp.gate_proj,8,32,8,128,1.0712890625,238.21875,234881024
+70,mlp.gate_proj,8,64,8,128,1.2529296875,231.109375,234881024
+70,mlp.gate_proj,8,128,8,128,1.4150390625,227.5546875,234881024
+70,mlp.up_proj,2,32,8,128,83.4375,70.21875,234881024
+70,mlp.up_proj,2,64,8,128,97.125,63.109375,234881024
+70,mlp.up_proj,2,128,8,128,109.5625,59.5546875,234881024
+70,mlp.up_proj,3,32,8,128,35.6875,98.21875,234881024
+70,mlp.up_proj,3,64,8,128,41.5625,91.109375,234881024
+70,mlp.up_proj,3,128,8,128,46.875,87.5546875,234881024
+70,mlp.up_proj,4,32,8,128,16.640625,126.21875,234881024
+70,mlp.up_proj,4,64,8,128,19.40625,119.109375,234881024
+70,mlp.up_proj,4,128,8,128,21.859375,115.5546875,234881024
+70,mlp.up_proj,8,32,8,128,1.01171875,238.21875,234881024
+70,mlp.up_proj,8,64,8,128,1.1806640625,231.109375,234881024
+70,mlp.up_proj,8,128,8,128,1.326171875,227.5546875,234881024
+70,mlp.down_proj,2,32,8,128,82.875,70.21875,234881024
+70,mlp.down_proj,2,64,8,128,96.4375,63.109375,234881024
+70,mlp.down_proj,2,128,8,128,108.6875,59.5546875,234881024
+70,mlp.down_proj,3,32,8,128,35.4375,98.21875,234881024
+70,mlp.down_proj,3,64,8,128,41.25,91.109375,234881024
+70,mlp.down_proj,3,128,8,128,46.5,87.5546875,234881024
+70,mlp.down_proj,4,32,8,128,16.515625,126.21875,234881024
+70,mlp.down_proj,4,64,8,128,19.265625,119.109375,234881024
+70,mlp.down_proj,4,128,8,128,21.6875,115.5546875,234881024
+70,mlp.down_proj,8,32,8,128,1.0048828125,238.21875,234881024
+70,mlp.down_proj,8,64,8,128,1.171875,231.109375,234881024
+70,mlp.down_proj,8,128,8,128,1.3154296875,227.5546875,234881024
+71,self_attn.q_proj,2,32,8,128,39.53125,20.0625,67108864
+71,self_attn.q_proj,2,64,8,128,46.46875,18.03125,67108864
+71,self_attn.q_proj,2,128,8,128,53.5625,17.015625,67108864
+71,self_attn.q_proj,3,32,8,128,16.90625,28.0625,67108864
+71,self_attn.q_proj,3,64,8,128,19.9375,26.03125,67108864
+71,self_attn.q_proj,3,128,8,128,23.046875,25.015625,67108864
+71,self_attn.q_proj,4,32,8,128,7.890625,36.0625,67108864
+71,self_attn.q_proj,4,64,8,128,9.296875,34.03125,67108864
+71,self_attn.q_proj,4,128,8,128,10.75,33.015625,67108864
+71,self_attn.q_proj,8,32,8,128,0.477783203125,68.0625,67108864
+71,self_attn.q_proj,8,64,8,128,0.5625,66.03125,67108864
+71,self_attn.q_proj,8,128,8,128,0.6484375,65.015625,67108864
+71,self_attn.v_proj,2,32,8,128,19.265625,2.5078125,8388608
+71,self_attn.v_proj,2,64,8,128,22.546875,2.25390625,8388608
+71,self_attn.v_proj,2,128,8,128,25.546875,2.126953125,8388608
+71,self_attn.v_proj,3,32,8,128,8.2421875,3.5078125,8388608
+71,self_attn.v_proj,3,64,8,128,9.6484375,3.25390625,8388608
+71,self_attn.v_proj,3,128,8,128,10.9296875,3.126953125,8388608
+71,self_attn.v_proj,4,32,8,128,3.841796875,4.5078125,8388608
+71,self_attn.v_proj,4,64,8,128,4.5,4.25390625,8388608
+71,self_attn.v_proj,4,128,8,128,5.1015625,4.126953125,8388608
+71,self_attn.v_proj,8,32,8,128,0.2327880859375,8.5078125,8388608
+71,self_attn.v_proj,8,64,8,128,0.272216796875,8.25390625,8388608
+71,self_attn.v_proj,8,128,8,128,0.307373046875,8.126953125,8388608
+71,self_attn.k_proj,2,32,8,128,22.703125,2.5078125,8388608
+71,self_attn.k_proj,2,64,8,128,27.296875,2.25390625,8388608
+71,self_attn.k_proj,2,128,8,128,31.359375,2.126953125,8388608
+71,self_attn.k_proj,3,32,8,128,9.75,3.5078125,8388608
+71,self_attn.k_proj,3,64,8,128,11.8125,3.25390625,8388608
+71,self_attn.k_proj,3,128,8,128,13.7734375,3.126953125,8388608
+71,self_attn.k_proj,4,32,8,128,4.546875,4.5078125,8388608
+71,self_attn.k_proj,4,64,8,128,5.51171875,4.25390625,8388608
+71,self_attn.k_proj,4,128,8,128,6.42578125,4.126953125,8388608
+71,self_attn.k_proj,8,32,8,128,0.275390625,8.5078125,8388608
+71,self_attn.k_proj,8,64,8,128,0.33349609375,8.25390625,8388608
+71,self_attn.k_proj,8,128,8,128,0.38720703125,8.126953125,8388608
+71,self_attn.o_proj,2,32,8,128,42.59375,20.0625,67108864
+71,self_attn.o_proj,2,64,8,128,49.625,18.03125,67108864
+71,self_attn.o_proj,2,128,8,128,55.96875,17.015625,67108864
+71,self_attn.o_proj,3,32,8,128,18.234375,28.0625,67108864
+71,self_attn.o_proj,3,64,8,128,21.265625,26.03125,67108864
+71,self_attn.o_proj,3,128,8,128,24.015625,25.015625,67108864
+71,self_attn.o_proj,4,32,8,128,8.5078125,36.0625,67108864
+71,self_attn.o_proj,4,64,8,128,9.9296875,34.03125,67108864
+71,self_attn.o_proj,4,128,8,128,11.21875,33.015625,67108864
+71,self_attn.o_proj,8,32,8,128,0.51513671875,68.0625,67108864
+71,self_attn.o_proj,8,64,8,128,0.6005859375,66.03125,67108864
+71,self_attn.o_proj,8,128,8,128,0.67626953125,65.015625,67108864
+71,mlp.gate_proj,2,32,8,128,88.1875,70.21875,234881024
+71,mlp.gate_proj,2,64,8,128,103.125,63.109375,234881024
+71,mlp.gate_proj,2,128,8,128,116.6875,59.5546875,234881024
+71,mlp.gate_proj,3,32,8,128,37.75,98.21875,234881024
+71,mlp.gate_proj,3,64,8,128,44.09375,91.109375,234881024
+71,mlp.gate_proj,3,128,8,128,50.0,87.5546875,234881024
+71,mlp.gate_proj,4,32,8,128,17.59375,126.21875,234881024
+71,mlp.gate_proj,4,64,8,128,20.578125,119.109375,234881024
+71,mlp.gate_proj,4,128,8,128,23.3125,115.5546875,234881024
+71,mlp.gate_proj,8,32,8,128,1.0703125,238.21875,234881024
+71,mlp.gate_proj,8,64,8,128,1.2529296875,231.109375,234881024
+71,mlp.gate_proj,8,128,8,128,1.4150390625,227.5546875,234881024
+71,mlp.up_proj,2,32,8,128,83.5,70.21875,234881024
+71,mlp.up_proj,2,64,8,128,97.25,63.109375,234881024
+71,mlp.up_proj,2,128,8,128,109.6875,59.5546875,234881024
+71,mlp.up_proj,3,32,8,128,35.71875,98.21875,234881024
+71,mlp.up_proj,3,64,8,128,41.625,91.109375,234881024
+71,mlp.up_proj,3,128,8,128,46.9375,87.5546875,234881024
+71,mlp.up_proj,4,32,8,128,16.65625,126.21875,234881024
+71,mlp.up_proj,4,64,8,128,19.4375,119.109375,234881024
+71,mlp.up_proj,4,128,8,128,21.890625,115.5546875,234881024
+71,mlp.up_proj,8,32,8,128,1.0126953125,238.21875,234881024
+71,mlp.up_proj,8,64,8,128,1.1826171875,231.109375,234881024
+71,mlp.up_proj,8,128,8,128,1.3291015625,227.5546875,234881024
+71,mlp.down_proj,2,32,8,128,82.875,70.21875,234881024
+71,mlp.down_proj,2,64,8,128,96.4375,63.109375,234881024
+71,mlp.down_proj,2,128,8,128,108.6875,59.5546875,234881024
+71,mlp.down_proj,3,32,8,128,35.4375,98.21875,234881024
+71,mlp.down_proj,3,64,8,128,41.28125,91.109375,234881024
+71,mlp.down_proj,3,128,8,128,46.46875,87.5546875,234881024
+71,mlp.down_proj,4,32,8,128,16.53125,126.21875,234881024
+71,mlp.down_proj,4,64,8,128,19.265625,119.109375,234881024
+71,mlp.down_proj,4,128,8,128,21.6875,115.5546875,234881024
+71,mlp.down_proj,8,32,8,128,1.0048828125,238.21875,234881024
+71,mlp.down_proj,8,64,8,128,1.171875,231.109375,234881024
+71,mlp.down_proj,8,128,8,128,1.31640625,227.5546875,234881024
+72,self_attn.q_proj,2,32,8,128,40.96875,20.0625,67108864
+72,self_attn.q_proj,2,64,8,128,48.15625,18.03125,67108864
+72,self_attn.q_proj,2,128,8,128,55.125,17.015625,67108864
+72,self_attn.q_proj,3,32,8,128,17.515625,28.0625,67108864
+72,self_attn.q_proj,3,64,8,128,20.625,26.03125,67108864
+72,self_attn.q_proj,3,128,8,128,23.65625,25.015625,67108864
+72,self_attn.q_proj,4,32,8,128,8.1796875,36.0625,67108864
+72,self_attn.q_proj,4,64,8,128,9.625,34.03125,67108864
+72,self_attn.q_proj,4,128,8,128,11.0390625,33.015625,67108864
+72,self_attn.q_proj,8,32,8,128,0.4951171875,68.0625,67108864
+72,self_attn.q_proj,8,64,8,128,0.58251953125,66.03125,67108864
+72,self_attn.q_proj,8,128,8,128,0.66552734375,65.015625,67108864
+72,self_attn.v_proj,2,32,8,128,19.03125,2.5078125,8388608
+72,self_attn.v_proj,2,64,8,128,22.203125,2.25390625,8388608
+72,self_attn.v_proj,2,128,8,128,25.09375,2.126953125,8388608
+72,self_attn.v_proj,3,32,8,128,8.140625,3.5078125,8388608
+72,self_attn.v_proj,3,64,8,128,9.5078125,3.25390625,8388608
+72,self_attn.v_proj,3,128,8,128,10.7421875,3.126953125,8388608
+72,self_attn.v_proj,4,32,8,128,3.796875,4.5078125,8388608
+72,self_attn.v_proj,4,64,8,128,4.43359375,4.25390625,8388608
+72,self_attn.v_proj,4,128,8,128,5.01171875,4.126953125,8388608
+72,self_attn.v_proj,8,32,8,128,0.22998046875,8.5078125,8388608
+72,self_attn.v_proj,8,64,8,128,0.268310546875,8.25390625,8388608
+72,self_attn.v_proj,8,128,8,128,0.30224609375,8.126953125,8388608
+72,self_attn.k_proj,2,32,8,128,23.71875,2.5078125,8388608
+72,self_attn.k_proj,2,64,8,128,28.5,2.25390625,8388608
+72,self_attn.k_proj,2,128,8,128,32.625,2.126953125,8388608
+72,self_attn.k_proj,3,32,8,128,10.1875,3.5078125,8388608
+72,self_attn.k_proj,3,64,8,128,12.328125,3.25390625,8388608
+72,self_attn.k_proj,3,128,8,128,14.3203125,3.126953125,8388608
+72,self_attn.k_proj,4,32,8,128,4.75390625,4.5078125,8388608
+72,self_attn.k_proj,4,64,8,128,5.75390625,4.25390625,8388608
+72,self_attn.k_proj,4,128,8,128,6.6875,4.126953125,8388608
+72,self_attn.k_proj,8,32,8,128,0.287841796875,8.5078125,8388608
+72,self_attn.k_proj,8,64,8,128,0.34814453125,8.25390625,8388608
+72,self_attn.k_proj,8,128,8,128,0.403076171875,8.126953125,8388608
+72,self_attn.o_proj,2,32,8,128,42.71875,20.0625,67108864
+72,self_attn.o_proj,2,64,8,128,49.75,18.03125,67108864
+72,self_attn.o_proj,2,128,8,128,56.09375,17.015625,67108864
+72,self_attn.o_proj,3,32,8,128,18.265625,28.0625,67108864
+72,self_attn.o_proj,3,64,8,128,21.296875,26.03125,67108864
+72,self_attn.o_proj,3,128,8,128,24.015625,25.015625,67108864
+72,self_attn.o_proj,4,32,8,128,8.5234375,36.0625,67108864
+72,self_attn.o_proj,4,64,8,128,9.9375,34.03125,67108864
+72,self_attn.o_proj,4,128,8,128,11.2109375,33.015625,67108864
+72,self_attn.o_proj,8,32,8,128,0.51611328125,68.0625,67108864
+72,self_attn.o_proj,8,64,8,128,0.60107421875,66.03125,67108864
+72,self_attn.o_proj,8,128,8,128,0.67578125,65.015625,67108864
+72,mlp.gate_proj,2,32,8,128,88.0,70.21875,234881024
+72,mlp.gate_proj,2,64,8,128,102.875,63.109375,234881024
+72,mlp.gate_proj,2,128,8,128,116.4375,59.5546875,234881024
+72,mlp.gate_proj,3,32,8,128,37.65625,98.21875,234881024
+72,mlp.gate_proj,3,64,8,128,44.0,91.109375,234881024
+72,mlp.gate_proj,3,128,8,128,49.875,87.5546875,234881024
+72,mlp.gate_proj,4,32,8,128,17.5625,126.21875,234881024
+72,mlp.gate_proj,4,64,8,128,20.546875,119.109375,234881024
+72,mlp.gate_proj,4,128,8,128,23.28125,115.5546875,234881024
+72,mlp.gate_proj,8,32,8,128,1.068359375,238.21875,234881024
+72,mlp.gate_proj,8,64,8,128,1.25,231.109375,234881024
+72,mlp.gate_proj,8,128,8,128,1.4111328125,227.5546875,234881024
+72,mlp.up_proj,2,32,8,128,83.75,70.21875,234881024
+72,mlp.up_proj,2,64,8,128,97.5,63.109375,234881024
+72,mlp.up_proj,2,128,8,128,110.0,59.5546875,234881024
+72,mlp.up_proj,3,32,8,128,35.8125,98.21875,234881024
+72,mlp.up_proj,3,64,8,128,41.71875,91.109375,234881024
+72,mlp.up_proj,3,128,8,128,47.03125,87.5546875,234881024
+72,mlp.up_proj,4,32,8,128,16.6875,126.21875,234881024
+72,mlp.up_proj,4,64,8,128,19.484375,119.109375,234881024
+72,mlp.up_proj,4,128,8,128,21.953125,115.5546875,234881024
+72,mlp.up_proj,8,32,8,128,1.015625,238.21875,234881024
+72,mlp.up_proj,8,64,8,128,1.1845703125,231.109375,234881024
+72,mlp.up_proj,8,128,8,128,1.33203125,227.5546875,234881024
+72,mlp.down_proj,2,32,8,128,83.0625,70.21875,234881024
+72,mlp.down_proj,2,64,8,128,96.6875,63.109375,234881024
+72,mlp.down_proj,2,128,8,128,109.0,59.5546875,234881024
+72,mlp.down_proj,3,32,8,128,35.53125,98.21875,234881024
+72,mlp.down_proj,3,64,8,128,41.375,91.109375,234881024
+72,mlp.down_proj,3,128,8,128,46.59375,87.5546875,234881024
+72,mlp.down_proj,4,32,8,128,16.5625,126.21875,234881024
+72,mlp.down_proj,4,64,8,128,19.3125,119.109375,234881024
+72,mlp.down_proj,4,128,8,128,21.75,115.5546875,234881024
+72,mlp.down_proj,8,32,8,128,1.0078125,238.21875,234881024
+72,mlp.down_proj,8,64,8,128,1.1748046875,231.109375,234881024
+72,mlp.down_proj,8,128,8,128,1.3193359375,227.5546875,234881024
+73,self_attn.q_proj,2,32,8,128,38.75,20.0625,67108864
+73,self_attn.q_proj,2,64,8,128,45.4375,18.03125,67108864
+73,self_attn.q_proj,2,128,8,128,52.0625,17.015625,67108864
+73,self_attn.q_proj,3,32,8,128,16.578125,28.0625,67108864
+73,self_attn.q_proj,3,64,8,128,19.46875,26.03125,67108864
+73,self_attn.q_proj,3,128,8,128,22.328125,25.015625,67108864
+73,self_attn.q_proj,4,32,8,128,7.734375,36.0625,67108864
+73,self_attn.q_proj,4,64,8,128,9.0859375,34.03125,67108864
+73,self_attn.q_proj,4,128,8,128,10.4296875,33.015625,67108864
+73,self_attn.q_proj,8,32,8,128,0.46826171875,68.0625,67108864
+73,self_attn.q_proj,8,64,8,128,0.5498046875,66.03125,67108864
+73,self_attn.q_proj,8,128,8,128,0.62890625,65.015625,67108864
+73,self_attn.v_proj,2,32,8,128,20.296875,2.5078125,8388608
+73,self_attn.v_proj,2,64,8,128,23.703125,2.25390625,8388608
+73,self_attn.v_proj,2,128,8,128,26.796875,2.126953125,8388608
+73,self_attn.v_proj,3,32,8,128,8.6796875,3.5078125,8388608
+73,self_attn.v_proj,3,64,8,128,10.140625,3.25390625,8388608
+73,self_attn.v_proj,3,128,8,128,11.4609375,3.126953125,8388608
+73,self_attn.v_proj,4,32,8,128,4.046875,4.5078125,8388608
+73,self_attn.v_proj,4,64,8,128,4.73046875,4.25390625,8388608
+73,self_attn.v_proj,4,128,8,128,5.3515625,4.126953125,8388608
+73,self_attn.v_proj,8,32,8,128,0.2452392578125,8.5078125,8388608
+73,self_attn.v_proj,8,64,8,128,0.2861328125,8.25390625,8388608
+73,self_attn.v_proj,8,128,8,128,0.322509765625,8.126953125,8388608
+73,self_attn.k_proj,2,32,8,128,22.453125,2.5078125,8388608
+73,self_attn.k_proj,2,64,8,128,26.921875,2.25390625,8388608
+73,self_attn.k_proj,2,128,8,128,30.921875,2.126953125,8388608
+73,self_attn.k_proj,3,32,8,128,9.640625,3.5078125,8388608
+73,self_attn.k_proj,3,64,8,128,11.625,3.25390625,8388608
+73,self_attn.k_proj,3,128,8,128,13.5078125,3.126953125,8388608
+73,self_attn.k_proj,4,32,8,128,4.49609375,4.5078125,8388608
+73,self_attn.k_proj,4,64,8,128,5.4296875,4.25390625,8388608
+73,self_attn.k_proj,4,128,8,128,6.3046875,4.126953125,8388608
+73,self_attn.k_proj,8,32,8,128,0.272216796875,8.5078125,8388608
+73,self_attn.k_proj,8,64,8,128,0.328369140625,8.25390625,8388608
+73,self_attn.k_proj,8,128,8,128,0.3798828125,8.126953125,8388608
+73,self_attn.o_proj,2,32,8,128,42.46875,20.0625,67108864
+73,self_attn.o_proj,2,64,8,128,49.4375,18.03125,67108864
+73,self_attn.o_proj,2,128,8,128,55.75,17.015625,67108864
+73,self_attn.o_proj,3,32,8,128,18.171875,28.0625,67108864
+73,self_attn.o_proj,3,64,8,128,21.1875,26.03125,67108864
+73,self_attn.o_proj,3,128,8,128,23.9375,25.015625,67108864
+73,self_attn.o_proj,4,32,8,128,8.4765625,36.0625,67108864
+73,self_attn.o_proj,4,64,8,128,9.890625,34.03125,67108864
+73,self_attn.o_proj,4,128,8,128,11.1796875,33.015625,67108864
+73,self_attn.o_proj,8,32,8,128,0.51318359375,68.0625,67108864
+73,self_attn.o_proj,8,64,8,128,0.59814453125,66.03125,67108864
+73,self_attn.o_proj,8,128,8,128,0.67333984375,65.015625,67108864
+73,mlp.gate_proj,2,32,8,128,87.9375,70.21875,234881024
+73,mlp.gate_proj,2,64,8,128,102.8125,63.109375,234881024
+73,mlp.gate_proj,2,128,8,128,116.375,59.5546875,234881024
+73,mlp.gate_proj,3,32,8,128,37.625,98.21875,234881024
+73,mlp.gate_proj,3,64,8,128,43.96875,91.109375,234881024
+73,mlp.gate_proj,3,128,8,128,49.875,87.5546875,234881024
+73,mlp.gate_proj,4,32,8,128,17.546875,126.21875,234881024
+73,mlp.gate_proj,4,64,8,128,20.53125,119.109375,234881024
+73,mlp.gate_proj,4,128,8,128,23.265625,115.5546875,234881024
+73,mlp.gate_proj,8,32,8,128,1.0673828125,238.21875,234881024
+73,mlp.gate_proj,8,64,8,128,1.25,231.109375,234881024
+73,mlp.gate_proj,8,128,8,128,1.4111328125,227.5546875,234881024
+73,mlp.up_proj,2,32,8,128,83.9375,70.21875,234881024
+73,mlp.up_proj,2,64,8,128,97.75,63.109375,234881024
+73,mlp.up_proj,2,128,8,128,110.25,59.5546875,234881024
+73,mlp.up_proj,3,32,8,128,35.90625,98.21875,234881024
+73,mlp.up_proj,3,64,8,128,41.8125,91.109375,234881024
+73,mlp.up_proj,3,128,8,128,47.1875,87.5546875,234881024
+73,mlp.up_proj,4,32,8,128,16.734375,126.21875,234881024
+73,mlp.up_proj,4,64,8,128,19.53125,119.109375,234881024
+73,mlp.up_proj,4,128,8,128,22.015625,115.5546875,234881024
+73,mlp.up_proj,8,32,8,128,1.0185546875,238.21875,234881024
+73,mlp.up_proj,8,64,8,128,1.1884765625,231.109375,234881024
+73,mlp.up_proj,8,128,8,128,1.3359375,227.5546875,234881024
+73,mlp.down_proj,2,32,8,128,83.25,70.21875,234881024
+73,mlp.down_proj,2,64,8,128,96.875,63.109375,234881024
+73,mlp.down_proj,2,128,8,128,109.25,59.5546875,234881024
+73,mlp.down_proj,3,32,8,128,35.59375,98.21875,234881024
+73,mlp.down_proj,3,64,8,128,41.46875,91.109375,234881024
+73,mlp.down_proj,3,128,8,128,46.71875,87.5546875,234881024
+73,mlp.down_proj,4,32,8,128,16.59375,126.21875,234881024
+73,mlp.down_proj,4,64,8,128,19.359375,119.109375,234881024
+73,mlp.down_proj,4,128,8,128,21.796875,115.5546875,234881024
+73,mlp.down_proj,8,32,8,128,1.009765625,238.21875,234881024
+73,mlp.down_proj,8,64,8,128,1.177734375,231.109375,234881024
+73,mlp.down_proj,8,128,8,128,1.322265625,227.5546875,234881024
+74,self_attn.q_proj,2,32,8,128,42.78125,20.0625,67108864
+74,self_attn.q_proj,2,64,8,128,50.59375,18.03125,67108864
+74,self_attn.q_proj,2,128,8,128,58.28125,17.015625,67108864
+74,self_attn.q_proj,3,32,8,128,18.3125,28.0625,67108864
+74,self_attn.q_proj,3,64,8,128,21.71875,26.03125,67108864
+74,self_attn.q_proj,3,128,8,128,25.125,25.015625,67108864
+74,self_attn.q_proj,4,32,8,128,8.546875,36.0625,67108864
+74,self_attn.q_proj,4,64,8,128,10.1328125,34.03125,67108864
+74,self_attn.q_proj,4,128,8,128,11.7265625,33.015625,67108864
+74,self_attn.q_proj,8,32,8,128,0.517578125,68.0625,67108864
+74,self_attn.q_proj,8,64,8,128,0.61328125,66.03125,67108864
+74,self_attn.q_proj,8,128,8,128,0.70703125,65.015625,67108864
+74,self_attn.v_proj,2,32,8,128,19.25,2.5078125,8388608
+74,self_attn.v_proj,2,64,8,128,22.5625,2.25390625,8388608
+74,self_attn.v_proj,2,128,8,128,25.5625,2.126953125,8388608
+74,self_attn.v_proj,3,32,8,128,8.2265625,3.5078125,8388608
+74,self_attn.v_proj,3,64,8,128,9.6484375,3.25390625,8388608
+74,self_attn.v_proj,3,128,8,128,10.9453125,3.126953125,8388608
+74,self_attn.v_proj,4,32,8,128,3.837890625,4.5078125,8388608
+74,self_attn.v_proj,4,64,8,128,4.5,4.25390625,8388608
+74,self_attn.v_proj,4,128,8,128,5.10546875,4.126953125,8388608
+74,self_attn.v_proj,8,32,8,128,0.2325439453125,8.5078125,8388608
+74,self_attn.v_proj,8,64,8,128,0.2724609375,8.25390625,8388608
+74,self_attn.v_proj,8,128,8,128,0.3076171875,8.126953125,8388608
+74,self_attn.k_proj,2,32,8,128,23.296875,2.5078125,8388608
+74,self_attn.k_proj,2,64,8,128,27.96875,2.25390625,8388608
+74,self_attn.k_proj,2,128,8,128,32.0625,2.126953125,8388608
+74,self_attn.k_proj,3,32,8,128,10.0078125,3.5078125,8388608
+74,self_attn.k_proj,3,64,8,128,12.1171875,3.25390625,8388608
+74,self_attn.k_proj,3,128,8,128,14.0859375,3.126953125,8388608
+74,self_attn.k_proj,4,32,8,128,4.671875,4.5078125,8388608
+74,self_attn.k_proj,4,64,8,128,5.65625,4.25390625,8388608
+74,self_attn.k_proj,4,128,8,128,6.578125,4.126953125,8388608
+74,self_attn.k_proj,8,32,8,128,0.28271484375,8.5078125,8388608
+74,self_attn.k_proj,8,64,8,128,0.342041015625,8.25390625,8388608
+74,self_attn.k_proj,8,128,8,128,0.396484375,8.126953125,8388608
+74,self_attn.o_proj,2,32,8,128,42.6875,20.0625,67108864
+74,self_attn.o_proj,2,64,8,128,49.8125,18.03125,67108864
+74,self_attn.o_proj,2,128,8,128,56.3125,17.015625,67108864
+74,self_attn.o_proj,3,32,8,128,18.28125,28.0625,67108864
+74,self_attn.o_proj,3,64,8,128,21.359375,26.03125,67108864
+74,self_attn.o_proj,3,128,8,128,24.234375,25.015625,67108864
+74,self_attn.o_proj,4,32,8,128,8.5234375,36.0625,67108864
+74,self_attn.o_proj,4,64,8,128,9.9765625,34.03125,67108864
+74,self_attn.o_proj,4,128,8,128,11.3203125,33.015625,67108864
+74,self_attn.o_proj,8,32,8,128,0.5166015625,68.0625,67108864
+74,self_attn.o_proj,8,64,8,128,0.603515625,66.03125,67108864
+74,self_attn.o_proj,8,128,8,128,0.68212890625,65.015625,67108864
+74,mlp.gate_proj,2,32,8,128,87.8125,70.21875,234881024
+74,mlp.gate_proj,2,64,8,128,102.6875,63.109375,234881024
+74,mlp.gate_proj,2,128,8,128,116.1875,59.5546875,234881024
+74,mlp.gate_proj,3,32,8,128,37.5625,98.21875,234881024
+74,mlp.gate_proj,3,64,8,128,43.90625,91.109375,234881024
+74,mlp.gate_proj,3,128,8,128,49.78125,87.5546875,234881024
+74,mlp.gate_proj,4,32,8,128,17.515625,126.21875,234881024
+74,mlp.gate_proj,4,64,8,128,20.5,119.109375,234881024
+74,mlp.gate_proj,4,128,8,128,23.234375,115.5546875,234881024
+74,mlp.gate_proj,8,32,8,128,1.06640625,238.21875,234881024
+74,mlp.gate_proj,8,64,8,128,1.248046875,231.109375,234881024
+74,mlp.gate_proj,8,128,8,128,1.4091796875,227.5546875,234881024
+74,mlp.up_proj,2,32,8,128,84.0,70.21875,234881024
+74,mlp.up_proj,2,64,8,128,97.875,63.109375,234881024
+74,mlp.up_proj,2,128,8,128,110.4375,59.5546875,234881024
+74,mlp.up_proj,3,32,8,128,35.9375,98.21875,234881024
+74,mlp.up_proj,3,64,8,128,41.875,91.109375,234881024
+74,mlp.up_proj,3,128,8,128,47.25,87.5546875,234881024
+74,mlp.up_proj,4,32,8,128,16.75,126.21875,234881024
+74,mlp.up_proj,4,64,8,128,19.5625,119.109375,234881024
+74,mlp.up_proj,4,128,8,128,22.046875,115.5546875,234881024
+74,mlp.up_proj,8,32,8,128,1.01953125,238.21875,234881024
+74,mlp.up_proj,8,64,8,128,1.189453125,231.109375,234881024
+74,mlp.up_proj,8,128,8,128,1.337890625,227.5546875,234881024
+74,mlp.down_proj,2,32,8,128,83.3125,70.21875,234881024
+74,mlp.down_proj,2,64,8,128,97.0,63.109375,234881024
+74,mlp.down_proj,2,128,8,128,109.3125,59.5546875,234881024
+74,mlp.down_proj,3,32,8,128,35.625,98.21875,234881024
+74,mlp.down_proj,3,64,8,128,41.5,91.109375,234881024
+74,mlp.down_proj,3,128,8,128,46.75,87.5546875,234881024
+74,mlp.down_proj,4,32,8,128,16.609375,126.21875,234881024
+74,mlp.down_proj,4,64,8,128,19.375,119.109375,234881024
+74,mlp.down_proj,4,128,8,128,21.828125,115.5546875,234881024
+74,mlp.down_proj,8,32,8,128,1.0107421875,238.21875,234881024
+74,mlp.down_proj,8,64,8,128,1.1787109375,231.109375,234881024
+74,mlp.down_proj,8,128,8,128,1.32421875,227.5546875,234881024
+75,self_attn.q_proj,2,32,8,128,40.1875,20.0625,67108864
+75,self_attn.q_proj,2,64,8,128,47.375,18.03125,67108864
+75,self_attn.q_proj,2,128,8,128,54.375,17.015625,67108864
+75,self_attn.q_proj,3,32,8,128,17.203125,28.0625,67108864
+75,self_attn.q_proj,3,64,8,128,20.296875,26.03125,67108864
+75,self_attn.q_proj,3,128,8,128,23.34375,25.015625,67108864
+75,self_attn.q_proj,4,32,8,128,8.03125,36.0625,67108864
+75,self_attn.q_proj,4,64,8,128,9.4765625,34.03125,67108864
+75,self_attn.q_proj,4,128,8,128,10.90625,33.015625,67108864
+75,self_attn.q_proj,8,32,8,128,0.486083984375,68.0625,67108864
+75,self_attn.q_proj,8,64,8,128,0.57373046875,66.03125,67108864
+75,self_attn.q_proj,8,128,8,128,0.65771484375,65.015625,67108864
+75,self_attn.v_proj,2,32,8,128,21.140625,2.5078125,8388608
+75,self_attn.v_proj,2,64,8,128,24.875,2.25390625,8388608
+75,self_attn.v_proj,2,128,8,128,28.375,2.126953125,8388608
+75,self_attn.v_proj,3,32,8,128,9.0390625,3.5078125,8388608
+75,self_attn.v_proj,3,64,8,128,10.640625,3.25390625,8388608
+75,self_attn.v_proj,3,128,8,128,12.140625,3.126953125,8388608
+75,self_attn.v_proj,4,32,8,128,4.21484375,4.5078125,8388608
+75,self_attn.v_proj,4,64,8,128,4.96875,4.25390625,8388608
+75,self_attn.v_proj,4,128,8,128,5.6640625,4.126953125,8388608
+75,self_attn.v_proj,8,32,8,128,0.25537109375,8.5078125,8388608
+75,self_attn.v_proj,8,64,8,128,0.300537109375,8.25390625,8388608
+75,self_attn.v_proj,8,128,8,128,0.341552734375,8.126953125,8388608
+75,self_attn.k_proj,2,32,8,128,21.78125,2.5078125,8388608
+75,self_attn.k_proj,2,64,8,128,26.15625,2.25390625,8388608
+75,self_attn.k_proj,2,128,8,128,30.046875,2.126953125,8388608
+75,self_attn.k_proj,3,32,8,128,9.3515625,3.5078125,8388608
+75,self_attn.k_proj,3,64,8,128,11.296875,3.25390625,8388608
+75,self_attn.k_proj,3,128,8,128,13.1328125,3.126953125,8388608
+75,self_attn.k_proj,4,32,8,128,4.359375,4.5078125,8388608
+75,self_attn.k_proj,4,64,8,128,5.26953125,4.25390625,8388608
+75,self_attn.k_proj,4,128,8,128,6.12890625,4.126953125,8388608
+75,self_attn.k_proj,8,32,8,128,0.26416015625,8.5078125,8388608
+75,self_attn.k_proj,8,64,8,128,0.31884765625,8.25390625,8388608
+75,self_attn.k_proj,8,128,8,128,0.36962890625,8.126953125,8388608
+75,self_attn.o_proj,2,32,8,128,42.78125,20.0625,67108864
+75,self_attn.o_proj,2,64,8,128,49.90625,18.03125,67108864
+75,self_attn.o_proj,2,128,8,128,56.40625,17.015625,67108864
+75,self_attn.o_proj,3,32,8,128,18.328125,28.0625,67108864
+75,self_attn.o_proj,3,64,8,128,21.421875,26.03125,67108864
+75,self_attn.o_proj,3,128,8,128,24.265625,25.015625,67108864
+75,self_attn.o_proj,4,32,8,128,8.5546875,36.0625,67108864
+75,self_attn.o_proj,4,64,8,128,10.0078125,34.03125,67108864
+75,self_attn.o_proj,4,128,8,128,11.3515625,33.015625,67108864
+75,self_attn.o_proj,8,32,8,128,0.51806640625,68.0625,67108864
+75,self_attn.o_proj,8,64,8,128,0.60498046875,66.03125,67108864
+75,self_attn.o_proj,8,128,8,128,0.68408203125,65.015625,67108864
+75,mlp.gate_proj,2,32,8,128,87.875,70.21875,234881024
+75,mlp.gate_proj,2,64,8,128,102.8125,63.109375,234881024
+75,mlp.gate_proj,2,128,8,128,116.375,59.5546875,234881024
+75,mlp.gate_proj,3,32,8,128,37.625,98.21875,234881024
+75,mlp.gate_proj,3,64,8,128,44.0,91.109375,234881024
+75,mlp.gate_proj,3,128,8,128,49.875,87.5546875,234881024
+75,mlp.gate_proj,4,32,8,128,17.53125,126.21875,234881024
+75,mlp.gate_proj,4,64,8,128,20.53125,119.109375,234881024
+75,mlp.gate_proj,4,128,8,128,23.28125,115.5546875,234881024
+75,mlp.gate_proj,8,32,8,128,1.0673828125,238.21875,234881024
+75,mlp.gate_proj,8,64,8,128,1.25,231.109375,234881024
+75,mlp.gate_proj,8,128,8,128,1.412109375,227.5546875,234881024
+75,mlp.up_proj,2,32,8,128,84.0625,70.21875,234881024
+75,mlp.up_proj,2,64,8,128,98.0,63.109375,234881024
+75,mlp.up_proj,2,128,8,128,110.625,59.5546875,234881024
+75,mlp.up_proj,3,32,8,128,35.96875,98.21875,234881024
+75,mlp.up_proj,3,64,8,128,41.9375,91.109375,234881024
+75,mlp.up_proj,3,128,8,128,47.34375,87.5546875,234881024
+75,mlp.up_proj,4,32,8,128,16.765625,126.21875,234881024
+75,mlp.up_proj,4,64,8,128,19.59375,119.109375,234881024
+75,mlp.up_proj,4,128,8,128,22.09375,115.5546875,234881024
+75,mlp.up_proj,8,32,8,128,1.0205078125,238.21875,234881024
+75,mlp.up_proj,8,64,8,128,1.19140625,231.109375,234881024
+75,mlp.up_proj,8,128,8,128,1.33984375,227.5546875,234881024
+75,mlp.down_proj,2,32,8,128,83.25,70.21875,234881024
+75,mlp.down_proj,2,64,8,128,96.9375,63.109375,234881024
+75,mlp.down_proj,2,128,8,128,109.3125,59.5546875,234881024
+75,mlp.down_proj,3,32,8,128,35.59375,98.21875,234881024
+75,mlp.down_proj,3,64,8,128,41.5,91.109375,234881024
+75,mlp.down_proj,3,128,8,128,46.78125,87.5546875,234881024
+75,mlp.down_proj,4,32,8,128,16.609375,126.21875,234881024
+75,mlp.down_proj,4,64,8,128,19.375,119.109375,234881024
+75,mlp.down_proj,4,128,8,128,21.84375,115.5546875,234881024
+75,mlp.down_proj,8,32,8,128,1.009765625,238.21875,234881024
+75,mlp.down_proj,8,64,8,128,1.1787109375,231.109375,234881024
+75,mlp.down_proj,8,128,8,128,1.326171875,227.5546875,234881024
+76,self_attn.q_proj,2,32,8,128,43.75,20.0625,67108864
+76,self_attn.q_proj,2,64,8,128,51.34375,18.03125,67108864
+76,self_attn.q_proj,2,128,8,128,58.9375,17.015625,67108864
+76,self_attn.q_proj,3,32,8,128,18.71875,28.0625,67108864
+76,self_attn.q_proj,3,64,8,128,21.984375,26.03125,67108864
+76,self_attn.q_proj,3,128,8,128,25.296875,25.015625,67108864
+76,self_attn.q_proj,4,32,8,128,8.734375,36.0625,67108864
+76,self_attn.q_proj,4,64,8,128,10.265625,34.03125,67108864
+76,self_attn.q_proj,4,128,8,128,11.8046875,33.015625,67108864
+76,self_attn.q_proj,8,32,8,128,0.52880859375,68.0625,67108864
+76,self_attn.q_proj,8,64,8,128,0.62109375,66.03125,67108864
+76,self_attn.q_proj,8,128,8,128,0.7119140625,65.015625,67108864
+76,self_attn.v_proj,2,32,8,128,22.40625,2.5078125,8388608
+76,self_attn.v_proj,2,64,8,128,26.40625,2.25390625,8388608
+76,self_attn.v_proj,2,128,8,128,30.15625,2.126953125,8388608
+76,self_attn.v_proj,3,32,8,128,9.578125,3.5078125,8388608
+76,self_attn.v_proj,3,64,8,128,11.3046875,3.25390625,8388608
+76,self_attn.v_proj,3,128,8,128,12.9140625,3.126953125,8388608
+76,self_attn.v_proj,4,32,8,128,4.46875,4.5078125,8388608
+76,self_attn.v_proj,4,64,8,128,5.2734375,4.25390625,8388608
+76,self_attn.v_proj,4,128,8,128,6.0234375,4.126953125,8388608
+76,self_attn.v_proj,8,32,8,128,0.270751953125,8.5078125,8388608
+76,self_attn.v_proj,8,64,8,128,0.319091796875,8.25390625,8388608
+76,self_attn.v_proj,8,128,8,128,0.363037109375,8.126953125,8388608
+76,self_attn.k_proj,2,32,8,128,22.375,2.5078125,8388608
+76,self_attn.k_proj,2,64,8,128,26.765625,2.25390625,8388608
+76,self_attn.k_proj,2,128,8,128,30.671875,2.126953125,8388608
+76,self_attn.k_proj,3,32,8,128,9.5859375,3.5078125,8388608
+76,self_attn.k_proj,3,64,8,128,11.5078125,3.25390625,8388608
+76,self_attn.k_proj,3,128,8,128,13.265625,3.126953125,8388608
+76,self_attn.k_proj,4,32,8,128,4.47265625,4.5078125,8388608
+76,self_attn.k_proj,4,64,8,128,5.37109375,4.25390625,8388608
+76,self_attn.k_proj,4,128,8,128,6.19140625,4.126953125,8388608
+76,self_attn.k_proj,8,32,8,128,0.270751953125,8.5078125,8388608
+76,self_attn.k_proj,8,64,8,128,0.324951171875,8.25390625,8388608
+76,self_attn.k_proj,8,128,8,128,0.373046875,8.126953125,8388608
+76,self_attn.o_proj,2,32,8,128,46.375,20.0625,67108864
+76,self_attn.o_proj,2,64,8,128,54.03125,18.03125,67108864
+76,self_attn.o_proj,2,128,8,128,60.96875,17.015625,67108864
+76,self_attn.o_proj,3,32,8,128,19.828125,28.0625,67108864
+76,self_attn.o_proj,3,64,8,128,23.140625,26.03125,67108864
+76,self_attn.o_proj,3,128,8,128,26.140625,25.015625,67108864
+76,self_attn.o_proj,4,32,8,128,9.25,36.0625,67108864
+76,self_attn.o_proj,4,64,8,128,10.796875,34.03125,67108864
+76,self_attn.o_proj,4,128,8,128,12.2109375,33.015625,67108864
+76,self_attn.o_proj,8,32,8,128,0.560546875,68.0625,67108864
+76,self_attn.o_proj,8,64,8,128,0.65283203125,66.03125,67108864
+76,self_attn.o_proj,8,128,8,128,0.7353515625,65.015625,67108864
+76,mlp.gate_proj,2,32,8,128,88.5,70.21875,234881024
+76,mlp.gate_proj,2,64,8,128,103.5625,63.109375,234881024
+76,mlp.gate_proj,2,128,8,128,117.375,59.5546875,234881024
+76,mlp.gate_proj,3,32,8,128,37.875,98.21875,234881024
+76,mlp.gate_proj,3,64,8,128,44.3125,91.109375,234881024
+76,mlp.gate_proj,3,128,8,128,50.3125,87.5546875,234881024
+76,mlp.gate_proj,4,32,8,128,17.65625,126.21875,234881024
+76,mlp.gate_proj,4,64,8,128,20.6875,119.109375,234881024
+76,mlp.gate_proj,4,128,8,128,23.484375,115.5546875,234881024
+76,mlp.gate_proj,8,32,8,128,1.07421875,238.21875,234881024
+76,mlp.gate_proj,8,64,8,128,1.2587890625,231.109375,234881024
+76,mlp.gate_proj,8,128,8,128,1.423828125,227.5546875,234881024
+76,mlp.up_proj,2,32,8,128,84.25,70.21875,234881024
+76,mlp.up_proj,2,64,8,128,98.25,63.109375,234881024
+76,mlp.up_proj,2,128,8,128,111.0,59.5546875,234881024
+76,mlp.up_proj,3,32,8,128,36.03125,98.21875,234881024
+76,mlp.up_proj,3,64,8,128,42.0625,91.109375,234881024
+76,mlp.up_proj,3,128,8,128,47.53125,87.5546875,234881024
+76,mlp.up_proj,4,32,8,128,16.796875,126.21875,234881024
+76,mlp.up_proj,4,64,8,128,19.640625,119.109375,234881024
+76,mlp.up_proj,4,128,8,128,22.203125,115.5546875,234881024
+76,mlp.up_proj,8,32,8,128,1.0224609375,238.21875,234881024
+76,mlp.up_proj,8,64,8,128,1.1943359375,231.109375,234881024
+76,mlp.up_proj,8,128,8,128,1.345703125,227.5546875,234881024
+76,mlp.down_proj,2,32,8,128,83.125,70.21875,234881024
+76,mlp.down_proj,2,64,8,128,96.8125,63.109375,234881024
+76,mlp.down_proj,2,128,8,128,109.3125,59.5546875,234881024
+76,mlp.down_proj,3,32,8,128,35.5625,98.21875,234881024
+76,mlp.down_proj,3,64,8,128,41.46875,91.109375,234881024
+76,mlp.down_proj,3,128,8,128,46.84375,87.5546875,234881024
+76,mlp.down_proj,4,32,8,128,16.578125,126.21875,234881024
+76,mlp.down_proj,4,64,8,128,19.375,119.109375,234881024
+76,mlp.down_proj,4,128,8,128,21.859375,115.5546875,234881024
+76,mlp.down_proj,8,32,8,128,1.0087890625,238.21875,234881024
+76,mlp.down_proj,8,64,8,128,1.1787109375,231.109375,234881024
+76,mlp.down_proj,8,128,8,128,1.3271484375,227.5546875,234881024
+77,self_attn.q_proj,2,32,8,128,41.84375,20.0625,67108864
+77,self_attn.q_proj,2,64,8,128,49.3125,18.03125,67108864
+77,self_attn.q_proj,2,128,8,128,56.8125,17.015625,67108864
+77,self_attn.q_proj,3,32,8,128,17.921875,28.0625,67108864
+77,self_attn.q_proj,3,64,8,128,21.140625,26.03125,67108864
+77,self_attn.q_proj,3,128,8,128,24.4375,25.015625,67108864
+77,self_attn.q_proj,4,32,8,128,8.359375,36.0625,67108864
+77,self_attn.q_proj,4,64,8,128,9.875,34.03125,67108864
+77,self_attn.q_proj,4,128,8,128,11.40625,33.015625,67108864
+77,self_attn.q_proj,8,32,8,128,0.50634765625,68.0625,67108864
+77,self_attn.q_proj,8,64,8,128,0.59716796875,66.03125,67108864
+77,self_attn.q_proj,8,128,8,128,0.6875,65.015625,67108864
+77,self_attn.v_proj,2,32,8,128,21.484375,2.5078125,8388608
+77,self_attn.v_proj,2,64,8,128,25.28125,2.25390625,8388608
+77,self_attn.v_proj,2,128,8,128,28.765625,2.126953125,8388608
+77,self_attn.v_proj,3,32,8,128,9.1875,3.5078125,8388608
+77,self_attn.v_proj,3,64,8,128,10.8203125,3.25390625,8388608
+77,self_attn.v_proj,3,128,8,128,12.3125,3.126953125,8388608
+77,self_attn.v_proj,4,32,8,128,4.28515625,4.5078125,8388608
+77,self_attn.v_proj,4,64,8,128,5.046875,4.25390625,8388608
+77,self_attn.v_proj,4,128,8,128,5.74609375,4.126953125,8388608
+77,self_attn.v_proj,8,32,8,128,0.259521484375,8.5078125,8388608
+77,self_attn.v_proj,8,64,8,128,0.305419921875,8.25390625,8388608
+77,self_attn.v_proj,8,128,8,128,0.346435546875,8.126953125,8388608
+77,self_attn.k_proj,2,32,8,128,22.234375,2.5078125,8388608
+77,self_attn.k_proj,2,64,8,128,26.75,2.25390625,8388608
+77,self_attn.k_proj,2,128,8,128,30.828125,2.126953125,8388608
+77,self_attn.k_proj,3,32,8,128,9.546875,3.5078125,8388608
+77,self_attn.k_proj,3,64,8,128,11.5546875,3.25390625,8388608
+77,self_attn.k_proj,3,128,8,128,13.46875,3.126953125,8388608
+77,self_attn.k_proj,4,32,8,128,4.45703125,4.5078125,8388608
+77,self_attn.k_proj,4,64,8,128,5.390625,4.25390625,8388608
+77,self_attn.k_proj,4,128,8,128,6.28125,4.126953125,8388608
+77,self_attn.k_proj,8,32,8,128,0.269775390625,8.5078125,8388608
+77,self_attn.k_proj,8,64,8,128,0.326171875,8.25390625,8388608
+77,self_attn.k_proj,8,128,8,128,0.378662109375,8.126953125,8388608
+77,self_attn.o_proj,2,32,8,128,43.8125,20.0625,67108864
+77,self_attn.o_proj,2,64,8,128,51.09375,18.03125,67108864
+77,self_attn.o_proj,2,128,8,128,57.75,17.015625,67108864
+77,self_attn.o_proj,3,32,8,128,18.75,28.0625,67108864
+77,self_attn.o_proj,3,64,8,128,21.90625,26.03125,67108864
+77,self_attn.o_proj,3,128,8,128,24.828125,25.015625,67108864
+77,self_attn.o_proj,4,32,8,128,8.75,36.0625,67108864
+77,self_attn.o_proj,4,64,8,128,10.2265625,34.03125,67108864
+77,self_attn.o_proj,4,128,8,128,11.5859375,33.015625,67108864
+77,self_attn.o_proj,8,32,8,128,0.52978515625,68.0625,67108864
+77,self_attn.o_proj,8,64,8,128,0.61865234375,66.03125,67108864
+77,self_attn.o_proj,8,128,8,128,0.6982421875,65.015625,67108864
+77,mlp.gate_proj,2,32,8,128,88.8125,70.21875,234881024
+77,mlp.gate_proj,2,64,8,128,103.9375,63.109375,234881024
+77,mlp.gate_proj,2,128,8,128,117.75,59.5546875,234881024
+77,mlp.gate_proj,3,32,8,128,38.03125,98.21875,234881024
+77,mlp.gate_proj,3,64,8,128,44.5,91.109375,234881024
+77,mlp.gate_proj,3,128,8,128,50.46875,87.5546875,234881024
+77,mlp.gate_proj,4,32,8,128,17.734375,126.21875,234881024
+77,mlp.gate_proj,4,64,8,128,20.765625,119.109375,234881024
+77,mlp.gate_proj,4,128,8,128,23.578125,115.5546875,234881024
+77,mlp.gate_proj,8,32,8,128,1.0791015625,238.21875,234881024
+77,mlp.gate_proj,8,64,8,128,1.263671875,231.109375,234881024
+77,mlp.gate_proj,8,128,8,128,1.4296875,227.5546875,234881024
+77,mlp.up_proj,2,32,8,128,84.75,70.21875,234881024
+77,mlp.up_proj,2,64,8,128,98.8125,63.109375,234881024
+77,mlp.up_proj,2,128,8,128,111.625,59.5546875,234881024
+77,mlp.up_proj,3,32,8,128,36.25,98.21875,234881024
+77,mlp.up_proj,3,64,8,128,42.28125,91.109375,234881024
+77,mlp.up_proj,3,128,8,128,47.8125,87.5546875,234881024
+77,mlp.up_proj,4,32,8,128,16.90625,126.21875,234881024
+77,mlp.up_proj,4,64,8,128,19.75,119.109375,234881024
+77,mlp.up_proj,4,128,8,128,22.328125,115.5546875,234881024
+77,mlp.up_proj,8,32,8,128,1.0283203125,238.21875,234881024
+77,mlp.up_proj,8,64,8,128,1.2021484375,231.109375,234881024
+77,mlp.up_proj,8,128,8,128,1.353515625,227.5546875,234881024
+77,mlp.down_proj,2,32,8,128,83.625,70.21875,234881024
+77,mlp.down_proj,2,64,8,128,97.5,63.109375,234881024
+77,mlp.down_proj,2,128,8,128,110.3125,59.5546875,234881024
+77,mlp.down_proj,3,32,8,128,35.78125,98.21875,234881024
+77,mlp.down_proj,3,64,8,128,41.8125,91.109375,234881024
+77,mlp.down_proj,3,128,8,128,47.375,87.5546875,234881024
+77,mlp.down_proj,4,32,8,128,16.703125,126.21875,234881024
+77,mlp.down_proj,4,64,8,128,19.546875,119.109375,234881024
+77,mlp.down_proj,4,128,8,128,22.15625,115.5546875,234881024
+77,mlp.down_proj,8,32,8,128,1.0166015625,238.21875,234881024
+77,mlp.down_proj,8,64,8,128,1.189453125,231.109375,234881024
+77,mlp.down_proj,8,128,8,128,1.3447265625,227.5546875,234881024
+78,self_attn.q_proj,2,32,8,128,40.78125,20.0625,67108864
+78,self_attn.q_proj,2,64,8,128,48.09375,18.03125,67108864
+78,self_attn.q_proj,2,128,8,128,55.0625,17.015625,67108864
+78,self_attn.q_proj,3,32,8,128,17.46875,28.0625,67108864
+78,self_attn.q_proj,3,64,8,128,20.640625,26.03125,67108864
+78,self_attn.q_proj,3,128,8,128,23.734375,25.015625,67108864
+78,self_attn.q_proj,4,32,8,128,8.1484375,36.0625,67108864
+78,self_attn.q_proj,4,64,8,128,9.6328125,34.03125,67108864
+78,self_attn.q_proj,4,128,8,128,11.0859375,33.015625,67108864
+78,self_attn.q_proj,8,32,8,128,0.493408203125,68.0625,67108864
+78,self_attn.q_proj,8,64,8,128,0.5830078125,66.03125,67108864
+78,self_attn.q_proj,8,128,8,128,0.66796875,65.015625,67108864
+78,self_attn.v_proj,2,32,8,128,19.75,2.5078125,8388608
+78,self_attn.v_proj,2,64,8,128,23.25,2.25390625,8388608
+78,self_attn.v_proj,2,128,8,128,26.515625,2.126953125,8388608
+78,self_attn.v_proj,3,32,8,128,8.4453125,3.5078125,8388608
+78,self_attn.v_proj,3,64,8,128,9.953125,3.25390625,8388608
+78,self_attn.v_proj,3,128,8,128,11.359375,3.126953125,8388608
+78,self_attn.v_proj,4,32,8,128,3.939453125,4.5078125,8388608
+78,self_attn.v_proj,4,64,8,128,4.64453125,4.25390625,8388608
+78,self_attn.v_proj,4,128,8,128,5.30078125,4.126953125,8388608
+78,self_attn.v_proj,8,32,8,128,0.238525390625,8.5078125,8388608
+78,self_attn.v_proj,8,64,8,128,0.281005859375,8.25390625,8388608
+78,self_attn.v_proj,8,128,8,128,0.319580078125,8.126953125,8388608
+78,self_attn.k_proj,2,32,8,128,22.203125,2.5078125,8388608
+78,self_attn.k_proj,2,64,8,128,26.671875,2.25390625,8388608
+78,self_attn.k_proj,2,128,8,128,30.546875,2.126953125,8388608
+78,self_attn.k_proj,3,32,8,128,9.53125,3.5078125,8388608
+78,self_attn.k_proj,3,64,8,128,11.515625,3.25390625,8388608
+78,self_attn.k_proj,3,128,8,128,13.3359375,3.126953125,8388608
+78,self_attn.k_proj,4,32,8,128,4.4453125,4.5078125,8388608
+78,self_attn.k_proj,4,64,8,128,5.37109375,4.25390625,8388608
+78,self_attn.k_proj,4,128,8,128,6.23046875,4.126953125,8388608
+78,self_attn.k_proj,8,32,8,128,0.269287109375,8.5078125,8388608
+78,self_attn.k_proj,8,64,8,128,0.324951171875,8.25390625,8388608
+78,self_attn.k_proj,8,128,8,128,0.375244140625,8.126953125,8388608
+78,self_attn.o_proj,2,32,8,128,41.0625,20.0625,67108864
+78,self_attn.o_proj,2,64,8,128,47.875,18.03125,67108864
+78,self_attn.o_proj,2,128,8,128,54.125,17.015625,67108864
+78,self_attn.o_proj,3,32,8,128,17.59375,28.0625,67108864
+78,self_attn.o_proj,3,64,8,128,20.5625,26.03125,67108864
+78,self_attn.o_proj,3,128,8,128,23.3125,25.015625,67108864
+78,self_attn.o_proj,4,32,8,128,8.21875,36.0625,67108864
+78,self_attn.o_proj,4,64,8,128,9.625,34.03125,67108864
+78,self_attn.o_proj,4,128,8,128,10.9375,33.015625,67108864
+78,self_attn.o_proj,8,32,8,128,0.4990234375,68.0625,67108864
+78,self_attn.o_proj,8,64,8,128,0.5849609375,66.03125,67108864
+78,self_attn.o_proj,8,128,8,128,0.6640625,65.015625,67108864
+78,mlp.gate_proj,2,32,8,128,90.125,70.21875,234881024
+78,mlp.gate_proj,2,64,8,128,105.6875,63.109375,234881024
+78,mlp.gate_proj,2,128,8,128,119.875,59.5546875,234881024
+78,mlp.gate_proj,3,32,8,128,38.59375,98.21875,234881024
+78,mlp.gate_proj,3,64,8,128,45.21875,91.109375,234881024
+78,mlp.gate_proj,3,128,8,128,51.5,87.5546875,234881024
+78,mlp.gate_proj,4,32,8,128,18.0,126.21875,234881024
+78,mlp.gate_proj,4,64,8,128,21.109375,119.109375,234881024
+78,mlp.gate_proj,4,128,8,128,24.03125,115.5546875,234881024
+78,mlp.gate_proj,8,32,8,128,1.0947265625,238.21875,234881024
+78,mlp.gate_proj,8,64,8,128,1.28515625,231.109375,234881024
+78,mlp.gate_proj,8,128,8,128,1.45703125,227.5546875,234881024
+78,mlp.up_proj,2,32,8,128,85.5,70.21875,234881024
+78,mlp.up_proj,2,64,8,128,99.75,63.109375,234881024
+78,mlp.up_proj,2,128,8,128,112.625,59.5546875,234881024
+78,mlp.up_proj,3,32,8,128,36.59375,98.21875,234881024
+78,mlp.up_proj,3,64,8,128,42.71875,91.109375,234881024
+78,mlp.up_proj,3,128,8,128,48.28125,87.5546875,234881024
+78,mlp.up_proj,4,32,8,128,17.0625,126.21875,234881024
+78,mlp.up_proj,4,64,8,128,19.9375,119.109375,234881024
+78,mlp.up_proj,4,128,8,128,22.53125,115.5546875,234881024
+78,mlp.up_proj,8,32,8,128,1.0380859375,238.21875,234881024
+78,mlp.up_proj,8,64,8,128,1.2138671875,231.109375,234881024
+78,mlp.up_proj,8,128,8,128,1.3671875,227.5546875,234881024
+78,mlp.down_proj,2,32,8,128,84.125,70.21875,234881024
+78,mlp.down_proj,2,64,8,128,98.375,63.109375,234881024
+78,mlp.down_proj,2,128,8,128,111.5,59.5546875,234881024
+78,mlp.down_proj,3,32,8,128,36.0625,98.21875,234881024
+78,mlp.down_proj,3,64,8,128,42.28125,91.109375,234881024
+78,mlp.down_proj,3,128,8,128,48.15625,87.5546875,234881024
+78,mlp.down_proj,4,32,8,128,16.828125,126.21875,234881024
+78,mlp.down_proj,4,64,8,128,19.765625,119.109375,234881024
+78,mlp.down_proj,4,128,8,128,22.515625,115.5546875,234881024
+78,mlp.down_proj,8,32,8,128,1.0234375,238.21875,234881024
+78,mlp.down_proj,8,64,8,128,1.2021484375,231.109375,234881024
+78,mlp.down_proj,8,128,8,128,1.3671875,227.5546875,234881024
+79,self_attn.q_proj,2,32,8,128,39.71875,20.0625,67108864
+79,self_attn.q_proj,2,64,8,128,46.875,18.03125,67108864
+79,self_attn.q_proj,2,128,8,128,53.8125,17.015625,67108864
+79,self_attn.q_proj,3,32,8,128,17.015625,28.0625,67108864
+79,self_attn.q_proj,3,64,8,128,20.15625,26.03125,67108864
+79,self_attn.q_proj,3,128,8,128,23.296875,25.015625,67108864
+79,self_attn.q_proj,4,32,8,128,7.94140625,36.0625,67108864
+79,self_attn.q_proj,4,64,8,128,9.40625,34.03125,67108864
+79,self_attn.q_proj,4,128,8,128,10.875,33.015625,67108864
+79,self_attn.q_proj,8,32,8,128,0.48095703125,68.0625,67108864
+79,self_attn.q_proj,8,64,8,128,0.5693359375,66.03125,67108864
+79,self_attn.q_proj,8,128,8,128,0.65576171875,65.015625,67108864
+79,self_attn.v_proj,2,32,8,128,16.671875,2.5078125,8388608
+79,self_attn.v_proj,2,64,8,128,19.859375,2.25390625,8388608
+79,self_attn.v_proj,2,128,8,128,22.859375,2.126953125,8388608
+79,self_attn.v_proj,3,32,8,128,7.13671875,3.5078125,8388608
+79,self_attn.v_proj,3,64,8,128,8.515625,3.25390625,8388608
+79,self_attn.v_proj,3,128,8,128,9.828125,3.126953125,8388608
+79,self_attn.v_proj,4,32,8,128,3.330078125,4.5078125,8388608
+79,self_attn.v_proj,4,64,8,128,3.974609375,4.25390625,8388608
+79,self_attn.v_proj,4,128,8,128,4.5859375,4.126953125,8388608
+79,self_attn.v_proj,8,32,8,128,0.20166015625,8.5078125,8388608
+79,self_attn.v_proj,8,64,8,128,0.240478515625,8.25390625,8388608
+79,self_attn.v_proj,8,128,8,128,0.276611328125,8.126953125,8388608
+79,self_attn.k_proj,2,32,8,128,22.890625,2.5078125,8388608
+79,self_attn.k_proj,2,64,8,128,27.640625,2.25390625,8388608
+79,self_attn.k_proj,2,128,8,128,31.828125,2.126953125,8388608
+79,self_attn.k_proj,3,32,8,128,9.828125,3.5078125,8388608
+79,self_attn.k_proj,3,64,8,128,11.9609375,3.25390625,8388608
+79,self_attn.k_proj,3,128,8,128,13.984375,3.126953125,8388608
+79,self_attn.k_proj,4,32,8,128,4.5859375,4.5078125,8388608
+79,self_attn.k_proj,4,64,8,128,5.58203125,4.25390625,8388608
+79,self_attn.k_proj,4,128,8,128,6.52734375,4.126953125,8388608
+79,self_attn.k_proj,8,32,8,128,0.277587890625,8.5078125,8388608
+79,self_attn.k_proj,8,64,8,128,0.337646484375,8.25390625,8388608
+79,self_attn.k_proj,8,128,8,128,0.393310546875,8.126953125,8388608
+79,self_attn.o_proj,2,32,8,128,35.375,20.0625,67108864
+79,self_attn.o_proj,2,64,8,128,41.625,18.03125,67108864
+79,self_attn.o_proj,2,128,8,128,47.6875,17.015625,67108864
+79,self_attn.o_proj,3,32,8,128,15.2109375,28.0625,67108864
+79,self_attn.o_proj,3,64,8,128,18.03125,26.03125,67108864
+79,self_attn.o_proj,3,128,8,128,20.890625,25.015625,67108864
+79,self_attn.o_proj,4,32,8,128,7.1015625,36.0625,67108864
+79,self_attn.o_proj,4,64,8,128,8.421875,34.03125,67108864
+79,self_attn.o_proj,4,128,8,128,9.78125,33.015625,67108864
+79,self_attn.o_proj,8,32,8,128,0.4296875,68.0625,67108864
+79,self_attn.o_proj,8,64,8,128,0.509765625,66.03125,67108864
+79,self_attn.o_proj,8,128,8,128,0.59033203125,65.015625,67108864
+79,mlp.gate_proj,2,32,8,128,92.375,70.21875,234881024
+79,mlp.gate_proj,2,64,8,128,108.5,63.109375,234881024
+79,mlp.gate_proj,2,128,8,128,123.375,59.5546875,234881024
+79,mlp.gate_proj,3,32,8,128,39.5625,98.21875,234881024
+79,mlp.gate_proj,3,64,8,128,46.46875,91.109375,234881024
+79,mlp.gate_proj,3,128,8,128,53.0,87.5546875,234881024
+79,mlp.gate_proj,4,32,8,128,18.453125,126.21875,234881024
+79,mlp.gate_proj,4,64,8,128,21.671875,119.109375,234881024
+79,mlp.gate_proj,4,128,8,128,24.734375,115.5546875,234881024
+79,mlp.gate_proj,8,32,8,128,1.1220703125,238.21875,234881024
+79,mlp.gate_proj,8,64,8,128,1.3193359375,231.109375,234881024
+79,mlp.gate_proj,8,128,8,128,1.5,227.5546875,234881024
+79,mlp.up_proj,2,32,8,128,88.1875,70.21875,234881024
+79,mlp.up_proj,2,64,8,128,103.25,63.109375,234881024
+79,mlp.up_proj,2,128,8,128,116.875,59.5546875,234881024
+79,mlp.up_proj,3,32,8,128,37.78125,98.21875,234881024
+79,mlp.up_proj,3,64,8,128,44.25,91.109375,234881024
+79,mlp.up_proj,3,128,8,128,50.21875,87.5546875,234881024
+79,mlp.up_proj,4,32,8,128,17.609375,126.21875,234881024
+79,mlp.up_proj,4,64,8,128,20.65625,119.109375,234881024
+79,mlp.up_proj,4,128,8,128,23.453125,115.5546875,234881024
+79,mlp.up_proj,8,32,8,128,1.0712890625,238.21875,234881024
+79,mlp.up_proj,8,64,8,128,1.2568359375,231.109375,234881024
+79,mlp.up_proj,8,128,8,128,1.421875,227.5546875,234881024
+79,mlp.down_proj,2,32,8,128,84.375,70.21875,234881024
+79,mlp.down_proj,2,64,8,128,98.875,63.109375,234881024
+79,mlp.down_proj,2,128,8,128,112.4375,59.5546875,234881024
+79,mlp.down_proj,3,32,8,128,36.28125,98.21875,234881024
+79,mlp.down_proj,3,64,8,128,42.71875,91.109375,234881024
+79,mlp.down_proj,3,128,8,128,49.0,87.5546875,234881024
+79,mlp.down_proj,4,32,8,128,16.96875,126.21875,234881024
+79,mlp.down_proj,4,64,8,128,20.046875,119.109375,234881024
+79,mlp.down_proj,4,128,8,128,23.0625,115.5546875,234881024
+79,mlp.down_proj,8,32,8,128,1.033203125,238.21875,234881024
+79,mlp.down_proj,8,64,8,128,1.22265625,231.109375,234881024
+79,mlp.down_proj,8,128,8,128,1.404296875,227.5546875,234881024
diff --git a/lm-quant-toolkit/src/data/fnorm-Llama-3-70B.csv b/lm-quant-toolkit/src/data/fnorm-Llama-3-70B.csv
new file mode 100644
index 0000000000000000000000000000000000000000..8f0de488b9765b2f1532050ae52c51f6a7c0d7f3
--- /dev/null
+++ b/lm-quant-toolkit/src/data/fnorm-Llama-3-70B.csv
@@ -0,0 +1,6721 @@
+layer,module,nbit1,gsize1,nbit2,gsize2,fnorm,memmb,params
+0,self_attn.q_proj,2,32,8,128,412.3119201660156,20.0625,67108864
+0,self_attn.q_proj,2,64,8,128,494.06134033203125,18.03125,67108864
+0,self_attn.q_proj,2,128,8,128,645.670654296875,17.015625,67108864
+0,self_attn.q_proj,3,32,8,128,175.69534301757812,28.0625,67108864
+0,self_attn.q_proj,3,64,8,128,217.89419555664062,26.03125,67108864
+0,self_attn.q_proj,3,128,8,128,269.4341735839844,25.015625,67108864
+0,self_attn.q_proj,4,32,8,128,83.59564208984375,36.0625,67108864
+0,self_attn.q_proj,4,64,8,128,105.31353759765625,34.03125,67108864
+0,self_attn.q_proj,4,128,8,128,130.7145233154297,33.015625,67108864
+0,self_attn.q_proj,8,32,8,128,5.112150192260742,68.0625,67108864
+0,self_attn.q_proj,8,64,8,128,6.596481800079346,66.03125,67108864
+0,self_attn.q_proj,8,128,8,128,8.151016235351562,65.015625,67108864
+0,self_attn.v_proj,2,32,8,128,552.514892578125,2.5078125,8388608
+0,self_attn.v_proj,2,64,8,128,673.5861206054688,2.25390625,8388608
+0,self_attn.v_proj,2,128,8,128,813.2427368164062,2.126953125,8388608
+0,self_attn.v_proj,3,32,8,128,233.45310974121094,3.5078125,8388608
+0,self_attn.v_proj,3,64,8,128,283.3836669921875,3.25390625,8388608
+0,self_attn.v_proj,3,128,8,128,330.1102600097656,3.126953125,8388608
+0,self_attn.v_proj,4,32,8,128,109.21837615966797,4.5078125,8388608
+0,self_attn.v_proj,4,64,8,128,135.6373291015625,4.25390625,8388608
+0,self_attn.v_proj,4,128,8,128,154.72120666503906,4.126953125,8388608
+0,self_attn.v_proj,8,32,8,128,6.378791809082031,8.5078125,8388608
+0,self_attn.v_proj,8,64,8,128,7.946229934692383,8.25390625,8388608
+0,self_attn.v_proj,8,128,8,128,9.187578201293945,8.126953125,8388608
+0,self_attn.k_proj,2,32,8,128,301.5050964355469,2.5078125,8388608
+0,self_attn.k_proj,2,64,8,128,371.7862548828125,2.25390625,8388608
+0,self_attn.k_proj,2,128,8,128,457.86767578125,2.126953125,8388608
+0,self_attn.k_proj,3,32,8,128,111.52918243408203,3.5078125,8388608
+0,self_attn.k_proj,3,64,8,128,152.079833984375,3.25390625,8388608
+0,self_attn.k_proj,3,128,8,128,176.66111755371094,3.126953125,8388608
+0,self_attn.k_proj,4,32,8,128,52.89655303955078,4.5078125,8388608
+0,self_attn.k_proj,4,64,8,128,71.72728729248047,4.25390625,8388608
+0,self_attn.k_proj,4,128,8,128,86.68524932861328,4.126953125,8388608
+0,self_attn.k_proj,8,32,8,128,3.164714813232422,8.5078125,8388608
+0,self_attn.k_proj,8,64,8,128,4.091573238372803,8.25390625,8388608
+0,self_attn.k_proj,8,128,8,128,4.915741920471191,8.126953125,8388608
+0,self_attn.o_proj,2,32,8,128,20.486906051635742,20.0625,67108864
+0,self_attn.o_proj,2,64,8,128,24.122081756591797,18.03125,67108864
+0,self_attn.o_proj,2,128,8,128,27.524822235107422,17.015625,67108864
+0,self_attn.o_proj,3,32,8,128,8.80359935760498,28.0625,67108864
+0,self_attn.o_proj,3,64,8,128,10.44011402130127,26.03125,67108864
+0,self_attn.o_proj,3,128,8,128,12.033958435058594,25.015625,67108864
+0,self_attn.o_proj,4,32,8,128,4.115697383880615,36.0625,67108864
+0,self_attn.o_proj,4,64,8,128,4.8837890625,34.03125,67108864
+0,self_attn.o_proj,4,128,8,128,5.65684175491333,33.015625,67108864
+0,self_attn.o_proj,8,32,8,128,0.2500344514846802,68.0625,67108864
+0,self_attn.o_proj,8,64,8,128,0.2967023551464081,66.03125,67108864
+0,self_attn.o_proj,8,128,8,128,0.34286898374557495,65.015625,67108864
+0,mlp.gate_proj,2,32,8,128,528.693359375,70.21875,234881024
+0,mlp.gate_proj,2,64,8,128,662.457763671875,63.109375,234881024
+0,mlp.gate_proj,2,128,8,128,836.6117553710938,59.5546875,234881024
+0,mlp.gate_proj,3,32,8,128,231.4826202392578,98.21875,234881024
+0,mlp.gate_proj,3,64,8,128,293.496826171875,91.109375,234881024
+0,mlp.gate_proj,3,128,8,128,364.10333251953125,87.5546875,234881024
+0,mlp.gate_proj,4,32,8,128,107.9271011352539,126.21875,234881024
+0,mlp.gate_proj,4,64,8,128,141.55003356933594,119.109375,234881024
+0,mlp.gate_proj,4,128,8,128,178.57774353027344,115.5546875,234881024
+0,mlp.gate_proj,8,32,8,128,6.366543292999268,238.21875,234881024
+0,mlp.gate_proj,8,64,8,128,8.489840507507324,231.109375,234881024
+0,mlp.gate_proj,8,128,8,128,10.665663719177246,227.5546875,234881024
+0,mlp.up_proj,2,32,8,128,540.5574340820312,70.21875,234881024
+0,mlp.up_proj,2,64,8,128,693.4840087890625,63.109375,234881024
+0,mlp.up_proj,2,128,8,128,855.2263793945312,59.5546875,234881024
+0,mlp.up_proj,3,32,8,128,232.68177795410156,98.21875,234881024
+0,mlp.up_proj,3,64,8,128,298.15606689453125,91.109375,234881024
+0,mlp.up_proj,3,128,8,128,368.7686462402344,87.5546875,234881024
+0,mlp.up_proj,4,32,8,128,107.39473724365234,126.21875,234881024
+0,mlp.up_proj,4,64,8,128,140.44207763671875,119.109375,234881024
+0,mlp.up_proj,4,128,8,128,174.3599853515625,115.5546875,234881024
+0,mlp.up_proj,8,32,8,128,6.347901821136475,238.21875,234881024
+0,mlp.up_proj,8,64,8,128,8.305356979370117,231.109375,234881024
+0,mlp.up_proj,8,128,8,128,10.445475578308105,227.5546875,234881024
+0,mlp.down_proj,2,32,8,128,50.11188507080078,70.21875,234881024
+0,mlp.down_proj,2,64,8,128,59.521156311035156,63.109375,234881024
+0,mlp.down_proj,2,128,8,128,68.16854095458984,59.5546875,234881024
+0,mlp.down_proj,3,32,8,128,21.645414352416992,98.21875,234881024
+0,mlp.down_proj,3,64,8,128,25.623926162719727,91.109375,234881024
+0,mlp.down_proj,3,128,8,128,29.846948623657227,87.5546875,234881024
+0,mlp.down_proj,4,32,8,128,10.076683044433594,126.21875,234881024
+0,mlp.down_proj,4,64,8,128,12.003569602966309,119.109375,234881024
+0,mlp.down_proj,4,128,8,128,13.943437576293945,115.5546875,234881024
+0,mlp.down_proj,8,32,8,128,0.6148910522460938,238.21875,234881024
+0,mlp.down_proj,8,64,8,128,0.7330105304718018,231.109375,234881024
+0,mlp.down_proj,8,128,8,128,0.8453769087791443,227.5546875,234881024
+1,self_attn.q_proj,2,32,8,128,50.44818115234375,20.0625,67108864
+1,self_attn.q_proj,2,64,8,128,59.71316146850586,18.03125,67108864
+1,self_attn.q_proj,2,128,8,128,68.00524139404297,17.015625,67108864
+1,self_attn.q_proj,3,32,8,128,23.32946014404297,28.0625,67108864
+1,self_attn.q_proj,3,64,8,128,28.296142578125,26.03125,67108864
+1,self_attn.q_proj,3,128,8,128,33.35201644897461,25.015625,67108864
+1,self_attn.q_proj,4,32,8,128,11.077507972717285,36.0625,67108864
+1,self_attn.q_proj,4,64,8,128,13.634126663208008,34.03125,67108864
+1,self_attn.q_proj,4,128,8,128,16.273012161254883,33.015625,67108864
+1,self_attn.q_proj,8,32,8,128,0.6772661209106445,68.0625,67108864
+1,self_attn.q_proj,8,64,8,128,0.8383092880249023,66.03125,67108864
+1,self_attn.q_proj,8,128,8,128,1.0027586221694946,65.015625,67108864
+1,self_attn.v_proj,2,32,8,128,55.781917572021484,2.5078125,8388608
+1,self_attn.v_proj,2,64,8,128,66.45654296875,2.25390625,8388608
+1,self_attn.v_proj,2,128,8,128,80.87865447998047,2.126953125,8388608
+1,self_attn.v_proj,3,32,8,128,24.29590606689453,3.5078125,8388608
+1,self_attn.v_proj,3,64,8,128,29.16925811767578,3.25390625,8388608
+1,self_attn.v_proj,3,128,8,128,34.53425216674805,3.126953125,8388608
+1,self_attn.v_proj,4,32,8,128,11.365776062011719,4.5078125,8388608
+1,self_attn.v_proj,4,64,8,128,13.7473783493042,4.25390625,8388608
+1,self_attn.v_proj,4,128,8,128,16.28279685974121,4.126953125,8388608
+1,self_attn.v_proj,8,32,8,128,0.689895510673523,8.5078125,8388608
+1,self_attn.v_proj,8,64,8,128,0.8493680357933044,8.25390625,8388608
+1,self_attn.v_proj,8,128,8,128,0.9994275569915771,8.126953125,8388608
+1,self_attn.k_proj,2,32,8,128,32.102294921875,2.5078125,8388608
+1,self_attn.k_proj,2,64,8,128,39.16419219970703,2.25390625,8388608
+1,self_attn.k_proj,2,128,8,128,44.60996627807617,2.126953125,8388608
+1,self_attn.k_proj,3,32,8,128,14.31464672088623,3.5078125,8388608
+1,self_attn.k_proj,3,64,8,128,17.433761596679688,3.25390625,8388608
+1,self_attn.k_proj,3,128,8,128,20.324657440185547,3.126953125,8388608
+1,self_attn.k_proj,4,32,8,128,6.7171630859375,4.5078125,8388608
+1,self_attn.k_proj,4,64,8,128,8.22674560546875,4.25390625,8388608
+1,self_attn.k_proj,4,128,8,128,9.837124824523926,4.126953125,8388608
+1,self_attn.k_proj,8,32,8,128,0.41478386521339417,8.5078125,8388608
+1,self_attn.k_proj,8,64,8,128,0.5109174251556396,8.25390625,8388608
+1,self_attn.k_proj,8,128,8,128,0.5985282063484192,8.126953125,8388608
+1,self_attn.o_proj,2,32,8,128,30.81624984741211,20.0625,67108864
+1,self_attn.o_proj,2,64,8,128,36.1392822265625,18.03125,67108864
+1,self_attn.o_proj,2,128,8,128,41.13111877441406,17.015625,67108864
+1,self_attn.o_proj,3,32,8,128,13.190587997436523,28.0625,67108864
+1,self_attn.o_proj,3,64,8,128,15.54847526550293,26.03125,67108864
+1,self_attn.o_proj,3,128,8,128,17.76421356201172,25.015625,67108864
+1,self_attn.o_proj,4,32,8,128,6.158023357391357,36.0625,67108864
+1,self_attn.o_proj,4,64,8,128,7.257264137268066,34.03125,67108864
+1,self_attn.o_proj,4,128,8,128,8.30705451965332,33.015625,67108864
+1,self_attn.o_proj,8,32,8,128,0.37453365325927734,68.0625,67108864
+1,self_attn.o_proj,8,64,8,128,0.44094663858413696,66.03125,67108864
+1,self_attn.o_proj,8,128,8,128,0.5031557679176331,65.015625,67108864
+1,mlp.gate_proj,2,32,8,128,66.9371566772461,70.21875,234881024
+1,mlp.gate_proj,2,64,8,128,78.47749328613281,63.109375,234881024
+1,mlp.gate_proj,2,128,8,128,89.2430191040039,59.5546875,234881024
+1,mlp.gate_proj,3,32,8,128,28.776317596435547,98.21875,234881024
+1,mlp.gate_proj,3,64,8,128,34.084781646728516,91.109375,234881024
+1,mlp.gate_proj,3,128,8,128,39.016265869140625,87.5546875,234881024
+1,mlp.gate_proj,4,32,8,128,13.459394454956055,126.21875,234881024
+1,mlp.gate_proj,4,64,8,128,15.959356307983398,119.109375,234881024
+1,mlp.gate_proj,4,128,8,128,18.372594833374023,115.5546875,234881024
+1,mlp.gate_proj,8,32,8,128,0.8213920593261719,238.21875,234881024
+1,mlp.gate_proj,8,64,8,128,0.9727925062179565,231.109375,234881024
+1,mlp.gate_proj,8,128,8,128,1.118391990661621,227.5546875,234881024
+1,mlp.up_proj,2,32,8,128,65.8187255859375,70.21875,234881024
+1,mlp.up_proj,2,64,8,128,76.99362182617188,63.109375,234881024
+1,mlp.up_proj,2,128,8,128,87.26081085205078,59.5546875,234881024
+1,mlp.up_proj,3,32,8,128,28.261022567749023,98.21875,234881024
+1,mlp.up_proj,3,64,8,128,33.30111312866211,91.109375,234881024
+1,mlp.up_proj,3,128,8,128,37.95240020751953,87.5546875,234881024
+1,mlp.up_proj,4,32,8,128,13.194257736206055,126.21875,234881024
+1,mlp.up_proj,4,64,8,128,15.561827659606934,119.109375,234881024
+1,mlp.up_proj,4,128,8,128,17.78076171875,115.5546875,234881024
+1,mlp.up_proj,8,32,8,128,0.8047879934310913,238.21875,234881024
+1,mlp.up_proj,8,64,8,128,0.9480494260787964,231.109375,234881024
+1,mlp.up_proj,8,128,8,128,1.0807156562805176,227.5546875,234881024
+1,mlp.down_proj,2,32,8,128,64.37713623046875,70.21875,234881024
+1,mlp.down_proj,2,64,8,128,74.91549682617188,63.109375,234881024
+1,mlp.down_proj,2,128,8,128,85.05756378173828,59.5546875,234881024
+1,mlp.down_proj,3,32,8,128,27.42173194885254,98.21875,234881024
+1,mlp.down_proj,3,64,8,128,32.25284194946289,91.109375,234881024
+1,mlp.down_proj,3,128,8,128,36.47598648071289,87.5546875,234881024
+1,mlp.down_proj,4,32,8,128,12.78183364868164,126.21875,234881024
+1,mlp.down_proj,4,64,8,128,15.032769203186035,119.109375,234881024
+1,mlp.down_proj,4,128,8,128,17.045616149902344,115.5546875,234881024
+1,mlp.down_proj,8,32,8,128,0.781058132648468,238.21875,234881024
+1,mlp.down_proj,8,64,8,128,0.9162397384643555,231.109375,234881024
+1,mlp.down_proj,8,128,8,128,1.037436842918396,227.5546875,234881024
+2,self_attn.q_proj,2,32,8,128,36.18125534057617,20.0625,67108864
+2,self_attn.q_proj,2,64,8,128,42.776939392089844,18.03125,67108864
+2,self_attn.q_proj,2,128,8,128,49.6442756652832,17.015625,67108864
+2,self_attn.q_proj,3,32,8,128,15.999823570251465,28.0625,67108864
+2,self_attn.q_proj,3,64,8,128,19.161495208740234,26.03125,67108864
+2,self_attn.q_proj,3,128,8,128,22.67094612121582,25.015625,67108864
+2,self_attn.q_proj,4,32,8,128,7.550156116485596,36.0625,67108864
+2,self_attn.q_proj,4,64,8,128,9.08881664276123,34.03125,67108864
+2,self_attn.q_proj,4,128,8,128,10.778512954711914,33.015625,67108864
+2,self_attn.q_proj,8,32,8,128,0.4585033357143402,68.0625,67108864
+2,self_attn.q_proj,8,64,8,128,0.552882194519043,66.03125,67108864
+2,self_attn.q_proj,8,128,8,128,0.6538348197937012,65.015625,67108864
+2,self_attn.v_proj,2,32,8,128,16.826086044311523,2.5078125,8388608
+2,self_attn.v_proj,2,64,8,128,19.873353958129883,2.25390625,8388608
+2,self_attn.v_proj,2,128,8,128,21.750499725341797,2.126953125,8388608
+2,self_attn.v_proj,3,32,8,128,7.644562721252441,3.5078125,8388608
+2,self_attn.v_proj,3,64,8,128,9.364161491394043,3.25390625,8388608
+2,self_attn.v_proj,3,128,8,128,10.468008995056152,3.126953125,8388608
+2,self_attn.v_proj,4,32,8,128,3.6289472579956055,4.5078125,8388608
+2,self_attn.v_proj,4,64,8,128,4.412881851196289,4.25390625,8388608
+2,self_attn.v_proj,4,128,8,128,5.081174373626709,4.126953125,8388608
+2,self_attn.v_proj,8,32,8,128,0.22135600447654724,8.5078125,8388608
+2,self_attn.v_proj,8,64,8,128,0.2701008915901184,8.25390625,8388608
+2,self_attn.v_proj,8,128,8,128,0.30767932534217834,8.126953125,8388608
+2,self_attn.k_proj,2,32,8,128,21.240495681762695,2.5078125,8388608
+2,self_attn.k_proj,2,64,8,128,25.363584518432617,2.25390625,8388608
+2,self_attn.k_proj,2,128,8,128,28.962146759033203,2.126953125,8388608
+2,self_attn.k_proj,3,32,8,128,9.333046913146973,3.5078125,8388608
+2,self_attn.k_proj,3,64,8,128,11.353588104248047,3.25390625,8388608
+2,self_attn.k_proj,3,128,8,128,13.322310447692871,3.126953125,8388608
+2,self_attn.k_proj,4,32,8,128,4.396079063415527,4.5078125,8388608
+2,self_attn.k_proj,4,64,8,128,5.400269985198975,4.25390625,8388608
+2,self_attn.k_proj,4,128,8,128,6.343028545379639,4.126953125,8388608
+2,self_attn.k_proj,8,32,8,128,0.26645657420158386,8.5078125,8388608
+2,self_attn.k_proj,8,64,8,128,0.3277994990348816,8.25390625,8388608
+2,self_attn.k_proj,8,128,8,128,0.3872666656970978,8.126953125,8388608
+2,self_attn.o_proj,2,32,8,128,33.539276123046875,20.0625,67108864
+2,self_attn.o_proj,2,64,8,128,39.40746307373047,18.03125,67108864
+2,self_attn.o_proj,2,128,8,128,44.96276092529297,17.015625,67108864
+2,self_attn.o_proj,3,32,8,128,14.3578519821167,28.0625,67108864
+2,self_attn.o_proj,3,64,8,128,16.931194305419922,26.03125,67108864
+2,self_attn.o_proj,3,128,8,128,19.398406982421875,25.015625,67108864
+2,self_attn.o_proj,4,32,8,128,6.696835041046143,36.0625,67108864
+2,self_attn.o_proj,4,64,8,128,7.9038920402526855,34.03125,67108864
+2,self_attn.o_proj,4,128,8,128,9.051676750183105,33.015625,67108864
+2,self_attn.o_proj,8,32,8,128,0.40709659457206726,68.0625,67108864
+2,self_attn.o_proj,8,64,8,128,0.479561448097229,66.03125,67108864
+2,self_attn.o_proj,8,128,8,128,0.5476164221763611,65.015625,67108864
+2,mlp.gate_proj,2,32,8,128,74.68174743652344,70.21875,234881024
+2,mlp.gate_proj,2,64,8,128,88.08162689208984,63.109375,234881024
+2,mlp.gate_proj,2,128,8,128,99.73556518554688,59.5546875,234881024
+2,mlp.gate_proj,3,32,8,128,32.40260696411133,98.21875,234881024
+2,mlp.gate_proj,3,64,8,128,38.25828170776367,91.109375,234881024
+2,mlp.gate_proj,3,128,8,128,43.98045349121094,87.5546875,234881024
+2,mlp.gate_proj,4,32,8,128,15.109156608581543,126.21875,234881024
+2,mlp.gate_proj,4,64,8,128,17.88209342956543,119.109375,234881024
+2,mlp.gate_proj,4,128,8,128,20.5338077545166,115.5546875,234881024
+2,mlp.gate_proj,8,32,8,128,0.9203056693077087,238.21875,234881024
+2,mlp.gate_proj,8,64,8,128,1.08951735496521,231.109375,234881024
+2,mlp.gate_proj,8,128,8,128,1.2481461763381958,227.5546875,234881024
+2,mlp.up_proj,2,32,8,128,74.16036224365234,70.21875,234881024
+2,mlp.up_proj,2,64,8,128,87.14720916748047,63.109375,234881024
+2,mlp.up_proj,2,128,8,128,98.42633056640625,59.5546875,234881024
+2,mlp.up_proj,3,32,8,128,32.143218994140625,98.21875,234881024
+2,mlp.up_proj,3,64,8,128,37.91081619262695,91.109375,234881024
+2,mlp.up_proj,3,128,8,128,43.5013542175293,87.5546875,234881024
+2,mlp.up_proj,4,32,8,128,15.009377479553223,126.21875,234881024
+2,mlp.up_proj,4,64,8,128,17.72980308532715,119.109375,234881024
+2,mlp.up_proj,4,128,8,128,20.337448120117188,115.5546875,234881024
+2,mlp.up_proj,8,32,8,128,0.9133980870246887,238.21875,234881024
+2,mlp.up_proj,8,64,8,128,1.0799469947814941,231.109375,234881024
+2,mlp.up_proj,8,128,8,128,1.2356910705566406,227.5546875,234881024
+2,mlp.down_proj,2,32,8,128,66.19136047363281,70.21875,234881024
+2,mlp.down_proj,2,64,8,128,77.13362884521484,63.109375,234881024
+2,mlp.down_proj,2,128,8,128,87.41912078857422,59.5546875,234881024
+2,mlp.down_proj,3,32,8,128,28.28021812438965,98.21875,234881024
+2,mlp.down_proj,3,64,8,128,33.092498779296875,91.109375,234881024
+2,mlp.down_proj,3,128,8,128,37.39693069458008,87.5546875,234881024
+2,mlp.down_proj,4,32,8,128,13.173892974853516,126.21875,234881024
+2,mlp.down_proj,4,64,8,128,15.469603538513184,119.109375,234881024
+2,mlp.down_proj,4,128,8,128,17.469661712646484,115.5546875,234881024
+2,mlp.down_proj,8,32,8,128,0.8046118021011353,238.21875,234881024
+2,mlp.down_proj,8,64,8,128,0.9430937767028809,231.109375,234881024
+2,mlp.down_proj,8,128,8,128,1.0628658533096313,227.5546875,234881024
+3,self_attn.q_proj,2,32,8,128,59.74952697753906,20.0625,67108864
+3,self_attn.q_proj,2,64,8,128,69.99409484863281,18.03125,67108864
+3,self_attn.q_proj,2,128,8,128,80.77509307861328,17.015625,67108864
+3,self_attn.q_proj,3,32,8,128,26.031709671020508,28.0625,67108864
+3,self_attn.q_proj,3,64,8,128,31.799169540405273,26.03125,67108864
+3,self_attn.q_proj,3,128,8,128,37.41286087036133,25.015625,67108864
+3,self_attn.q_proj,4,32,8,128,12.278810501098633,36.0625,67108864
+3,self_attn.q_proj,4,64,8,128,14.868700981140137,34.03125,67108864
+3,self_attn.q_proj,4,128,8,128,17.717914581298828,33.015625,67108864
+3,self_attn.q_proj,8,32,8,128,0.7531364560127258,68.0625,67108864
+3,self_attn.q_proj,8,64,8,128,0.9166735410690308,66.03125,67108864
+3,self_attn.q_proj,8,128,8,128,1.0996769666671753,65.015625,67108864
+3,self_attn.v_proj,2,32,8,128,26.890459060668945,2.5078125,8388608
+3,self_attn.v_proj,2,64,8,128,33.49943542480469,2.25390625,8388608
+3,self_attn.v_proj,2,128,8,128,39.12443923950195,2.126953125,8388608
+3,self_attn.v_proj,3,32,8,128,11.768305778503418,3.5078125,8388608
+3,self_attn.v_proj,3,64,8,128,14.810311317443848,3.25390625,8388608
+3,self_attn.v_proj,3,128,8,128,16.931617736816406,3.126953125,8388608
+3,self_attn.v_proj,4,32,8,128,5.343806266784668,4.5078125,8388608
+3,self_attn.v_proj,4,64,8,128,6.639118671417236,4.25390625,8388608
+3,self_attn.v_proj,4,128,8,128,7.810975074768066,4.126953125,8388608
+3,self_attn.v_proj,8,32,8,128,0.3310251533985138,8.5078125,8388608
+3,self_attn.v_proj,8,64,8,128,0.4091964662075043,8.25390625,8388608
+3,self_attn.v_proj,8,128,8,128,0.49247467517852783,8.126953125,8388608
+3,self_attn.k_proj,2,32,8,128,41.04716110229492,2.5078125,8388608
+3,self_attn.k_proj,2,64,8,128,52.908409118652344,2.25390625,8388608
+3,self_attn.k_proj,2,128,8,128,61.78349685668945,2.126953125,8388608
+3,self_attn.k_proj,3,32,8,128,17.374799728393555,3.5078125,8388608
+3,self_attn.k_proj,3,64,8,128,21.949769973754883,3.25390625,8388608
+3,self_attn.k_proj,3,128,8,128,26.153705596923828,3.126953125,8388608
+3,self_attn.k_proj,4,32,8,128,8.270864486694336,4.5078125,8388608
+3,self_attn.k_proj,4,64,8,128,10.586869239807129,4.25390625,8388608
+3,self_attn.k_proj,4,128,8,128,12.516523361206055,4.126953125,8388608
+3,self_attn.k_proj,8,32,8,128,0.5329487919807434,8.5078125,8388608
+3,self_attn.k_proj,8,64,8,128,0.6887435913085938,8.25390625,8388608
+3,self_attn.k_proj,8,128,8,128,0.841251015663147,8.126953125,8388608
+3,self_attn.o_proj,2,32,8,128,32.52632522583008,20.0625,67108864
+3,self_attn.o_proj,2,64,8,128,38.12694549560547,18.03125,67108864
+3,self_attn.o_proj,2,128,8,128,43.39473342895508,17.015625,67108864
+3,self_attn.o_proj,3,32,8,128,13.913260459899902,28.0625,67108864
+3,self_attn.o_proj,3,64,8,128,16.362455368041992,26.03125,67108864
+3,self_attn.o_proj,3,128,8,128,18.669137954711914,25.015625,67108864
+3,self_attn.o_proj,4,32,8,128,6.489813327789307,36.0625,67108864
+3,self_attn.o_proj,4,64,8,128,7.636290550231934,34.03125,67108864
+3,self_attn.o_proj,4,128,8,128,8.71917724609375,33.015625,67108864
+3,self_attn.o_proj,8,32,8,128,0.3946588933467865,68.0625,67108864
+3,self_attn.o_proj,8,64,8,128,0.46359798312187195,66.03125,67108864
+3,self_attn.o_proj,8,128,8,128,0.5274143218994141,65.015625,67108864
+3,mlp.gate_proj,2,32,8,128,81.97745513916016,70.21875,234881024
+3,mlp.gate_proj,2,64,8,128,96.88933563232422,63.109375,234881024
+3,mlp.gate_proj,2,128,8,128,109.73867797851562,59.5546875,234881024
+3,mlp.gate_proj,3,32,8,128,37.173152923583984,98.21875,234881024
+3,mlp.gate_proj,3,64,8,128,44.051551818847656,91.109375,234881024
+3,mlp.gate_proj,3,128,8,128,50.829071044921875,87.5546875,234881024
+3,mlp.gate_proj,4,32,8,128,17.741239547729492,126.21875,234881024
+3,mlp.gate_proj,4,64,8,128,21.477657318115234,119.109375,234881024
+3,mlp.gate_proj,4,128,8,128,25.150115966796875,115.5546875,234881024
+3,mlp.gate_proj,8,32,8,128,1.0917351245880127,238.21875,234881024
+3,mlp.gate_proj,8,64,8,128,1.3305259943008423,231.109375,234881024
+3,mlp.gate_proj,8,128,8,128,1.5777264833450317,227.5546875,234881024
+3,mlp.up_proj,2,32,8,128,80.28726959228516,70.21875,234881024
+3,mlp.up_proj,2,64,8,128,94.3073501586914,63.109375,234881024
+3,mlp.up_proj,2,128,8,128,105.85498809814453,59.5546875,234881024
+3,mlp.up_proj,3,32,8,128,35.896671295166016,98.21875,234881024
+3,mlp.up_proj,3,64,8,128,42.38737487792969,91.109375,234881024
+3,mlp.up_proj,3,128,8,128,48.25473403930664,87.5546875,234881024
+3,mlp.up_proj,4,32,8,128,17.134450912475586,126.21875,234881024
+3,mlp.up_proj,4,64,8,128,20.500944137573242,119.109375,234881024
+3,mlp.up_proj,4,128,8,128,23.506977081298828,115.5546875,234881024
+3,mlp.up_proj,8,32,8,128,1.0457408428192139,238.21875,234881024
+3,mlp.up_proj,8,64,8,128,1.254599928855896,231.109375,234881024
+3,mlp.up_proj,8,128,8,128,1.4484914541244507,227.5546875,234881024
+3,mlp.down_proj,2,32,8,128,66.43341827392578,70.21875,234881024
+3,mlp.down_proj,2,64,8,128,77.40484619140625,63.109375,234881024
+3,mlp.down_proj,2,128,8,128,87.68379974365234,59.5546875,234881024
+3,mlp.down_proj,3,32,8,128,28.39859962463379,98.21875,234881024
+3,mlp.down_proj,3,64,8,128,33.21863555908203,91.109375,234881024
+3,mlp.down_proj,3,128,8,128,37.52996826171875,87.5546875,234881024
+3,mlp.down_proj,4,32,8,128,13.23177433013916,126.21875,234881024
+3,mlp.down_proj,4,64,8,128,15.536585807800293,119.109375,234881024
+3,mlp.down_proj,4,128,8,128,17.540103912353516,115.5546875,234881024
+3,mlp.down_proj,8,32,8,128,0.8086262345314026,238.21875,234881024
+3,mlp.down_proj,8,64,8,128,0.9483247995376587,231.109375,234881024
+3,mlp.down_proj,8,128,8,128,1.0692405700683594,227.5546875,234881024
+4,self_attn.q_proj,2,32,8,128,40.417938232421875,20.0625,67108864
+4,self_attn.q_proj,2,64,8,128,48.398468017578125,18.03125,67108864
+4,self_attn.q_proj,2,128,8,128,56.16194152832031,17.015625,67108864
+4,self_attn.q_proj,3,32,8,128,17.43234634399414,28.0625,67108864
+4,self_attn.q_proj,3,64,8,128,21.09022331237793,26.03125,67108864
+4,self_attn.q_proj,3,128,8,128,24.868576049804688,25.015625,67108864
+4,self_attn.q_proj,4,32,8,128,8.143096923828125,36.0625,67108864
+4,self_attn.q_proj,4,64,8,128,9.838781356811523,34.03125,67108864
+4,self_attn.q_proj,4,128,8,128,11.621025085449219,33.015625,67108864
+4,self_attn.q_proj,8,32,8,128,0.49425601959228516,68.0625,67108864
+4,self_attn.q_proj,8,64,8,128,0.5974262356758118,66.03125,67108864
+4,self_attn.q_proj,8,128,8,128,0.7037109732627869,65.015625,67108864
+4,self_attn.v_proj,2,32,8,128,13.315932273864746,2.5078125,8388608
+4,self_attn.v_proj,2,64,8,128,15.642918586730957,2.25390625,8388608
+4,self_attn.v_proj,2,128,8,128,17.78481101989746,2.126953125,8388608
+4,self_attn.v_proj,3,32,8,128,5.695431232452393,3.5078125,8388608
+4,self_attn.v_proj,3,64,8,128,6.693916320800781,3.25390625,8388608
+4,self_attn.v_proj,3,128,8,128,7.610927581787109,3.126953125,8388608
+4,self_attn.v_proj,4,32,8,128,2.65805721282959,4.5078125,8388608
+4,self_attn.v_proj,4,64,8,128,3.1244704723358154,4.25390625,8388608
+4,self_attn.v_proj,4,128,8,128,3.5546014308929443,4.126953125,8388608
+4,self_attn.v_proj,8,32,8,128,0.16089312732219696,8.5078125,8388608
+4,self_attn.v_proj,8,64,8,128,0.18893873691558838,8.25390625,8388608
+4,self_attn.v_proj,8,128,8,128,0.21413554251194,8.126953125,8388608
+4,self_attn.k_proj,2,32,8,128,24.426607131958008,2.5078125,8388608
+4,self_attn.k_proj,2,64,8,128,29.545495986938477,2.25390625,8388608
+4,self_attn.k_proj,2,128,8,128,34.10772705078125,2.126953125,8388608
+4,self_attn.k_proj,3,32,8,128,10.701102256774902,3.5078125,8388608
+4,self_attn.k_proj,3,64,8,128,13.286202430725098,3.25390625,8388608
+4,self_attn.k_proj,3,128,8,128,15.895528793334961,3.126953125,8388608
+4,self_attn.k_proj,4,32,8,128,4.996914863586426,4.5078125,8388608
+4,self_attn.k_proj,4,64,8,128,6.208497047424316,4.25390625,8388608
+4,self_attn.k_proj,4,128,8,128,7.449280738830566,4.126953125,8388608
+4,self_attn.k_proj,8,32,8,128,0.30231577157974243,8.5078125,8388608
+4,self_attn.k_proj,8,64,8,128,0.37551403045654297,8.25390625,8388608
+4,self_attn.k_proj,8,128,8,128,0.44878920912742615,8.126953125,8388608
+4,self_attn.o_proj,2,32,8,128,33.46279525756836,20.0625,67108864
+4,self_attn.o_proj,2,64,8,128,39.5308952331543,18.03125,67108864
+4,self_attn.o_proj,2,128,8,128,45.300724029541016,17.015625,67108864
+4,self_attn.o_proj,3,32,8,128,14.39267349243164,28.0625,67108864
+4,self_attn.o_proj,3,64,8,128,17.141189575195312,26.03125,67108864
+4,self_attn.o_proj,3,128,8,128,19.904890060424805,25.015625,67108864
+4,self_attn.o_proj,4,32,8,128,6.716916084289551,36.0625,67108864
+4,self_attn.o_proj,4,64,8,128,8.016361236572266,34.03125,67108864
+4,self_attn.o_proj,4,128,8,128,9.308857917785645,33.015625,67108864
+4,self_attn.o_proj,8,32,8,128,0.4082787036895752,68.0625,67108864
+4,self_attn.o_proj,8,64,8,128,0.48643389344215393,66.03125,67108864
+4,self_attn.o_proj,8,128,8,128,0.5634787082672119,65.015625,67108864
+4,mlp.gate_proj,2,32,8,128,74.04476165771484,70.21875,234881024
+4,mlp.gate_proj,2,64,8,128,87.19051361083984,63.109375,234881024
+4,mlp.gate_proj,2,128,8,128,98.40513610839844,59.5546875,234881024
+4,mlp.gate_proj,3,32,8,128,32.50069808959961,98.21875,234881024
+4,mlp.gate_proj,3,64,8,128,38.26327133178711,91.109375,234881024
+4,mlp.gate_proj,3,128,8,128,43.67420959472656,87.5546875,234881024
+4,mlp.gate_proj,4,32,8,128,15.346501350402832,126.21875,234881024
+4,mlp.gate_proj,4,64,8,128,18.17471694946289,119.109375,234881024
+4,mlp.gate_proj,4,128,8,128,20.74964714050293,115.5546875,234881024
+4,mlp.gate_proj,8,32,8,128,0.9440214037895203,238.21875,234881024
+4,mlp.gate_proj,8,64,8,128,1.12307608127594,231.109375,234881024
+4,mlp.gate_proj,8,128,8,128,1.2887871265411377,227.5546875,234881024
+4,mlp.up_proj,2,32,8,128,73.14556884765625,70.21875,234881024
+4,mlp.up_proj,2,64,8,128,85.77298736572266,63.109375,234881024
+4,mlp.up_proj,2,128,8,128,97.05268096923828,59.5546875,234881024
+4,mlp.up_proj,3,32,8,128,32.05057144165039,98.21875,234881024
+4,mlp.up_proj,3,64,8,128,37.6307487487793,91.109375,234881024
+4,mlp.up_proj,3,128,8,128,43.007991790771484,87.5546875,234881024
+4,mlp.up_proj,4,32,8,128,15.122846603393555,126.21875,234881024
+4,mlp.up_proj,4,64,8,128,17.872774124145508,119.109375,234881024
+4,mlp.up_proj,4,128,8,128,20.433687210083008,115.5546875,234881024
+4,mlp.up_proj,8,32,8,128,0.9284446835517883,238.21875,234881024
+4,mlp.up_proj,8,64,8,128,1.100246787071228,231.109375,234881024
+4,mlp.up_proj,8,128,8,128,1.2656745910644531,227.5546875,234881024
+4,mlp.down_proj,2,32,8,128,66.78948974609375,70.21875,234881024
+4,mlp.down_proj,2,64,8,128,77.82302856445312,63.109375,234881024
+4,mlp.down_proj,2,128,8,128,88.12859344482422,59.5546875,234881024
+4,mlp.down_proj,3,32,8,128,28.564115524291992,98.21875,234881024
+4,mlp.down_proj,3,64,8,128,33.37190628051758,91.109375,234881024
+4,mlp.down_proj,3,128,8,128,37.67279815673828,87.5546875,234881024
+4,mlp.down_proj,4,32,8,128,13.303967475891113,126.21875,234881024
+4,mlp.down_proj,4,64,8,128,15.613908767700195,119.109375,234881024
+4,mlp.down_proj,4,128,8,128,17.596881866455078,115.5546875,234881024
+4,mlp.down_proj,8,32,8,128,0.8125475645065308,238.21875,234881024
+4,mlp.down_proj,8,64,8,128,0.9520935416221619,231.109375,234881024
+4,mlp.down_proj,8,128,8,128,1.071336269378662,227.5546875,234881024
+5,self_attn.q_proj,2,32,8,128,43.253013610839844,20.0625,67108864
+5,self_attn.q_proj,2,64,8,128,51.1600456237793,18.03125,67108864
+5,self_attn.q_proj,2,128,8,128,59.10588455200195,17.015625,67108864
+5,self_attn.q_proj,3,32,8,128,18.5592041015625,28.0625,67108864
+5,self_attn.q_proj,3,64,8,128,22.0830135345459,26.03125,67108864
+5,self_attn.q_proj,3,128,8,128,25.662431716918945,25.015625,67108864
+5,self_attn.q_proj,4,32,8,128,8.66601848602295,36.0625,67108864
+5,self_attn.q_proj,4,64,8,128,10.314270973205566,34.03125,67108864
+5,self_attn.q_proj,4,128,8,128,11.998252868652344,33.015625,67108864
+5,self_attn.q_proj,8,32,8,128,0.5263553261756897,68.0625,67108864
+5,self_attn.q_proj,8,64,8,128,0.6262632012367249,66.03125,67108864
+5,self_attn.q_proj,8,128,8,128,0.7271850109100342,65.015625,67108864
+5,self_attn.v_proj,2,32,8,128,14.525792121887207,2.5078125,8388608
+5,self_attn.v_proj,2,64,8,128,17.008825302124023,2.25390625,8388608
+5,self_attn.v_proj,2,128,8,128,19.226329803466797,2.126953125,8388608
+5,self_attn.v_proj,3,32,8,128,6.252191543579102,3.5078125,8388608
+5,self_attn.v_proj,3,64,8,128,7.29576301574707,3.25390625,8388608
+5,self_attn.v_proj,3,128,8,128,8.294096946716309,3.126953125,8388608
+5,self_attn.v_proj,4,32,8,128,2.9260787963867188,4.5078125,8388608
+5,self_attn.v_proj,4,64,8,128,3.440277576446533,4.25390625,8388608
+5,self_attn.v_proj,4,128,8,128,3.900209903717041,4.126953125,8388608
+5,self_attn.v_proj,8,32,8,128,0.17675410211086273,8.5078125,8388608
+5,self_attn.v_proj,8,64,8,128,0.20790831744670868,8.25390625,8388608
+5,self_attn.v_proj,8,128,8,128,0.23370029032230377,8.126953125,8388608
+5,self_attn.k_proj,2,32,8,128,25.716524124145508,2.5078125,8388608
+5,self_attn.k_proj,2,64,8,128,30.584775924682617,2.25390625,8388608
+5,self_attn.k_proj,2,128,8,128,34.740901947021484,2.126953125,8388608
+5,self_attn.k_proj,3,32,8,128,11.230887413024902,3.5078125,8388608
+5,self_attn.k_proj,3,64,8,128,13.518305778503418,3.25390625,8388608
+5,self_attn.k_proj,3,128,8,128,15.672306060791016,3.126953125,8388608
+5,self_attn.k_proj,4,32,8,128,5.242802143096924,4.5078125,8388608
+5,self_attn.k_proj,4,64,8,128,6.39649772644043,4.25390625,8388608
+5,self_attn.k_proj,4,128,8,128,7.41140079498291,4.126953125,8388608
+5,self_attn.k_proj,8,32,8,128,0.31905442476272583,8.5078125,8388608
+5,self_attn.k_proj,8,64,8,128,0.38645926117897034,8.25390625,8388608
+5,self_attn.k_proj,8,128,8,128,0.44737255573272705,8.126953125,8388608
+5,self_attn.o_proj,2,32,8,128,33.18539047241211,20.0625,67108864
+5,self_attn.o_proj,2,64,8,128,38.77861785888672,18.03125,67108864
+5,self_attn.o_proj,2,128,8,128,43.95132064819336,17.015625,67108864
+5,self_attn.o_proj,3,32,8,128,14.185256004333496,28.0625,67108864
+5,self_attn.o_proj,3,64,8,128,16.610177993774414,26.03125,67108864
+5,self_attn.o_proj,3,128,8,128,18.842233657836914,25.015625,67108864
+5,self_attn.o_proj,4,32,8,128,6.616762161254883,36.0625,67108864
+5,self_attn.o_proj,4,64,8,128,7.753121852874756,34.03125,67108864
+5,self_attn.o_proj,4,128,8,128,8.788284301757812,33.015625,67108864
+5,self_attn.o_proj,8,32,8,128,0.40223515033721924,68.0625,67108864
+5,self_attn.o_proj,8,64,8,128,0.47035595774650574,66.03125,67108864
+5,self_attn.o_proj,8,128,8,128,0.53166264295578,65.015625,67108864
+5,mlp.gate_proj,2,32,8,128,159.06866455078125,70.21875,234881024
+5,mlp.gate_proj,2,64,8,128,190.4473114013672,63.109375,234881024
+5,mlp.gate_proj,2,128,8,128,221.09178161621094,59.5546875,234881024
+5,mlp.gate_proj,3,32,8,128,68.55345153808594,98.21875,234881024
+5,mlp.gate_proj,3,64,8,128,83.2413330078125,91.109375,234881024
+5,mlp.gate_proj,3,128,8,128,96.70982360839844,87.5546875,234881024
+5,mlp.gate_proj,4,32,8,128,31.736398696899414,126.21875,234881024
+5,mlp.gate_proj,4,64,8,128,38.69285202026367,119.109375,234881024
+5,mlp.gate_proj,4,128,8,128,45.25963592529297,115.5546875,234881024
+5,mlp.gate_proj,8,32,8,128,1.934840202331543,238.21875,234881024
+5,mlp.gate_proj,8,64,8,128,2.358738660812378,231.109375,234881024
+5,mlp.gate_proj,8,128,8,128,2.7292733192443848,227.5546875,234881024
+5,mlp.up_proj,2,32,8,128,158.83888244628906,70.21875,234881024
+5,mlp.up_proj,2,64,8,128,188.7771759033203,63.109375,234881024
+5,mlp.up_proj,2,128,8,128,216.08700561523438,59.5546875,234881024
+5,mlp.up_proj,3,32,8,128,68.91569519042969,98.21875,234881024
+5,mlp.up_proj,3,64,8,128,83.48359680175781,91.109375,234881024
+5,mlp.up_proj,3,128,8,128,96.03461456298828,87.5546875,234881024
+5,mlp.up_proj,4,32,8,128,31.88117218017578,126.21875,234881024
+5,mlp.up_proj,4,64,8,128,38.80200958251953,119.109375,234881024
+5,mlp.up_proj,4,128,8,128,45.709632873535156,115.5546875,234881024
+5,mlp.up_proj,8,32,8,128,1.936833143234253,238.21875,234881024
+5,mlp.up_proj,8,64,8,128,2.36116886138916,231.109375,234881024
+5,mlp.up_proj,8,128,8,128,2.758185625076294,227.5546875,234881024
+5,mlp.down_proj,2,32,8,128,66.00739288330078,70.21875,234881024
+5,mlp.down_proj,2,64,8,128,76.76351165771484,63.109375,234881024
+5,mlp.down_proj,2,128,8,128,86.87688446044922,59.5546875,234881024
+5,mlp.down_proj,3,32,8,128,28.18328857421875,98.21875,234881024
+5,mlp.down_proj,3,64,8,128,32.92206954956055,91.109375,234881024
+5,mlp.down_proj,3,128,8,128,37.06754684448242,87.5546875,234881024
+5,mlp.down_proj,4,32,8,128,13.128169059753418,126.21875,234881024
+5,mlp.down_proj,4,64,8,128,15.379554748535156,119.109375,234881024
+5,mlp.down_proj,4,128,8,128,17.317325592041016,115.5546875,234881024
+5,mlp.down_proj,8,32,8,128,0.8019521832466125,238.21875,234881024
+5,mlp.down_proj,8,64,8,128,0.937972366809845,231.109375,234881024
+5,mlp.down_proj,8,128,8,128,1.0542681217193604,227.5546875,234881024
+6,self_attn.q_proj,2,32,8,128,45.95602798461914,20.0625,67108864
+6,self_attn.q_proj,2,64,8,128,54.402976989746094,18.03125,67108864
+6,self_attn.q_proj,2,128,8,128,62.79246520996094,17.015625,67108864
+6,self_attn.q_proj,3,32,8,128,19.69997215270996,28.0625,67108864
+6,self_attn.q_proj,3,64,8,128,23.445499420166016,26.03125,67108864
+6,self_attn.q_proj,3,128,8,128,27.22785186767578,25.015625,67108864
+6,self_attn.q_proj,4,32,8,128,9.191225051879883,36.0625,67108864
+6,self_attn.q_proj,4,64,8,128,10.942652702331543,34.03125,67108864
+6,self_attn.q_proj,4,128,8,128,12.708148956298828,33.015625,67108864
+6,self_attn.q_proj,8,32,8,128,0.5581753849983215,68.0625,67108864
+6,self_attn.q_proj,8,64,8,128,0.664516031742096,66.03125,67108864
+6,self_attn.q_proj,8,128,8,128,0.7693008780479431,65.015625,67108864
+6,self_attn.v_proj,2,32,8,128,11.765523910522461,2.5078125,8388608
+6,self_attn.v_proj,2,64,8,128,13.734859466552734,2.25390625,8388608
+6,self_attn.v_proj,2,128,8,128,15.518104553222656,2.126953125,8388608
+6,self_attn.v_proj,3,32,8,128,5.032236576080322,3.5078125,8388608
+6,self_attn.v_proj,3,64,8,128,5.87481164932251,3.25390625,8388608
+6,self_attn.v_proj,3,128,8,128,6.639275550842285,3.126953125,8388608
+6,self_attn.v_proj,4,32,8,128,2.3478448390960693,4.5078125,8388608
+6,self_attn.v_proj,4,64,8,128,2.742417812347412,4.25390625,8388608
+6,self_attn.v_proj,4,128,8,128,3.099691152572632,4.126953125,8388608
+6,self_attn.v_proj,8,32,8,128,0.14212337136268616,8.5078125,8388608
+6,self_attn.v_proj,8,64,8,128,0.1659134179353714,8.25390625,8388608
+6,self_attn.v_proj,8,128,8,128,0.1867447942495346,8.126953125,8388608
+6,self_attn.k_proj,2,32,8,128,27.068862915039062,2.5078125,8388608
+6,self_attn.k_proj,2,64,8,128,32.45933151245117,2.25390625,8388608
+6,self_attn.k_proj,2,128,8,128,36.93017578125,2.126953125,8388608
+6,self_attn.k_proj,3,32,8,128,11.74743938446045,3.5078125,8388608
+6,self_attn.k_proj,3,64,8,128,14.283638954162598,3.25390625,8388608
+6,self_attn.k_proj,3,128,8,128,16.701330184936523,3.126953125,8388608
+6,self_attn.k_proj,4,32,8,128,5.484091281890869,4.5078125,8388608
+6,self_attn.k_proj,4,64,8,128,6.674204349517822,4.25390625,8388608
+6,self_attn.k_proj,4,128,8,128,7.808329105377197,4.126953125,8388608
+6,self_attn.k_proj,8,32,8,128,0.3319609761238098,8.5078125,8388608
+6,self_attn.k_proj,8,64,8,128,0.4035075604915619,8.25390625,8388608
+6,self_attn.k_proj,8,128,8,128,0.4703173041343689,8.126953125,8388608
+6,self_attn.o_proj,2,32,8,128,32.796142578125,20.0625,67108864
+6,self_attn.o_proj,2,64,8,128,38.25432586669922,18.03125,67108864
+6,self_attn.o_proj,2,128,8,128,43.27894592285156,17.015625,67108864
+6,self_attn.o_proj,3,32,8,128,14.018450736999512,28.0625,67108864
+6,self_attn.o_proj,3,64,8,128,16.38478660583496,26.03125,67108864
+6,self_attn.o_proj,3,128,8,128,18.515296936035156,25.015625,67108864
+6,self_attn.o_proj,4,32,8,128,6.538758754730225,36.0625,67108864
+6,self_attn.o_proj,4,64,8,128,7.642812252044678,34.03125,67108864
+6,self_attn.o_proj,4,128,8,128,8.642255783081055,33.015625,67108864
+6,self_attn.o_proj,8,32,8,128,0.39750564098358154,68.0625,67108864
+6,self_attn.o_proj,8,64,8,128,0.4638553261756897,66.03125,67108864
+6,self_attn.o_proj,8,128,8,128,0.5227326154708862,65.015625,67108864
+6,mlp.gate_proj,2,32,8,128,69.22300720214844,70.21875,234881024
+6,mlp.gate_proj,2,64,8,128,81.18342590332031,63.109375,234881024
+6,mlp.gate_proj,2,128,8,128,92.07108306884766,59.5546875,234881024
+6,mlp.gate_proj,3,32,8,128,29.866518020629883,98.21875,234881024
+6,mlp.gate_proj,3,64,8,128,34.960262298583984,91.109375,234881024
+6,mlp.gate_proj,3,128,8,128,39.74114227294922,87.5546875,234881024
+6,mlp.gate_proj,4,32,8,128,13.930682182312012,126.21875,234881024
+6,mlp.gate_proj,4,64,8,128,16.407533645629883,119.109375,234881024
+6,mlp.gate_proj,4,128,8,128,18.623361587524414,115.5546875,234881024
+6,mlp.gate_proj,8,32,8,128,0.8500714302062988,238.21875,234881024
+6,mlp.gate_proj,8,64,8,128,1.0039230585098267,231.109375,234881024
+6,mlp.gate_proj,8,128,8,128,1.1346209049224854,227.5546875,234881024
+6,mlp.up_proj,2,32,8,128,67.37158203125,70.21875,234881024
+6,mlp.up_proj,2,64,8,128,78.69816589355469,63.109375,234881024
+6,mlp.up_proj,2,128,8,128,89.2208023071289,59.5546875,234881024
+6,mlp.up_proj,3,32,8,128,29.01270866394043,98.21875,234881024
+6,mlp.up_proj,3,64,8,128,33.95350646972656,91.109375,234881024
+6,mlp.up_proj,3,128,8,128,38.48289108276367,87.5546875,234881024
+6,mlp.up_proj,4,32,8,128,13.5328369140625,126.21875,234881024
+6,mlp.up_proj,4,64,8,128,15.956012725830078,119.109375,234881024
+6,mlp.up_proj,4,128,8,128,18.04596519470215,115.5546875,234881024
+6,mlp.up_proj,8,32,8,128,0.8260284066200256,238.21875,234881024
+6,mlp.up_proj,8,64,8,128,0.9741737842559814,231.109375,234881024
+6,mlp.up_proj,8,128,8,128,1.0993338823318481,227.5546875,234881024
+6,mlp.down_proj,2,32,8,128,65.82149505615234,70.21875,234881024
+6,mlp.down_proj,2,64,8,128,76.52638244628906,63.109375,234881024
+6,mlp.down_proj,2,128,8,128,86.52789306640625,59.5546875,234881024
+6,mlp.down_proj,3,32,8,128,28.09271812438965,98.21875,234881024
+6,mlp.down_proj,3,64,8,128,32.81765365600586,91.109375,234881024
+6,mlp.down_proj,3,128,8,128,36.92912292480469,87.5546875,234881024
+6,mlp.down_proj,4,32,8,128,13.08680534362793,126.21875,234881024
+6,mlp.down_proj,4,64,8,128,15.325726509094238,119.109375,234881024
+6,mlp.down_proj,4,128,8,128,17.244062423706055,115.5546875,234881024
+6,mlp.down_proj,8,32,8,128,0.7993874549865723,238.21875,234881024
+6,mlp.down_proj,8,64,8,128,0.9342942237854004,231.109375,234881024
+6,mlp.down_proj,8,128,8,128,1.0498114824295044,227.5546875,234881024
+7,self_attn.q_proj,2,32,8,128,43.74844741821289,20.0625,67108864
+7,self_attn.q_proj,2,64,8,128,51.68952941894531,18.03125,67108864
+7,self_attn.q_proj,2,128,8,128,59.64290237426758,17.015625,67108864
+7,self_attn.q_proj,3,32,8,128,18.747230529785156,28.0625,67108864
+7,self_attn.q_proj,3,64,8,128,22.271623611450195,26.03125,67108864
+7,self_attn.q_proj,3,128,8,128,25.84452247619629,25.015625,67108864
+7,self_attn.q_proj,4,32,8,128,8.749650955200195,36.0625,67108864
+7,self_attn.q_proj,4,64,8,128,10.390202522277832,34.03125,67108864
+7,self_attn.q_proj,4,128,8,128,12.06707763671875,33.015625,67108864
+7,self_attn.q_proj,8,32,8,128,0.5313504934310913,68.0625,67108864
+7,self_attn.q_proj,8,64,8,128,0.6309176683425903,66.03125,67108864
+7,self_attn.q_proj,8,128,8,128,0.7307023406028748,65.015625,67108864
+7,self_attn.v_proj,2,32,8,128,12.871381759643555,2.5078125,8388608
+7,self_attn.v_proj,2,64,8,128,15.083501815795898,2.25390625,8388608
+7,self_attn.v_proj,2,128,8,128,17.09899139404297,2.126953125,8388608
+7,self_attn.v_proj,3,32,8,128,5.506316661834717,3.5078125,8388608
+7,self_attn.v_proj,3,64,8,128,6.45461368560791,3.25390625,8388608
+7,self_attn.v_proj,3,128,8,128,7.323793411254883,3.126953125,8388608
+7,self_attn.v_proj,4,32,8,128,2.5688445568084717,4.5078125,8388608
+7,self_attn.v_proj,4,64,8,128,3.012552499771118,4.25390625,8388608
+7,self_attn.v_proj,4,128,8,128,3.417985677719116,4.126953125,8388608
+7,self_attn.v_proj,8,32,8,128,0.155520960688591,8.5078125,8388608
+7,self_attn.v_proj,8,64,8,128,0.18220360577106476,8.25390625,8388608
+7,self_attn.v_proj,8,128,8,128,0.20585329830646515,8.126953125,8388608
+7,self_attn.k_proj,2,32,8,128,25.67827606201172,2.5078125,8388608
+7,self_attn.k_proj,2,64,8,128,30.84294319152832,2.25390625,8388608
+7,self_attn.k_proj,2,128,8,128,35.17082977294922,2.126953125,8388608
+7,self_attn.k_proj,3,32,8,128,11.155869483947754,3.5078125,8388608
+7,self_attn.k_proj,3,64,8,128,13.619274139404297,3.25390625,8388608
+7,self_attn.k_proj,3,128,8,128,15.998790740966797,3.126953125,8388608
+7,self_attn.k_proj,4,32,8,128,5.206180572509766,4.5078125,8388608
+7,self_attn.k_proj,4,64,8,128,6.3588175773620605,4.25390625,8388608
+7,self_attn.k_proj,4,128,8,128,7.475996494293213,4.126953125,8388608
+7,self_attn.k_proj,8,32,8,128,0.3149857223033905,8.5078125,8388608
+7,self_attn.k_proj,8,64,8,128,0.38452520966529846,8.25390625,8388608
+7,self_attn.k_proj,8,128,8,128,0.4503993093967438,8.126953125,8388608
+7,self_attn.o_proj,2,32,8,128,32.7458610534668,20.0625,67108864
+7,self_attn.o_proj,2,64,8,128,38.221736907958984,18.03125,67108864
+7,self_attn.o_proj,2,128,8,128,43.26932144165039,17.015625,67108864
+7,self_attn.o_proj,3,32,8,128,13.996316909790039,28.0625,67108864
+7,self_attn.o_proj,3,64,8,128,16.366836547851562,26.03125,67108864
+7,self_attn.o_proj,3,128,8,128,18.52035140991211,25.015625,67108864
+7,self_attn.o_proj,4,32,8,128,6.528789520263672,36.0625,67108864
+7,self_attn.o_proj,4,64,8,128,7.6375885009765625,34.03125,67108864
+7,self_attn.o_proj,4,128,8,128,8.644270896911621,33.015625,67108864
+7,self_attn.o_proj,8,32,8,128,0.39701780676841736,68.0625,67108864
+7,self_attn.o_proj,8,64,8,128,0.46361806988716125,66.03125,67108864
+7,self_attn.o_proj,8,128,8,128,0.522871196269989,65.015625,67108864
+7,mlp.gate_proj,2,32,8,128,68.1837158203125,70.21875,234881024
+7,mlp.gate_proj,2,64,8,128,80.00153350830078,63.109375,234881024
+7,mlp.gate_proj,2,128,8,128,91.12039947509766,59.5546875,234881024
+7,mlp.gate_proj,3,32,8,128,29.253034591674805,98.21875,234881024
+7,mlp.gate_proj,3,64,8,128,34.307376861572266,91.109375,234881024
+7,mlp.gate_proj,3,128,8,128,39.023681640625,87.5546875,234881024
+7,mlp.gate_proj,4,32,8,128,13.623880386352539,126.21875,234881024
+7,mlp.gate_proj,4,64,8,128,16.083417892456055,119.109375,234881024
+7,mlp.gate_proj,4,128,8,128,18.23673439025879,115.5546875,234881024
+7,mlp.gate_proj,8,32,8,128,0.8316003680229187,238.21875,234881024
+7,mlp.gate_proj,8,64,8,128,0.9820075035095215,231.109375,234881024
+7,mlp.gate_proj,8,128,8,128,1.1091071367263794,227.5546875,234881024
+7,mlp.up_proj,2,32,8,128,66.09703826904297,70.21875,234881024
+7,mlp.up_proj,2,64,8,128,77.14047241210938,63.109375,234881024
+7,mlp.up_proj,2,128,8,128,87.51713562011719,59.5546875,234881024
+7,mlp.up_proj,3,32,8,128,28.276100158691406,98.21875,234881024
+7,mlp.up_proj,3,64,8,128,33.15349197387695,91.109375,234881024
+7,mlp.up_proj,3,128,8,128,37.52778244018555,87.5546875,234881024
+7,mlp.up_proj,4,32,8,128,13.17590618133545,126.21875,234881024
+7,mlp.up_proj,4,64,8,128,15.513771057128906,119.109375,234881024
+7,mlp.up_proj,4,128,8,128,17.546300888061523,115.5546875,234881024
+7,mlp.up_proj,8,32,8,128,0.8045756220817566,238.21875,234881024
+7,mlp.up_proj,8,64,8,128,0.9456003904342651,231.109375,234881024
+7,mlp.up_proj,8,128,8,128,1.06758451461792,227.5546875,234881024
+7,mlp.down_proj,2,32,8,128,65.45753479003906,70.21875,234881024
+7,mlp.down_proj,2,64,8,128,76.05559539794922,63.109375,234881024
+7,mlp.down_proj,2,128,8,128,85.98831939697266,59.5546875,234881024
+7,mlp.down_proj,3,32,8,128,27.934539794921875,98.21875,234881024
+7,mlp.down_proj,3,64,8,128,32.6642951965332,91.109375,234881024
+7,mlp.down_proj,3,128,8,128,36.7711067199707,87.5546875,234881024
+7,mlp.down_proj,4,32,8,128,13.014552116394043,126.21875,234881024
+7,mlp.down_proj,4,64,8,128,15.2509183883667,119.109375,234881024
+7,mlp.down_proj,4,128,8,128,17.184749603271484,115.5546875,234881024
+7,mlp.down_proj,8,32,8,128,0.7950599193572998,238.21875,234881024
+7,mlp.down_proj,8,64,8,128,0.9298620223999023,231.109375,234881024
+7,mlp.down_proj,8,128,8,128,1.0459222793579102,227.5546875,234881024
+8,self_attn.q_proj,2,32,8,128,45.72865295410156,20.0625,67108864
+8,self_attn.q_proj,2,64,8,128,53.94852066040039,18.03125,67108864
+8,self_attn.q_proj,2,128,8,128,62.1143798828125,17.015625,67108864
+8,self_attn.q_proj,3,32,8,128,19.572649002075195,28.0625,67108864
+8,self_attn.q_proj,3,64,8,128,23.177040100097656,26.03125,67108864
+8,self_attn.q_proj,3,128,8,128,26.750885009765625,25.015625,67108864
+8,self_attn.q_proj,4,32,8,128,9.132364273071289,36.0625,67108864
+8,self_attn.q_proj,4,64,8,128,10.815155029296875,34.03125,67108864
+8,self_attn.q_proj,4,128,8,128,12.486407279968262,33.015625,67108864
+8,self_attn.q_proj,8,32,8,128,0.5546671748161316,68.0625,67108864
+8,self_attn.q_proj,8,64,8,128,0.6568783521652222,66.03125,67108864
+8,self_attn.q_proj,8,128,8,128,0.7563000321388245,65.015625,67108864
+8,self_attn.v_proj,2,32,8,128,12.15407943725586,2.5078125,8388608
+8,self_attn.v_proj,2,64,8,128,14.170284271240234,2.25390625,8388608
+8,self_attn.v_proj,2,128,8,128,15.990191459655762,2.126953125,8388608
+8,self_attn.v_proj,3,32,8,128,5.198652267456055,3.5078125,8388608
+8,self_attn.v_proj,3,64,8,128,6.063883304595947,3.25390625,8388608
+8,self_attn.v_proj,3,128,8,128,6.847848415374756,3.126953125,8388608
+8,self_attn.v_proj,4,32,8,128,2.425276279449463,4.5078125,8388608
+8,self_attn.v_proj,4,64,8,128,2.829632520675659,4.25390625,8388608
+8,self_attn.v_proj,4,128,8,128,3.1955137252807617,4.126953125,8388608
+8,self_attn.v_proj,8,32,8,128,0.14685645699501038,8.5078125,8388608
+8,self_attn.v_proj,8,64,8,128,0.1710994839668274,8.25390625,8388608
+8,self_attn.v_proj,8,128,8,128,0.19255059957504272,8.126953125,8388608
+8,self_attn.k_proj,2,32,8,128,28.5035457611084,2.5078125,8388608
+8,self_attn.k_proj,2,64,8,128,34.309425354003906,2.25390625,8388608
+8,self_attn.k_proj,2,128,8,128,39.41565704345703,2.126953125,8388608
+8,self_attn.k_proj,3,32,8,128,12.343832969665527,3.5078125,8388608
+8,self_attn.k_proj,3,64,8,128,15.063261985778809,3.25390625,8388608
+8,self_attn.k_proj,3,128,8,128,17.719131469726562,3.126953125,8388608
+8,self_attn.k_proj,4,32,8,128,5.762792110443115,4.5078125,8388608
+8,self_attn.k_proj,4,64,8,128,7.032728672027588,4.25390625,8388608
+8,self_attn.k_proj,4,128,8,128,8.280454635620117,4.126953125,8388608
+8,self_attn.k_proj,8,32,8,128,0.3487890958786011,8.5078125,8388608
+8,self_attn.k_proj,8,64,8,128,0.42517638206481934,8.25390625,8388608
+8,self_attn.k_proj,8,128,8,128,0.49883177876472473,8.126953125,8388608
+8,self_attn.o_proj,2,32,8,128,33.79190444946289,20.0625,67108864
+8,self_attn.o_proj,2,64,8,128,39.48871994018555,18.03125,67108864
+8,self_attn.o_proj,2,128,8,128,44.75978469848633,17.015625,67108864
+8,self_attn.o_proj,3,32,8,128,14.451738357543945,28.0625,67108864
+8,self_attn.o_proj,3,64,8,128,16.91231918334961,26.03125,67108864
+8,self_attn.o_proj,3,128,8,128,19.1943359375,25.015625,67108864
+8,self_attn.o_proj,4,32,8,128,6.740121841430664,36.0625,67108864
+8,self_attn.o_proj,4,64,8,128,7.895496845245361,34.03125,67108864
+8,self_attn.o_proj,4,128,8,128,8.952275276184082,33.015625,67108864
+8,self_attn.o_proj,8,32,8,128,0.4097226858139038,68.0625,67108864
+8,self_attn.o_proj,8,64,8,128,0.4790648818016052,66.03125,67108864
+8,self_attn.o_proj,8,128,8,128,0.5416540503501892,65.015625,67108864
+8,mlp.gate_proj,2,32,8,128,74.89094543457031,70.21875,234881024
+8,mlp.gate_proj,2,64,8,128,88.10325622558594,63.109375,234881024
+8,mlp.gate_proj,2,128,8,128,100.32807159423828,59.5546875,234881024
+8,mlp.gate_proj,3,32,8,128,32.437904357910156,98.21875,234881024
+8,mlp.gate_proj,3,64,8,128,38.39273452758789,91.109375,234881024
+8,mlp.gate_proj,3,128,8,128,43.902565002441406,87.5546875,234881024
+8,mlp.gate_proj,4,32,8,128,15.163054466247559,126.21875,234881024
+8,mlp.gate_proj,4,64,8,128,18.041866302490234,119.109375,234881024
+8,mlp.gate_proj,4,128,8,128,20.607418060302734,115.5546875,234881024
+8,mlp.gate_proj,8,32,8,128,0.9245887994766235,238.21875,234881024
+8,mlp.gate_proj,8,64,8,128,1.1001396179199219,231.109375,234881024
+8,mlp.gate_proj,8,128,8,128,1.2534157037734985,227.5546875,234881024
+8,mlp.up_proj,2,32,8,128,72.66354370117188,70.21875,234881024
+8,mlp.up_proj,2,64,8,128,85.25971221923828,63.109375,234881024
+8,mlp.up_proj,2,128,8,128,96.85344696044922,59.5546875,234881024
+8,mlp.up_proj,3,32,8,128,31.469615936279297,98.21875,234881024
+8,mlp.up_proj,3,64,8,128,37.22433090209961,91.109375,234881024
+8,mlp.up_proj,3,128,8,128,42.42769241333008,87.5546875,234881024
+8,mlp.up_proj,4,32,8,128,14.703145980834961,126.21875,234881024
+8,mlp.up_proj,4,64,8,128,17.468461990356445,119.109375,234881024
+8,mlp.up_proj,4,128,8,128,19.93691062927246,115.5546875,234881024
+8,mlp.up_proj,8,32,8,128,0.8955661654472351,238.21875,234881024
+8,mlp.up_proj,8,64,8,128,1.0628725290298462,231.109375,234881024
+8,mlp.up_proj,8,128,8,128,1.2127257585525513,227.5546875,234881024
+8,mlp.down_proj,2,32,8,128,65.49151611328125,70.21875,234881024
+8,mlp.down_proj,2,64,8,128,76.14095306396484,63.109375,234881024
+8,mlp.down_proj,2,128,8,128,86.16998291015625,59.5546875,234881024
+8,mlp.down_proj,3,32,8,128,27.940214157104492,98.21875,234881024
+8,mlp.down_proj,3,64,8,128,32.67463684082031,91.109375,234881024
+8,mlp.down_proj,3,128,8,128,36.795677185058594,87.5546875,234881024
+8,mlp.down_proj,4,32,8,128,13.015117645263672,126.21875,234881024
+8,mlp.down_proj,4,64,8,128,15.25265884399414,119.109375,234881024
+8,mlp.down_proj,4,128,8,128,17.195833206176758,115.5546875,234881024
+8,mlp.down_proj,8,32,8,128,0.7950452566146851,238.21875,234881024
+8,mlp.down_proj,8,64,8,128,0.9298548698425293,231.109375,234881024
+8,mlp.down_proj,8,128,8,128,1.0458652973175049,227.5546875,234881024
+9,self_attn.q_proj,2,32,8,128,45.744144439697266,20.0625,67108864
+9,self_attn.q_proj,2,64,8,128,53.94227981567383,18.03125,67108864
+9,self_attn.q_proj,2,128,8,128,62.38959884643555,17.015625,67108864
+9,self_attn.q_proj,3,32,8,128,19.599485397338867,28.0625,67108864
+9,self_attn.q_proj,3,64,8,128,23.214609146118164,26.03125,67108864
+9,self_attn.q_proj,3,128,8,128,26.983135223388672,25.015625,67108864
+9,self_attn.q_proj,4,32,8,128,9.14249038696289,36.0625,67108864
+9,self_attn.q_proj,4,64,8,128,10.8329439163208,34.03125,67108864
+9,self_attn.q_proj,4,128,8,128,12.597217559814453,33.015625,67108864
+9,self_attn.q_proj,8,32,8,128,0.555483341217041,68.0625,67108864
+9,self_attn.q_proj,8,64,8,128,0.6582139134407043,66.03125,67108864
+9,self_attn.q_proj,8,128,8,128,0.7630046606063843,65.015625,67108864
+9,self_attn.v_proj,2,32,8,128,12.721362113952637,2.5078125,8388608
+9,self_attn.v_proj,2,64,8,128,14.8699369430542,2.25390625,8388608
+9,self_attn.v_proj,2,128,8,128,16.819732666015625,2.126953125,8388608
+9,self_attn.v_proj,3,32,8,128,5.440305709838867,3.5078125,8388608
+9,self_attn.v_proj,3,64,8,128,6.363877773284912,3.25390625,8388608
+9,self_attn.v_proj,3,128,8,128,7.200989723205566,3.126953125,8388608
+9,self_attn.v_proj,4,32,8,128,2.5382540225982666,4.5078125,8388608
+9,self_attn.v_proj,4,64,8,128,2.968956708908081,4.25390625,8388608
+9,self_attn.v_proj,4,128,8,128,3.363720655441284,4.126953125,8388608
+9,self_attn.v_proj,8,32,8,128,0.15372319519519806,8.5078125,8388608
+9,self_attn.v_proj,8,64,8,128,0.17961916327476501,8.25390625,8388608
+9,self_attn.v_proj,8,128,8,128,0.20253467559814453,8.126953125,8388608
+9,self_attn.k_proj,2,32,8,128,26.42189598083496,2.5078125,8388608
+9,self_attn.k_proj,2,64,8,128,31.615352630615234,2.25390625,8388608
+9,self_attn.k_proj,2,128,8,128,36.22029495239258,2.126953125,8388608
+9,self_attn.k_proj,3,32,8,128,11.376606941223145,3.5078125,8388608
+9,self_attn.k_proj,3,64,8,128,13.745140075683594,3.25390625,8388608
+9,self_attn.k_proj,3,128,8,128,16.012033462524414,3.126953125,8388608
+9,self_attn.k_proj,4,32,8,128,5.314411163330078,4.5078125,8388608
+9,self_attn.k_proj,4,64,8,128,6.414425373077393,4.25390625,8388608
+9,self_attn.k_proj,4,128,8,128,7.477529525756836,4.126953125,8388608
+9,self_attn.k_proj,8,32,8,128,0.32158344984054565,8.5078125,8388608
+9,self_attn.k_proj,8,64,8,128,0.3880411982536316,8.25390625,8388608
+9,self_attn.k_proj,8,128,8,128,0.4507000744342804,8.126953125,8388608
+9,self_attn.o_proj,2,32,8,128,33.76325607299805,20.0625,67108864
+9,self_attn.o_proj,2,64,8,128,39.530357360839844,18.03125,67108864
+9,self_attn.o_proj,2,128,8,128,44.890201568603516,17.015625,67108864
+9,self_attn.o_proj,3,32,8,128,14.439377784729004,28.0625,67108864
+9,self_attn.o_proj,3,64,8,128,16.930442810058594,26.03125,67108864
+9,self_attn.o_proj,3,128,8,128,19.259347915649414,25.015625,67108864
+9,self_attn.o_proj,4,32,8,128,6.73428201675415,36.0625,67108864
+9,self_attn.o_proj,4,64,8,128,7.904836177825928,34.03125,67108864
+9,self_attn.o_proj,4,128,8,128,8.983747482299805,33.015625,67108864
+9,self_attn.o_proj,8,32,8,128,0.4093394875526428,68.0625,67108864
+9,self_attn.o_proj,8,64,8,128,0.479574590921402,66.03125,67108864
+9,self_attn.o_proj,8,128,8,128,0.5435936450958252,65.015625,67108864
+9,mlp.gate_proj,2,32,8,128,142.41015625,70.21875,234881024
+9,mlp.gate_proj,2,64,8,128,173.82757568359375,63.109375,234881024
+9,mlp.gate_proj,2,128,8,128,199.448974609375,59.5546875,234881024
+9,mlp.gate_proj,3,32,8,128,61.30406188964844,98.21875,234881024
+9,mlp.gate_proj,3,64,8,128,75.36935424804688,91.109375,234881024
+9,mlp.gate_proj,3,128,8,128,89.3721694946289,87.5546875,234881024
+9,mlp.gate_proj,4,32,8,128,28.98422622680664,126.21875,234881024
+9,mlp.gate_proj,4,64,8,128,35.85440444946289,119.109375,234881024
+9,mlp.gate_proj,4,128,8,128,41.79305648803711,115.5546875,234881024
+9,mlp.gate_proj,8,32,8,128,1.7943781614303589,238.21875,234881024
+9,mlp.gate_proj,8,64,8,128,2.2386634349823,231.109375,234881024
+9,mlp.gate_proj,8,128,8,128,2.676602363586426,227.5546875,234881024
+9,mlp.up_proj,2,32,8,128,139.359375,70.21875,234881024
+9,mlp.up_proj,2,64,8,128,165.7630157470703,63.109375,234881024
+9,mlp.up_proj,2,128,8,128,190.1451873779297,59.5546875,234881024
+9,mlp.up_proj,3,32,8,128,60.169776916503906,98.21875,234881024
+9,mlp.up_proj,3,64,8,128,72.91801452636719,91.109375,234881024
+9,mlp.up_proj,3,128,8,128,85.9036636352539,87.5546875,234881024
+9,mlp.up_proj,4,32,8,128,28.197317123413086,126.21875,234881024
+9,mlp.up_proj,4,64,8,128,34.85843276977539,119.109375,234881024
+9,mlp.up_proj,4,128,8,128,40.8798942565918,115.5546875,234881024
+9,mlp.up_proj,8,32,8,128,1.7541924715042114,238.21875,234881024
+9,mlp.up_proj,8,64,8,128,2.196526050567627,231.109375,234881024
+9,mlp.up_proj,8,128,8,128,2.6011335849761963,227.5546875,234881024
+9,mlp.down_proj,2,32,8,128,65.73462677001953,70.21875,234881024
+9,mlp.down_proj,2,64,8,128,76.44322967529297,63.109375,234881024
+9,mlp.down_proj,2,128,8,128,86.4638671875,59.5546875,234881024
+9,mlp.down_proj,3,32,8,128,28.058053970336914,98.21875,234881024
+9,mlp.down_proj,3,64,8,128,32.79258346557617,91.109375,234881024
+9,mlp.down_proj,3,128,8,128,36.93619918823242,87.5546875,234881024
+9,mlp.down_proj,4,32,8,128,13.069445610046387,126.21875,234881024
+9,mlp.down_proj,4,64,8,128,15.315149307250977,119.109375,234881024
+9,mlp.down_proj,4,128,8,128,17.250225067138672,115.5546875,234881024
+9,mlp.down_proj,8,32,8,128,0.7983719110488892,238.21875,234881024
+9,mlp.down_proj,8,64,8,128,0.9339200854301453,231.109375,234881024
+9,mlp.down_proj,8,128,8,128,1.0499272346496582,227.5546875,234881024
+10,self_attn.q_proj,2,32,8,128,47.14068603515625,20.0625,67108864
+10,self_attn.q_proj,2,64,8,128,55.589046478271484,18.03125,67108864
+10,self_attn.q_proj,2,128,8,128,64.31179809570312,17.015625,67108864
+10,self_attn.q_proj,3,32,8,128,20.20406723022461,28.0625,67108864
+10,self_attn.q_proj,3,64,8,128,23.906391143798828,26.03125,67108864
+10,self_attn.q_proj,3,128,8,128,27.77785301208496,25.015625,67108864
+10,self_attn.q_proj,4,32,8,128,9.422062873840332,36.0625,67108864
+10,self_attn.q_proj,4,64,8,128,11.16172981262207,34.03125,67108864
+10,self_attn.q_proj,4,128,8,128,12.966678619384766,33.015625,67108864
+10,self_attn.q_proj,8,32,8,128,0.5725036263465881,68.0625,67108864
+10,self_attn.q_proj,8,64,8,128,0.6778901815414429,66.03125,67108864
+10,self_attn.q_proj,8,128,8,128,0.7850294709205627,65.015625,67108864
+10,self_attn.v_proj,2,32,8,128,11.197010040283203,2.5078125,8388608
+10,self_attn.v_proj,2,64,8,128,13.087926864624023,2.25390625,8388608
+10,self_attn.v_proj,2,128,8,128,14.809041023254395,2.126953125,8388608
+10,self_attn.v_proj,3,32,8,128,4.789017677307129,3.5078125,8388608
+10,self_attn.v_proj,3,64,8,128,5.6010966300964355,3.25390625,8388608
+10,self_attn.v_proj,3,128,8,128,6.337094783782959,3.126953125,8388608
+10,self_attn.v_proj,4,32,8,128,2.235208749771118,4.5078125,8388608
+10,self_attn.v_proj,4,64,8,128,2.6131129264831543,4.25390625,8388608
+10,self_attn.v_proj,4,128,8,128,2.959289789199829,4.126953125,8388608
+10,self_attn.v_proj,8,32,8,128,0.1353050023317337,8.5078125,8388608
+10,self_attn.v_proj,8,64,8,128,0.15808051824569702,8.25390625,8388608
+10,self_attn.v_proj,8,128,8,128,0.17831480503082275,8.126953125,8388608
+10,self_attn.k_proj,2,32,8,128,29.74114990234375,2.5078125,8388608
+10,self_attn.k_proj,2,64,8,128,35.666168212890625,2.25390625,8388608
+10,self_attn.k_proj,2,128,8,128,40.49492645263672,2.126953125,8388608
+10,self_attn.k_proj,3,32,8,128,12.971846580505371,3.5078125,8388608
+10,self_attn.k_proj,3,64,8,128,15.866230964660645,3.25390625,8388608
+10,self_attn.k_proj,3,128,8,128,18.66630744934082,3.126953125,8388608
+10,self_attn.k_proj,4,32,8,128,6.054455757141113,4.5078125,8388608
+10,self_attn.k_proj,4,64,8,128,7.405684471130371,4.25390625,8388608
+10,self_attn.k_proj,4,128,8,128,8.722984313964844,4.126953125,8388608
+10,self_attn.k_proj,8,32,8,128,0.36622342467308044,8.5078125,8388608
+10,self_attn.k_proj,8,64,8,128,0.44772791862487793,8.25390625,8388608
+10,self_attn.k_proj,8,128,8,128,0.5256458520889282,8.126953125,8388608
+10,self_attn.o_proj,2,32,8,128,32.55480194091797,20.0625,67108864
+10,self_attn.o_proj,2,64,8,128,37.96102523803711,18.03125,67108864
+10,self_attn.o_proj,2,128,8,128,42.890541076660156,17.015625,67108864
+10,self_attn.o_proj,3,32,8,128,13.908888816833496,28.0625,67108864
+10,self_attn.o_proj,3,64,8,128,16.246912002563477,26.03125,67108864
+10,self_attn.o_proj,3,128,8,128,18.34898567199707,25.015625,67108864
+10,self_attn.o_proj,4,32,8,128,6.488602638244629,36.0625,67108864
+10,self_attn.o_proj,4,64,8,128,7.578617572784424,34.03125,67108864
+10,self_attn.o_proj,4,128,8,128,8.564111709594727,33.015625,67108864
+10,self_attn.o_proj,8,32,8,128,0.39453575015068054,68.0625,67108864
+10,self_attn.o_proj,8,64,8,128,0.460052490234375,66.03125,67108864
+10,self_attn.o_proj,8,128,8,128,0.5181158781051636,65.015625,67108864
+10,mlp.gate_proj,2,32,8,128,68.98130798339844,70.21875,234881024
+10,mlp.gate_proj,2,64,8,128,80.78734588623047,63.109375,234881024
+10,mlp.gate_proj,2,128,8,128,91.6896743774414,59.5546875,234881024
+10,mlp.gate_proj,3,32,8,128,29.608379364013672,98.21875,234881024
+10,mlp.gate_proj,3,64,8,128,34.61275863647461,91.109375,234881024
+10,mlp.gate_proj,3,128,8,128,39.23339080810547,87.5546875,234881024
+10,mlp.gate_proj,4,32,8,128,13.78589153289795,126.21875,234881024
+10,mlp.gate_proj,4,64,8,128,16.21443748474121,119.109375,234881024
+10,mlp.gate_proj,4,128,8,128,18.336498260498047,115.5546875,234881024
+10,mlp.gate_proj,8,32,8,128,0.8414453268051147,238.21875,234881024
+10,mlp.gate_proj,8,64,8,128,0.990977942943573,231.109375,234881024
+10,mlp.gate_proj,8,128,8,128,1.1153910160064697,227.5546875,234881024
+10,mlp.up_proj,2,32,8,128,66.5533676147461,70.21875,234881024
+10,mlp.up_proj,2,64,8,128,77.57467651367188,63.109375,234881024
+10,mlp.up_proj,2,128,8,128,87.87248229980469,59.5546875,234881024
+10,mlp.up_proj,3,32,8,128,28.47943115234375,98.21875,234881024
+10,mlp.up_proj,3,64,8,128,33.30709457397461,91.109375,234881024
+10,mlp.up_proj,3,128,8,128,37.61131286621094,87.5546875,234881024
+10,mlp.up_proj,4,32,8,128,13.267901420593262,126.21875,234881024
+10,mlp.up_proj,4,64,8,128,15.586748123168945,119.109375,234881024
+10,mlp.up_proj,4,128,8,128,17.581785202026367,115.5546875,234881024
+10,mlp.up_proj,8,32,8,128,0.810247540473938,238.21875,234881024
+10,mlp.up_proj,8,64,8,128,0.9502071738243103,231.109375,234881024
+10,mlp.up_proj,8,128,8,128,1.0699801445007324,227.5546875,234881024
+10,mlp.down_proj,2,32,8,128,66.35365295410156,70.21875,234881024
+10,mlp.down_proj,2,64,8,128,77.18567657470703,63.109375,234881024
+10,mlp.down_proj,2,128,8,128,87.27243041992188,59.5546875,234881024
+10,mlp.down_proj,3,32,8,128,28.34636688232422,98.21875,234881024
+10,mlp.down_proj,3,64,8,128,33.08552932739258,91.109375,234881024
+10,mlp.down_proj,3,128,8,128,37.25356674194336,87.5546875,234881024
+10,mlp.down_proj,4,32,8,128,13.202288627624512,126.21875,234881024
+10,mlp.down_proj,4,64,8,128,15.466925621032715,119.109375,234881024
+10,mlp.down_proj,4,128,8,128,17.396480560302734,115.5546875,234881024
+10,mlp.down_proj,8,32,8,128,0.806340217590332,238.21875,234881024
+10,mlp.down_proj,8,64,8,128,0.9429009556770325,231.109375,234881024
+10,mlp.down_proj,8,128,8,128,1.0587960481643677,227.5546875,234881024
+11,self_attn.q_proj,2,32,8,128,47.767738342285156,20.0625,67108864
+11,self_attn.q_proj,2,64,8,128,56.40552520751953,18.03125,67108864
+11,self_attn.q_proj,2,128,8,128,64.79865264892578,17.015625,67108864
+11,self_attn.q_proj,3,32,8,128,20.451942443847656,28.0625,67108864
+11,self_attn.q_proj,3,64,8,128,24.19611167907715,26.03125,67108864
+11,self_attn.q_proj,3,128,8,128,27.862035751342773,25.015625,67108864
+11,self_attn.q_proj,4,32,8,128,9.539348602294922,36.0625,67108864
+11,self_attn.q_proj,4,64,8,128,11.303452491760254,34.03125,67108864
+11,self_attn.q_proj,4,128,8,128,13.00934886932373,33.015625,67108864
+11,self_attn.q_proj,8,32,8,128,0.5797201991081238,68.0625,67108864
+11,self_attn.q_proj,8,64,8,128,0.686638355255127,66.03125,67108864
+11,self_attn.q_proj,8,128,8,128,0.7877641916275024,65.015625,67108864
+11,self_attn.v_proj,2,32,8,128,11.666751861572266,2.5078125,8388608
+11,self_attn.v_proj,2,64,8,128,13.598281860351562,2.25390625,8388608
+11,self_attn.v_proj,2,128,8,128,15.32270622253418,2.126953125,8388608
+11,self_attn.v_proj,3,32,8,128,5.001588821411133,3.5078125,8388608
+11,self_attn.v_proj,3,64,8,128,5.834916591644287,3.25390625,8388608
+11,self_attn.v_proj,3,128,8,128,6.579465389251709,3.126953125,8388608
+11,self_attn.v_proj,4,32,8,128,2.334949016571045,4.5078125,8388608
+11,self_attn.v_proj,4,64,8,128,2.726151466369629,4.25390625,8388608
+11,self_attn.v_proj,4,128,8,128,3.0753023624420166,4.126953125,8388608
+11,self_attn.v_proj,8,32,8,128,0.1413068175315857,8.5078125,8388608
+11,self_attn.v_proj,8,64,8,128,0.16494955122470856,8.25390625,8388608
+11,self_attn.v_proj,8,128,8,128,0.18565921485424042,8.126953125,8388608
+11,self_attn.k_proj,2,32,8,128,30.501800537109375,2.5078125,8388608
+11,self_attn.k_proj,2,64,8,128,36.524513244628906,2.25390625,8388608
+11,self_attn.k_proj,2,128,8,128,41.789894104003906,2.126953125,8388608
+11,self_attn.k_proj,3,32,8,128,13.224815368652344,3.5078125,8388608
+11,self_attn.k_proj,3,64,8,128,16.006458282470703,3.25390625,8388608
+11,self_attn.k_proj,3,128,8,128,18.649927139282227,3.126953125,8388608
+11,self_attn.k_proj,4,32,8,128,6.174098968505859,4.5078125,8388608
+11,self_attn.k_proj,4,64,8,128,7.477685928344727,4.25390625,8388608
+11,self_attn.k_proj,4,128,8,128,8.725138664245605,4.126953125,8388608
+11,self_attn.k_proj,8,32,8,128,0.3738352358341217,8.5078125,8388608
+11,self_attn.k_proj,8,64,8,128,0.4530104994773865,8.25390625,8388608
+11,self_attn.k_proj,8,128,8,128,0.5279937982559204,8.126953125,8388608
+11,self_attn.o_proj,2,32,8,128,33.46442794799805,20.0625,67108864
+11,self_attn.o_proj,2,64,8,128,38.971405029296875,18.03125,67108864
+11,self_attn.o_proj,2,128,8,128,43.96527099609375,17.015625,67108864
+11,self_attn.o_proj,3,32,8,128,14.307816505432129,28.0625,67108864
+11,self_attn.o_proj,3,64,8,128,16.680299758911133,26.03125,67108864
+11,self_attn.o_proj,3,128,8,128,18.8251953125,25.015625,67108864
+11,self_attn.o_proj,4,32,8,128,6.672486305236816,36.0625,67108864
+11,self_attn.o_proj,4,64,8,128,7.785013675689697,34.03125,67108864
+11,self_attn.o_proj,4,128,8,128,8.779267311096191,33.015625,67108864
+11,self_attn.o_proj,8,32,8,128,0.40569835901260376,68.0625,67108864
+11,self_attn.o_proj,8,64,8,128,0.47240838408470154,66.03125,67108864
+11,self_attn.o_proj,8,128,8,128,0.5310494303703308,65.015625,67108864
+11,mlp.gate_proj,2,32,8,128,71.75784301757812,70.21875,234881024
+11,mlp.gate_proj,2,64,8,128,83.99767303466797,63.109375,234881024
+11,mlp.gate_proj,2,128,8,128,95.28524017333984,59.5546875,234881024
+11,mlp.gate_proj,3,32,8,128,30.81612205505371,98.21875,234881024
+11,mlp.gate_proj,3,64,8,128,36.135948181152344,91.109375,234881024
+11,mlp.gate_proj,3,128,8,128,40.96001052856445,87.5546875,234881024
+11,mlp.gate_proj,4,32,8,128,14.358738899230957,126.21875,234881024
+11,mlp.gate_proj,4,64,8,128,16.89822006225586,119.109375,234881024
+11,mlp.gate_proj,4,128,8,128,19.172298431396484,115.5546875,234881024
+11,mlp.gate_proj,8,32,8,128,0.8768954277038574,238.21875,234881024
+11,mlp.gate_proj,8,64,8,128,1.0320100784301758,231.109375,234881024
+11,mlp.gate_proj,8,128,8,128,1.1669518947601318,227.5546875,234881024
+11,mlp.up_proj,2,32,8,128,69.36652374267578,70.21875,234881024
+11,mlp.up_proj,2,64,8,128,80.97540283203125,63.109375,234881024
+11,mlp.up_proj,2,128,8,128,91.56926727294922,59.5546875,234881024
+11,mlp.up_proj,3,32,8,128,29.73406219482422,98.21875,234881024
+11,mlp.up_proj,3,64,8,128,34.84250259399414,91.109375,234881024
+11,mlp.up_proj,3,128,8,128,39.43355178833008,87.5546875,234881024
+11,mlp.up_proj,4,32,8,128,13.872541427612305,126.21875,234881024
+11,mlp.up_proj,4,64,8,128,16.308040618896484,119.109375,234881024
+11,mlp.up_proj,4,128,8,128,18.469900131225586,115.5546875,234881024
+11,mlp.up_proj,8,32,8,128,0.8470475077629089,238.21875,234881024
+11,mlp.up_proj,8,64,8,128,0.9942194819450378,231.109375,234881024
+11,mlp.up_proj,8,128,8,128,1.122808814048767,227.5546875,234881024
+11,mlp.down_proj,2,32,8,128,66.81829071044922,70.21875,234881024
+11,mlp.down_proj,2,64,8,128,77.74824523925781,63.109375,234881024
+11,mlp.down_proj,2,128,8,128,87.90068817138672,59.5546875,234881024
+11,mlp.down_proj,3,32,8,128,28.561784744262695,98.21875,234881024
+11,mlp.down_proj,3,64,8,128,33.30084228515625,91.109375,234881024
+11,mlp.down_proj,3,128,8,128,37.48672866821289,87.5546875,234881024
+11,mlp.down_proj,4,32,8,128,13.302050590515137,126.21875,234881024
+11,mlp.down_proj,4,64,8,128,15.578849792480469,119.109375,234881024
+11,mlp.down_proj,4,128,8,128,17.510520935058594,115.5546875,234881024
+11,mlp.down_proj,8,32,8,128,0.8122872114181519,238.21875,234881024
+11,mlp.down_proj,8,64,8,128,0.9497777819633484,231.109375,234881024
+11,mlp.down_proj,8,128,8,128,1.0653804540634155,227.5546875,234881024
+12,self_attn.q_proj,2,32,8,128,45.970252990722656,20.0625,67108864
+12,self_attn.q_proj,2,64,8,128,54.733154296875,18.03125,67108864
+12,self_attn.q_proj,2,128,8,128,63.23746871948242,17.015625,67108864
+12,self_attn.q_proj,3,32,8,128,19.70374870300293,28.0625,67108864
+12,self_attn.q_proj,3,64,8,128,23.57019805908203,26.03125,67108864
+12,self_attn.q_proj,3,128,8,128,27.379981994628906,25.015625,67108864
+12,self_attn.q_proj,4,32,8,128,9.1904935836792,36.0625,67108864
+12,self_attn.q_proj,4,64,8,128,11.001408576965332,34.03125,67108864
+12,self_attn.q_proj,4,128,8,128,12.773686408996582,33.015625,67108864
+12,self_attn.q_proj,8,32,8,128,0.5582144260406494,68.0625,67108864
+12,self_attn.q_proj,8,64,8,128,0.6679866313934326,66.03125,67108864
+12,self_attn.q_proj,8,128,8,128,0.7734392285346985,65.015625,67108864
+12,self_attn.v_proj,2,32,8,128,11.925644874572754,2.5078125,8388608
+12,self_attn.v_proj,2,64,8,128,13.90407657623291,2.25390625,8388608
+12,self_attn.v_proj,2,128,8,128,15.69293212890625,2.126953125,8388608
+12,self_attn.v_proj,3,32,8,128,5.097866535186768,3.5078125,8388608
+12,self_attn.v_proj,3,64,8,128,5.948110103607178,3.25390625,8388608
+12,self_attn.v_proj,3,128,8,128,6.715869903564453,3.126953125,8388608
+12,self_attn.v_proj,4,32,8,128,2.378878355026245,4.5078125,8388608
+12,self_attn.v_proj,4,64,8,128,2.7749738693237305,4.25390625,8388608
+12,self_attn.v_proj,4,128,8,128,3.1339030265808105,4.126953125,8388608
+12,self_attn.v_proj,8,32,8,128,0.1439635306596756,8.5078125,8388608
+12,self_attn.v_proj,8,64,8,128,0.16780085861682892,8.25390625,8388608
+12,self_attn.v_proj,8,128,8,128,0.1888420730829239,8.126953125,8388608
+12,self_attn.k_proj,2,32,8,128,28.83409881591797,2.5078125,8388608
+12,self_attn.k_proj,2,64,8,128,34.81871032714844,2.25390625,8388608
+12,self_attn.k_proj,2,128,8,128,39.84324264526367,2.126953125,8388608
+12,self_attn.k_proj,3,32,8,128,12.634294509887695,3.5078125,8388608
+12,self_attn.k_proj,3,64,8,128,15.654329299926758,3.25390625,8388608
+12,self_attn.k_proj,3,128,8,128,18.688997268676758,3.126953125,8388608
+12,self_attn.k_proj,4,32,8,128,5.896311283111572,4.5078125,8388608
+12,self_attn.k_proj,4,64,8,128,7.311997413635254,4.25390625,8388608
+12,self_attn.k_proj,4,128,8,128,8.737876892089844,4.126953125,8388608
+12,self_attn.k_proj,8,32,8,128,0.3568747341632843,8.5078125,8388608
+12,self_attn.k_proj,8,64,8,128,0.4419971704483032,8.25390625,8388608
+12,self_attn.k_proj,8,128,8,128,0.526684045791626,8.126953125,8388608
+12,self_attn.o_proj,2,32,8,128,33.648521423339844,20.0625,67108864
+12,self_attn.o_proj,2,64,8,128,39.29265594482422,18.03125,67108864
+12,self_attn.o_proj,2,128,8,128,44.4813117980957,17.015625,67108864
+12,self_attn.o_proj,3,32,8,128,14.388138771057129,28.0625,67108864
+12,self_attn.o_proj,3,64,8,128,16.819339752197266,26.03125,67108864
+12,self_attn.o_proj,3,128,8,128,19.04666519165039,25.015625,67108864
+12,self_attn.o_proj,4,32,8,128,6.709958076477051,36.0625,67108864
+12,self_attn.o_proj,4,64,8,128,7.851311683654785,34.03125,67108864
+12,self_attn.o_proj,4,128,8,128,8.887606620788574,33.015625,67108864
+12,self_attn.o_proj,8,32,8,128,0.4080013036727905,68.0625,67108864
+12,self_attn.o_proj,8,64,8,128,0.47638407349586487,66.03125,67108864
+12,self_attn.o_proj,8,128,8,128,0.5377086400985718,65.015625,67108864
+12,mlp.gate_proj,2,32,8,128,69.25147247314453,70.21875,234881024
+12,mlp.gate_proj,2,64,8,128,81.35717010498047,63.109375,234881024
+12,mlp.gate_proj,2,128,8,128,92.57120513916016,59.5546875,234881024
+12,mlp.gate_proj,3,32,8,128,29.71519660949707,98.21875,234881024
+12,mlp.gate_proj,3,64,8,128,34.81437301635742,91.109375,234881024
+12,mlp.gate_proj,3,128,8,128,39.620758056640625,87.5546875,234881024
+12,mlp.gate_proj,4,32,8,128,13.83614444732666,126.21875,234881024
+12,mlp.gate_proj,4,64,8,128,16.301456451416016,119.109375,234881024
+12,mlp.gate_proj,4,128,8,128,18.49755859375,115.5546875,234881024
+12,mlp.gate_proj,8,32,8,128,0.8445115685462952,238.21875,234881024
+12,mlp.gate_proj,8,64,8,128,0.9973483085632324,231.109375,234881024
+12,mlp.gate_proj,8,128,8,128,1.1254737377166748,227.5546875,234881024
+12,mlp.up_proj,2,32,8,128,66.6687240600586,70.21875,234881024
+12,mlp.up_proj,2,64,8,128,77.78538513183594,63.109375,234881024
+12,mlp.up_proj,2,128,8,128,88.19718170166016,59.5546875,234881024
+12,mlp.up_proj,3,32,8,128,28.506656646728516,98.21875,234881024
+12,mlp.up_proj,3,64,8,128,33.34957504272461,91.109375,234881024
+12,mlp.up_proj,3,128,8,128,37.68560028076172,87.5546875,234881024
+12,mlp.up_proj,4,32,8,128,13.279901504516602,126.21875,234881024
+12,mlp.up_proj,4,64,8,128,15.604920387268066,119.109375,234881024
+12,mlp.up_proj,4,128,8,128,17.60580825805664,115.5546875,234881024
+12,mlp.up_proj,8,32,8,128,0.8110992312431335,238.21875,234881024
+12,mlp.up_proj,8,64,8,128,0.951673686504364,231.109375,234881024
+12,mlp.up_proj,8,128,8,128,1.0720009803771973,227.5546875,234881024
+12,mlp.down_proj,2,32,8,128,66.17350006103516,70.21875,234881024
+12,mlp.down_proj,2,64,8,128,76.96678161621094,63.109375,234881024
+12,mlp.down_proj,2,128,8,128,87.02436065673828,59.5546875,234881024
+12,mlp.down_proj,3,32,8,128,28.261655807495117,98.21875,234881024
+12,mlp.down_proj,3,64,8,128,32.99079132080078,91.109375,234881024
+12,mlp.down_proj,3,128,8,128,37.13298797607422,87.5546875,234881024
+12,mlp.down_proj,4,32,8,128,13.163488388061523,126.21875,234881024
+12,mlp.down_proj,4,64,8,128,15.417389869689941,119.109375,234881024
+12,mlp.down_proj,4,128,8,128,17.342052459716797,115.5546875,234881024
+12,mlp.down_proj,8,32,8,128,0.803975522518158,238.21875,234881024
+12,mlp.down_proj,8,64,8,128,0.94005286693573,231.109375,234881024
+12,mlp.down_proj,8,128,8,128,1.0553438663482666,227.5546875,234881024
+13,self_attn.q_proj,2,32,8,128,49.25015640258789,20.0625,67108864
+13,self_attn.q_proj,2,64,8,128,58.07244110107422,18.03125,67108864
+13,self_attn.q_proj,2,128,8,128,66.85311889648438,17.015625,67108864
+13,self_attn.q_proj,3,32,8,128,21.109045028686523,28.0625,67108864
+13,self_attn.q_proj,3,64,8,128,24.923521041870117,26.03125,67108864
+13,self_attn.q_proj,3,128,8,128,28.77683448791504,25.015625,67108864
+13,self_attn.q_proj,4,32,8,128,9.843463897705078,36.0625,67108864
+13,self_attn.q_proj,4,64,8,128,11.635232925415039,34.03125,67108864
+13,self_attn.q_proj,4,128,8,128,13.424088478088379,33.015625,67108864
+13,self_attn.q_proj,8,32,8,128,0.5981475114822388,68.0625,67108864
+13,self_attn.q_proj,8,64,8,128,0.7073034048080444,66.03125,67108864
+13,self_attn.q_proj,8,128,8,128,0.8130020499229431,65.015625,67108864
+13,self_attn.v_proj,2,32,8,128,11.035346031188965,2.5078125,8388608
+13,self_attn.v_proj,2,64,8,128,12.886762619018555,2.25390625,8388608
+13,self_attn.v_proj,2,128,8,128,14.561017990112305,2.126953125,8388608
+13,self_attn.v_proj,3,32,8,128,4.718418121337891,3.5078125,8388608
+13,self_attn.v_proj,3,64,8,128,5.512012481689453,3.25390625,8388608
+13,self_attn.v_proj,3,128,8,128,6.229242324829102,3.126953125,8388608
+13,self_attn.v_proj,4,32,8,128,2.201510429382324,4.5078125,8388608
+13,self_attn.v_proj,4,64,8,128,2.5720157623291016,4.25390625,8388608
+13,self_attn.v_proj,4,128,8,128,2.9074878692626953,4.126953125,8388608
+13,self_attn.v_proj,8,32,8,128,0.13324934244155884,8.5078125,8388608
+13,self_attn.v_proj,8,64,8,128,0.15555711090564728,8.25390625,8388608
+13,self_attn.v_proj,8,128,8,128,0.17520011961460114,8.126953125,8388608
+13,self_attn.k_proj,2,32,8,128,32.54615020751953,2.5078125,8388608
+13,self_attn.k_proj,2,64,8,128,39.335060119628906,2.25390625,8388608
+13,self_attn.k_proj,2,128,8,128,44.98366165161133,2.126953125,8388608
+13,self_attn.k_proj,3,32,8,128,14.185782432556152,3.5078125,8388608
+13,self_attn.k_proj,3,64,8,128,17.453325271606445,3.25390625,8388608
+13,self_attn.k_proj,3,128,8,128,20.650190353393555,3.126953125,8388608
+13,self_attn.k_proj,4,32,8,128,6.618594646453857,4.5078125,8388608
+13,self_attn.k_proj,4,64,8,128,8.152591705322266,4.25390625,8388608
+13,self_attn.k_proj,4,128,8,128,9.649757385253906,4.126953125,8388608
+13,self_attn.k_proj,8,32,8,128,0.4005458950996399,8.5078125,8388608
+13,self_attn.k_proj,8,64,8,128,0.49274417757987976,8.25390625,8388608
+13,self_attn.k_proj,8,128,8,128,0.5815945863723755,8.126953125,8388608
+13,self_attn.o_proj,2,32,8,128,33.44627380371094,20.0625,67108864
+13,self_attn.o_proj,2,64,8,128,39.05561828613281,18.03125,67108864
+13,self_attn.o_proj,2,128,8,128,44.21564865112305,17.015625,67108864
+13,self_attn.o_proj,3,32,8,128,14.298200607299805,28.0625,67108864
+13,self_attn.o_proj,3,64,8,128,16.71733856201172,26.03125,67108864
+13,self_attn.o_proj,3,128,8,128,18.94234275817871,25.015625,67108864
+13,self_attn.o_proj,4,32,8,128,6.668493747711182,36.0625,67108864
+13,self_attn.o_proj,4,64,8,128,7.80316686630249,34.03125,67108864
+13,self_attn.o_proj,4,128,8,128,8.834348678588867,33.015625,67108864
+13,self_attn.o_proj,8,32,8,128,0.4054746925830841,68.0625,67108864
+13,self_attn.o_proj,8,64,8,128,0.4735655188560486,66.03125,67108864
+13,self_attn.o_proj,8,128,8,128,0.5344103574752808,65.015625,67108864
+13,mlp.gate_proj,2,32,8,128,69.03190612792969,70.21875,234881024
+13,mlp.gate_proj,2,64,8,128,81.01173400878906,63.109375,234881024
+13,mlp.gate_proj,2,128,8,128,92.14631652832031,59.5546875,234881024
+13,mlp.gate_proj,3,32,8,128,29.597579956054688,98.21875,234881024
+13,mlp.gate_proj,3,64,8,128,34.64426040649414,91.109375,234881024
+13,mlp.gate_proj,3,128,8,128,39.37221145629883,87.5546875,234881024
+13,mlp.gate_proj,4,32,8,128,13.775668144226074,126.21875,234881024
+13,mlp.gate_proj,4,64,8,128,16.220352172851562,119.109375,234881024
+13,mlp.gate_proj,4,128,8,128,18.368988037109375,115.5546875,234881024
+13,mlp.gate_proj,8,32,8,128,0.8409239649772644,238.21875,234881024
+13,mlp.gate_proj,8,64,8,128,0.991669774055481,231.109375,234881024
+13,mlp.gate_proj,8,128,8,128,1.1175488233566284,227.5546875,234881024
+13,mlp.up_proj,2,32,8,128,66.7879638671875,70.21875,234881024
+13,mlp.up_proj,2,64,8,128,77.96022033691406,63.109375,234881024
+13,mlp.up_proj,2,128,8,128,88.4473876953125,59.5546875,234881024
+13,mlp.up_proj,3,32,8,128,28.552574157714844,98.21875,234881024
+13,mlp.up_proj,3,64,8,128,33.40336608886719,91.109375,234881024
+13,mlp.up_proj,3,128,8,128,37.746620178222656,87.5546875,234881024
+13,mlp.up_proj,4,32,8,128,13.298750877380371,126.21875,234881024
+13,mlp.up_proj,4,64,8,128,15.629368782043457,119.109375,234881024
+13,mlp.up_proj,4,128,8,128,17.63620376586914,115.5546875,234881024
+13,mlp.up_proj,8,32,8,128,0.8120517730712891,238.21875,234881024
+13,mlp.up_proj,8,64,8,128,0.9528236389160156,231.109375,234881024
+13,mlp.up_proj,8,128,8,128,1.0728344917297363,227.5546875,234881024
+13,mlp.down_proj,2,32,8,128,66.42121124267578,70.21875,234881024
+13,mlp.down_proj,2,64,8,128,77.27613067626953,63.109375,234881024
+13,mlp.down_proj,2,128,8,128,87.41708374023438,59.5546875,234881024
+13,mlp.down_proj,3,32,8,128,28.37793731689453,98.21875,234881024
+13,mlp.down_proj,3,64,8,128,33.11520767211914,91.109375,234881024
+13,mlp.down_proj,3,128,8,128,37.28759002685547,87.5546875,234881024
+13,mlp.down_proj,4,32,8,128,13.217059135437012,126.21875,234881024
+13,mlp.down_proj,4,64,8,128,15.481522560119629,119.109375,234881024
+13,mlp.down_proj,4,128,8,128,17.4088191986084,115.5546875,234881024
+13,mlp.down_proj,8,32,8,128,0.8071901202201843,238.21875,234881024
+13,mlp.down_proj,8,64,8,128,0.9440082311630249,231.109375,234881024
+13,mlp.down_proj,8,128,8,128,1.0594356060028076,227.5546875,234881024
+14,self_attn.q_proj,2,32,8,128,49.55269241333008,20.0625,67108864
+14,self_attn.q_proj,2,64,8,128,58.64954376220703,18.03125,67108864
+14,self_attn.q_proj,2,128,8,128,67.902587890625,17.015625,67108864
+14,self_attn.q_proj,3,32,8,128,21.257667541503906,28.0625,67108864
+14,self_attn.q_proj,3,64,8,128,25.224964141845703,26.03125,67108864
+14,self_attn.q_proj,3,128,8,128,29.407995223999023,25.015625,67108864
+14,self_attn.q_proj,4,32,8,128,9.91373348236084,36.0625,67108864
+14,self_attn.q_proj,4,64,8,128,11.775971412658691,34.03125,67108864
+14,self_attn.q_proj,4,128,8,128,13.71927261352539,33.015625,67108864
+14,self_attn.q_proj,8,32,8,128,0.6024156212806702,68.0625,67108864
+14,self_attn.q_proj,8,64,8,128,0.7157009243965149,66.03125,67108864
+14,self_attn.q_proj,8,128,8,128,0.830431342124939,65.015625,67108864
+14,self_attn.v_proj,2,32,8,128,10.771615982055664,2.5078125,8388608
+14,self_attn.v_proj,2,64,8,128,12.641671180725098,2.25390625,8388608
+14,self_attn.v_proj,2,128,8,128,14.363245964050293,2.126953125,8388608
+14,self_attn.v_proj,3,32,8,128,4.607372760772705,3.5078125,8388608
+14,self_attn.v_proj,3,64,8,128,5.408575057983398,3.25390625,8388608
+14,self_attn.v_proj,3,128,8,128,6.141956806182861,3.126953125,8388608
+14,self_attn.v_proj,4,32,8,128,2.1491665840148926,4.5078125,8388608
+14,self_attn.v_proj,4,64,8,128,2.5239756107330322,4.25390625,8388608
+14,self_attn.v_proj,4,128,8,128,2.8668816089630127,4.126953125,8388608
+14,self_attn.v_proj,8,32,8,128,0.13016216456890106,8.5078125,8388608
+14,self_attn.v_proj,8,64,8,128,0.15261071920394897,8.25390625,8388608
+14,self_attn.v_proj,8,128,8,128,0.17280392348766327,8.126953125,8388608
+14,self_attn.k_proj,2,32,8,128,30.60262680053711,2.5078125,8388608
+14,self_attn.k_proj,2,64,8,128,36.77401351928711,2.25390625,8388608
+14,self_attn.k_proj,2,128,8,128,41.902408599853516,2.126953125,8388608
+14,self_attn.k_proj,3,32,8,128,13.323531150817871,3.5078125,8388608
+14,self_attn.k_proj,3,64,8,128,16.3106689453125,3.25390625,8388608
+14,self_attn.k_proj,3,128,8,128,19.221939086914062,3.126953125,8388608
+14,self_attn.k_proj,4,32,8,128,6.221292495727539,4.5078125,8388608
+14,self_attn.k_proj,4,64,8,128,7.614718914031982,4.25390625,8388608
+14,self_attn.k_proj,4,128,8,128,8.983316421508789,4.126953125,8388608
+14,self_attn.k_proj,8,32,8,128,0.3762752413749695,8.5078125,8388608
+14,self_attn.k_proj,8,64,8,128,0.4601679742336273,8.25390625,8388608
+14,self_attn.k_proj,8,128,8,128,0.5410057306289673,8.126953125,8388608
+14,self_attn.o_proj,2,32,8,128,32.464168548583984,20.0625,67108864
+14,self_attn.o_proj,2,64,8,128,37.81532287597656,18.03125,67108864
+14,self_attn.o_proj,2,128,8,128,42.7044563293457,17.015625,67108864
+14,self_attn.o_proj,3,32,8,128,13.870448112487793,28.0625,67108864
+14,self_attn.o_proj,3,64,8,128,16.1888427734375,26.03125,67108864
+14,self_attn.o_proj,3,128,8,128,18.266937255859375,25.015625,67108864
+14,self_attn.o_proj,4,32,8,128,6.469182014465332,36.0625,67108864
+14,self_attn.o_proj,4,64,8,128,7.553121089935303,34.03125,67108864
+14,self_attn.o_proj,4,128,8,128,8.52535343170166,33.015625,67108864
+14,self_attn.o_proj,8,32,8,128,0.3933440148830414,68.0625,67108864
+14,self_attn.o_proj,8,64,8,128,0.4583703577518463,66.03125,67108864
+14,self_attn.o_proj,8,128,8,128,0.515684962272644,65.015625,67108864
+14,mlp.gate_proj,2,32,8,128,69.46522521972656,70.21875,234881024
+14,mlp.gate_proj,2,64,8,128,81.52556610107422,63.109375,234881024
+14,mlp.gate_proj,2,128,8,128,92.62480926513672,59.5546875,234881024
+14,mlp.gate_proj,3,32,8,128,29.790237426757812,98.21875,234881024
+14,mlp.gate_proj,3,64,8,128,34.86628341674805,91.109375,234881024
+14,mlp.gate_proj,3,128,8,128,39.59559631347656,87.5546875,234881024
+14,mlp.gate_proj,4,32,8,128,13.867393493652344,126.21875,234881024
+14,mlp.gate_proj,4,64,8,128,16.312068939208984,119.109375,234881024
+14,mlp.gate_proj,4,128,8,128,18.489431381225586,115.5546875,234881024
+14,mlp.gate_proj,8,32,8,128,0.8463444113731384,238.21875,234881024
+14,mlp.gate_proj,8,64,8,128,0.998119592666626,231.109375,234881024
+14,mlp.gate_proj,8,128,8,128,1.123921513557434,227.5546875,234881024
+14,mlp.up_proj,2,32,8,128,66.89312744140625,70.21875,234881024
+14,mlp.up_proj,2,64,8,128,78.0659408569336,63.109375,234881024
+14,mlp.up_proj,2,128,8,128,88.52325439453125,59.5546875,234881024
+14,mlp.up_proj,3,32,8,128,28.603185653686523,98.21875,234881024
+14,mlp.up_proj,3,64,8,128,33.44575500488281,91.109375,234881024
+14,mlp.up_proj,3,128,8,128,37.79161071777344,87.5546875,234881024
+14,mlp.up_proj,4,32,8,128,13.320771217346191,126.21875,234881024
+14,mlp.up_proj,4,64,8,128,15.64788818359375,119.109375,234881024
+14,mlp.up_proj,4,128,8,128,17.647743225097656,115.5546875,234881024
+14,mlp.up_proj,8,32,8,128,0.8134067058563232,238.21875,234881024
+14,mlp.up_proj,8,64,8,128,0.9541072249412537,231.109375,234881024
+14,mlp.up_proj,8,128,8,128,1.0740185976028442,227.5546875,234881024
+14,mlp.down_proj,2,32,8,128,66.69234466552734,70.21875,234881024
+14,mlp.down_proj,2,64,8,128,77.62615966796875,63.109375,234881024
+14,mlp.down_proj,2,128,8,128,87.81917572021484,59.5546875,234881024
+14,mlp.down_proj,3,32,8,128,28.506433486938477,98.21875,234881024
+14,mlp.down_proj,3,64,8,128,33.254634857177734,91.109375,234881024
+14,mlp.down_proj,3,128,8,128,37.46749496459961,87.5546875,234881024
+14,mlp.down_proj,4,32,8,128,13.275007247924805,126.21875,234881024
+14,mlp.down_proj,4,64,8,128,15.556656837463379,119.109375,234881024
+14,mlp.down_proj,4,128,8,128,17.48824119567871,115.5546875,234881024
+14,mlp.down_proj,8,32,8,128,0.8106511235237122,238.21875,234881024
+14,mlp.down_proj,8,64,8,128,0.948485255241394,231.109375,234881024
+14,mlp.down_proj,8,128,8,128,1.0642961263656616,227.5546875,234881024
+15,self_attn.q_proj,2,32,8,128,49.02880096435547,20.0625,67108864
+15,self_attn.q_proj,2,64,8,128,57.864864349365234,18.03125,67108864
+15,self_attn.q_proj,2,128,8,128,66.77290344238281,17.015625,67108864
+15,self_attn.q_proj,3,32,8,128,21.002965927124023,28.0625,67108864
+15,self_attn.q_proj,3,64,8,128,24.83354949951172,26.03125,67108864
+15,self_attn.q_proj,3,128,8,128,28.762001037597656,25.015625,67108864
+15,self_attn.q_proj,4,32,8,128,9.794624328613281,36.0625,67108864
+15,self_attn.q_proj,4,64,8,128,11.592740058898926,34.03125,67108864
+15,self_attn.q_proj,4,128,8,128,13.415684700012207,33.015625,67108864
+15,self_attn.q_proj,8,32,8,128,0.5952129364013672,68.0625,67108864
+15,self_attn.q_proj,8,64,8,128,0.7045966982841492,66.03125,67108864
+15,self_attn.q_proj,8,128,8,128,0.8124611377716064,65.015625,67108864
+15,self_attn.v_proj,2,32,8,128,11.293058395385742,2.5078125,8388608
+15,self_attn.v_proj,2,64,8,128,13.216156959533691,2.25390625,8388608
+15,self_attn.v_proj,2,128,8,128,14.970909118652344,2.126953125,8388608
+15,self_attn.v_proj,3,32,8,128,4.829313278198242,3.5078125,8388608
+15,self_attn.v_proj,3,64,8,128,5.65459680557251,3.25390625,8388608
+15,self_attn.v_proj,3,128,8,128,6.402256488800049,3.126953125,8388608
+15,self_attn.v_proj,4,32,8,128,2.2535667419433594,4.5078125,8388608
+15,self_attn.v_proj,4,64,8,128,2.6384201049804688,4.25390625,8388608
+15,self_attn.v_proj,4,128,8,128,2.990255355834961,4.126953125,8388608
+15,self_attn.v_proj,8,32,8,128,0.1363966464996338,8.5078125,8388608
+15,self_attn.v_proj,8,64,8,128,0.1595510095357895,8.25390625,8388608
+15,self_attn.v_proj,8,128,8,128,0.18013429641723633,8.126953125,8388608
+15,self_attn.k_proj,2,32,8,128,30.47816276550293,2.5078125,8388608
+15,self_attn.k_proj,2,64,8,128,36.61341094970703,2.25390625,8388608
+15,self_attn.k_proj,2,128,8,128,41.760231018066406,2.126953125,8388608
+15,self_attn.k_proj,3,32,8,128,13.20106029510498,3.5078125,8388608
+15,self_attn.k_proj,3,64,8,128,16.070430755615234,3.25390625,8388608
+15,self_attn.k_proj,3,128,8,128,18.823087692260742,3.126953125,8388608
+15,self_attn.k_proj,4,32,8,128,6.158248424530029,4.5078125,8388608
+15,self_attn.k_proj,4,64,8,128,7.501287937164307,4.25390625,8388608
+15,self_attn.k_proj,4,128,8,128,8.787373542785645,4.126953125,8388608
+15,self_attn.k_proj,8,32,8,128,0.37275221943855286,8.5078125,8388608
+15,self_attn.k_proj,8,64,8,128,0.4534023106098175,8.25390625,8388608
+15,self_attn.k_proj,8,128,8,128,0.5293384194374084,8.126953125,8388608
+15,self_attn.o_proj,2,32,8,128,33.02671432495117,20.0625,67108864
+15,self_attn.o_proj,2,64,8,128,38.4486083984375,18.03125,67108864
+15,self_attn.o_proj,2,128,8,128,43.374549865722656,17.015625,67108864
+15,self_attn.o_proj,3,32,8,128,14.113816261291504,28.0625,67108864
+15,self_attn.o_proj,3,64,8,128,16.454504013061523,26.03125,67108864
+15,self_attn.o_proj,3,128,8,128,18.550046920776367,25.015625,67108864
+15,self_attn.o_proj,4,32,8,128,6.582306861877441,36.0625,67108864
+15,self_attn.o_proj,4,64,8,128,7.680431365966797,34.03125,67108864
+15,self_attn.o_proj,4,128,8,128,8.65785026550293,33.015625,67108864
+15,self_attn.o_proj,8,32,8,128,0.4002125859260559,68.0625,67108864
+15,self_attn.o_proj,8,64,8,128,0.46597400307655334,66.03125,67108864
+15,self_attn.o_proj,8,128,8,128,0.5237253904342651,65.015625,67108864
+15,mlp.gate_proj,2,32,8,128,70.5521240234375,70.21875,234881024
+15,mlp.gate_proj,2,64,8,128,82.91805267333984,63.109375,234881024
+15,mlp.gate_proj,2,128,8,128,94.15438842773438,59.5546875,234881024
+15,mlp.gate_proj,3,32,8,128,30.28624725341797,98.21875,234881024
+15,mlp.gate_proj,3,64,8,128,35.42639923095703,91.109375,234881024
+15,mlp.gate_proj,3,128,8,128,40.30752944946289,87.5546875,234881024
+15,mlp.gate_proj,4,32,8,128,14.095834732055664,126.21875,234881024
+15,mlp.gate_proj,4,64,8,128,16.56496810913086,119.109375,234881024
+15,mlp.gate_proj,4,128,8,128,18.803739547729492,115.5546875,234881024
+15,mlp.gate_proj,8,32,8,128,0.8603140711784363,238.21875,234881024
+15,mlp.gate_proj,8,64,8,128,1.0132313966751099,231.109375,234881024
+15,mlp.gate_proj,8,128,8,128,1.1434966325759888,227.5546875,234881024
+15,mlp.up_proj,2,32,8,128,67.12837982177734,70.21875,234881024
+15,mlp.up_proj,2,64,8,128,78.33838653564453,63.109375,234881024
+15,mlp.up_proj,2,128,8,128,88.82623291015625,59.5546875,234881024
+15,mlp.up_proj,3,32,8,128,28.712894439697266,98.21875,234881024
+15,mlp.up_proj,3,64,8,128,33.54962921142578,91.109375,234881024
+15,mlp.up_proj,3,128,8,128,37.903011322021484,87.5546875,234881024
+15,mlp.up_proj,4,32,8,128,13.371227264404297,126.21875,234881024
+15,mlp.up_proj,4,64,8,128,15.70266056060791,119.109375,234881024
+15,mlp.up_proj,4,128,8,128,17.695453643798828,115.5546875,234881024
+15,mlp.up_proj,8,32,8,128,0.816433310508728,238.21875,234881024
+15,mlp.up_proj,8,64,8,128,0.957400918006897,231.109375,234881024
+15,mlp.up_proj,8,128,8,128,1.0769476890563965,227.5546875,234881024
+15,mlp.down_proj,2,32,8,128,66.71514129638672,70.21875,234881024
+15,mlp.down_proj,2,64,8,128,77.63180541992188,63.109375,234881024
+15,mlp.down_proj,2,128,8,128,87.75765228271484,59.5546875,234881024
+15,mlp.down_proj,3,32,8,128,28.514732360839844,98.21875,234881024
+15,mlp.down_proj,3,64,8,128,33.25006103515625,91.109375,234881024
+15,mlp.down_proj,3,128,8,128,37.43309020996094,87.5546875,234881024
+15,mlp.down_proj,4,32,8,128,13.280423164367676,126.21875,234881024
+15,mlp.down_proj,4,64,8,128,15.552584648132324,119.109375,234881024
+15,mlp.down_proj,4,128,8,128,17.480363845825195,115.5546875,234881024
+15,mlp.down_proj,8,32,8,128,0.8109672665596008,238.21875,234881024
+15,mlp.down_proj,8,64,8,128,0.9483130574226379,231.109375,234881024
+15,mlp.down_proj,8,128,8,128,1.0635464191436768,227.5546875,234881024
+16,self_attn.q_proj,2,32,8,128,50.310821533203125,20.0625,67108864
+16,self_attn.q_proj,2,64,8,128,59.42587661743164,18.03125,67108864
+16,self_attn.q_proj,2,128,8,128,68.41814422607422,17.015625,67108864
+16,self_attn.q_proj,3,32,8,128,21.558940887451172,28.0625,67108864
+16,self_attn.q_proj,3,64,8,128,25.499467849731445,26.03125,67108864
+16,self_attn.q_proj,3,128,8,128,29.49885368347168,25.015625,67108864
+16,self_attn.q_proj,4,32,8,128,10.05441665649414,36.0625,67108864
+16,self_attn.q_proj,4,64,8,128,11.89987850189209,34.03125,67108864
+16,self_attn.q_proj,4,128,8,128,13.762334823608398,33.015625,67108864
+16,self_attn.q_proj,8,32,8,128,0.6111400723457336,68.0625,67108864
+16,self_attn.q_proj,8,64,8,128,0.7232654094696045,66.03125,67108864
+16,self_attn.q_proj,8,128,8,128,0.8330042958259583,65.015625,67108864
+16,self_attn.v_proj,2,32,8,128,11.12975025177002,2.5078125,8388608
+16,self_attn.v_proj,2,64,8,128,12.965991973876953,2.25390625,8388608
+16,self_attn.v_proj,2,128,8,128,14.626153945922852,2.126953125,8388608
+16,self_attn.v_proj,3,32,8,128,4.758610725402832,3.5078125,8388608
+16,self_attn.v_proj,3,64,8,128,5.545928955078125,3.25390625,8388608
+16,self_attn.v_proj,3,128,8,128,6.255888938903809,3.126953125,8388608
+16,self_attn.v_proj,4,32,8,128,2.219667673110962,4.5078125,8388608
+16,self_attn.v_proj,4,64,8,128,2.5888612270355225,4.25390625,8388608
+16,self_attn.v_proj,4,128,8,128,2.9207077026367188,4.126953125,8388608
+16,self_attn.v_proj,8,32,8,128,0.13442517817020416,8.5078125,8388608
+16,self_attn.v_proj,8,64,8,128,0.15657876431941986,8.25390625,8388608
+16,self_attn.v_proj,8,128,8,128,0.17596407234668732,8.126953125,8388608
+16,self_attn.k_proj,2,32,8,128,29.379011154174805,2.5078125,8388608
+16,self_attn.k_proj,2,64,8,128,35.0977783203125,2.25390625,8388608
+16,self_attn.k_proj,2,128,8,128,40.17096710205078,2.126953125,8388608
+16,self_attn.k_proj,3,32,8,128,12.5999116897583,3.5078125,8388608
+16,self_attn.k_proj,3,64,8,128,15.13239574432373,3.25390625,8388608
+16,self_attn.k_proj,3,128,8,128,17.53217315673828,3.126953125,8388608
+16,self_attn.k_proj,4,32,8,128,5.882595539093018,4.5078125,8388608
+16,self_attn.k_proj,4,64,8,128,7.063272476196289,4.25390625,8388608
+16,self_attn.k_proj,4,128,8,128,8.191452980041504,4.126953125,8388608
+16,self_attn.k_proj,8,32,8,128,0.35603103041648865,8.5078125,8388608
+16,self_attn.k_proj,8,64,8,128,0.4270090162754059,8.25390625,8388608
+16,self_attn.k_proj,8,128,8,128,0.4932229220867157,8.126953125,8388608
+16,self_attn.o_proj,2,32,8,128,33.346073150634766,20.0625,67108864
+16,self_attn.o_proj,2,64,8,128,38.88410186767578,18.03125,67108864
+16,self_attn.o_proj,2,128,8,128,43.9342155456543,17.015625,67108864
+16,self_attn.o_proj,3,32,8,128,14.251571655273438,28.0625,67108864
+16,self_attn.o_proj,3,64,8,128,16.638931274414062,26.03125,67108864
+16,self_attn.o_proj,3,128,8,128,18.811643600463867,25.015625,67108864
+16,self_attn.o_proj,4,32,8,128,6.648558139801025,36.0625,67108864
+16,self_attn.o_proj,4,64,8,128,7.766978740692139,34.03125,67108864
+16,self_attn.o_proj,4,128,8,128,8.777078628540039,33.015625,67108864
+16,self_attn.o_proj,8,32,8,128,0.4042188823223114,68.0625,67108864
+16,self_attn.o_proj,8,64,8,128,0.4712960720062256,66.03125,67108864
+16,self_attn.o_proj,8,128,8,128,0.530884861946106,65.015625,67108864
+16,mlp.gate_proj,2,32,8,128,71.30467987060547,70.21875,234881024
+16,mlp.gate_proj,2,64,8,128,83.95096588134766,63.109375,234881024
+16,mlp.gate_proj,2,128,8,128,95.39571380615234,59.5546875,234881024
+16,mlp.gate_proj,3,32,8,128,30.632850646972656,98.21875,234881024
+16,mlp.gate_proj,3,64,8,128,35.864341735839844,91.109375,234881024
+16,mlp.gate_proj,3,128,8,128,40.901180267333984,87.5546875,234881024
+16,mlp.gate_proj,4,32,8,128,14.256300926208496,126.21875,234881024
+16,mlp.gate_proj,4,64,8,128,16.762351989746094,119.109375,234881024
+16,mlp.gate_proj,4,128,8,128,19.08058738708496,115.5546875,234881024
+16,mlp.gate_proj,8,32,8,128,0.8700228929519653,238.21875,234881024
+16,mlp.gate_proj,8,64,8,128,1.024910569190979,231.109375,234881024
+16,mlp.gate_proj,8,128,8,128,1.1602004766464233,227.5546875,234881024
+16,mlp.up_proj,2,32,8,128,67.5016098022461,70.21875,234881024
+16,mlp.up_proj,2,64,8,128,78.80122375488281,63.109375,234881024
+16,mlp.up_proj,2,128,8,128,89.3873062133789,59.5546875,234881024
+16,mlp.up_proj,3,32,8,128,28.88479232788086,98.21875,234881024
+16,mlp.up_proj,3,64,8,128,33.74320983886719,91.109375,234881024
+16,mlp.up_proj,3,128,8,128,38.12447738647461,87.5546875,234881024
+16,mlp.up_proj,4,32,8,128,13.450243949890137,126.21875,234881024
+16,mlp.up_proj,4,64,8,128,15.80029582977295,119.109375,234881024
+16,mlp.up_proj,4,128,8,128,17.80415153503418,115.5546875,234881024
+16,mlp.up_proj,8,32,8,128,0.8212133646011353,238.21875,234881024
+16,mlp.up_proj,8,64,8,128,0.9636436700820923,231.109375,234881024
+16,mlp.up_proj,8,128,8,128,1.0836187601089478,227.5546875,234881024
+16,mlp.down_proj,2,32,8,128,66.95897674560547,70.21875,234881024
+16,mlp.down_proj,2,64,8,128,77.96288299560547,63.109375,234881024
+16,mlp.down_proj,2,128,8,128,88.2173080444336,59.5546875,234881024
+16,mlp.down_proj,3,32,8,128,28.63302230834961,98.21875,234881024
+16,mlp.down_proj,3,64,8,128,33.39550018310547,91.109375,234881024
+16,mlp.down_proj,3,128,8,128,37.63404846191406,87.5546875,234881024
+16,mlp.down_proj,4,32,8,128,13.334043502807617,126.21875,234881024
+16,mlp.down_proj,4,64,8,128,15.627598762512207,119.109375,234881024
+16,mlp.down_proj,4,128,8,128,17.571325302124023,115.5546875,234881024
+16,mlp.down_proj,8,32,8,128,0.8142887353897095,238.21875,234881024
+16,mlp.down_proj,8,64,8,128,0.9530476927757263,231.109375,234881024
+16,mlp.down_proj,8,128,8,128,1.069318175315857,227.5546875,234881024
+17,self_attn.q_proj,2,32,8,128,51.631832122802734,20.0625,67108864
+17,self_attn.q_proj,2,64,8,128,61.066680908203125,18.03125,67108864
+17,self_attn.q_proj,2,128,8,128,70.4135513305664,17.015625,67108864
+17,self_attn.q_proj,3,32,8,128,22.118371963500977,28.0625,67108864
+17,self_attn.q_proj,3,64,8,128,26.184022903442383,26.03125,67108864
+17,self_attn.q_proj,3,128,8,128,30.31125831604004,25.015625,67108864
+17,self_attn.q_proj,4,32,8,128,10.315722465515137,36.0625,67108864
+17,self_attn.q_proj,4,64,8,128,12.215842247009277,34.03125,67108864
+17,self_attn.q_proj,4,128,8,128,14.138089179992676,33.015625,67108864
+17,self_attn.q_proj,8,32,8,128,0.6268228888511658,68.0625,67108864
+17,self_attn.q_proj,8,64,8,128,0.7425990104675293,66.03125,67108864
+17,self_attn.q_proj,8,128,8,128,0.8560142517089844,65.015625,67108864
+17,self_attn.v_proj,2,32,8,128,10.17674732208252,2.5078125,8388608
+17,self_attn.v_proj,2,64,8,128,11.891242027282715,2.25390625,8388608
+17,self_attn.v_proj,2,128,8,128,13.456775665283203,2.126953125,8388608
+17,self_attn.v_proj,3,32,8,128,4.351374626159668,3.5078125,8388608
+17,self_attn.v_proj,3,64,8,128,5.089025497436523,3.25390625,8388608
+17,self_attn.v_proj,3,128,8,128,5.756843566894531,3.126953125,8388608
+17,self_attn.v_proj,4,32,8,128,2.030363082885742,4.5078125,8388608
+17,self_attn.v_proj,4,64,8,128,2.374021053314209,4.25390625,8388608
+17,self_attn.v_proj,4,128,8,128,2.687563419342041,4.126953125,8388608
+17,self_attn.v_proj,8,32,8,128,0.12294828146696091,8.5078125,8388608
+17,self_attn.v_proj,8,64,8,128,0.1435958445072174,8.25390625,8388608
+17,self_attn.v_proj,8,128,8,128,0.16188985109329224,8.126953125,8388608
+17,self_attn.k_proj,2,32,8,128,31.90034294128418,2.5078125,8388608
+17,self_attn.k_proj,2,64,8,128,38.152076721191406,2.25390625,8388608
+17,self_attn.k_proj,2,128,8,128,43.609066009521484,2.126953125,8388608
+17,self_attn.k_proj,3,32,8,128,13.7058687210083,3.5078125,8388608
+17,self_attn.k_proj,3,64,8,128,16.500383377075195,3.25390625,8388608
+17,self_attn.k_proj,3,128,8,128,19.14496612548828,3.126953125,8388608
+17,self_attn.k_proj,4,32,8,128,6.395565986633301,4.5078125,8388608
+17,self_attn.k_proj,4,64,8,128,7.699107646942139,4.25390625,8388608
+17,self_attn.k_proj,4,128,8,128,8.93569278717041,4.126953125,8388608
+17,self_attn.k_proj,8,32,8,128,0.3868349492549896,8.5078125,8388608
+17,self_attn.k_proj,8,64,8,128,0.46538108587265015,8.25390625,8388608
+17,self_attn.k_proj,8,128,8,128,0.5385136604309082,8.126953125,8388608
+17,self_attn.o_proj,2,32,8,128,32.61872863769531,20.0625,67108864
+17,self_attn.o_proj,2,64,8,128,38.004634857177734,18.03125,67108864
+17,self_attn.o_proj,2,128,8,128,42.92573547363281,17.015625,67108864
+17,self_attn.o_proj,3,32,8,128,13.938030242919922,28.0625,67108864
+17,self_attn.o_proj,3,64,8,128,16.272544860839844,26.03125,67108864
+17,self_attn.o_proj,3,128,8,128,18.36614227294922,25.015625,67108864
+17,self_attn.o_proj,4,32,8,128,6.502345085144043,36.0625,67108864
+17,self_attn.o_proj,4,64,8,128,7.592833518981934,34.03125,67108864
+17,self_attn.o_proj,4,128,8,128,8.572738647460938,33.015625,67108864
+17,self_attn.o_proj,8,32,8,128,0.39533862471580505,68.0625,67108864
+17,self_attn.o_proj,8,64,8,128,0.4607778489589691,66.03125,67108864
+17,self_attn.o_proj,8,128,8,128,0.5186139941215515,65.015625,67108864
+17,mlp.gate_proj,2,32,8,128,71.27976989746094,70.21875,234881024
+17,mlp.gate_proj,2,64,8,128,83.81795501708984,63.109375,234881024
+17,mlp.gate_proj,2,128,8,128,95.13545227050781,59.5546875,234881024
+17,mlp.gate_proj,3,32,8,128,30.616682052612305,98.21875,234881024
+17,mlp.gate_proj,3,64,8,128,35.801761627197266,91.109375,234881024
+17,mlp.gate_proj,3,128,8,128,40.76245880126953,87.5546875,234881024
+17,mlp.gate_proj,4,32,8,128,14.248665809631348,126.21875,234881024
+17,mlp.gate_proj,4,64,8,128,16.732898712158203,119.109375,234881024
+17,mlp.gate_proj,4,128,8,128,19.015371322631836,115.5546875,234881024
+17,mlp.gate_proj,8,32,8,128,0.8695554733276367,238.21875,234881024
+17,mlp.gate_proj,8,64,8,128,1.023230791091919,231.109375,234881024
+17,mlp.gate_proj,8,128,8,128,1.1565591096878052,227.5546875,234881024
+17,mlp.up_proj,2,32,8,128,67.53092193603516,70.21875,234881024
+17,mlp.up_proj,2,64,8,128,78.78026580810547,63.109375,234881024
+17,mlp.up_proj,2,128,8,128,89.25162506103516,59.5546875,234881024
+17,mlp.up_proj,3,32,8,128,28.89960479736328,98.21875,234881024
+17,mlp.up_proj,3,64,8,128,33.735260009765625,91.109375,234881024
+17,mlp.up_proj,3,128,8,128,38.09262466430664,87.5546875,234881024
+17,mlp.up_proj,4,32,8,128,13.457110404968262,126.21875,234881024
+17,mlp.up_proj,4,64,8,128,15.794622421264648,119.109375,234881024
+17,mlp.up_proj,4,128,8,128,17.787395477294922,115.5546875,234881024
+17,mlp.up_proj,8,32,8,128,0.821583092212677,238.21875,234881024
+17,mlp.up_proj,8,64,8,128,0.9632319808006287,231.109375,234881024
+17,mlp.up_proj,8,128,8,128,1.0822772979736328,227.5546875,234881024
+17,mlp.down_proj,2,32,8,128,67.25837707519531,70.21875,234881024
+17,mlp.down_proj,2,64,8,128,78.31805419921875,63.109375,234881024
+17,mlp.down_proj,2,128,8,128,88.62403106689453,59.5546875,234881024
+17,mlp.down_proj,3,32,8,128,28.771207809448242,98.21875,234881024
+17,mlp.down_proj,3,64,8,128,33.53458786010742,91.109375,234881024
+17,mlp.down_proj,3,128,8,128,37.794132232666016,87.5546875,234881024
+17,mlp.down_proj,4,32,8,128,13.39718246459961,126.21875,234881024
+17,mlp.down_proj,4,64,8,128,15.699185371398926,119.109375,234881024
+17,mlp.down_proj,4,128,8,128,17.645389556884766,115.5546875,234881024
+17,mlp.down_proj,8,32,8,128,0.8180246949195862,238.21875,234881024
+17,mlp.down_proj,8,64,8,128,0.9574726223945618,231.109375,234881024
+17,mlp.down_proj,8,128,8,128,1.0738341808319092,227.5546875,234881024
+18,self_attn.q_proj,2,32,8,128,52.59255599975586,20.0625,67108864
+18,self_attn.q_proj,2,64,8,128,62.40863800048828,18.03125,67108864
+18,self_attn.q_proj,2,128,8,128,72.23614501953125,17.015625,67108864
+18,self_attn.q_proj,3,32,8,128,22.555204391479492,28.0625,67108864
+18,self_attn.q_proj,3,64,8,128,26.788002014160156,26.03125,67108864
+18,self_attn.q_proj,3,128,8,128,31.23375129699707,25.015625,67108864
+18,self_attn.q_proj,4,32,8,128,10.517943382263184,36.0625,67108864
+18,self_attn.q_proj,4,64,8,128,12.502656936645508,34.03125,67108864
+18,self_attn.q_proj,4,128,8,128,14.572912216186523,33.015625,67108864
+18,self_attn.q_proj,8,32,8,128,0.6392547488212585,68.0625,67108864
+18,self_attn.q_proj,8,64,8,128,0.7597793340682983,66.03125,67108864
+18,self_attn.q_proj,8,128,8,128,0.8818164467811584,65.015625,67108864
+18,self_attn.v_proj,2,32,8,128,10.399649620056152,2.5078125,8388608
+18,self_attn.v_proj,2,64,8,128,12.178648948669434,2.25390625,8388608
+18,self_attn.v_proj,2,128,8,128,13.80640983581543,2.126953125,8388608
+18,self_attn.v_proj,3,32,8,128,4.446982383728027,3.5078125,8388608
+18,self_attn.v_proj,3,64,8,128,5.213193893432617,3.25390625,8388608
+18,self_attn.v_proj,3,128,8,128,5.909554958343506,3.126953125,8388608
+18,self_attn.v_proj,4,32,8,128,2.0744380950927734,4.5078125,8388608
+18,self_attn.v_proj,4,64,8,128,2.4318230152130127,4.25390625,8388608
+18,self_attn.v_proj,4,128,8,128,2.757657051086426,4.126953125,8388608
+18,self_attn.v_proj,8,32,8,128,0.12565304338932037,8.5078125,8388608
+18,self_attn.v_proj,8,64,8,128,0.14706619083881378,8.25390625,8388608
+18,self_attn.v_proj,8,128,8,128,0.16619838774204254,8.126953125,8388608
+18,self_attn.k_proj,2,32,8,128,30.838287353515625,2.5078125,8388608
+18,self_attn.k_proj,2,64,8,128,36.830562591552734,2.25390625,8388608
+18,self_attn.k_proj,2,128,8,128,42.20684051513672,2.126953125,8388608
+18,self_attn.k_proj,3,32,8,128,13.21714973449707,3.5078125,8388608
+18,self_attn.k_proj,3,64,8,128,15.844949722290039,3.25390625,8388608
+18,self_attn.k_proj,3,128,8,128,18.327537536621094,3.126953125,8388608
+18,self_attn.k_proj,4,32,8,128,6.166986465454102,4.5078125,8388608
+18,self_attn.k_proj,4,64,8,128,7.394495964050293,4.25390625,8388608
+18,self_attn.k_proj,4,128,8,128,8.554437637329102,4.126953125,8388608
+18,self_attn.k_proj,8,32,8,128,0.37323716282844543,8.5078125,8388608
+18,self_attn.k_proj,8,64,8,128,0.44686952233314514,8.25390625,8388608
+18,self_attn.k_proj,8,128,8,128,0.5152238011360168,8.126953125,8388608
+18,self_attn.o_proj,2,32,8,128,32.53702926635742,20.0625,67108864
+18,self_attn.o_proj,2,64,8,128,37.874359130859375,18.03125,67108864
+18,self_attn.o_proj,2,128,8,128,42.70774459838867,17.015625,67108864
+18,self_attn.o_proj,3,32,8,128,13.903160095214844,28.0625,67108864
+18,self_attn.o_proj,3,64,8,128,16.215290069580078,26.03125,67108864
+18,self_attn.o_proj,3,128,8,128,18.267932891845703,25.015625,67108864
+18,self_attn.o_proj,4,32,8,128,6.485110759735107,36.0625,67108864
+18,self_attn.o_proj,4,64,8,128,7.5635552406311035,34.03125,67108864
+18,self_attn.o_proj,4,128,8,128,8.530006408691406,33.015625,67108864
+18,self_attn.o_proj,8,32,8,128,0.3943660259246826,68.0625,67108864
+18,self_attn.o_proj,8,64,8,128,0.4591262936592102,66.03125,67108864
+18,self_attn.o_proj,8,128,8,128,0.5157896280288696,65.015625,67108864
+18,mlp.gate_proj,2,32,8,128,72.03632354736328,70.21875,234881024
+18,mlp.gate_proj,2,64,8,128,84.69075775146484,63.109375,234881024
+18,mlp.gate_proj,2,128,8,128,96.06310272216797,59.5546875,234881024
+18,mlp.gate_proj,3,32,8,128,30.957969665527344,98.21875,234881024
+18,mlp.gate_proj,3,64,8,128,36.162925720214844,91.109375,234881024
+18,mlp.gate_proj,3,128,8,128,41.15971374511719,87.5546875,234881024
+18,mlp.gate_proj,4,32,8,128,14.403627395629883,126.21875,234881024
+18,mlp.gate_proj,4,64,8,128,16.895828247070312,119.109375,234881024
+18,mlp.gate_proj,4,128,8,128,19.198261260986328,115.5546875,234881024
+18,mlp.gate_proj,8,32,8,128,0.8791666030883789,238.21875,234881024
+18,mlp.gate_proj,8,64,8,128,1.03290593624115,231.109375,234881024
+18,mlp.gate_proj,8,128,8,128,1.1677472591400146,227.5546875,234881024
+18,mlp.up_proj,2,32,8,128,67.44025421142578,70.21875,234881024
+18,mlp.up_proj,2,64,8,128,78.59921264648438,63.109375,234881024
+18,mlp.up_proj,2,128,8,128,89.00941467285156,59.5546875,234881024
+18,mlp.up_proj,3,32,8,128,28.855478286743164,98.21875,234881024
+18,mlp.up_proj,3,64,8,128,33.6644287109375,91.109375,234881024
+18,mlp.up_proj,3,128,8,128,37.98163986206055,87.5546875,234881024
+18,mlp.up_proj,4,32,8,128,13.43778133392334,126.21875,234881024
+18,mlp.up_proj,4,64,8,128,15.75947380065918,119.109375,234881024
+18,mlp.up_proj,4,128,8,128,17.731069564819336,115.5546875,234881024
+18,mlp.up_proj,8,32,8,128,0.8204500079154968,238.21875,234881024
+18,mlp.up_proj,8,64,8,128,0.9611146450042725,231.109375,234881024
+18,mlp.up_proj,8,128,8,128,1.0792654752731323,227.5546875,234881024
+18,mlp.down_proj,2,32,8,128,67.35299682617188,70.21875,234881024
+18,mlp.down_proj,2,64,8,128,78.43480682373047,63.109375,234881024
+18,mlp.down_proj,2,128,8,128,88.74095153808594,59.5546875,234881024
+18,mlp.down_proj,3,32,8,128,28.818437576293945,98.21875,234881024
+18,mlp.down_proj,3,64,8,128,33.593040466308594,91.109375,234881024
+18,mlp.down_proj,3,128,8,128,37.879112243652344,87.5546875,234881024
+18,mlp.down_proj,4,32,8,128,13.419743537902832,126.21875,234881024
+18,mlp.down_proj,4,64,8,128,15.728473663330078,119.109375,234881024
+18,mlp.down_proj,4,128,8,128,17.683727264404297,115.5546875,234881024
+18,mlp.down_proj,8,32,8,128,0.8193972706794739,238.21875,234881024
+18,mlp.down_proj,8,64,8,128,0.9592167139053345,231.109375,234881024
+18,mlp.down_proj,8,128,8,128,1.0762253999710083,227.5546875,234881024
+19,self_attn.q_proj,2,32,8,128,51.91263198852539,20.0625,67108864
+19,self_attn.q_proj,2,64,8,128,61.68068313598633,18.03125,67108864
+19,self_attn.q_proj,2,128,8,128,71.31692504882812,17.015625,67108864
+19,self_attn.q_proj,3,32,8,128,22.260133743286133,28.0625,67108864
+19,self_attn.q_proj,3,64,8,128,26.477550506591797,26.03125,67108864
+19,self_attn.q_proj,3,128,8,128,30.8136043548584,25.015625,67108864
+19,self_attn.q_proj,4,32,8,128,10.381484985351562,36.0625,67108864
+19,self_attn.q_proj,4,64,8,128,12.357270240783691,34.03125,67108864
+19,self_attn.q_proj,4,128,8,128,14.37234115600586,33.015625,67108864
+19,self_attn.q_proj,8,32,8,128,0.6308198571205139,68.0625,67108864
+19,self_attn.q_proj,8,64,8,128,0.7508780360221863,66.03125,67108864
+19,self_attn.q_proj,8,128,8,128,0.8699560761451721,65.015625,67108864
+19,self_attn.v_proj,2,32,8,128,11.920948028564453,2.5078125,8388608
+19,self_attn.v_proj,2,64,8,128,13.999704360961914,2.25390625,8388608
+19,self_attn.v_proj,2,128,8,128,15.911893844604492,2.126953125,8388608
+19,self_attn.v_proj,3,32,8,128,5.097856521606445,3.5078125,8388608
+19,self_attn.v_proj,3,64,8,128,5.987442970275879,3.25390625,8388608
+19,self_attn.v_proj,3,128,8,128,6.813611030578613,3.126953125,8388608
+19,self_attn.v_proj,4,32,8,128,2.3785390853881836,4.5078125,8388608
+19,self_attn.v_proj,4,64,8,128,2.7954697608947754,4.25390625,8388608
+19,self_attn.v_proj,4,128,8,128,3.1784417629241943,4.126953125,8388608
+19,self_attn.v_proj,8,32,8,128,0.14399676024913788,8.5078125,8388608
+19,self_attn.v_proj,8,64,8,128,0.16900722682476044,8.25390625,8388608
+19,self_attn.v_proj,8,128,8,128,0.19144342839717865,8.126953125,8388608
+19,self_attn.k_proj,2,32,8,128,30.985157012939453,2.5078125,8388608
+19,self_attn.k_proj,2,64,8,128,37.013240814208984,2.25390625,8388608
+19,self_attn.k_proj,2,128,8,128,42.361236572265625,2.126953125,8388608
+19,self_attn.k_proj,3,32,8,128,13.294069290161133,3.5078125,8388608
+19,self_attn.k_proj,3,64,8,128,15.937882423400879,3.25390625,8388608
+19,self_attn.k_proj,3,128,8,128,18.418926239013672,3.126953125,8388608
+19,self_attn.k_proj,4,32,8,128,6.198727607727051,4.5078125,8388608
+19,self_attn.k_proj,4,64,8,128,7.437555313110352,4.25390625,8388608
+19,self_attn.k_proj,4,128,8,128,8.591792106628418,4.126953125,8388608
+19,self_attn.k_proj,8,32,8,128,0.3752766251564026,8.5078125,8388608
+19,self_attn.k_proj,8,64,8,128,0.44972512125968933,8.25390625,8388608
+19,self_attn.k_proj,8,128,8,128,0.5179126858711243,8.126953125,8388608
+19,self_attn.o_proj,2,32,8,128,34.50767517089844,20.0625,67108864
+19,self_attn.o_proj,2,64,8,128,40.19588851928711,18.03125,67108864
+19,self_attn.o_proj,2,128,8,128,45.33636474609375,17.015625,67108864
+19,self_attn.o_proj,3,32,8,128,14.758105278015137,28.0625,67108864
+19,self_attn.o_proj,3,64,8,128,17.193201065063477,26.03125,67108864
+19,self_attn.o_proj,3,128,8,128,19.396921157836914,25.015625,67108864
+19,self_attn.o_proj,4,32,8,128,6.882750034332275,36.0625,67108864
+19,self_attn.o_proj,4,64,8,128,8.028165817260742,34.03125,67108864
+19,self_attn.o_proj,4,128,8,128,9.0474214553833,33.015625,67108864
+19,self_attn.o_proj,8,32,8,128,0.4182819426059723,68.0625,67108864
+19,self_attn.o_proj,8,64,8,128,0.4870538115501404,66.03125,67108864
+19,self_attn.o_proj,8,128,8,128,0.5475350618362427,65.015625,67108864
+19,mlp.gate_proj,2,32,8,128,73.16341400146484,70.21875,234881024
+19,mlp.gate_proj,2,64,8,128,86.16093444824219,63.109375,234881024
+19,mlp.gate_proj,2,128,8,128,97.71698760986328,59.5546875,234881024
+19,mlp.gate_proj,3,32,8,128,31.463123321533203,98.21875,234881024
+19,mlp.gate_proj,3,64,8,128,36.77312469482422,91.109375,234881024
+19,mlp.gate_proj,3,128,8,128,41.930931091308594,87.5546875,234881024
+19,mlp.gate_proj,4,32,8,128,14.637476921081543,126.21875,234881024
+19,mlp.gate_proj,4,64,8,128,17.17462730407715,119.109375,234881024
+19,mlp.gate_proj,4,128,8,128,19.5492000579834,115.5546875,234881024
+19,mlp.gate_proj,8,32,8,128,0.8934692144393921,238.21875,234881024
+19,mlp.gate_proj,8,64,8,128,1.0492820739746094,231.109375,234881024
+19,mlp.gate_proj,8,128,8,128,1.1887071132659912,227.5546875,234881024
+19,mlp.up_proj,2,32,8,128,67.23274230957031,70.21875,234881024
+19,mlp.up_proj,2,64,8,128,78.30107879638672,63.109375,234881024
+19,mlp.up_proj,2,128,8,128,88.58267974853516,59.5546875,234881024
+19,mlp.up_proj,3,32,8,128,28.75506019592285,98.21875,234881024
+19,mlp.up_proj,3,64,8,128,33.538795471191406,91.109375,234881024
+19,mlp.up_proj,3,128,8,128,37.81147384643555,87.5546875,234881024
+19,mlp.up_proj,4,32,8,128,13.39201831817627,126.21875,234881024
+19,mlp.up_proj,4,64,8,128,15.693902015686035,119.109375,234881024
+19,mlp.up_proj,4,128,8,128,17.654075622558594,115.5546875,234881024
+19,mlp.up_proj,8,32,8,128,0.8176634311676025,238.21875,234881024
+19,mlp.up_proj,8,64,8,128,0.9570653438568115,231.109375,234881024
+19,mlp.up_proj,8,128,8,128,1.0745723247528076,227.5546875,234881024
+19,mlp.down_proj,2,32,8,128,67.50320434570312,70.21875,234881024
+19,mlp.down_proj,2,64,8,128,78.6197509765625,63.109375,234881024
+19,mlp.down_proj,2,128,8,128,88.89871978759766,59.5546875,234881024
+19,mlp.down_proj,3,32,8,128,28.882736206054688,98.21875,234881024
+19,mlp.down_proj,3,64,8,128,33.65791702270508,91.109375,234881024
+19,mlp.down_proj,3,128,8,128,37.927711486816406,87.5546875,234881024
+19,mlp.down_proj,4,32,8,128,13.449456214904785,126.21875,234881024
+19,mlp.down_proj,4,64,8,128,15.75979995727539,119.109375,234881024
+19,mlp.down_proj,4,128,8,128,17.722213745117188,115.5546875,234881024
+19,mlp.down_proj,8,32,8,128,0.8212486505508423,238.21875,234881024
+19,mlp.down_proj,8,64,8,128,0.961070716381073,231.109375,234881024
+19,mlp.down_proj,8,128,8,128,1.077902913093567,227.5546875,234881024
+20,self_attn.q_proj,2,32,8,128,47.50260543823242,20.0625,67108864
+20,self_attn.q_proj,2,64,8,128,55.51136779785156,18.03125,67108864
+20,self_attn.q_proj,2,128,8,128,63.34025192260742,17.015625,67108864
+20,self_attn.q_proj,3,32,8,128,20.31348419189453,28.0625,67108864
+20,self_attn.q_proj,3,64,8,128,23.75819969177246,26.03125,67108864
+20,self_attn.q_proj,3,128,8,128,27.099708557128906,25.015625,67108864
+20,self_attn.q_proj,4,32,8,128,9.472949981689453,36.0625,67108864
+20,self_attn.q_proj,4,64,8,128,11.093342781066895,34.03125,67108864
+20,self_attn.q_proj,4,128,8,128,12.645990371704102,33.015625,67108864
+20,self_attn.q_proj,8,32,8,128,0.5757126808166504,68.0625,67108864
+20,self_attn.q_proj,8,64,8,128,0.6736910939216614,66.03125,67108864
+20,self_attn.q_proj,8,128,8,128,0.7659715414047241,65.015625,67108864
+20,self_attn.v_proj,2,32,8,128,11.411632537841797,2.5078125,8388608
+20,self_attn.v_proj,2,64,8,128,13.401925086975098,2.25390625,8388608
+20,self_attn.v_proj,2,128,8,128,15.225994110107422,2.126953125,8388608
+20,self_attn.v_proj,3,32,8,128,4.878335952758789,3.5078125,8388608
+20,self_attn.v_proj,3,64,8,128,5.731382846832275,3.25390625,8388608
+20,self_attn.v_proj,3,128,8,128,6.511422157287598,3.126953125,8388608
+20,self_attn.v_proj,4,32,8,128,2.276078701019287,4.5078125,8388608
+20,self_attn.v_proj,4,64,8,128,2.674957036972046,4.25390625,8388608
+20,self_attn.v_proj,4,128,8,128,3.041455030441284,4.126953125,8388608
+20,self_attn.v_proj,8,32,8,128,0.1378287523984909,8.5078125,8388608
+20,self_attn.v_proj,8,64,8,128,0.16174660623073578,8.25390625,8388608
+20,self_attn.v_proj,8,128,8,128,0.18319904804229736,8.126953125,8388608
+20,self_attn.k_proj,2,32,8,128,33.53535461425781,2.5078125,8388608
+20,self_attn.k_proj,2,64,8,128,40.125247955322266,2.25390625,8388608
+20,self_attn.k_proj,2,128,8,128,45.93983459472656,2.126953125,8388608
+20,self_attn.k_proj,3,32,8,128,14.396808624267578,3.5078125,8388608
+20,self_attn.k_proj,3,64,8,128,17.296878814697266,3.25390625,8388608
+20,self_attn.k_proj,3,128,8,128,19.97919464111328,3.126953125,8388608
+20,self_attn.k_proj,4,32,8,128,6.718997001647949,4.5078125,8388608
+20,self_attn.k_proj,4,64,8,128,8.073019981384277,4.25390625,8388608
+20,self_attn.k_proj,4,128,8,128,9.326210021972656,4.126953125,8388608
+20,self_attn.k_proj,8,32,8,128,0.40645691752433777,8.5078125,8388608
+20,self_attn.k_proj,8,64,8,128,0.48805341124534607,8.25390625,8388608
+20,self_attn.k_proj,8,128,8,128,0.5620036721229553,8.126953125,8388608
+20,self_attn.o_proj,2,32,8,128,34.181419372558594,20.0625,67108864
+20,self_attn.o_proj,2,64,8,128,39.831539154052734,18.03125,67108864
+20,self_attn.o_proj,2,128,8,128,44.96343231201172,17.015625,67108864
+20,self_attn.o_proj,3,32,8,128,14.618040084838867,28.0625,67108864
+20,self_attn.o_proj,3,64,8,128,17.044902801513672,26.03125,67108864
+20,self_attn.o_proj,3,128,8,128,19.234046936035156,25.015625,67108864
+20,self_attn.o_proj,4,32,8,128,6.817072868347168,36.0625,67108864
+20,self_attn.o_proj,4,64,8,128,7.957001686096191,34.03125,67108864
+20,self_attn.o_proj,4,128,8,128,8.977210998535156,33.015625,67108864
+20,self_attn.o_proj,8,32,8,128,0.41432157158851624,68.0625,67108864
+20,self_attn.o_proj,8,64,8,128,0.4826522767543793,66.03125,67108864
+20,self_attn.o_proj,8,128,8,128,0.5429307222366333,65.015625,67108864
+20,mlp.gate_proj,2,32,8,128,72.93668365478516,70.21875,234881024
+20,mlp.gate_proj,2,64,8,128,85.74252319335938,63.109375,234881024
+20,mlp.gate_proj,2,128,8,128,97.14581298828125,59.5546875,234881024
+20,mlp.gate_proj,3,32,8,128,31.358304977416992,98.21875,234881024
+20,mlp.gate_proj,3,64,8,128,36.59581756591797,91.109375,234881024
+20,mlp.gate_proj,3,128,8,128,41.65080642700195,87.5546875,234881024
+20,mlp.gate_proj,4,32,8,128,14.590651512145996,126.21875,234881024
+20,mlp.gate_proj,4,64,8,128,17.089847564697266,119.109375,234881024
+20,mlp.gate_proj,4,128,8,128,19.423892974853516,115.5546875,234881024
+20,mlp.gate_proj,8,32,8,128,0.8906527161598206,238.21875,234881024
+20,mlp.gate_proj,8,64,8,128,1.0447826385498047,231.109375,234881024
+20,mlp.gate_proj,8,128,8,128,1.1810166835784912,227.5546875,234881024
+20,mlp.up_proj,2,32,8,128,67.83265686035156,70.21875,234881024
+20,mlp.up_proj,2,64,8,128,79.0323486328125,63.109375,234881024
+20,mlp.up_proj,2,128,8,128,89.38765716552734,59.5546875,234881024
+20,mlp.up_proj,3,32,8,128,29.034212112426758,98.21875,234881024
+20,mlp.up_proj,3,64,8,128,33.83763122558594,91.109375,234881024
+20,mlp.up_proj,3,128,8,128,38.136077880859375,87.5546875,234881024
+20,mlp.up_proj,4,32,8,128,13.518312454223633,126.21875,234881024
+20,mlp.up_proj,4,64,8,128,15.845868110656738,119.109375,234881024
+20,mlp.up_proj,4,128,8,128,17.8106746673584,115.5546875,234881024
+20,mlp.up_proj,8,32,8,128,0.8254263401031494,238.21875,234881024
+20,mlp.up_proj,8,64,8,128,0.9665279388427734,231.109375,234881024
+20,mlp.up_proj,8,128,8,128,1.084122896194458,227.5546875,234881024
+20,mlp.down_proj,2,32,8,128,67.9695816040039,70.21875,234881024
+20,mlp.down_proj,2,64,8,128,79.1846694946289,63.109375,234881024
+20,mlp.down_proj,2,128,8,128,89.61698150634766,59.5546875,234881024
+20,mlp.down_proj,3,32,8,128,29.098194122314453,98.21875,234881024
+20,mlp.down_proj,3,64,8,128,33.8963508605957,91.109375,234881024
+20,mlp.down_proj,3,128,8,128,38.20039367675781,87.5546875,234881024
+20,mlp.down_proj,4,32,8,128,13.54868221282959,126.21875,234881024
+20,mlp.down_proj,4,64,8,128,15.880101203918457,119.109375,234881024
+20,mlp.down_proj,4,128,8,128,17.839256286621094,115.5546875,234881024
+20,mlp.down_proj,8,32,8,128,0.8272631168365479,238.21875,234881024
+20,mlp.down_proj,8,64,8,128,0.9684608578681946,231.109375,234881024
+20,mlp.down_proj,8,128,8,128,1.085929274559021,227.5546875,234881024
+21,self_attn.q_proj,2,32,8,128,50.066139221191406,20.0625,67108864
+21,self_attn.q_proj,2,64,8,128,58.76658630371094,18.03125,67108864
+21,self_attn.q_proj,2,128,8,128,67.51437377929688,17.015625,67108864
+21,self_attn.q_proj,3,32,8,128,21.44175148010254,28.0625,67108864
+21,self_attn.q_proj,3,64,8,128,25.168209075927734,26.03125,67108864
+21,self_attn.q_proj,3,128,8,128,28.97112464904785,25.015625,67108864
+21,self_attn.q_proj,4,32,8,128,9.997611999511719,36.0625,67108864
+21,self_attn.q_proj,4,64,8,128,11.75015926361084,34.03125,67108864
+21,self_attn.q_proj,4,128,8,128,13.517221450805664,33.015625,67108864
+21,self_attn.q_proj,8,32,8,128,0.6075009703636169,68.0625,67108864
+21,self_attn.q_proj,8,64,8,128,0.7141141295433044,66.03125,67108864
+21,self_attn.q_proj,8,128,8,128,0.8184065222740173,65.015625,67108864
+21,self_attn.v_proj,2,32,8,128,10.314805030822754,2.5078125,8388608
+21,self_attn.v_proj,2,64,8,128,12.170238494873047,2.25390625,8388608
+21,self_attn.v_proj,2,128,8,128,13.904681205749512,2.126953125,8388608
+21,self_attn.v_proj,3,32,8,128,4.4141316413879395,3.5078125,8388608
+21,self_attn.v_proj,3,64,8,128,5.208975791931152,3.25390625,8388608
+21,self_attn.v_proj,3,128,8,128,5.950900077819824,3.126953125,8388608
+21,self_attn.v_proj,4,32,8,128,2.058828830718994,4.5078125,8388608
+21,self_attn.v_proj,4,64,8,128,2.429746627807617,4.25390625,8388608
+21,self_attn.v_proj,4,128,8,128,2.777170181274414,4.126953125,8388608
+21,self_attn.v_proj,8,32,8,128,0.12459807842969894,8.5078125,8388608
+21,self_attn.v_proj,8,64,8,128,0.14693237841129303,8.25390625,8388608
+21,self_attn.v_proj,8,128,8,128,0.1673242151737213,8.126953125,8388608
+21,self_attn.k_proj,2,32,8,128,33.77277374267578,2.5078125,8388608
+21,self_attn.k_proj,2,64,8,128,40.270687103271484,2.25390625,8388608
+21,self_attn.k_proj,2,128,8,128,45.982051849365234,2.126953125,8388608
+21,self_attn.k_proj,3,32,8,128,14.507922172546387,3.5078125,8388608
+21,self_attn.k_proj,3,64,8,128,17.370420455932617,3.25390625,8388608
+21,self_attn.k_proj,3,128,8,128,20.04755973815918,3.126953125,8388608
+21,self_attn.k_proj,4,32,8,128,6.771570205688477,4.5078125,8388608
+21,self_attn.k_proj,4,64,8,128,8.10830020904541,4.25390625,8388608
+21,self_attn.k_proj,4,128,8,128,9.361713409423828,4.126953125,8388608
+21,self_attn.k_proj,8,32,8,128,0.40966057777404785,8.5078125,8388608
+21,self_attn.k_proj,8,64,8,128,0.49005240201950073,8.25390625,8388608
+21,self_attn.k_proj,8,128,8,128,0.5639031529426575,8.126953125,8388608
+21,self_attn.o_proj,2,32,8,128,33.11326599121094,20.0625,67108864
+21,self_attn.o_proj,2,64,8,128,38.561458587646484,18.03125,67108864
+21,self_attn.o_proj,2,128,8,128,43.50345230102539,17.015625,67108864
+21,self_attn.o_proj,3,32,8,128,14.153902053833008,28.0625,67108864
+21,self_attn.o_proj,3,64,8,128,16.50600814819336,26.03125,67108864
+21,self_attn.o_proj,3,128,8,128,18.61819076538086,25.015625,67108864
+21,self_attn.o_proj,4,32,8,128,6.6011810302734375,36.0625,67108864
+21,self_attn.o_proj,4,64,8,128,7.70367431640625,34.03125,67108864
+21,self_attn.o_proj,4,128,8,128,8.689971923828125,33.015625,67108864
+21,self_attn.o_proj,8,32,8,128,0.4013918340206146,68.0625,67108864
+21,self_attn.o_proj,8,64,8,128,0.4674668610095978,66.03125,67108864
+21,self_attn.o_proj,8,128,8,128,0.525651216506958,65.015625,67108864
+21,mlp.gate_proj,2,32,8,128,73.16300964355469,70.21875,234881024
+21,mlp.gate_proj,2,64,8,128,86.01152801513672,63.109375,234881024
+21,mlp.gate_proj,2,128,8,128,97.38836669921875,59.5546875,234881024
+21,mlp.gate_proj,3,32,8,128,31.463926315307617,98.21875,234881024
+21,mlp.gate_proj,3,64,8,128,36.704994201660156,91.109375,234881024
+21,mlp.gate_proj,3,128,8,128,41.772159576416016,87.5546875,234881024
+21,mlp.gate_proj,4,32,8,128,14.638014793395996,126.21875,234881024
+21,mlp.gate_proj,4,64,8,128,17.140975952148438,119.109375,234881024
+21,mlp.gate_proj,4,128,8,128,19.475421905517578,115.5546875,234881024
+21,mlp.gate_proj,8,32,8,128,0.893546998500824,238.21875,234881024
+21,mlp.gate_proj,8,64,8,128,1.0477420091629028,231.109375,234881024
+21,mlp.gate_proj,8,128,8,128,1.1847013235092163,227.5546875,234881024
+21,mlp.up_proj,2,32,8,128,67.81924438476562,70.21875,234881024
+21,mlp.up_proj,2,64,8,128,78.98265838623047,63.109375,234881024
+21,mlp.up_proj,2,128,8,128,89.31256866455078,59.5546875,234881024
+21,mlp.up_proj,3,32,8,128,29.028831481933594,98.21875,234881024
+21,mlp.up_proj,3,64,8,128,33.81718063354492,91.109375,234881024
+21,mlp.up_proj,3,128,8,128,38.095943450927734,87.5546875,234881024
+21,mlp.up_proj,4,32,8,128,13.516995429992676,126.21875,234881024
+21,mlp.up_proj,4,64,8,128,15.836551666259766,119.109375,234881024
+21,mlp.up_proj,4,128,8,128,17.79637336730957,115.5546875,234881024
+21,mlp.up_proj,8,32,8,128,0.8253889679908752,238.21875,234881024
+21,mlp.up_proj,8,64,8,128,0.9659287333488464,231.109375,234881024
+21,mlp.up_proj,8,128,8,128,1.0832128524780273,227.5546875,234881024
+21,mlp.down_proj,2,32,8,128,68.0453872680664,70.21875,234881024
+21,mlp.down_proj,2,64,8,128,79.27557373046875,63.109375,234881024
+21,mlp.down_proj,2,128,8,128,89.68626403808594,59.5546875,234881024
+21,mlp.down_proj,3,32,8,128,29.132936477661133,98.21875,234881024
+21,mlp.down_proj,3,64,8,128,33.93088912963867,91.109375,234881024
+21,mlp.down_proj,3,128,8,128,38.240806579589844,87.5546875,234881024
+21,mlp.down_proj,4,32,8,128,13.565064430236816,126.21875,234881024
+21,mlp.down_proj,4,64,8,128,15.896529197692871,119.109375,234881024
+21,mlp.down_proj,4,128,8,128,17.860828399658203,115.5546875,234881024
+21,mlp.down_proj,8,32,8,128,0.8282769918441772,238.21875,234881024
+21,mlp.down_proj,8,64,8,128,0.969706654548645,231.109375,234881024
+21,mlp.down_proj,8,128,8,128,1.0871803760528564,227.5546875,234881024
+22,self_attn.q_proj,2,32,8,128,49.0333251953125,20.0625,67108864
+22,self_attn.q_proj,2,64,8,128,57.58620834350586,18.03125,67108864
+22,self_attn.q_proj,2,128,8,128,66.29540252685547,17.015625,67108864
+22,self_attn.q_proj,3,32,8,128,20.998470306396484,28.0625,67108864
+22,self_attn.q_proj,3,64,8,128,24.695531845092773,26.03125,67108864
+22,self_attn.q_proj,3,128,8,128,28.50078582763672,25.015625,67108864
+22,self_attn.q_proj,4,32,8,128,9.79373836517334,36.0625,67108864
+22,self_attn.q_proj,4,64,8,128,11.530495643615723,34.03125,67108864
+22,self_attn.q_proj,4,128,8,128,13.291687965393066,33.015625,67108864
+22,self_attn.q_proj,8,32,8,128,0.5951723456382751,68.0625,67108864
+22,self_attn.q_proj,8,64,8,128,0.7006952166557312,66.03125,67108864
+22,self_attn.q_proj,8,128,8,128,0.8049605488777161,65.015625,67108864
+22,self_attn.v_proj,2,32,8,128,12.17402458190918,2.5078125,8388608
+22,self_attn.v_proj,2,64,8,128,14.270858764648438,2.25390625,8388608
+22,self_attn.v_proj,2,128,8,128,16.17725372314453,2.126953125,8388608
+22,self_attn.v_proj,3,32,8,128,5.205935955047607,3.5078125,8388608
+22,self_attn.v_proj,3,64,8,128,6.10429573059082,3.25390625,8388608
+22,self_attn.v_proj,3,128,8,128,6.924643039703369,3.126953125,8388608
+22,self_attn.v_proj,4,32,8,128,2.428332805633545,4.5078125,8388608
+22,self_attn.v_proj,4,64,8,128,2.8483965396881104,4.25390625,8388608
+22,self_attn.v_proj,4,128,8,128,3.2292377948760986,4.126953125,8388608
+22,self_attn.v_proj,8,32,8,128,0.14698553085327148,8.5078125,8388608
+22,self_attn.v_proj,8,64,8,128,0.1722264587879181,8.25390625,8388608
+22,self_attn.v_proj,8,128,8,128,0.19458496570587158,8.126953125,8388608
+22,self_attn.k_proj,2,32,8,128,33.71762466430664,2.5078125,8388608
+22,self_attn.k_proj,2,64,8,128,40.137874603271484,2.25390625,8388608
+22,self_attn.k_proj,2,128,8,128,45.671607971191406,2.126953125,8388608
+22,self_attn.k_proj,3,32,8,128,14.483540534973145,3.5078125,8388608
+22,self_attn.k_proj,3,64,8,128,17.305959701538086,3.25390625,8388608
+22,self_attn.k_proj,3,128,8,128,19.912616729736328,3.126953125,8388608
+22,self_attn.k_proj,4,32,8,128,6.758945465087891,4.5078125,8388608
+22,self_attn.k_proj,4,64,8,128,8.0787935256958,4.25390625,8388608
+22,self_attn.k_proj,4,128,8,128,9.29574966430664,4.126953125,8388608
+22,self_attn.k_proj,8,32,8,128,0.4088408648967743,8.5078125,8388608
+22,self_attn.k_proj,8,64,8,128,0.48839548230171204,8.25390625,8388608
+22,self_attn.k_proj,8,128,8,128,0.5598931312561035,8.126953125,8388608
+22,self_attn.o_proj,2,32,8,128,35.02137756347656,20.0625,67108864
+22,self_attn.o_proj,2,64,8,128,40.79725646972656,18.03125,67108864
+22,self_attn.o_proj,2,128,8,128,45.98588180541992,17.015625,67108864
+22,self_attn.o_proj,3,32,8,128,14.981328964233398,28.0625,67108864
+22,self_attn.o_proj,3,64,8,128,17.452945709228516,26.03125,67108864
+22,self_attn.o_proj,3,128,8,128,19.68682861328125,25.015625,67108864
+22,self_attn.o_proj,4,32,8,128,6.986776351928711,36.0625,67108864
+22,self_attn.o_proj,4,64,8,128,8.147902488708496,34.03125,67108864
+22,self_attn.o_proj,4,128,8,128,9.186807632446289,33.015625,67108864
+22,self_attn.o_proj,8,32,8,128,0.4245470464229584,68.0625,67108864
+22,self_attn.o_proj,8,64,8,128,0.4943808615207672,66.03125,67108864
+22,self_attn.o_proj,8,128,8,128,0.5559003353118896,65.015625,67108864
+22,mlp.gate_proj,2,32,8,128,73.89613342285156,70.21875,234881024
+22,mlp.gate_proj,2,64,8,128,86.89825439453125,63.109375,234881024
+22,mlp.gate_proj,2,128,8,128,98.32536315917969,59.5546875,234881024
+22,mlp.gate_proj,3,32,8,128,31.786226272583008,98.21875,234881024
+22,mlp.gate_proj,3,64,8,128,37.06299591064453,91.109375,234881024
+22,mlp.gate_proj,3,128,8,128,42.17759704589844,87.5546875,234881024
+22,mlp.gate_proj,4,32,8,128,14.786981582641602,126.21875,234881024
+22,mlp.gate_proj,4,64,8,128,17.306238174438477,119.109375,234881024
+22,mlp.gate_proj,4,128,8,128,19.66209602355957,115.5546875,234881024
+22,mlp.gate_proj,8,32,8,128,0.9027595520019531,238.21875,234881024
+22,mlp.gate_proj,8,64,8,128,1.057629942893982,231.109375,234881024
+22,mlp.gate_proj,8,128,8,128,1.1958508491516113,227.5546875,234881024
+22,mlp.up_proj,2,32,8,128,67.68273162841797,70.21875,234881024
+22,mlp.up_proj,2,64,8,128,78.81617736816406,63.109375,234881024
+22,mlp.up_proj,2,128,8,128,89.12592315673828,59.5546875,234881024
+22,mlp.up_proj,3,32,8,128,28.966690063476562,98.21875,234881024
+22,mlp.up_proj,3,64,8,128,33.75699234008789,91.109375,234881024
+22,mlp.up_proj,3,128,8,128,38.02843475341797,87.5546875,234881024
+22,mlp.up_proj,4,32,8,128,13.489409446716309,126.21875,234881024
+22,mlp.up_proj,4,64,8,128,15.805479049682617,119.109375,234881024
+22,mlp.up_proj,4,128,8,128,17.767690658569336,115.5546875,234881024
+22,mlp.up_proj,8,32,8,128,0.8236560821533203,238.21875,234881024
+22,mlp.up_proj,8,64,8,128,0.9639034271240234,231.109375,234881024
+22,mlp.up_proj,8,128,8,128,1.0811716318130493,227.5546875,234881024
+22,mlp.down_proj,2,32,8,128,67.95698547363281,70.21875,234881024
+22,mlp.down_proj,2,64,8,128,79.1680679321289,63.109375,234881024
+22,mlp.down_proj,2,128,8,128,89.5595474243164,59.5546875,234881024
+22,mlp.down_proj,3,32,8,128,29.09288787841797,98.21875,234881024
+22,mlp.down_proj,3,64,8,128,33.8816032409668,91.109375,234881024
+22,mlp.down_proj,3,128,8,128,38.19157409667969,87.5546875,234881024
+22,mlp.down_proj,4,32,8,128,13.544726371765137,126.21875,234881024
+22,mlp.down_proj,4,64,8,128,15.86958122253418,119.109375,234881024
+22,mlp.down_proj,4,128,8,128,17.83646011352539,115.5546875,234881024
+22,mlp.down_proj,8,32,8,128,0.8270640969276428,238.21875,234881024
+22,mlp.down_proj,8,64,8,128,0.9680188298225403,231.109375,234881024
+22,mlp.down_proj,8,128,8,128,1.0854259729385376,227.5546875,234881024
+23,self_attn.q_proj,2,32,8,128,48.95332336425781,20.0625,67108864
+23,self_attn.q_proj,2,64,8,128,57.35587692260742,18.03125,67108864
+23,self_attn.q_proj,2,128,8,128,65.69770812988281,17.015625,67108864
+23,self_attn.q_proj,3,32,8,128,20.95292854309082,28.0625,67108864
+23,self_attn.q_proj,3,64,8,128,24.56814193725586,26.03125,67108864
+23,self_attn.q_proj,3,128,8,128,28.163707733154297,25.015625,67108864
+23,self_attn.q_proj,4,32,8,128,9.773574829101562,36.0625,67108864
+23,self_attn.q_proj,4,64,8,128,11.471784591674805,34.03125,67108864
+23,self_attn.q_proj,4,128,8,128,13.139389038085938,33.015625,67108864
+23,self_attn.q_proj,8,32,8,128,0.5939086675643921,68.0625,67108864
+23,self_attn.q_proj,8,64,8,128,0.6970583200454712,66.03125,67108864
+23,self_attn.q_proj,8,128,8,128,0.795661985874176,65.015625,67108864
+23,self_attn.v_proj,2,32,8,128,11.557222366333008,2.5078125,8388608
+23,self_attn.v_proj,2,64,8,128,13.600379943847656,2.25390625,8388608
+23,self_attn.v_proj,2,128,8,128,15.4859619140625,2.126953125,8388608
+23,self_attn.v_proj,3,32,8,128,4.942420482635498,3.5078125,8388608
+23,self_attn.v_proj,3,64,8,128,5.817290306091309,3.25390625,8388608
+23,self_attn.v_proj,3,128,8,128,6.6255388259887695,3.126953125,8388608
+23,self_attn.v_proj,4,32,8,128,2.305755376815796,4.5078125,8388608
+23,self_attn.v_proj,4,64,8,128,2.714259386062622,4.25390625,8388608
+23,self_attn.v_proj,4,128,8,128,3.09171724319458,4.126953125,8388608
+23,self_attn.v_proj,8,32,8,128,0.13959766924381256,8.5078125,8388608
+23,self_attn.v_proj,8,64,8,128,0.16420796513557434,8.25390625,8388608
+23,self_attn.v_proj,8,128,8,128,0.18633045256137848,8.126953125,8388608
+23,self_attn.k_proj,2,32,8,128,34.49933624267578,2.5078125,8388608
+23,self_attn.k_proj,2,64,8,128,41.13096618652344,2.25390625,8388608
+23,self_attn.k_proj,2,128,8,128,46.86101531982422,2.126953125,8388608
+23,self_attn.k_proj,3,32,8,128,14.828217506408691,3.5078125,8388608
+23,self_attn.k_proj,3,64,8,128,17.759111404418945,3.25390625,8388608
+23,self_attn.k_proj,3,128,8,128,20.487688064575195,3.126953125,8388608
+23,self_attn.k_proj,4,32,8,128,6.920931816101074,4.5078125,8388608
+23,self_attn.k_proj,4,64,8,128,8.288797378540039,4.25390625,8388608
+23,self_attn.k_proj,4,128,8,128,9.55774974822998,4.126953125,8388608
+23,self_attn.k_proj,8,32,8,128,0.41864943504333496,8.5078125,8388608
+23,self_attn.k_proj,8,64,8,128,0.5011165142059326,8.25390625,8388608
+23,self_attn.k_proj,8,128,8,128,0.5759000778198242,8.126953125,8388608
+23,self_attn.o_proj,2,32,8,128,34.400882720947266,20.0625,67108864
+23,self_attn.o_proj,2,64,8,128,40.0746955871582,18.03125,67108864
+23,self_attn.o_proj,2,128,8,128,45.22163391113281,17.015625,67108864
+23,self_attn.o_proj,3,32,8,128,14.71457290649414,28.0625,67108864
+23,self_attn.o_proj,3,64,8,128,17.151464462280273,26.03125,67108864
+23,self_attn.o_proj,3,128,8,128,19.35689926147461,25.015625,67108864
+23,self_attn.o_proj,4,32,8,128,6.862423419952393,36.0625,67108864
+23,self_attn.o_proj,4,64,8,128,8.010310173034668,34.03125,67108864
+23,self_attn.o_proj,4,128,8,128,9.033561706542969,33.015625,67108864
+23,self_attn.o_proj,8,32,8,128,0.4170621931552887,68.0625,67108864
+23,self_attn.o_proj,8,64,8,128,0.4857839047908783,66.03125,67108864
+23,self_attn.o_proj,8,128,8,128,0.5465974807739258,65.015625,67108864
+23,mlp.gate_proj,2,32,8,128,74.1429443359375,70.21875,234881024
+23,mlp.gate_proj,2,64,8,128,87.2253646850586,63.109375,234881024
+23,mlp.gate_proj,2,128,8,128,98.73306274414062,59.5546875,234881024
+23,mlp.gate_proj,3,32,8,128,31.898380279541016,98.21875,234881024
+23,mlp.gate_proj,3,64,8,128,37.20298385620117,91.109375,234881024
+23,mlp.gate_proj,3,128,8,128,42.3560676574707,87.5546875,234881024
+23,mlp.gate_proj,4,32,8,128,14.839951515197754,126.21875,234881024
+23,mlp.gate_proj,4,64,8,128,17.368871688842773,119.109375,234881024
+23,mlp.gate_proj,4,128,8,128,19.7391414642334,115.5546875,234881024
+23,mlp.gate_proj,8,32,8,128,0.9059820175170898,238.21875,234881024
+23,mlp.gate_proj,8,64,8,128,1.061463713645935,231.109375,234881024
+23,mlp.gate_proj,8,128,8,128,1.2006580829620361,227.5546875,234881024
+23,mlp.up_proj,2,32,8,128,67.61155700683594,70.21875,234881024
+23,mlp.up_proj,2,64,8,128,78.70379638671875,63.109375,234881024
+23,mlp.up_proj,2,128,8,128,88.9647445678711,59.5546875,234881024
+23,mlp.up_proj,3,32,8,128,28.937255859375,98.21875,234881024
+23,mlp.up_proj,3,64,8,128,33.70938491821289,91.109375,234881024
+23,mlp.up_proj,3,128,8,128,37.97586441040039,87.5546875,234881024
+23,mlp.up_proj,4,32,8,128,13.473372459411621,126.21875,234881024
+23,mlp.up_proj,4,64,8,128,15.784368515014648,119.109375,234881024
+23,mlp.up_proj,4,128,8,128,17.73467445373535,115.5546875,234881024
+23,mlp.up_proj,8,32,8,128,0.8228102922439575,238.21875,234881024
+23,mlp.up_proj,8,64,8,128,0.9625856876373291,231.109375,234881024
+23,mlp.up_proj,8,128,8,128,1.079542636871338,227.5546875,234881024
+23,mlp.down_proj,2,32,8,128,67.89913177490234,70.21875,234881024
+23,mlp.down_proj,2,64,8,128,79.08594512939453,63.109375,234881024
+23,mlp.down_proj,2,128,8,128,89.42433166503906,59.5546875,234881024
+23,mlp.down_proj,3,32,8,128,29.06273078918457,98.21875,234881024
+23,mlp.down_proj,3,64,8,128,33.848114013671875,91.109375,234881024
+23,mlp.down_proj,3,128,8,128,38.119632720947266,87.5546875,234881024
+23,mlp.down_proj,4,32,8,128,13.531939506530762,126.21875,234881024
+23,mlp.down_proj,4,64,8,128,15.85130786895752,119.109375,234881024
+23,mlp.down_proj,4,128,8,128,17.803142547607422,115.5546875,234881024
+23,mlp.down_proj,8,32,8,128,0.8262805342674255,238.21875,234881024
+23,mlp.down_proj,8,64,8,128,0.9668347835540771,231.109375,234881024
+23,mlp.down_proj,8,128,8,128,1.0837453603744507,227.5546875,234881024
+24,self_attn.q_proj,2,32,8,128,46.72608947753906,20.0625,67108864
+24,self_attn.q_proj,2,64,8,128,55.0274772644043,18.03125,67108864
+24,self_attn.q_proj,2,128,8,128,63.36747360229492,17.015625,67108864
+24,self_attn.q_proj,3,32,8,128,20.021081924438477,28.0625,67108864
+24,self_attn.q_proj,3,64,8,128,23.67501449584961,26.03125,67108864
+24,self_attn.q_proj,3,128,8,128,27.352802276611328,25.015625,67108864
+24,self_attn.q_proj,4,32,8,128,9.339029312133789,36.0625,67108864
+24,self_attn.q_proj,4,64,8,128,11.05237102508545,34.03125,67108864
+24,self_attn.q_proj,4,128,8,128,12.770557403564453,33.015625,67108864
+24,self_attn.q_proj,8,32,8,128,0.5673384070396423,68.0625,67108864
+24,self_attn.q_proj,8,64,8,128,0.6713024973869324,66.03125,67108864
+24,self_attn.q_proj,8,128,8,128,0.7732316255569458,65.015625,67108864
+24,self_attn.v_proj,2,32,8,128,12.398767471313477,2.5078125,8388608
+24,self_attn.v_proj,2,64,8,128,14.579327583312988,2.25390625,8388608
+24,self_attn.v_proj,2,128,8,128,16.575864791870117,2.126953125,8388608
+24,self_attn.v_proj,3,32,8,128,5.301939964294434,3.5078125,8388608
+24,self_attn.v_proj,3,64,8,128,6.237957000732422,3.25390625,8388608
+24,self_attn.v_proj,3,128,8,128,7.096917629241943,3.126953125,8388608
+24,self_attn.v_proj,4,32,8,128,2.4741358757019043,4.5078125,8388608
+24,self_attn.v_proj,4,64,8,128,2.911261558532715,4.25390625,8388608
+24,self_attn.v_proj,4,128,8,128,3.312446117401123,4.126953125,8388608
+24,self_attn.v_proj,8,32,8,128,0.1497873067855835,8.5078125,8388608
+24,self_attn.v_proj,8,64,8,128,0.1760377287864685,8.25390625,8388608
+24,self_attn.v_proj,8,128,8,128,0.19949668645858765,8.126953125,8388608
+24,self_attn.k_proj,2,32,8,128,34.48940658569336,2.5078125,8388608
+24,self_attn.k_proj,2,64,8,128,41.43503189086914,2.25390625,8388608
+24,self_attn.k_proj,2,128,8,128,47.28764343261719,2.126953125,8388608
+24,self_attn.k_proj,3,32,8,128,14.898791313171387,3.5078125,8388608
+24,self_attn.k_proj,3,64,8,128,18.101438522338867,3.25390625,8388608
+24,self_attn.k_proj,3,128,8,128,21.153045654296875,3.126953125,8388608
+24,self_attn.k_proj,4,32,8,128,6.955023288726807,4.5078125,8388608
+24,self_attn.k_proj,4,64,8,128,8.449739456176758,4.25390625,8388608
+24,self_attn.k_proj,4,128,8,128,9.879576683044434,4.126953125,8388608
+24,self_attn.k_proj,8,32,8,128,0.4209446310997009,8.5078125,8388608
+24,self_attn.k_proj,8,64,8,128,0.51088947057724,8.25390625,8388608
+24,self_attn.k_proj,8,128,8,128,0.595433235168457,8.126953125,8388608
+24,self_attn.o_proj,2,32,8,128,34.694698333740234,20.0625,67108864
+24,self_attn.o_proj,2,64,8,128,40.57896041870117,18.03125,67108864
+24,self_attn.o_proj,2,128,8,128,45.97595977783203,17.015625,67108864
+24,self_attn.o_proj,3,32,8,128,14.866718292236328,28.0625,67108864
+24,self_attn.o_proj,3,64,8,128,17.426631927490234,26.03125,67108864
+24,self_attn.o_proj,3,128,8,128,19.840778350830078,25.015625,67108864
+24,self_attn.o_proj,4,32,8,128,6.935458660125732,36.0625,67108864
+24,self_attn.o_proj,4,64,8,128,8.141594886779785,34.03125,67108864
+24,self_attn.o_proj,4,128,8,128,9.266505241394043,33.015625,67108864
+24,self_attn.o_proj,8,32,8,128,0.42144566774368286,68.0625,67108864
+24,self_attn.o_proj,8,64,8,128,0.493855357170105,66.03125,67108864
+24,self_attn.o_proj,8,128,8,128,0.5608824491500854,65.015625,67108864
+24,mlp.gate_proj,2,32,8,128,73.83680725097656,70.21875,234881024
+24,mlp.gate_proj,2,64,8,128,86.78517150878906,63.109375,234881024
+24,mlp.gate_proj,2,128,8,128,98.18970489501953,59.5546875,234881024
+24,mlp.gate_proj,3,32,8,128,31.760698318481445,98.21875,234881024
+24,mlp.gate_proj,3,64,8,128,37.03117370605469,91.109375,234881024
+24,mlp.gate_proj,3,128,8,128,42.1378173828125,87.5546875,234881024
+24,mlp.gate_proj,4,32,8,128,14.776872634887695,126.21875,234881024
+24,mlp.gate_proj,4,64,8,128,17.296966552734375,119.109375,234881024
+24,mlp.gate_proj,4,128,8,128,19.64874267578125,115.5546875,234881024
+24,mlp.gate_proj,8,32,8,128,0.9020965099334717,238.21875,234881024
+24,mlp.gate_proj,8,64,8,128,1.0568231344223022,231.109375,234881024
+24,mlp.gate_proj,8,128,8,128,1.1950596570968628,227.5546875,234881024
+24,mlp.up_proj,2,32,8,128,68.20198822021484,70.21875,234881024
+24,mlp.up_proj,2,64,8,128,79.43785858154297,63.109375,234881024
+24,mlp.up_proj,2,128,8,128,89.83841705322266,59.5546875,234881024
+24,mlp.up_proj,3,32,8,128,29.206405639648438,98.21875,234881024
+24,mlp.up_proj,3,64,8,128,34.010196685791016,91.109375,234881024
+24,mlp.up_proj,3,128,8,128,38.32136535644531,87.5546875,234881024
+24,mlp.up_proj,4,32,8,128,13.598177909851074,126.21875,234881024
+24,mlp.up_proj,4,64,8,128,15.934477806091309,119.109375,234881024
+24,mlp.up_proj,4,128,8,128,17.898771286010742,115.5546875,234881024
+24,mlp.up_proj,8,32,8,128,0.8302566409111023,238.21875,234881024
+24,mlp.up_proj,8,64,8,128,0.9719550013542175,231.109375,234881024
+24,mlp.up_proj,8,128,8,128,1.0894569158554077,227.5546875,234881024
+24,mlp.down_proj,2,32,8,128,68.25871276855469,70.21875,234881024
+24,mlp.down_proj,2,64,8,128,79.52396392822266,63.109375,234881024
+24,mlp.down_proj,2,128,8,128,89.9458999633789,59.5546875,234881024
+24,mlp.down_proj,3,32,8,128,29.230743408203125,98.21875,234881024
+24,mlp.down_proj,3,64,8,128,34.02956771850586,91.109375,234881024
+24,mlp.down_proj,3,128,8,128,38.35895919799805,87.5546875,234881024
+24,mlp.down_proj,4,32,8,128,13.610882759094238,126.21875,234881024
+24,mlp.down_proj,4,64,8,128,15.947091102600098,119.109375,234881024
+24,mlp.down_proj,4,128,8,128,17.91204261779785,115.5546875,234881024
+24,mlp.down_proj,8,32,8,128,0.830915629863739,238.21875,234881024
+24,mlp.down_proj,8,64,8,128,0.9728398323059082,231.109375,234881024
+24,mlp.down_proj,8,128,8,128,1.0903218984603882,227.5546875,234881024
+25,self_attn.q_proj,2,32,8,128,47.60781478881836,20.0625,67108864
+25,self_attn.q_proj,2,64,8,128,55.861778259277344,18.03125,67108864
+25,self_attn.q_proj,2,128,8,128,64.1476058959961,17.015625,67108864
+25,self_attn.q_proj,3,32,8,128,20.385061264038086,28.0625,67108864
+25,self_attn.q_proj,3,64,8,128,23.97136688232422,26.03125,67108864
+25,self_attn.q_proj,3,128,8,128,27.533632278442383,25.015625,67108864
+25,self_attn.q_proj,4,32,8,128,9.506421089172363,36.0625,67108864
+25,self_attn.q_proj,4,64,8,128,11.192293167114258,34.03125,67108864
+25,self_attn.q_proj,4,128,8,128,12.848099708557129,33.015625,67108864
+25,self_attn.q_proj,8,32,8,128,0.5777422189712524,68.0625,67108864
+25,self_attn.q_proj,8,64,8,128,0.6798167824745178,66.03125,67108864
+25,self_attn.q_proj,8,128,8,128,0.7783313393592834,65.015625,67108864
+25,self_attn.v_proj,2,32,8,128,12.110830307006836,2.5078125,8388608
+25,self_attn.v_proj,2,64,8,128,14.246417045593262,2.25390625,8388608
+25,self_attn.v_proj,2,128,8,128,16.169309616088867,2.126953125,8388608
+25,self_attn.v_proj,3,32,8,128,5.177356719970703,3.5078125,8388608
+25,self_attn.v_proj,3,64,8,128,6.091578483581543,3.25390625,8388608
+25,self_attn.v_proj,3,128,8,128,6.922048568725586,3.126953125,8388608
+25,self_attn.v_proj,4,32,8,128,2.4162817001342773,4.5078125,8388608
+25,self_attn.v_proj,4,64,8,128,2.8430593013763428,4.25390625,8388608
+25,self_attn.v_proj,4,128,8,128,3.2282891273498535,4.126953125,8388608
+25,self_attn.v_proj,8,32,8,128,0.14628539979457855,8.5078125,8388608
+25,self_attn.v_proj,8,64,8,128,0.17189115285873413,8.25390625,8388608
+25,self_attn.v_proj,8,128,8,128,0.19462408125400543,8.126953125,8388608
+25,self_attn.k_proj,2,32,8,128,35.376861572265625,2.5078125,8388608
+25,self_attn.k_proj,2,64,8,128,42.148475646972656,2.25390625,8388608
+25,self_attn.k_proj,2,128,8,128,47.86886215209961,2.126953125,8388608
+25,self_attn.k_proj,3,32,8,128,15.220548629760742,3.5078125,8388608
+25,self_attn.k_proj,3,64,8,128,18.23306655883789,3.25390625,8388608
+25,self_attn.k_proj,3,128,8,128,21.009967803955078,3.126953125,8388608
+25,self_attn.k_proj,4,32,8,128,7.1028313636779785,4.5078125,8388608
+25,self_attn.k_proj,4,64,8,128,8.508831024169922,4.25390625,8388608
+25,self_attn.k_proj,4,128,8,128,9.805822372436523,4.126953125,8388608
+25,self_attn.k_proj,8,32,8,128,0.42999157309532166,8.5078125,8388608
+25,self_attn.k_proj,8,64,8,128,0.5144612193107605,8.25390625,8388608
+25,self_attn.k_proj,8,128,8,128,0.5911647081375122,8.126953125,8388608
+25,self_attn.o_proj,2,32,8,128,34.601951599121094,20.0625,67108864
+25,self_attn.o_proj,2,64,8,128,40.3206901550293,18.03125,67108864
+25,self_attn.o_proj,2,128,8,128,45.474666595458984,17.015625,67108864
+25,self_attn.o_proj,3,32,8,128,14.804893493652344,28.0625,67108864
+25,self_attn.o_proj,3,64,8,128,17.258630752563477,26.03125,67108864
+25,self_attn.o_proj,3,128,8,128,19.484601974487305,25.015625,67108864
+25,self_attn.o_proj,4,32,8,128,6.906524181365967,36.0625,67108864
+25,self_attn.o_proj,4,64,8,128,8.061245918273926,34.03125,67108864
+25,self_attn.o_proj,4,128,8,128,9.095800399780273,33.015625,67108864
+25,self_attn.o_proj,8,32,8,128,0.4197379946708679,68.0625,67108864
+25,self_attn.o_proj,8,64,8,128,0.4890459179878235,66.03125,67108864
+25,self_attn.o_proj,8,128,8,128,0.550571084022522,65.015625,67108864
+25,mlp.gate_proj,2,32,8,128,73.99052429199219,70.21875,234881024
+25,mlp.gate_proj,2,64,8,128,86.94344329833984,63.109375,234881024
+25,mlp.gate_proj,2,128,8,128,98.33256530761719,59.5546875,234881024
+25,mlp.gate_proj,3,32,8,128,31.831628799438477,98.21875,234881024
+25,mlp.gate_proj,3,64,8,128,37.095035552978516,91.109375,234881024
+25,mlp.gate_proj,3,128,8,128,42.190277099609375,87.5546875,234881024
+25,mlp.gate_proj,4,32,8,128,14.810413360595703,126.21875,234881024
+25,mlp.gate_proj,4,64,8,128,17.326290130615234,119.109375,234881024
+25,mlp.gate_proj,4,128,8,128,19.663619995117188,115.5546875,234881024
+25,mlp.gate_proj,8,32,8,128,0.904346227645874,238.21875,234881024
+25,mlp.gate_proj,8,64,8,128,1.0589368343353271,231.109375,234881024
+25,mlp.gate_proj,8,128,8,128,1.1967191696166992,227.5546875,234881024
+25,mlp.up_proj,2,32,8,128,68.3187026977539,70.21875,234881024
+25,mlp.up_proj,2,64,8,128,79.5895004272461,63.109375,234881024
+25,mlp.up_proj,2,128,8,128,90.02964782714844,59.5546875,234881024
+25,mlp.up_proj,3,32,8,128,29.25966453552246,98.21875,234881024
+25,mlp.up_proj,3,64,8,128,34.06951141357422,91.109375,234881024
+25,mlp.up_proj,3,128,8,128,38.389869689941406,87.5546875,234881024
+25,mlp.up_proj,4,32,8,128,13.624835014343262,126.21875,234881024
+25,mlp.up_proj,4,64,8,128,15.96699047088623,119.109375,234881024
+25,mlp.up_proj,4,128,8,128,17.927520751953125,115.5546875,234881024
+25,mlp.up_proj,8,32,8,128,0.8320541977882385,238.21875,234881024
+25,mlp.up_proj,8,64,8,128,0.9743241667747498,231.109375,234881024
+25,mlp.up_proj,8,128,8,128,1.09199857711792,227.5546875,234881024
+25,mlp.down_proj,2,32,8,128,68.20001983642578,70.21875,234881024
+25,mlp.down_proj,2,64,8,128,79.45153045654297,63.109375,234881024
+25,mlp.down_proj,2,128,8,128,89.90258026123047,59.5546875,234881024
+25,mlp.down_proj,3,32,8,128,29.202898025512695,98.21875,234881024
+25,mlp.down_proj,3,64,8,128,34.00436782836914,91.109375,234881024
+25,mlp.down_proj,3,128,8,128,38.32203674316406,87.5546875,234881024
+25,mlp.down_proj,4,32,8,128,13.59666633605957,126.21875,234881024
+25,mlp.down_proj,4,64,8,128,15.929262161254883,119.109375,234881024
+25,mlp.down_proj,4,128,8,128,17.89276123046875,115.5546875,234881024
+25,mlp.down_proj,8,32,8,128,0.830154299736023,238.21875,234881024
+25,mlp.down_proj,8,64,8,128,0.9718079566955566,231.109375,234881024
+25,mlp.down_proj,8,128,8,128,1.0892091989517212,227.5546875,234881024
+26,self_attn.q_proj,2,32,8,128,47.083980560302734,20.0625,67108864
+26,self_attn.q_proj,2,64,8,128,55.33705139160156,18.03125,67108864
+26,self_attn.q_proj,2,128,8,128,63.41636657714844,17.015625,67108864
+26,self_attn.q_proj,3,32,8,128,20.144786834716797,28.0625,67108864
+26,self_attn.q_proj,3,64,8,128,23.72101593017578,26.03125,67108864
+26,self_attn.q_proj,3,128,8,128,27.18961524963379,25.015625,67108864
+26,self_attn.q_proj,4,32,8,128,9.39599895477295,36.0625,67108864
+26,self_attn.q_proj,4,64,8,128,11.073731422424316,34.03125,67108864
+26,self_attn.q_proj,4,128,8,128,12.688228607177734,33.015625,67108864
+26,self_attn.q_proj,8,32,8,128,0.5709197521209717,68.0625,67108864
+26,self_attn.q_proj,8,64,8,128,0.6726192831993103,66.03125,67108864
+26,self_attn.q_proj,8,128,8,128,0.7684031128883362,65.015625,67108864
+26,self_attn.v_proj,2,32,8,128,11.78567123413086,2.5078125,8388608
+26,self_attn.v_proj,2,64,8,128,13.816083908081055,2.25390625,8388608
+26,self_attn.v_proj,2,128,8,128,15.668251991271973,2.126953125,8388608
+26,self_attn.v_proj,3,32,8,128,5.038313865661621,3.5078125,8388608
+26,self_attn.v_proj,3,64,8,128,5.91035270690918,3.25390625,8388608
+26,self_attn.v_proj,3,128,8,128,6.709087371826172,3.126953125,8388608
+26,self_attn.v_proj,4,32,8,128,2.351661443710327,4.5078125,8388608
+26,self_attn.v_proj,4,64,8,128,2.7580389976501465,4.25390625,8388608
+26,self_attn.v_proj,4,128,8,128,3.128790855407715,4.126953125,8388608
+26,self_attn.v_proj,8,32,8,128,0.14238116145133972,8.5078125,8388608
+26,self_attn.v_proj,8,64,8,128,0.1667589545249939,8.25390625,8388608
+26,self_attn.v_proj,8,128,8,128,0.18855491280555725,8.126953125,8388608
+26,self_attn.k_proj,2,32,8,128,35.075374603271484,2.5078125,8388608
+26,self_attn.k_proj,2,64,8,128,42.050270080566406,2.25390625,8388608
+26,self_attn.k_proj,2,128,8,128,48.079776763916016,2.126953125,8388608
+26,self_attn.k_proj,3,32,8,128,15.111221313476562,3.5078125,8388608
+26,self_attn.k_proj,3,64,8,128,18.25792694091797,3.25390625,8388608
+26,self_attn.k_proj,3,128,8,128,21.239376068115234,3.126953125,8388608
+26,self_attn.k_proj,4,32,8,128,7.053283214569092,4.5078125,8388608
+26,self_attn.k_proj,4,64,8,128,8.521101951599121,4.25390625,8388608
+26,self_attn.k_proj,4,128,8,128,9.91661548614502,4.126953125,8388608
+26,self_attn.k_proj,8,32,8,128,0.4267183542251587,8.5078125,8388608
+26,self_attn.k_proj,8,64,8,128,0.5154181718826294,8.25390625,8388608
+26,self_attn.k_proj,8,128,8,128,0.5976796746253967,8.126953125,8388608
+26,self_attn.o_proj,2,32,8,128,34.46342468261719,20.0625,67108864
+26,self_attn.o_proj,2,64,8,128,40.16053771972656,18.03125,67108864
+26,self_attn.o_proj,2,128,8,128,45.31977081298828,17.015625,67108864
+26,self_attn.o_proj,3,32,8,128,14.745534896850586,28.0625,67108864
+26,self_attn.o_proj,3,64,8,128,17.193540573120117,26.03125,67108864
+26,self_attn.o_proj,3,128,8,128,19.428125381469727,25.015625,67108864
+26,self_attn.o_proj,4,32,8,128,6.877230644226074,36.0625,67108864
+26,self_attn.o_proj,4,64,8,128,8.030227661132812,34.03125,67108864
+26,self_attn.o_proj,4,128,8,128,9.068549156188965,33.015625,67108864
+26,self_attn.o_proj,8,32,8,128,0.4180542230606079,68.0625,67108864
+26,self_attn.o_proj,8,64,8,128,0.48714888095855713,66.03125,67108864
+26,self_attn.o_proj,8,128,8,128,0.5486733913421631,65.015625,67108864
+26,mlp.gate_proj,2,32,8,128,73.9573745727539,70.21875,234881024
+26,mlp.gate_proj,2,64,8,128,87.0776596069336,63.109375,234881024
+26,mlp.gate_proj,2,128,8,128,98.64891815185547,59.5546875,234881024
+26,mlp.gate_proj,3,32,8,128,31.826786041259766,98.21875,234881024
+26,mlp.gate_proj,3,64,8,128,37.16089630126953,91.109375,234881024
+26,mlp.gate_proj,3,128,8,128,42.3885383605957,87.5546875,234881024
+26,mlp.gate_proj,4,32,8,128,14.806824684143066,126.21875,234881024
+26,mlp.gate_proj,4,64,8,128,17.35426139831543,119.109375,234881024
+26,mlp.gate_proj,4,128,8,128,19.759292602539062,115.5546875,234881024
+26,mlp.gate_proj,8,32,8,128,0.9038394093513489,238.21875,234881024
+26,mlp.gate_proj,8,64,8,128,1.060187578201294,231.109375,234881024
+26,mlp.gate_proj,8,128,8,128,1.2012782096862793,227.5546875,234881024
+26,mlp.up_proj,2,32,8,128,68.42585754394531,70.21875,234881024
+26,mlp.up_proj,2,64,8,128,79.7314453125,63.109375,234881024
+26,mlp.up_proj,2,128,8,128,90.19134521484375,59.5546875,234881024
+26,mlp.up_proj,3,32,8,128,29.31565284729004,98.21875,234881024
+26,mlp.up_proj,3,64,8,128,34.13851547241211,91.109375,234881024
+26,mlp.up_proj,3,128,8,128,38.490379333496094,87.5546875,234881024
+26,mlp.up_proj,4,32,8,128,13.648061752319336,126.21875,234881024
+26,mlp.up_proj,4,64,8,128,16.00127410888672,119.109375,234881024
+26,mlp.up_proj,4,128,8,128,17.9852237701416,115.5546875,234881024
+26,mlp.up_proj,8,32,8,128,0.8333629369735718,238.21875,234881024
+26,mlp.up_proj,8,64,8,128,0.9761505722999573,231.109375,234881024
+26,mlp.up_proj,8,128,8,128,1.0944294929504395,227.5546875,234881024
+26,mlp.down_proj,2,32,8,128,68.05906677246094,70.21875,234881024
+26,mlp.down_proj,2,64,8,128,79.29278564453125,63.109375,234881024
+26,mlp.down_proj,2,128,8,128,89.69574737548828,59.5546875,234881024
+26,mlp.down_proj,3,32,8,128,29.144756317138672,98.21875,234881024
+26,mlp.down_proj,3,64,8,128,33.94838333129883,91.109375,234881024
+26,mlp.down_proj,3,128,8,128,38.278995513916016,87.5546875,234881024
+26,mlp.down_proj,4,32,8,128,13.569798469543457,126.21875,234881024
+26,mlp.down_proj,4,64,8,128,15.90481948852539,119.109375,234881024
+26,mlp.down_proj,4,128,8,128,17.881196975708008,115.5546875,234881024
+26,mlp.down_proj,8,32,8,128,0.82847660779953,238.21875,234881024
+26,mlp.down_proj,8,64,8,128,0.9701429605484009,231.109375,234881024
+26,mlp.down_proj,8,128,8,128,1.0881397724151611,227.5546875,234881024
+27,self_attn.q_proj,2,32,8,128,49.84476089477539,20.0625,67108864
+27,self_attn.q_proj,2,64,8,128,58.92628860473633,18.03125,67108864
+27,self_attn.q_proj,2,128,8,128,67.9470443725586,17.015625,67108864
+27,self_attn.q_proj,3,32,8,128,21.344709396362305,28.0625,67108864
+27,self_attn.q_proj,3,64,8,128,25.253633499145508,26.03125,67108864
+27,self_attn.q_proj,3,128,8,128,29.205476760864258,25.015625,67108864
+27,self_attn.q_proj,4,32,8,128,9.954429626464844,36.0625,67108864
+27,self_attn.q_proj,4,64,8,128,11.788969039916992,34.03125,67108864
+27,self_attn.q_proj,4,128,8,128,13.625144004821777,33.015625,67108864
+27,self_attn.q_proj,8,32,8,128,0.6048736572265625,68.0625,67108864
+27,self_attn.q_proj,8,64,8,128,0.716489315032959,66.03125,67108864
+27,self_attn.q_proj,8,128,8,128,0.8249659538269043,65.015625,67108864
+27,self_attn.v_proj,2,32,8,128,9.8643217086792,2.5078125,8388608
+27,self_attn.v_proj,2,64,8,128,11.589247703552246,2.25390625,8388608
+27,self_attn.v_proj,2,128,8,128,13.171034812927246,2.126953125,8388608
+27,self_attn.v_proj,3,32,8,128,4.216168403625488,3.5078125,8388608
+27,self_attn.v_proj,3,64,8,128,4.955900192260742,3.25390625,8388608
+27,self_attn.v_proj,3,128,8,128,5.635286331176758,3.126953125,8388608
+27,self_attn.v_proj,4,32,8,128,1.9673177003860474,4.5078125,8388608
+27,self_attn.v_proj,4,64,8,128,2.3129966259002686,4.25390625,8388608
+27,self_attn.v_proj,4,128,8,128,2.6298961639404297,4.126953125,8388608
+27,self_attn.v_proj,8,32,8,128,0.1190623939037323,8.5078125,8388608
+27,self_attn.v_proj,8,64,8,128,0.1398770809173584,8.25390625,8388608
+27,self_attn.v_proj,8,128,8,128,0.15852779150009155,8.126953125,8388608
+27,self_attn.k_proj,2,32,8,128,36.916648864746094,2.5078125,8388608
+27,self_attn.k_proj,2,64,8,128,44.39013671875,2.25390625,8388608
+27,self_attn.k_proj,2,128,8,128,50.8521842956543,2.126953125,8388608
+27,self_attn.k_proj,3,32,8,128,15.900928497314453,3.5078125,8388608
+27,self_attn.k_proj,3,64,8,128,19.273344039916992,3.25390625,8388608
+27,self_attn.k_proj,3,128,8,128,22.451162338256836,3.126953125,8388608
+27,self_attn.k_proj,4,32,8,128,7.4187445640563965,4.5078125,8388608
+27,self_attn.k_proj,4,64,8,128,8.993167877197266,4.25390625,8388608
+27,self_attn.k_proj,4,128,8,128,10.478439331054688,4.126953125,8388608
+27,self_attn.k_proj,8,32,8,128,0.44913873076438904,8.5078125,8388608
+27,self_attn.k_proj,8,64,8,128,0.5438657999038696,8.25390625,8388608
+27,self_attn.k_proj,8,128,8,128,0.6316400766372681,8.126953125,8388608
+27,self_attn.o_proj,2,32,8,128,33.25370788574219,20.0625,67108864
+27,self_attn.o_proj,2,64,8,128,38.76079177856445,18.03125,67108864
+27,self_attn.o_proj,2,128,8,128,43.79789733886719,17.015625,67108864
+27,self_attn.o_proj,3,32,8,128,14.223897933959961,28.0625,67108864
+27,self_attn.o_proj,3,64,8,128,16.61452865600586,26.03125,67108864
+27,self_attn.o_proj,3,128,8,128,18.800975799560547,25.015625,67108864
+27,self_attn.o_proj,4,32,8,128,6.635249137878418,36.0625,67108864
+27,self_attn.o_proj,4,64,8,128,7.757177352905273,34.03125,67108864
+27,self_attn.o_proj,4,128,8,128,8.776421546936035,33.015625,67108864
+27,self_attn.o_proj,8,32,8,128,0.4033682644367218,68.0625,67108864
+27,self_attn.o_proj,8,64,8,128,0.4707346558570862,66.03125,67108864
+27,self_attn.o_proj,8,128,8,128,0.5310478806495667,65.015625,67108864
+27,mlp.gate_proj,2,32,8,128,72.92655944824219,70.21875,234881024
+27,mlp.gate_proj,2,64,8,128,85.9267807006836,63.109375,234881024
+27,mlp.gate_proj,2,128,8,128,97.50157928466797,59.5546875,234881024
+27,mlp.gate_proj,3,32,8,128,31.361446380615234,98.21875,234881024
+27,mlp.gate_proj,3,64,8,128,36.67316818237305,91.109375,234881024
+27,mlp.gate_proj,3,128,8,128,41.848148345947266,87.5546875,234881024
+27,mlp.gate_proj,4,32,8,128,14.590837478637695,126.21875,234881024
+27,mlp.gate_proj,4,64,8,128,17.122400283813477,119.109375,234881024
+27,mlp.gate_proj,4,128,8,128,19.508316040039062,115.5546875,234881024
+27,mlp.gate_proj,8,32,8,128,0.8905929327011108,238.21875,234881024
+27,mlp.gate_proj,8,64,8,128,1.0466492176055908,231.109375,234881024
+27,mlp.gate_proj,8,128,8,128,1.1862571239471436,227.5546875,234881024
+27,mlp.up_proj,2,32,8,128,68.03720092773438,70.21875,234881024
+27,mlp.up_proj,2,64,8,128,79.25502014160156,63.109375,234881024
+27,mlp.up_proj,2,128,8,128,89.63394165039062,59.5546875,234881024
+27,mlp.up_proj,3,32,8,128,29.129423141479492,98.21875,234881024
+27,mlp.up_proj,3,64,8,128,33.93697738647461,91.109375,234881024
+27,mlp.up_proj,3,128,8,128,38.238319396972656,87.5546875,234881024
+27,mlp.up_proj,4,32,8,128,13.564035415649414,126.21875,234881024
+27,mlp.up_proj,4,64,8,128,15.896672248840332,119.109375,234881024
+27,mlp.up_proj,4,128,8,128,17.867677688598633,115.5546875,234881024
+27,mlp.up_proj,8,32,8,128,0.828164279460907,238.21875,234881024
+27,mlp.up_proj,8,64,8,128,0.9696444272994995,231.109375,234881024
+27,mlp.up_proj,8,128,8,128,1.0872124433517456,227.5546875,234881024
+27,mlp.down_proj,2,32,8,128,67.93889617919922,70.21875,234881024
+27,mlp.down_proj,2,64,8,128,79.14494323730469,63.109375,234881024
+27,mlp.down_proj,2,128,8,128,89.54010009765625,59.5546875,234881024
+27,mlp.down_proj,3,32,8,128,29.0874080657959,98.21875,234881024
+27,mlp.down_proj,3,64,8,128,33.88753128051758,91.109375,234881024
+27,mlp.down_proj,3,128,8,128,38.21411895751953,87.5546875,234881024
+27,mlp.down_proj,4,32,8,128,13.545812606811523,126.21875,234881024
+27,mlp.down_proj,4,64,8,128,15.874497413635254,119.109375,234881024
+27,mlp.down_proj,4,128,8,128,17.849069595336914,115.5546875,234881024
+27,mlp.down_proj,8,32,8,128,0.8270373344421387,238.21875,234881024
+27,mlp.down_proj,8,64,8,128,0.968384861946106,231.109375,234881024
+27,mlp.down_proj,8,128,8,128,1.0862085819244385,227.5546875,234881024
+28,self_attn.q_proj,2,32,8,128,48.146392822265625,20.0625,67108864
+28,self_attn.q_proj,2,64,8,128,56.60183334350586,18.03125,67108864
+28,self_attn.q_proj,2,128,8,128,65.12285614013672,17.015625,67108864
+28,self_attn.q_proj,3,32,8,128,20.607765197753906,28.0625,67108864
+28,self_attn.q_proj,3,64,8,128,24.267560958862305,26.03125,67108864
+28,self_attn.q_proj,3,128,8,128,27.955015182495117,25.015625,67108864
+28,self_attn.q_proj,4,32,8,128,9.611725807189941,36.0625,67108864
+28,self_attn.q_proj,4,64,8,128,11.33084774017334,34.03125,67108864
+28,self_attn.q_proj,4,128,8,128,13.046244621276855,33.015625,67108864
+28,self_attn.q_proj,8,32,8,128,0.5839813351631165,68.0625,67108864
+28,self_attn.q_proj,8,64,8,128,0.688347339630127,66.03125,67108864
+28,self_attn.q_proj,8,128,8,128,0.7899396419525146,65.015625,67108864
+28,self_attn.v_proj,2,32,8,128,10.467779159545898,2.5078125,8388608
+28,self_attn.v_proj,2,64,8,128,12.413727760314941,2.25390625,8388608
+28,self_attn.v_proj,2,128,8,128,14.21363353729248,2.126953125,8388608
+28,self_attn.v_proj,3,32,8,128,4.477202892303467,3.5078125,8388608
+28,self_attn.v_proj,3,64,8,128,5.315450668334961,3.25390625,8388608
+28,self_attn.v_proj,3,128,8,128,6.090766906738281,3.126953125,8388608
+28,self_attn.v_proj,4,32,8,128,2.089815855026245,4.5078125,8388608
+28,self_attn.v_proj,4,64,8,128,2.481015205383301,4.25390625,8388608
+28,self_attn.v_proj,4,128,8,128,2.8428783416748047,4.126953125,8388608
+28,self_attn.v_proj,8,32,8,128,0.12645933032035828,8.5078125,8388608
+28,self_attn.v_proj,8,64,8,128,0.1500316858291626,8.25390625,8388608
+28,self_attn.v_proj,8,128,8,128,0.17146465182304382,8.126953125,8388608
+28,self_attn.k_proj,2,32,8,128,35.672698974609375,2.5078125,8388608
+28,self_attn.k_proj,2,64,8,128,42.739906311035156,2.25390625,8388608
+28,self_attn.k_proj,2,128,8,128,48.75528335571289,2.126953125,8388608
+28,self_attn.k_proj,3,32,8,128,15.385613441467285,3.5078125,8388608
+28,self_attn.k_proj,3,64,8,128,18.568578720092773,3.25390625,8388608
+28,self_attn.k_proj,3,128,8,128,21.581567764282227,3.126953125,8388608
+28,self_attn.k_proj,4,32,8,128,7.180924415588379,4.5078125,8388608
+28,self_attn.k_proj,4,64,8,128,8.670633316040039,4.25390625,8388608
+28,self_attn.k_proj,4,128,8,128,10.076040267944336,4.126953125,8388608
+28,self_attn.k_proj,8,32,8,128,0.43443161249160767,8.5078125,8388608
+28,self_attn.k_proj,8,64,8,128,0.5241854190826416,8.25390625,8388608
+28,self_attn.k_proj,8,128,8,128,0.6072473526000977,8.126953125,8388608
+28,self_attn.o_proj,2,32,8,128,33.5322265625,20.0625,67108864
+28,self_attn.o_proj,2,64,8,128,39.06879806518555,18.03125,67108864
+28,self_attn.o_proj,2,128,8,128,44.086830139160156,17.015625,67108864
+28,self_attn.o_proj,3,32,8,128,14.35306453704834,28.0625,67108864
+28,self_attn.o_proj,3,64,8,128,16.74953269958496,26.03125,67108864
+28,self_attn.o_proj,3,128,8,128,18.939416885375977,25.015625,67108864
+28,self_attn.o_proj,4,32,8,128,6.69839334487915,36.0625,67108864
+28,self_attn.o_proj,4,64,8,128,7.8291215896606445,34.03125,67108864
+28,self_attn.o_proj,4,128,8,128,8.8527250289917,33.015625,67108864
+28,self_attn.o_proj,8,32,8,128,0.40731319785118103,68.0625,67108864
+28,self_attn.o_proj,8,64,8,128,0.4752146005630493,66.03125,67108864
+28,self_attn.o_proj,8,128,8,128,0.5360721349716187,65.015625,67108864
+28,mlp.gate_proj,2,32,8,128,73.00426483154297,70.21875,234881024
+28,mlp.gate_proj,2,64,8,128,86.17831420898438,63.109375,234881024
+28,mlp.gate_proj,2,128,8,128,98.02418518066406,59.5546875,234881024
+28,mlp.gate_proj,3,32,8,128,31.406492233276367,98.21875,234881024
+28,mlp.gate_proj,3,64,8,128,36.82604217529297,91.109375,234881024
+28,mlp.gate_proj,3,128,8,128,42.18220520019531,87.5546875,234881024
+28,mlp.gate_proj,4,32,8,128,14.613818168640137,126.21875,234881024
+28,mlp.gate_proj,4,64,8,128,17.191295623779297,119.109375,234881024
+28,mlp.gate_proj,4,128,8,128,19.67759132385254,115.5546875,234881024
+28,mlp.gate_proj,8,32,8,128,0.8919325470924377,238.21875,234881024
+28,mlp.gate_proj,8,64,8,128,1.0505988597869873,231.109375,234881024
+28,mlp.gate_proj,8,128,8,128,1.196032166481018,227.5546875,234881024
+28,mlp.up_proj,2,32,8,128,67.99238586425781,70.21875,234881024
+28,mlp.up_proj,2,64,8,128,79.22877502441406,63.109375,234881024
+28,mlp.up_proj,2,128,8,128,89.6084213256836,59.5546875,234881024
+28,mlp.up_proj,3,32,8,128,29.108850479125977,98.21875,234881024
+28,mlp.up_proj,3,64,8,128,33.920650482177734,91.109375,234881024
+28,mlp.up_proj,3,128,8,128,38.24842834472656,87.5546875,234881024
+28,mlp.up_proj,4,32,8,128,13.554842948913574,126.21875,234881024
+28,mlp.up_proj,4,64,8,128,15.889409065246582,119.109375,234881024
+28,mlp.up_proj,4,128,8,128,17.864017486572266,115.5546875,234881024
+28,mlp.up_proj,8,32,8,128,0.8276427388191223,238.21875,234881024
+28,mlp.up_proj,8,64,8,128,0.9692996740341187,231.109375,234881024
+28,mlp.up_proj,8,128,8,128,1.0871031284332275,227.5546875,234881024
+28,mlp.down_proj,2,32,8,128,67.8237533569336,70.21875,234881024
+28,mlp.down_proj,2,64,8,128,79.0003433227539,63.109375,234881024
+28,mlp.down_proj,2,128,8,128,89.37496185302734,59.5546875,234881024
+28,mlp.down_proj,3,32,8,128,29.031822204589844,98.21875,234881024
+28,mlp.down_proj,3,64,8,128,33.83279037475586,91.109375,234881024
+28,mlp.down_proj,3,128,8,128,38.144344329833984,87.5546875,234881024
+28,mlp.down_proj,4,32,8,128,13.520061492919922,126.21875,234881024
+28,mlp.down_proj,4,64,8,128,15.846943855285645,119.109375,234881024
+28,mlp.down_proj,4,128,8,128,17.822616577148438,115.5546875,234881024
+28,mlp.down_proj,8,32,8,128,0.8254652619361877,238.21875,234881024
+28,mlp.down_proj,8,64,8,128,0.9665296077728271,231.109375,234881024
+28,mlp.down_proj,8,128,8,128,1.0844069719314575,227.5546875,234881024
+29,self_attn.q_proj,2,32,8,128,47.52573776245117,20.0625,67108864
+29,self_attn.q_proj,2,64,8,128,55.95750427246094,18.03125,67108864
+29,self_attn.q_proj,2,128,8,128,64.52127075195312,17.015625,67108864
+29,self_attn.q_proj,3,32,8,128,20.335607528686523,28.0625,67108864
+29,self_attn.q_proj,3,64,8,128,23.986539840698242,26.03125,67108864
+29,self_attn.q_proj,3,128,8,128,27.690242767333984,25.015625,67108864
+29,self_attn.q_proj,4,32,8,128,9.484135627746582,36.0625,67108864
+29,self_attn.q_proj,4,64,8,128,11.202115058898926,34.03125,67108864
+29,self_attn.q_proj,4,128,8,128,12.921989440917969,33.015625,67108864
+29,self_attn.q_proj,8,32,8,128,0.5762859582901001,68.0625,67108864
+29,self_attn.q_proj,8,64,8,128,0.6802347302436829,66.03125,67108864
+29,self_attn.q_proj,8,128,8,128,0.7825019955635071,65.015625,67108864
+29,self_attn.v_proj,2,32,8,128,9.82052230834961,2.5078125,8388608
+29,self_attn.v_proj,2,64,8,128,11.577470779418945,2.25390625,8388608
+29,self_attn.v_proj,2,128,8,128,13.20482063293457,2.126953125,8388608
+29,self_attn.v_proj,3,32,8,128,4.1994404792785645,3.5078125,8388608
+29,self_attn.v_proj,3,64,8,128,4.955023765563965,3.25390625,8388608
+29,self_attn.v_proj,3,128,8,128,5.656523704528809,3.126953125,8388608
+29,self_attn.v_proj,4,32,8,128,1.9597570896148682,4.5078125,8388608
+29,self_attn.v_proj,4,64,8,128,2.311890125274658,4.25390625,8388608
+29,self_attn.v_proj,4,128,8,128,2.6397783756256104,4.126953125,8388608
+29,self_attn.v_proj,8,32,8,128,0.1186094805598259,8.5078125,8388608
+29,self_attn.v_proj,8,64,8,128,0.13978399336338043,8.25390625,8388608
+29,self_attn.v_proj,8,128,8,128,0.15908226370811462,8.126953125,8388608
+29,self_attn.k_proj,2,32,8,128,36.11398696899414,2.5078125,8388608
+29,self_attn.k_proj,2,64,8,128,43.17828369140625,2.25390625,8388608
+29,self_attn.k_proj,2,128,8,128,49.04681396484375,2.126953125,8388608
+29,self_attn.k_proj,3,32,8,128,15.59365177154541,3.5078125,8388608
+29,self_attn.k_proj,3,64,8,128,18.786762237548828,3.25390625,8388608
+29,self_attn.k_proj,3,128,8,128,21.765201568603516,3.126953125,8388608
+29,self_attn.k_proj,4,32,8,128,7.277462005615234,4.5078125,8388608
+29,self_attn.k_proj,4,64,8,128,8.768425941467285,4.25390625,8388608
+29,self_attn.k_proj,4,128,8,128,10.157294273376465,4.126953125,8388608
+29,self_attn.k_proj,8,32,8,128,0.4402463138103485,8.5078125,8388608
+29,self_attn.k_proj,8,64,8,128,0.530223548412323,8.25390625,8388608
+29,self_attn.k_proj,8,128,8,128,0.6123812794685364,8.126953125,8388608
+29,self_attn.o_proj,2,32,8,128,33.599361419677734,20.0625,67108864
+29,self_attn.o_proj,2,64,8,128,39.154178619384766,18.03125,67108864
+29,self_attn.o_proj,2,128,8,128,44.22515869140625,17.015625,67108864
+29,self_attn.o_proj,3,32,8,128,14.382062911987305,28.0625,67108864
+29,self_attn.o_proj,3,64,8,128,16.78978729248047,26.03125,67108864
+29,self_attn.o_proj,3,128,8,128,18.99472427368164,25.015625,67108864
+29,self_attn.o_proj,4,32,8,128,6.709314346313477,36.0625,67108864
+29,self_attn.o_proj,4,64,8,128,7.844139099121094,34.03125,67108864
+29,self_attn.o_proj,4,128,8,128,8.874648094177246,33.015625,67108864
+29,self_attn.o_proj,8,32,8,128,0.40792641043663025,68.0625,67108864
+29,self_attn.o_proj,8,64,8,128,0.4759964942932129,66.03125,67108864
+29,self_attn.o_proj,8,128,8,128,0.5372146964073181,65.015625,67108864
+29,mlp.gate_proj,2,32,8,128,72.67528533935547,70.21875,234881024
+29,mlp.gate_proj,2,64,8,128,85.6358413696289,63.109375,234881024
+29,mlp.gate_proj,2,128,8,128,97.27716827392578,59.5546875,234881024
+29,mlp.gate_proj,3,32,8,128,31.250316619873047,98.21875,234881024
+29,mlp.gate_proj,3,64,8,128,36.58244323730469,91.109375,234881024
+29,mlp.gate_proj,3,128,8,128,41.774471282958984,87.5546875,234881024
+29,mlp.gate_proj,4,32,8,128,14.540748596191406,126.21875,234881024
+29,mlp.gate_proj,4,64,8,128,17.0800838470459,119.109375,234881024
+29,mlp.gate_proj,4,128,8,128,19.48118782043457,115.5546875,234881024
+29,mlp.gate_proj,8,32,8,128,0.8875487446784973,238.21875,234881024
+29,mlp.gate_proj,8,64,8,128,1.0440508127212524,231.109375,234881024
+29,mlp.gate_proj,8,128,8,128,1.1846528053283691,227.5546875,234881024
+29,mlp.up_proj,2,32,8,128,67.83169555664062,70.21875,234881024
+29,mlp.up_proj,2,64,8,128,79.01787567138672,63.109375,234881024
+29,mlp.up_proj,2,128,8,128,89.37704467773438,59.5546875,234881024
+29,mlp.up_proj,3,32,8,128,29.037107467651367,98.21875,234881024
+29,mlp.up_proj,3,64,8,128,33.83564758300781,91.109375,234881024
+29,mlp.up_proj,3,128,8,128,38.14558029174805,87.5546875,234881024
+29,mlp.up_proj,4,32,8,128,13.519640922546387,126.21875,234881024
+29,mlp.up_proj,4,64,8,128,15.845134735107422,119.109375,234881024
+29,mlp.up_proj,4,128,8,128,17.814693450927734,115.5546875,234881024
+29,mlp.up_proj,8,32,8,128,0.8255332708358765,238.21875,234881024
+29,mlp.up_proj,8,64,8,128,0.966469407081604,231.109375,234881024
+29,mlp.up_proj,8,128,8,128,1.0840320587158203,227.5546875,234881024
+29,mlp.down_proj,2,32,8,128,67.5396728515625,70.21875,234881024
+29,mlp.down_proj,2,64,8,128,78.6578369140625,63.109375,234881024
+29,mlp.down_proj,2,128,8,128,88.93196868896484,59.5546875,234881024
+29,mlp.down_proj,3,32,8,128,28.901416778564453,98.21875,234881024
+29,mlp.down_proj,3,64,8,128,33.67726516723633,91.109375,234881024
+29,mlp.down_proj,3,128,8,128,37.93714904785156,87.5546875,234881024
+29,mlp.down_proj,4,32,8,128,13.458597183227539,126.21875,234881024
+29,mlp.down_proj,4,64,8,128,15.769915580749512,119.109375,234881024
+29,mlp.down_proj,4,128,8,128,17.722280502319336,115.5546875,234881024
+29,mlp.down_proj,8,32,8,128,0.8218253254890442,238.21875,234881024
+29,mlp.down_proj,8,64,8,128,0.9617174863815308,231.109375,234881024
+29,mlp.down_proj,8,128,8,128,1.0788441896438599,227.5546875,234881024
+30,self_attn.q_proj,2,32,8,128,49.54801940917969,20.0625,67108864
+30,self_attn.q_proj,2,64,8,128,58.529502868652344,18.03125,67108864
+30,self_attn.q_proj,2,128,8,128,67.39814758300781,17.015625,67108864
+30,self_attn.q_proj,3,32,8,128,21.24106788635254,28.0625,67108864
+30,self_attn.q_proj,3,64,8,128,25.12933921813965,26.03125,67108864
+30,self_attn.q_proj,3,128,8,128,29.059755325317383,25.015625,67108864
+30,self_attn.q_proj,4,32,8,128,9.90589714050293,36.0625,67108864
+30,self_attn.q_proj,4,64,8,128,11.73281478881836,34.03125,67108864
+30,self_attn.q_proj,4,128,8,128,13.559320449829102,33.015625,67108864
+30,self_attn.q_proj,8,32,8,128,0.6019740700721741,68.0625,67108864
+30,self_attn.q_proj,8,64,8,128,0.7132033705711365,66.03125,67108864
+30,self_attn.q_proj,8,128,8,128,0.8210058212280273,65.015625,67108864
+30,self_attn.v_proj,2,32,8,128,10.501734733581543,2.5078125,8388608
+30,self_attn.v_proj,2,64,8,128,12.380599021911621,2.25390625,8388608
+30,self_attn.v_proj,2,128,8,128,14.106791496276855,2.126953125,8388608
+30,self_attn.v_proj,3,32,8,128,4.493621349334717,3.5078125,8388608
+30,self_attn.v_proj,3,64,8,128,5.295505523681641,3.25390625,8388608
+30,self_attn.v_proj,3,128,8,128,6.0399322509765625,3.126953125,8388608
+30,self_attn.v_proj,4,32,8,128,2.0961320400238037,4.5078125,8388608
+30,self_attn.v_proj,4,64,8,128,2.4708356857299805,4.25390625,8388608
+30,self_attn.v_proj,4,128,8,128,2.81715989112854,4.126953125,8388608
+30,self_attn.v_proj,8,32,8,128,0.12688598036766052,8.5078125,8388608
+30,self_attn.v_proj,8,64,8,128,0.1494956612586975,8.25390625,8388608
+30,self_attn.v_proj,8,128,8,128,0.1698315441608429,8.126953125,8388608
+30,self_attn.k_proj,2,32,8,128,36.18784713745117,2.5078125,8388608
+30,self_attn.k_proj,2,64,8,128,43.26191711425781,2.25390625,8388608
+30,self_attn.k_proj,2,128,8,128,49.378170013427734,2.126953125,8388608
+30,self_attn.k_proj,3,32,8,128,15.574408531188965,3.5078125,8388608
+30,self_attn.k_proj,3,64,8,128,18.73681640625,3.25390625,8388608
+30,self_attn.k_proj,3,128,8,128,21.722909927368164,3.126953125,8388608
+30,self_attn.k_proj,4,32,8,128,7.263796806335449,4.5078125,8388608
+30,self_attn.k_proj,4,64,8,128,8.748115539550781,4.25390625,8388608
+30,self_attn.k_proj,4,128,8,128,10.14378833770752,4.126953125,8388608
+30,self_attn.k_proj,8,32,8,128,0.43958282470703125,8.5078125,8388608
+30,self_attn.k_proj,8,64,8,128,0.5287224054336548,8.25390625,8388608
+30,self_attn.k_proj,8,128,8,128,0.6110882759094238,8.126953125,8388608
+30,self_attn.o_proj,2,32,8,128,33.5689697265625,20.0625,67108864
+30,self_attn.o_proj,2,64,8,128,39.107704162597656,18.03125,67108864
+30,self_attn.o_proj,2,128,8,128,44.152652740478516,17.015625,67108864
+30,self_attn.o_proj,3,32,8,128,14.360038757324219,28.0625,67108864
+30,self_attn.o_proj,3,64,8,128,16.75472640991211,26.03125,67108864
+30,self_attn.o_proj,3,128,8,128,18.935224533081055,25.015625,67108864
+30,self_attn.o_proj,4,32,8,128,6.698476314544678,36.0625,67108864
+30,self_attn.o_proj,4,64,8,128,7.824026584625244,34.03125,67108864
+30,self_attn.o_proj,4,128,8,128,8.841182708740234,33.015625,67108864
+30,self_attn.o_proj,8,32,8,128,0.4071967303752899,68.0625,67108864
+30,self_attn.o_proj,8,64,8,128,0.47476452589035034,66.03125,67108864
+30,self_attn.o_proj,8,128,8,128,0.5349661111831665,65.015625,67108864
+30,mlp.gate_proj,2,32,8,128,72.37278747558594,70.21875,234881024
+30,mlp.gate_proj,2,64,8,128,85.42486572265625,63.109375,234881024
+30,mlp.gate_proj,2,128,8,128,97.20103454589844,59.5546875,234881024
+30,mlp.gate_proj,3,32,8,128,31.115589141845703,98.21875,234881024
+30,mlp.gate_proj,3,64,8,128,36.48292541503906,91.109375,234881024
+30,mlp.gate_proj,3,128,8,128,41.75707244873047,87.5546875,234881024
+30,mlp.gate_proj,4,32,8,128,14.47757625579834,126.21875,234881024
+30,mlp.gate_proj,4,64,8,128,17.037147521972656,119.109375,234881024
+30,mlp.gate_proj,4,128,8,128,19.472454071044922,115.5546875,234881024
+30,mlp.gate_proj,8,32,8,128,0.8836104273796082,238.21875,234881024
+30,mlp.gate_proj,8,64,8,128,1.0413802862167358,231.109375,234881024
+30,mlp.gate_proj,8,128,8,128,1.1836825609207153,227.5546875,234881024
+30,mlp.up_proj,2,32,8,128,67.71720123291016,70.21875,234881024
+30,mlp.up_proj,2,64,8,128,78.8940658569336,63.109375,234881024
+30,mlp.up_proj,2,128,8,128,89.26329040527344,59.5546875,234881024
+30,mlp.up_proj,3,32,8,128,28.981489181518555,98.21875,234881024
+30,mlp.up_proj,3,64,8,128,33.77913284301758,91.109375,234881024
+30,mlp.up_proj,3,128,8,128,38.085269927978516,87.5546875,234881024
+30,mlp.up_proj,4,32,8,128,13.493767738342285,126.21875,234881024
+30,mlp.up_proj,4,64,8,128,15.81805419921875,119.109375,234881024
+30,mlp.up_proj,4,128,8,128,17.780054092407227,115.5546875,234881024
+30,mlp.up_proj,8,32,8,128,0.8239632844924927,238.21875,234881024
+30,mlp.up_proj,8,64,8,128,0.9646780490875244,231.109375,234881024
+30,mlp.up_proj,8,128,8,128,1.082389235496521,227.5546875,234881024
+30,mlp.down_proj,2,32,8,128,67.42532348632812,70.21875,234881024
+30,mlp.down_proj,2,64,8,128,78.51761627197266,63.109375,234881024
+30,mlp.down_proj,2,128,8,128,88.82176971435547,59.5546875,234881024
+30,mlp.down_proj,3,32,8,128,28.85008430480957,98.21875,234881024
+30,mlp.down_proj,3,64,8,128,33.63211441040039,91.109375,234881024
+30,mlp.down_proj,3,128,8,128,37.91531753540039,87.5546875,234881024
+30,mlp.down_proj,4,32,8,128,13.435891151428223,126.21875,234881024
+30,mlp.down_proj,4,64,8,128,15.745800971984863,119.109375,234881024
+30,mlp.down_proj,4,128,8,128,17.720096588134766,115.5546875,234881024
+30,mlp.down_proj,8,32,8,128,0.8205734491348267,238.21875,234881024
+30,mlp.down_proj,8,64,8,128,0.9603773951530457,231.109375,234881024
+30,mlp.down_proj,8,128,8,128,1.0780234336853027,227.5546875,234881024
+31,self_attn.q_proj,2,32,8,128,51.14046096801758,20.0625,67108864
+31,self_attn.q_proj,2,64,8,128,60.59627151489258,18.03125,67108864
+31,self_attn.q_proj,2,128,8,128,70.117919921875,17.015625,67108864
+31,self_attn.q_proj,3,32,8,128,21.91644287109375,28.0625,67108864
+31,self_attn.q_proj,3,64,8,128,25.989696502685547,26.03125,67108864
+31,self_attn.q_proj,3,128,8,128,30.230167388916016,25.015625,67108864
+31,self_attn.q_proj,4,32,8,128,10.220741271972656,36.0625,67108864
+31,self_attn.q_proj,4,64,8,128,12.130699157714844,34.03125,67108864
+31,self_attn.q_proj,4,128,8,128,14.103045463562012,33.015625,67108864
+31,self_attn.q_proj,8,32,8,128,0.6211313009262085,68.0625,67108864
+31,self_attn.q_proj,8,64,8,128,0.7370679378509521,66.03125,67108864
+31,self_attn.q_proj,8,128,8,128,0.8538223505020142,65.015625,67108864
+31,self_attn.v_proj,2,32,8,128,8.923979759216309,2.5078125,8388608
+31,self_attn.v_proj,2,64,8,128,10.539978981018066,2.25390625,8388608
+31,self_attn.v_proj,2,128,8,128,12.041669845581055,2.126953125,8388608
+31,self_attn.v_proj,3,32,8,128,3.8189823627471924,3.5078125,8388608
+31,self_attn.v_proj,3,64,8,128,4.51163911819458,3.25390625,8388608
+31,self_attn.v_proj,3,128,8,128,5.160958290100098,3.126953125,8388608
+31,self_attn.v_proj,4,32,8,128,1.7813353538513184,4.5078125,8388608
+31,self_attn.v_proj,4,64,8,128,2.1056413650512695,4.25390625,8388608
+31,self_attn.v_proj,4,128,8,128,2.409059524536133,4.126953125,8388608
+31,self_attn.v_proj,8,32,8,128,0.10782914608716965,8.5078125,8388608
+31,self_attn.v_proj,8,64,8,128,0.12729644775390625,8.25390625,8388608
+31,self_attn.v_proj,8,128,8,128,0.145102858543396,8.126953125,8388608
+31,self_attn.k_proj,2,32,8,128,36.07902145385742,2.5078125,8388608
+31,self_attn.k_proj,2,64,8,128,43.4513053894043,2.25390625,8388608
+31,self_attn.k_proj,2,128,8,128,49.9744987487793,2.126953125,8388608
+31,self_attn.k_proj,3,32,8,128,15.556744575500488,3.5078125,8388608
+31,self_attn.k_proj,3,64,8,128,18.86691665649414,3.25390625,8388608
+31,self_attn.k_proj,3,128,8,128,22.073871612548828,3.126953125,8388608
+31,self_attn.k_proj,4,32,8,128,7.257084369659424,4.5078125,8388608
+31,self_attn.k_proj,4,64,8,128,8.805440902709961,4.25390625,8388608
+31,self_attn.k_proj,4,128,8,128,10.302979469299316,4.126953125,8388608
+31,self_attn.k_proj,8,32,8,128,0.43899354338645935,8.5078125,8388608
+31,self_attn.k_proj,8,64,8,128,0.5324029326438904,8.25390625,8388608
+31,self_attn.k_proj,8,128,8,128,0.6210692524909973,8.126953125,8388608
+31,self_attn.o_proj,2,32,8,128,32.363887786865234,20.0625,67108864
+31,self_attn.o_proj,2,64,8,128,37.68110656738281,18.03125,67108864
+31,self_attn.o_proj,2,128,8,128,42.53911209106445,17.015625,67108864
+31,self_attn.o_proj,3,32,8,128,13.837742805480957,28.0625,67108864
+31,self_attn.o_proj,3,64,8,128,16.157773971557617,26.03125,67108864
+31,self_attn.o_proj,3,128,8,128,18.259550094604492,25.015625,67108864
+31,self_attn.o_proj,4,32,8,128,6.457170009613037,36.0625,67108864
+31,self_attn.o_proj,4,64,8,128,7.543743133544922,34.03125,67108864
+31,self_attn.o_proj,4,128,8,128,8.531970024108887,33.015625,67108864
+31,self_attn.o_proj,8,32,8,128,0.3926047086715698,68.0625,67108864
+31,self_attn.o_proj,8,64,8,128,0.4579595923423767,66.03125,67108864
+31,self_attn.o_proj,8,128,8,128,0.5161820650100708,65.015625,67108864
+31,mlp.gate_proj,2,32,8,128,71.41051483154297,70.21875,234881024
+31,mlp.gate_proj,2,64,8,128,84.27467346191406,63.109375,234881024
+31,mlp.gate_proj,2,128,8,128,96.01087951660156,59.5546875,234881024
+31,mlp.gate_proj,3,32,8,128,30.6951847076416,98.21875,234881024
+31,mlp.gate_proj,3,64,8,128,36.033348083496094,91.109375,234881024
+31,mlp.gate_proj,3,128,8,128,41.31676483154297,87.5546875,234881024
+31,mlp.gate_proj,4,32,8,128,14.283645629882812,126.21875,234881024
+31,mlp.gate_proj,4,64,8,128,16.83727264404297,119.109375,234881024
+31,mlp.gate_proj,4,128,8,128,19.266122817993164,115.5546875,234881024
+31,mlp.gate_proj,8,32,8,128,0.8717745542526245,238.21875,234881024
+31,mlp.gate_proj,8,64,8,128,1.0292688608169556,231.109375,234881024
+31,mlp.gate_proj,8,128,8,128,1.1714969873428345,227.5546875,234881024
+31,mlp.up_proj,2,32,8,128,67.60722351074219,70.21875,234881024
+31,mlp.up_proj,2,64,8,128,78.74046325683594,63.109375,234881024
+31,mlp.up_proj,2,128,8,128,89.06258392333984,59.5546875,234881024
+31,mlp.up_proj,3,32,8,128,28.92757225036621,98.21875,234881024
+31,mlp.up_proj,3,64,8,128,33.71366500854492,91.109375,234881024
+31,mlp.up_proj,3,128,8,128,37.985103607177734,87.5546875,234881024
+31,mlp.up_proj,4,32,8,128,13.469902038574219,126.21875,234881024
+31,mlp.up_proj,4,64,8,128,15.783296585083008,119.109375,234881024
+31,mlp.up_proj,4,128,8,128,17.745813369750977,115.5546875,234881024
+31,mlp.up_proj,8,32,8,128,0.8225113749504089,238.21875,234881024
+31,mlp.up_proj,8,64,8,128,0.9625873565673828,231.109375,234881024
+31,mlp.up_proj,8,128,8,128,1.0798496007919312,227.5546875,234881024
+31,mlp.down_proj,2,32,8,128,67.47991180419922,70.21875,234881024
+31,mlp.down_proj,2,64,8,128,78.56333923339844,63.109375,234881024
+31,mlp.down_proj,2,128,8,128,88.82557678222656,59.5546875,234881024
+31,mlp.down_proj,3,32,8,128,28.873563766479492,98.21875,234881024
+31,mlp.down_proj,3,64,8,128,33.64424514770508,91.109375,234881024
+31,mlp.down_proj,3,128,8,128,37.91650390625,87.5546875,234881024
+31,mlp.down_proj,4,32,8,128,13.445062637329102,126.21875,234881024
+31,mlp.down_proj,4,64,8,128,15.749591827392578,119.109375,234881024
+31,mlp.down_proj,4,128,8,128,17.70409393310547,115.5546875,234881024
+31,mlp.down_proj,8,32,8,128,0.8210251331329346,238.21875,234881024
+31,mlp.down_proj,8,64,8,128,0.9605519771575928,231.109375,234881024
+31,mlp.down_proj,8,128,8,128,1.0774914026260376,227.5546875,234881024
+32,self_attn.q_proj,2,32,8,128,47.46177291870117,20.0625,67108864
+32,self_attn.q_proj,2,64,8,128,56.38787841796875,18.03125,67108864
+32,self_attn.q_proj,2,128,8,128,65.28728485107422,17.015625,67108864
+32,self_attn.q_proj,3,32,8,128,20.36381721496582,28.0625,67108864
+32,self_attn.q_proj,3,64,8,128,24.30384063720703,26.03125,67108864
+32,self_attn.q_proj,3,128,8,128,28.341981887817383,25.015625,67108864
+32,self_attn.q_proj,4,32,8,128,9.498432159423828,36.0625,67108864
+32,self_attn.q_proj,4,64,8,128,11.350874900817871,34.03125,67108864
+32,self_attn.q_proj,4,128,8,128,13.227127075195312,33.015625,67108864
+32,self_attn.q_proj,8,32,8,128,0.5771182775497437,68.0625,67108864
+32,self_attn.q_proj,8,64,8,128,0.6893654465675354,66.03125,67108864
+32,self_attn.q_proj,8,128,8,128,0.8005905747413635,65.015625,67108864
+32,self_attn.v_proj,2,32,8,128,11.10325813293457,2.5078125,8388608
+32,self_attn.v_proj,2,64,8,128,12.971238136291504,2.25390625,8388608
+32,self_attn.v_proj,2,128,8,128,14.666997909545898,2.126953125,8388608
+32,self_attn.v_proj,3,32,8,128,4.747076511383057,3.5078125,8388608
+32,self_attn.v_proj,3,64,8,128,5.550022602081299,3.25390625,8388608
+32,self_attn.v_proj,3,128,8,128,6.276065826416016,3.126953125,8388608
+32,self_attn.v_proj,4,32,8,128,2.2150888442993164,4.5078125,8388608
+32,self_attn.v_proj,4,64,8,128,2.5904500484466553,4.25390625,8388608
+32,self_attn.v_proj,4,128,8,128,2.9304351806640625,4.126953125,8388608
+32,self_attn.v_proj,8,32,8,128,0.13414107263088226,8.5078125,8388608
+32,self_attn.v_proj,8,64,8,128,0.1566787213087082,8.25390625,8388608
+32,self_attn.v_proj,8,128,8,128,0.17660056054592133,8.126953125,8388608
+32,self_attn.k_proj,2,32,8,128,33.55158615112305,2.5078125,8388608
+32,self_attn.k_proj,2,64,8,128,39.994327545166016,2.25390625,8388608
+32,self_attn.k_proj,2,128,8,128,45.54110336303711,2.126953125,8388608
+32,self_attn.k_proj,3,32,8,128,14.3914794921875,3.5078125,8388608
+32,self_attn.k_proj,3,64,8,128,17.22331428527832,3.25390625,8388608
+32,self_attn.k_proj,3,128,8,128,19.813596725463867,3.126953125,8388608
+32,self_attn.k_proj,4,32,8,128,6.714600563049316,4.5078125,8388608
+32,self_attn.k_proj,4,64,8,128,8.03504753112793,4.25390625,8388608
+32,self_attn.k_proj,4,128,8,128,9.25003433227539,4.126953125,8388608
+32,self_attn.k_proj,8,32,8,128,0.40655896067619324,8.5078125,8388608
+32,self_attn.k_proj,8,64,8,128,0.4858890175819397,8.25390625,8388608
+32,self_attn.k_proj,8,128,8,128,0.5573316216468811,8.126953125,8388608
+32,self_attn.o_proj,2,32,8,128,33.79617691040039,20.0625,67108864
+32,self_attn.o_proj,2,64,8,128,39.37616729736328,18.03125,67108864
+32,self_attn.o_proj,2,128,8,128,44.4664421081543,17.015625,67108864
+32,self_attn.o_proj,3,32,8,128,14.463912010192871,28.0625,67108864
+32,self_attn.o_proj,3,64,8,128,16.8792781829834,26.03125,67108864
+32,self_attn.o_proj,3,128,8,128,19.103565216064453,25.015625,67108864
+32,self_attn.o_proj,4,32,8,128,6.748192310333252,36.0625,67108864
+32,self_attn.o_proj,4,64,8,128,7.886838436126709,34.03125,67108864
+32,self_attn.o_proj,4,128,8,128,8.921226501464844,33.015625,67108864
+32,self_attn.o_proj,8,32,8,128,0.4102882444858551,68.0625,67108864
+32,self_attn.o_proj,8,64,8,128,0.47862517833709717,66.03125,67108864
+32,self_attn.o_proj,8,128,8,128,0.539973795413971,65.015625,67108864
+32,mlp.gate_proj,2,32,8,128,71.83802032470703,70.21875,234881024
+32,mlp.gate_proj,2,64,8,128,84.790283203125,63.109375,234881024
+32,mlp.gate_proj,2,128,8,128,96.56626892089844,59.5546875,234881024
+32,mlp.gate_proj,3,32,8,128,30.87574577331543,98.21875,234881024
+32,mlp.gate_proj,3,64,8,128,36.22426223754883,91.109375,234881024
+32,mlp.gate_proj,3,128,8,128,41.4808464050293,87.5546875,234881024
+32,mlp.gate_proj,4,32,8,128,14.368136405944824,126.21875,234881024
+32,mlp.gate_proj,4,64,8,128,16.92091941833496,119.109375,234881024
+32,mlp.gate_proj,4,128,8,128,19.34186553955078,115.5546875,234881024
+32,mlp.gate_proj,8,32,8,128,0.8768007755279541,238.21875,234881024
+32,mlp.gate_proj,8,64,8,128,1.0342847108840942,231.109375,234881024
+32,mlp.gate_proj,8,128,8,128,1.175828456878662,227.5546875,234881024
+32,mlp.up_proj,2,32,8,128,67.48066711425781,70.21875,234881024
+32,mlp.up_proj,2,64,8,128,78.59209442138672,63.109375,234881024
+32,mlp.up_proj,2,128,8,128,88.89799499511719,59.5546875,234881024
+32,mlp.up_proj,3,32,8,128,28.8720703125,98.21875,234881024
+32,mlp.up_proj,3,64,8,128,33.66029357910156,91.109375,234881024
+32,mlp.up_proj,3,128,8,128,37.94504928588867,87.5546875,234881024
+32,mlp.up_proj,4,32,8,128,13.444136619567871,126.21875,234881024
+32,mlp.up_proj,4,64,8,128,15.756120681762695,119.109375,234881024
+32,mlp.up_proj,4,128,8,128,17.71871566772461,115.5546875,234881024
+32,mlp.up_proj,8,32,8,128,0.8209574222564697,238.21875,234881024
+32,mlp.up_proj,8,64,8,128,0.9609847664833069,231.109375,234881024
+32,mlp.up_proj,8,128,8,128,1.0784614086151123,227.5546875,234881024
+32,mlp.down_proj,2,32,8,128,67.28137969970703,70.21875,234881024
+32,mlp.down_proj,2,64,8,128,78.33758544921875,63.109375,234881024
+32,mlp.down_proj,2,128,8,128,88.57029724121094,59.5546875,234881024
+32,mlp.down_proj,3,32,8,128,28.783994674682617,98.21875,234881024
+32,mlp.down_proj,3,64,8,128,33.55759048461914,91.109375,234881024
+32,mlp.down_proj,3,128,8,128,37.81536865234375,87.5546875,234881024
+32,mlp.down_proj,4,32,8,128,13.40473461151123,126.21875,234881024
+32,mlp.down_proj,4,64,8,128,15.705506324768066,119.109375,234881024
+32,mlp.down_proj,4,128,8,128,17.664043426513672,115.5546875,234881024
+32,mlp.down_proj,8,32,8,128,0.8186801671981812,238.21875,234881024
+32,mlp.down_proj,8,64,8,128,0.9579216837882996,231.109375,234881024
+32,mlp.down_proj,8,128,8,128,1.0752801895141602,227.5546875,234881024
+33,self_attn.q_proj,2,32,8,128,50.138641357421875,20.0625,67108864
+33,self_attn.q_proj,2,64,8,128,59.381534576416016,18.03125,67108864
+33,self_attn.q_proj,2,128,8,128,68.66699981689453,17.015625,67108864
+33,self_attn.q_proj,3,32,8,128,21.47183609008789,28.0625,67108864
+33,self_attn.q_proj,3,64,8,128,25.449115753173828,26.03125,67108864
+33,self_attn.q_proj,3,128,8,128,29.528066635131836,25.015625,67108864
+33,self_attn.q_proj,4,32,8,128,10.015103340148926,36.0625,67108864
+33,self_attn.q_proj,4,64,8,128,11.881717681884766,34.03125,67108864
+33,self_attn.q_proj,4,128,8,128,13.774515151977539,33.015625,67108864
+33,self_attn.q_proj,8,32,8,128,0.6086939573287964,68.0625,67108864
+33,self_attn.q_proj,8,64,8,128,0.7220991849899292,66.03125,67108864
+33,self_attn.q_proj,8,128,8,128,0.8340964317321777,65.015625,67108864
+33,self_attn.v_proj,2,32,8,128,8.89248275756836,2.5078125,8388608
+33,self_attn.v_proj,2,64,8,128,10.398435592651367,2.25390625,8388608
+33,self_attn.v_proj,2,128,8,128,11.77049732208252,2.126953125,8388608
+33,self_attn.v_proj,3,32,8,128,3.803224802017212,3.5078125,8388608
+33,self_attn.v_proj,3,64,8,128,4.449094295501709,3.25390625,8388608
+33,self_attn.v_proj,3,128,8,128,5.038853168487549,3.126953125,8388608
+33,self_attn.v_proj,4,32,8,128,1.7742490768432617,4.5078125,8388608
+33,self_attn.v_proj,4,64,8,128,2.076596260070801,4.25390625,8388608
+33,self_attn.v_proj,4,128,8,128,2.3516080379486084,4.126953125,8388608
+33,self_attn.v_proj,8,32,8,128,0.10741309821605682,8.5078125,8388608
+33,self_attn.v_proj,8,64,8,128,0.12562018632888794,8.25390625,8388608
+33,self_attn.v_proj,8,128,8,128,0.14171017706394196,8.126953125,8388608
+33,self_attn.k_proj,2,32,8,128,35.317684173583984,2.5078125,8388608
+33,self_attn.k_proj,2,64,8,128,42.390159606933594,2.25390625,8388608
+33,self_attn.k_proj,2,128,8,128,48.679500579833984,2.126953125,8388608
+33,self_attn.k_proj,3,32,8,128,15.175359725952148,3.5078125,8388608
+33,self_attn.k_proj,3,64,8,128,18.318532943725586,3.25390625,8388608
+33,self_attn.k_proj,3,128,8,128,21.298316955566406,3.126953125,8388608
+33,self_attn.k_proj,4,32,8,128,7.079436779022217,4.5078125,8388608
+33,self_attn.k_proj,4,64,8,128,8.54666519165039,4.25390625,8388608
+33,self_attn.k_proj,4,128,8,128,9.941571235656738,4.126953125,8388608
+33,self_attn.k_proj,8,32,8,128,0.42829668521881104,8.5078125,8388608
+33,self_attn.k_proj,8,64,8,128,0.5168045163154602,8.25390625,8388608
+33,self_attn.k_proj,8,128,8,128,0.5990022420883179,8.126953125,8388608
+33,self_attn.o_proj,2,32,8,128,32.7528190612793,20.0625,67108864
+33,self_attn.o_proj,2,64,8,128,38.17301940917969,18.03125,67108864
+33,self_attn.o_proj,2,128,8,128,43.13280487060547,17.015625,67108864
+33,self_attn.o_proj,3,32,8,128,14.019634246826172,28.0625,67108864
+33,self_attn.o_proj,3,64,8,128,16.38770866394043,26.03125,67108864
+33,self_attn.o_proj,3,128,8,128,18.546648025512695,25.015625,67108864
+33,self_attn.o_proj,4,32,8,128,6.541453838348389,36.0625,67108864
+33,self_attn.o_proj,4,64,8,128,7.652033805847168,34.03125,67108864
+33,self_attn.o_proj,4,128,8,128,8.669571876525879,33.015625,67108864
+33,self_attn.o_proj,8,32,8,128,0.39772629737854004,68.0625,67108864
+33,self_attn.o_proj,8,64,8,128,0.4644114375114441,66.03125,67108864
+33,self_attn.o_proj,8,128,8,128,0.5244633555412292,65.015625,67108864
+33,mlp.gate_proj,2,32,8,128,198.75314331054688,70.21875,234881024
+33,mlp.gate_proj,2,64,8,128,251.2969207763672,63.109375,234881024
+33,mlp.gate_proj,2,128,8,128,314.5469665527344,59.5546875,234881024
+33,mlp.gate_proj,3,32,8,128,85.32621002197266,98.21875,234881024
+33,mlp.gate_proj,3,64,8,128,111.6767807006836,91.109375,234881024
+33,mlp.gate_proj,3,128,8,128,151.51951599121094,87.5546875,234881024
+33,mlp.gate_proj,4,32,8,128,41.379703521728516,126.21875,234881024
+33,mlp.gate_proj,4,64,8,128,57.594993591308594,119.109375,234881024
+33,mlp.gate_proj,4,128,8,128,67.3125,115.5546875,234881024
+33,mlp.gate_proj,8,32,8,128,2.6202709674835205,238.21875,234881024
+33,mlp.gate_proj,8,64,8,128,3.5005998611450195,231.109375,234881024
+33,mlp.gate_proj,8,128,8,128,4.537593841552734,227.5546875,234881024
+33,mlp.up_proj,2,32,8,128,204.9520263671875,70.21875,234881024
+33,mlp.up_proj,2,64,8,128,255.16822814941406,63.109375,234881024
+33,mlp.up_proj,2,128,8,128,304.4066467285156,59.5546875,234881024
+33,mlp.up_proj,3,32,8,128,87.73062133789062,98.21875,234881024
+33,mlp.up_proj,3,64,8,128,112.67892456054688,91.109375,234881024
+33,mlp.up_proj,3,128,8,128,132.76768493652344,87.5546875,234881024
+33,mlp.up_proj,4,32,8,128,41.085853576660156,126.21875,234881024
+33,mlp.up_proj,4,64,8,128,52.52837371826172,119.109375,234881024
+33,mlp.up_proj,4,128,8,128,65.60658264160156,115.5546875,234881024
+33,mlp.up_proj,8,32,8,128,2.5572216510772705,238.21875,234881024
+33,mlp.up_proj,8,64,8,128,3.3381834030151367,231.109375,234881024
+33,mlp.up_proj,8,128,8,128,4.186249256134033,227.5546875,234881024
+33,mlp.down_proj,2,32,8,128,67.26222229003906,70.21875,234881024
+33,mlp.down_proj,2,64,8,128,78.2949447631836,63.109375,234881024
+33,mlp.down_proj,2,128,8,128,88.56038665771484,59.5546875,234881024
+33,mlp.down_proj,3,32,8,128,28.773548126220703,98.21875,234881024
+33,mlp.down_proj,3,64,8,128,33.54150390625,91.109375,234881024
+33,mlp.down_proj,3,128,8,128,37.79460525512695,87.5546875,234881024
+33,mlp.down_proj,4,32,8,128,13.399627685546875,126.21875,234881024
+33,mlp.down_proj,4,64,8,128,15.698665618896484,119.109375,234881024
+33,mlp.down_proj,4,128,8,128,17.65961456298828,115.5546875,234881024
+33,mlp.down_proj,8,32,8,128,0.8182861804962158,238.21875,234881024
+33,mlp.down_proj,8,64,8,128,0.9574892520904541,231.109375,234881024
+33,mlp.down_proj,8,128,8,128,1.0745689868927002,227.5546875,234881024
+34,self_attn.q_proj,2,32,8,128,50.96077346801758,20.0625,67108864
+34,self_attn.q_proj,2,64,8,128,60.222084045410156,18.03125,67108864
+34,self_attn.q_proj,2,128,8,128,69.51188659667969,17.015625,67108864
+34,self_attn.q_proj,3,32,8,128,21.829702377319336,28.0625,67108864
+34,self_attn.q_proj,3,64,8,128,25.815223693847656,26.03125,67108864
+34,self_attn.q_proj,3,128,8,128,29.902870178222656,25.015625,67108864
+34,self_attn.q_proj,4,32,8,128,10.181685447692871,36.0625,67108864
+34,self_attn.q_proj,4,64,8,128,12.049577713012695,34.03125,67108864
+34,self_attn.q_proj,4,128,8,128,13.949464797973633,33.015625,67108864
+34,self_attn.q_proj,8,32,8,128,0.6187719106674194,68.0625,67108864
+34,self_attn.q_proj,8,64,8,128,0.7323747277259827,66.03125,67108864
+34,self_attn.q_proj,8,128,8,128,0.844765305519104,65.015625,67108864
+34,self_attn.v_proj,2,32,8,128,9.924267768859863,2.5078125,8388608
+34,self_attn.v_proj,2,64,8,128,11.675955772399902,2.25390625,8388608
+34,self_attn.v_proj,2,128,8,128,13.283502578735352,2.126953125,8388608
+34,self_attn.v_proj,3,32,8,128,4.243849277496338,3.5078125,8388608
+34,self_attn.v_proj,3,64,8,128,4.995206832885742,3.25390625,8388608
+34,self_attn.v_proj,3,128,8,128,5.686900615692139,3.126953125,8388608
+34,self_attn.v_proj,4,32,8,128,1.9803414344787598,4.5078125,8388608
+34,self_attn.v_proj,4,64,8,128,2.3305394649505615,4.25390625,8388608
+34,self_attn.v_proj,4,128,8,128,2.6536929607391357,4.126953125,8388608
+34,self_attn.v_proj,8,32,8,128,0.11987567692995071,8.5078125,8388608
+34,self_attn.v_proj,8,64,8,128,0.14095653593540192,8.25390625,8388608
+34,self_attn.v_proj,8,128,8,128,0.15990754961967468,8.126953125,8388608
+34,self_attn.k_proj,2,32,8,128,35.022361755371094,2.5078125,8388608
+34,self_attn.k_proj,2,64,8,128,41.75425720214844,2.25390625,8388608
+34,self_attn.k_proj,2,128,8,128,47.78679275512695,2.126953125,8388608
+34,self_attn.k_proj,3,32,8,128,15.018684387207031,3.5078125,8388608
+34,self_attn.k_proj,3,64,8,128,17.950963973999023,3.25390625,8388608
+34,self_attn.k_proj,3,128,8,128,20.69633674621582,3.126953125,8388608
+34,self_attn.k_proj,4,32,8,128,7.008457183837891,4.5078125,8388608
+34,self_attn.k_proj,4,64,8,128,8.382545471191406,4.25390625,8388608
+34,self_attn.k_proj,4,128,8,128,9.65855884552002,4.126953125,8388608
+34,self_attn.k_proj,8,32,8,128,0.42411482334136963,8.5078125,8388608
+34,self_attn.k_proj,8,64,8,128,0.5068756937980652,8.25390625,8388608
+34,self_attn.k_proj,8,128,8,128,0.5824387669563293,8.126953125,8388608
+34,self_attn.o_proj,2,32,8,128,33.00217819213867,20.0625,67108864
+34,self_attn.o_proj,2,64,8,128,38.47316360473633,18.03125,67108864
+34,self_attn.o_proj,2,128,8,128,43.52177810668945,17.015625,67108864
+34,self_attn.o_proj,3,32,8,128,14.130882263183594,28.0625,67108864
+34,self_attn.o_proj,3,64,8,128,16.520776748657227,26.03125,67108864
+34,self_attn.o_proj,3,128,8,128,18.713058471679688,25.015625,67108864
+34,self_attn.o_proj,4,32,8,128,6.596983909606934,36.0625,67108864
+34,self_attn.o_proj,4,64,8,128,7.722385406494141,34.03125,67108864
+34,self_attn.o_proj,4,128,8,128,8.76045036315918,33.015625,67108864
+34,self_attn.o_proj,8,32,8,128,0.4011198878288269,68.0625,67108864
+34,self_attn.o_proj,8,64,8,128,0.4688936173915863,66.03125,67108864
+34,self_attn.o_proj,8,128,8,128,0.5304512977600098,65.015625,67108864
+34,mlp.gate_proj,2,32,8,128,71.733642578125,70.21875,234881024
+34,mlp.gate_proj,2,64,8,128,84.75482177734375,63.109375,234881024
+34,mlp.gate_proj,2,128,8,128,96.61151885986328,59.5546875,234881024
+34,mlp.gate_proj,3,32,8,128,30.8321475982666,98.21875,234881024
+34,mlp.gate_proj,3,64,8,128,36.21010208129883,91.109375,234881024
+34,mlp.gate_proj,3,128,8,128,41.52918243408203,87.5546875,234881024
+34,mlp.gate_proj,4,32,8,128,14.347773551940918,126.21875,234881024
+34,mlp.gate_proj,4,64,8,128,16.915714263916016,119.109375,234881024
+34,mlp.gate_proj,4,128,8,128,19.362146377563477,115.5546875,234881024
+34,mlp.gate_proj,8,32,8,128,0.8755366802215576,238.21875,234881024
+34,mlp.gate_proj,8,64,8,128,1.0339595079421997,231.109375,234881024
+34,mlp.gate_proj,8,128,8,128,1.1771856546401978,227.5546875,234881024
+34,mlp.up_proj,2,32,8,128,67.70628356933594,70.21875,234881024
+34,mlp.up_proj,2,64,8,128,78.88849639892578,63.109375,234881024
+34,mlp.up_proj,2,128,8,128,89.24568939208984,59.5546875,234881024
+34,mlp.up_proj,3,32,8,128,28.97541618347168,98.21875,234881024
+34,mlp.up_proj,3,64,8,128,33.775428771972656,91.109375,234881024
+34,mlp.up_proj,3,128,8,128,38.082149505615234,87.5546875,234881024
+34,mlp.up_proj,4,32,8,128,13.49256420135498,126.21875,234881024
+34,mlp.up_proj,4,64,8,128,15.816307067871094,119.109375,234881024
+34,mlp.up_proj,4,128,8,128,17.77935028076172,115.5546875,234881024
+34,mlp.up_proj,8,32,8,128,0.8238152265548706,238.21875,234881024
+34,mlp.up_proj,8,64,8,128,0.9644780158996582,231.109375,234881024
+34,mlp.up_proj,8,128,8,128,1.0822468996047974,227.5546875,234881024
+34,mlp.down_proj,2,32,8,128,67.5384750366211,70.21875,234881024
+34,mlp.down_proj,2,64,8,128,78.65740203857422,63.109375,234881024
+34,mlp.down_proj,2,128,8,128,89.00469970703125,59.5546875,234881024
+34,mlp.down_proj,3,32,8,128,28.905487060546875,98.21875,234881024
+34,mlp.down_proj,3,64,8,128,33.69512939453125,91.109375,234881024
+34,mlp.down_proj,3,128,8,128,38.00348663330078,87.5546875,234881024
+34,mlp.down_proj,4,32,8,128,13.459750175476074,126.21875,234881024
+34,mlp.down_proj,4,64,8,128,15.779035568237305,119.109375,234881024
+34,mlp.down_proj,4,128,8,128,17.751209259033203,115.5546875,234881024
+34,mlp.down_proj,8,32,8,128,0.8220255374908447,238.21875,234881024
+34,mlp.down_proj,8,64,8,128,0.9624583721160889,231.109375,234881024
+34,mlp.down_proj,8,128,8,128,1.0804518461227417,227.5546875,234881024
+35,self_attn.q_proj,2,32,8,128,50.697486877441406,20.0625,67108864
+35,self_attn.q_proj,2,64,8,128,59.99409484863281,18.03125,67108864
+35,self_attn.q_proj,2,128,8,128,69.50666046142578,17.015625,67108864
+35,self_attn.q_proj,3,32,8,128,21.735816955566406,28.0625,67108864
+35,self_attn.q_proj,3,64,8,128,25.744043350219727,26.03125,67108864
+35,self_attn.q_proj,3,128,8,128,29.984115600585938,25.015625,67108864
+35,self_attn.q_proj,4,32,8,128,10.136711120605469,36.0625,67108864
+35,self_attn.q_proj,4,64,8,128,12.017094612121582,34.03125,67108864
+35,self_attn.q_proj,4,128,8,128,13.991418838500977,33.015625,67108864
+35,self_attn.q_proj,8,32,8,128,0.6158551573753357,68.0625,67108864
+35,self_attn.q_proj,8,64,8,128,0.7301482558250427,66.03125,67108864
+35,self_attn.q_proj,8,128,8,128,0.8464478850364685,65.015625,67108864
+35,self_attn.v_proj,2,32,8,128,9.42782974243164,2.5078125,8388608
+35,self_attn.v_proj,2,64,8,128,11.081932067871094,2.25390625,8388608
+35,self_attn.v_proj,2,128,8,128,12.605387687683105,2.126953125,8388608
+35,self_attn.v_proj,3,32,8,128,4.0323686599731445,3.5078125,8388608
+35,self_attn.v_proj,3,64,8,128,4.742734909057617,3.25390625,8388608
+35,self_attn.v_proj,3,128,8,128,5.399531364440918,3.126953125,8388608
+35,self_attn.v_proj,4,32,8,128,1.8825116157531738,4.5078125,8388608
+35,self_attn.v_proj,4,64,8,128,2.2136998176574707,4.25390625,8388608
+35,self_attn.v_proj,4,128,8,128,2.519371271133423,4.126953125,8388608
+35,self_attn.v_proj,8,32,8,128,0.11388057470321655,8.5078125,8388608
+35,self_attn.v_proj,8,64,8,128,0.1338469386100769,8.25390625,8388608
+35,self_attn.v_proj,8,128,8,128,0.15185941755771637,8.126953125,8388608
+35,self_attn.k_proj,2,32,8,128,36.811424255371094,2.5078125,8388608
+35,self_attn.k_proj,2,64,8,128,44.15408706665039,2.25390625,8388608
+35,self_attn.k_proj,2,128,8,128,50.452213287353516,2.126953125,8388608
+35,self_attn.k_proj,3,32,8,128,15.881057739257812,3.5078125,8388608
+35,self_attn.k_proj,3,64,8,128,19.1912899017334,3.25390625,8388608
+35,self_attn.k_proj,3,128,8,128,22.337966918945312,3.126953125,8388608
+35,self_attn.k_proj,4,32,8,128,7.408674240112305,4.5078125,8388608
+35,self_attn.k_proj,4,64,8,128,8.959287643432617,4.25390625,8388608
+35,self_attn.k_proj,4,128,8,128,10.435249328613281,4.126953125,8388608
+35,self_attn.k_proj,8,32,8,128,0.4480791985988617,8.5078125,8388608
+35,self_attn.k_proj,8,64,8,128,0.5415132641792297,8.25390625,8388608
+35,self_attn.k_proj,8,128,8,128,0.6285603046417236,8.126953125,8388608
+35,self_attn.o_proj,2,32,8,128,33.1751823425293,20.0625,67108864
+35,self_attn.o_proj,2,64,8,128,38.65250015258789,18.03125,67108864
+35,self_attn.o_proj,2,128,8,128,43.665000915527344,17.015625,67108864
+35,self_attn.o_proj,3,32,8,128,14.207647323608398,28.0625,67108864
+35,self_attn.o_proj,3,64,8,128,16.599445343017578,26.03125,67108864
+35,self_attn.o_proj,3,128,8,128,18.77910614013672,25.015625,67108864
+35,self_attn.o_proj,4,32,8,128,6.637192249298096,36.0625,67108864
+35,self_attn.o_proj,4,64,8,128,7.767022609710693,34.03125,67108864
+35,self_attn.o_proj,4,128,8,128,8.811430931091309,33.015625,67108864
+35,self_attn.o_proj,8,32,8,128,0.40366631746292114,68.0625,67108864
+35,self_attn.o_proj,8,64,8,128,0.47181516885757446,66.03125,67108864
+35,self_attn.o_proj,8,128,8,128,0.5335252285003662,65.015625,67108864
+35,mlp.gate_proj,2,32,8,128,71.31248474121094,70.21875,234881024
+35,mlp.gate_proj,2,64,8,128,84.13043212890625,63.109375,234881024
+35,mlp.gate_proj,2,128,8,128,95.86441802978516,59.5546875,234881024
+35,mlp.gate_proj,3,32,8,128,30.644149780273438,98.21875,234881024
+35,mlp.gate_proj,3,64,8,128,35.96042251586914,91.109375,234881024
+35,mlp.gate_proj,3,128,8,128,41.21429443359375,87.5546875,234881024
+35,mlp.gate_proj,4,32,8,128,14.260550498962402,126.21875,234881024
+35,mlp.gate_proj,4,64,8,128,16.802846908569336,119.109375,234881024
+35,mlp.gate_proj,4,128,8,128,19.222015380859375,115.5546875,234881024
+35,mlp.gate_proj,8,32,8,128,0.87044358253479,238.21875,234881024
+35,mlp.gate_proj,8,64,8,128,1.0273053646087646,231.109375,234881024
+35,mlp.gate_proj,8,128,8,128,1.1686172485351562,227.5546875,234881024
+35,mlp.up_proj,2,32,8,128,67.27732849121094,70.21875,234881024
+35,mlp.up_proj,2,64,8,128,78.3026123046875,63.109375,234881024
+35,mlp.up_proj,2,128,8,128,88.53294372558594,59.5546875,234881024
+35,mlp.up_proj,3,32,8,128,28.77227783203125,98.21875,234881024
+35,mlp.up_proj,3,64,8,128,33.539154052734375,91.109375,234881024
+35,mlp.up_proj,3,128,8,128,37.77756881713867,87.5546875,234881024
+35,mlp.up_proj,4,32,8,128,13.3992919921875,126.21875,234881024
+35,mlp.up_proj,4,64,8,128,15.693004608154297,119.109375,234881024
+35,mlp.up_proj,4,128,8,128,17.645854949951172,115.5546875,234881024
+35,mlp.up_proj,8,32,8,128,0.8182170987129211,238.21875,234881024
+35,mlp.up_proj,8,64,8,128,0.9570108652114868,231.109375,234881024
+35,mlp.up_proj,8,128,8,128,1.0737718343734741,227.5546875,234881024
+35,mlp.down_proj,2,32,8,128,67.5034408569336,70.21875,234881024
+35,mlp.down_proj,2,64,8,128,78.58964538574219,63.109375,234881024
+35,mlp.down_proj,2,128,8,128,88.85456085205078,59.5546875,234881024
+35,mlp.down_proj,3,32,8,128,28.882829666137695,98.21875,234881024
+35,mlp.down_proj,3,64,8,128,33.65208435058594,91.109375,234881024
+35,mlp.down_proj,3,128,8,128,37.89738845825195,87.5546875,234881024
+35,mlp.down_proj,4,32,8,128,13.449867248535156,126.21875,234881024
+35,mlp.down_proj,4,64,8,128,15.753365516662598,119.109375,234881024
+35,mlp.down_proj,4,128,8,128,17.709789276123047,115.5546875,234881024
+35,mlp.down_proj,8,32,8,128,0.8213303089141846,238.21875,234881024
+35,mlp.down_proj,8,64,8,128,0.9607887268066406,231.109375,234881024
+35,mlp.down_proj,8,128,8,128,1.0777109861373901,227.5546875,234881024
+36,self_attn.q_proj,2,32,8,128,48.99386215209961,20.0625,67108864
+36,self_attn.q_proj,2,64,8,128,57.95236587524414,18.03125,67108864
+36,self_attn.q_proj,2,128,8,128,67.12415313720703,17.015625,67108864
+36,self_attn.q_proj,3,32,8,128,21.01606559753418,28.0625,67108864
+36,self_attn.q_proj,3,64,8,128,24.928054809570312,26.03125,67108864
+36,self_attn.q_proj,3,128,8,128,29.03415298461914,25.015625,67108864
+36,self_attn.q_proj,4,32,8,128,9.801060676574707,36.0625,67108864
+36,self_attn.q_proj,4,64,8,128,11.638506889343262,34.03125,67108864
+36,self_attn.q_proj,4,128,8,128,13.54099178314209,33.015625,67108864
+36,self_attn.q_proj,8,32,8,128,0.5954850316047668,68.0625,67108864
+36,self_attn.q_proj,8,64,8,128,0.7072767019271851,66.03125,67108864
+36,self_attn.q_proj,8,128,8,128,0.8199697732925415,65.015625,67108864
+36,self_attn.v_proj,2,32,8,128,10.423274993896484,2.5078125,8388608
+36,self_attn.v_proj,2,64,8,128,12.213955879211426,2.25390625,8388608
+36,self_attn.v_proj,2,128,8,128,13.859861373901367,2.126953125,8388608
+36,self_attn.v_proj,3,32,8,128,4.4575395584106445,3.5078125,8388608
+36,self_attn.v_proj,3,64,8,128,5.22735071182251,3.25390625,8388608
+36,self_attn.v_proj,3,128,8,128,5.930144786834717,3.126953125,8388608
+36,self_attn.v_proj,4,32,8,128,2.0802104473114014,4.5078125,8388608
+36,self_attn.v_proj,4,64,8,128,2.4396064281463623,4.25390625,8388608
+36,self_attn.v_proj,4,128,8,128,2.768282890319824,4.126953125,8388608
+36,self_attn.v_proj,8,32,8,128,0.12586529552936554,8.5078125,8388608
+36,self_attn.v_proj,8,64,8,128,0.14747153222560883,8.25390625,8388608
+36,self_attn.v_proj,8,128,8,128,0.16680380702018738,8.126953125,8388608
+36,self_attn.k_proj,2,32,8,128,36.43983459472656,2.5078125,8388608
+36,self_attn.k_proj,2,64,8,128,43.791786193847656,2.25390625,8388608
+36,self_attn.k_proj,2,128,8,128,50.14414596557617,2.126953125,8388608
+36,self_attn.k_proj,3,32,8,128,15.703350067138672,3.5078125,8388608
+36,self_attn.k_proj,3,64,8,128,19.02281951904297,3.25390625,8388608
+36,self_attn.k_proj,3,128,8,128,22.172351837158203,3.126953125,8388608
+36,self_attn.k_proj,4,32,8,128,7.325253486633301,4.5078125,8388608
+36,self_attn.k_proj,4,64,8,128,8.878975868225098,4.25390625,8388608
+36,self_attn.k_proj,4,128,8,128,10.353686332702637,4.126953125,8388608
+36,self_attn.k_proj,8,32,8,128,0.44324591755867004,8.5078125,8388608
+36,self_attn.k_proj,8,64,8,128,0.5368437767028809,8.25390625,8388608
+36,self_attn.k_proj,8,128,8,128,0.6239434480667114,8.126953125,8388608
+36,self_attn.o_proj,2,32,8,128,33.52398681640625,20.0625,67108864
+36,self_attn.o_proj,2,64,8,128,39.079444885253906,18.03125,67108864
+36,self_attn.o_proj,2,128,8,128,44.16586685180664,17.015625,67108864
+36,self_attn.o_proj,3,32,8,128,14.343820571899414,28.0625,67108864
+36,self_attn.o_proj,3,64,8,128,16.759777069091797,26.03125,67108864
+36,self_attn.o_proj,3,128,8,128,18.974546432495117,25.015625,67108864
+36,self_attn.o_proj,4,32,8,128,6.6908650398254395,36.0625,67108864
+36,self_attn.o_proj,4,64,8,128,7.824714660644531,34.03125,67108864
+36,self_attn.o_proj,4,128,8,128,8.855690956115723,33.015625,67108864
+36,self_attn.o_proj,8,32,8,128,0.4067402780056,68.0625,67108864
+36,self_attn.o_proj,8,64,8,128,0.47474807500839233,66.03125,67108864
+36,self_attn.o_proj,8,128,8,128,0.5357546210289001,65.015625,67108864
+36,mlp.gate_proj,2,32,8,128,71.99796295166016,70.21875,234881024
+36,mlp.gate_proj,2,64,8,128,85.00530242919922,63.109375,234881024
+36,mlp.gate_proj,2,128,8,128,96.90571594238281,59.5546875,234881024
+36,mlp.gate_proj,3,32,8,128,30.95675277709961,98.21875,234881024
+36,mlp.gate_proj,3,64,8,128,36.34044647216797,91.109375,234881024
+36,mlp.gate_proj,3,128,8,128,41.7164306640625,87.5546875,234881024
+36,mlp.gate_proj,4,32,8,128,14.405339241027832,126.21875,234881024
+36,mlp.gate_proj,4,64,8,128,16.973711013793945,119.109375,234881024
+36,mlp.gate_proj,4,128,8,128,19.45402717590332,115.5546875,234881024
+36,mlp.gate_proj,8,32,8,128,0.8791212439537048,238.21875,234881024
+36,mlp.gate_proj,8,64,8,128,1.0373412370681763,231.109375,234881024
+36,mlp.gate_proj,8,128,8,128,1.1825097799301147,227.5546875,234881024
+36,mlp.up_proj,2,32,8,128,67.13861083984375,70.21875,234881024
+36,mlp.up_proj,2,64,8,128,78.14275360107422,63.109375,234881024
+36,mlp.up_proj,2,128,8,128,88.3395004272461,59.5546875,234881024
+36,mlp.up_proj,3,32,8,128,28.712299346923828,98.21875,234881024
+36,mlp.up_proj,3,64,8,128,33.47875213623047,91.109375,234881024
+36,mlp.up_proj,3,128,8,128,37.71246337890625,87.5546875,234881024
+36,mlp.up_proj,4,32,8,128,13.370915412902832,126.21875,234881024
+36,mlp.up_proj,4,64,8,128,15.663275718688965,119.109375,234881024
+36,mlp.up_proj,4,128,8,128,17.615253448486328,115.5546875,234881024
+36,mlp.up_proj,8,32,8,128,0.8164921998977661,238.21875,234881024
+36,mlp.up_proj,8,64,8,128,0.9551365375518799,231.109375,234881024
+36,mlp.up_proj,8,128,8,128,1.0719245672225952,227.5546875,234881024
+36,mlp.down_proj,2,32,8,128,67.38861846923828,70.21875,234881024
+36,mlp.down_proj,2,64,8,128,78.44713592529297,63.109375,234881024
+36,mlp.down_proj,2,128,8,128,88.6866226196289,59.5546875,234881024
+36,mlp.down_proj,3,32,8,128,28.834672927856445,98.21875,234881024
+36,mlp.down_proj,3,64,8,128,33.60419464111328,91.109375,234881024
+36,mlp.down_proj,3,128,8,128,37.87106704711914,87.5546875,234881024
+36,mlp.down_proj,4,32,8,128,13.427734375,126.21875,234881024
+36,mlp.down_proj,4,64,8,128,15.733452796936035,119.109375,234881024
+36,mlp.down_proj,4,128,8,128,17.694250106811523,115.5546875,234881024
+36,mlp.down_proj,8,32,8,128,0.8199701309204102,238.21875,234881024
+36,mlp.down_proj,8,64,8,128,0.9595968127250671,231.109375,234881024
+36,mlp.down_proj,8,128,8,128,1.0771989822387695,227.5546875,234881024
+37,self_attn.q_proj,2,32,8,128,49.57589340209961,20.0625,67108864
+37,self_attn.q_proj,2,64,8,128,58.23594665527344,18.03125,67108864
+37,self_attn.q_proj,2,128,8,128,67.07501983642578,17.015625,67108864
+37,self_attn.q_proj,3,32,8,128,21.22739028930664,28.0625,67108864
+37,self_attn.q_proj,3,64,8,128,24.958280563354492,26.03125,67108864
+37,self_attn.q_proj,3,128,8,128,28.813594818115234,25.015625,67108864
+37,self_attn.q_proj,4,32,8,128,9.899438858032227,36.0625,67108864
+37,self_attn.q_proj,4,64,8,128,11.652031898498535,34.03125,67108864
+37,self_attn.q_proj,4,128,8,128,13.43982219696045,33.015625,67108864
+37,self_attn.q_proj,8,32,8,128,0.6016361713409424,68.0625,67108864
+37,self_attn.q_proj,8,64,8,128,0.7082011103630066,66.03125,67108864
+37,self_attn.q_proj,8,128,8,128,0.8138288259506226,65.015625,67108864
+37,self_attn.v_proj,2,32,8,128,9.73326587677002,2.5078125,8388608
+37,self_attn.v_proj,2,64,8,128,11.38343620300293,2.25390625,8388608
+37,self_attn.v_proj,2,128,8,128,12.875391960144043,2.126953125,8388608
+37,self_attn.v_proj,3,32,8,128,4.160890579223633,3.5078125,8388608
+37,self_attn.v_proj,3,64,8,128,4.8671722412109375,3.25390625,8388608
+37,self_attn.v_proj,3,128,8,128,5.508688449859619,3.126953125,8388608
+37,self_attn.v_proj,4,32,8,128,1.9416882991790771,4.5078125,8388608
+37,self_attn.v_proj,4,64,8,128,2.271498918533325,4.25390625,8388608
+37,self_attn.v_proj,4,128,8,128,2.5708978176116943,4.126953125,8388608
+37,self_attn.v_proj,8,32,8,128,0.11753822863101959,8.5078125,8388608
+37,self_attn.v_proj,8,64,8,128,0.13737376034259796,8.25390625,8388608
+37,self_attn.v_proj,8,128,8,128,0.15499362349510193,8.126953125,8388608
+37,self_attn.k_proj,2,32,8,128,37.073265075683594,2.5078125,8388608
+37,self_attn.k_proj,2,64,8,128,44.21490478515625,2.25390625,8388608
+37,self_attn.k_proj,2,128,8,128,50.363521575927734,2.126953125,8388608
+37,self_attn.k_proj,3,32,8,128,15.910534858703613,3.5078125,8388608
+37,self_attn.k_proj,3,64,8,128,19.065753936767578,3.25390625,8388608
+37,self_attn.k_proj,3,128,8,128,21.948333740234375,3.126953125,8388608
+37,self_attn.k_proj,4,32,8,128,7.425031661987305,4.5078125,8388608
+37,self_attn.k_proj,4,64,8,128,8.897001266479492,4.25390625,8388608
+37,self_attn.k_proj,4,128,8,128,10.248730659484863,4.126953125,8388608
+37,self_attn.k_proj,8,32,8,128,0.44921255111694336,8.5078125,8388608
+37,self_attn.k_proj,8,64,8,128,0.5377213954925537,8.25390625,8388608
+37,self_attn.k_proj,8,128,8,128,0.617260217666626,8.126953125,8388608
+37,self_attn.o_proj,2,32,8,128,33.58125686645508,20.0625,67108864
+37,self_attn.o_proj,2,64,8,128,39.107032775878906,18.03125,67108864
+37,self_attn.o_proj,2,128,8,128,44.1419563293457,17.015625,67108864
+37,self_attn.o_proj,3,32,8,128,14.37392807006836,28.0625,67108864
+37,self_attn.o_proj,3,64,8,128,16.7641544342041,26.03125,67108864
+37,self_attn.o_proj,3,128,8,128,18.926284790039062,25.015625,67108864
+37,self_attn.o_proj,4,32,8,128,6.708399772644043,36.0625,67108864
+37,self_attn.o_proj,4,64,8,128,7.833434104919434,34.03125,67108864
+37,self_attn.o_proj,4,128,8,128,8.852543830871582,33.015625,67108864
+37,self_attn.o_proj,8,32,8,128,0.4079088568687439,68.0625,67108864
+37,self_attn.o_proj,8,64,8,128,0.4755544364452362,66.03125,67108864
+37,self_attn.o_proj,8,128,8,128,0.5360546708106995,65.015625,67108864
+37,mlp.gate_proj,2,32,8,128,71.81346893310547,70.21875,234881024
+37,mlp.gate_proj,2,64,8,128,84.5894546508789,63.109375,234881024
+37,mlp.gate_proj,2,128,8,128,96.19647216796875,59.5546875,234881024
+37,mlp.gate_proj,3,32,8,128,30.857757568359375,98.21875,234881024
+37,mlp.gate_proj,3,64,8,128,36.14644241333008,91.109375,234881024
+37,mlp.gate_proj,3,128,8,128,41.30503463745117,87.5546875,234881024
+37,mlp.gate_proj,4,32,8,128,14.360860824584961,126.21875,234881024
+37,mlp.gate_proj,4,64,8,128,16.88521957397461,119.109375,234881024
+37,mlp.gate_proj,4,128,8,128,19.267602920532227,115.5546875,234881024
+37,mlp.gate_proj,8,32,8,128,0.8765199780464172,238.21875,234881024
+37,mlp.gate_proj,8,64,8,128,1.032407522201538,231.109375,234881024
+37,mlp.gate_proj,8,128,8,128,1.171352744102478,227.5546875,234881024
+37,mlp.up_proj,2,32,8,128,67.16079711914062,70.21875,234881024
+37,mlp.up_proj,2,64,8,128,78.1637954711914,63.109375,234881024
+37,mlp.up_proj,2,128,8,128,88.36465454101562,59.5546875,234881024
+37,mlp.up_proj,3,32,8,128,28.72227668762207,98.21875,234881024
+37,mlp.up_proj,3,64,8,128,33.48160934448242,91.109375,234881024
+37,mlp.up_proj,3,128,8,128,37.69949722290039,87.5546875,234881024
+37,mlp.up_proj,4,32,8,128,13.375240325927734,126.21875,234881024
+37,mlp.up_proj,4,64,8,128,15.665079116821289,119.109375,234881024
+37,mlp.up_proj,4,128,8,128,17.617374420166016,115.5546875,234881024
+37,mlp.up_proj,8,32,8,128,0.8168332576751709,238.21875,234881024
+37,mlp.up_proj,8,64,8,128,0.955348789691925,231.109375,234881024
+37,mlp.up_proj,8,128,8,128,1.0720200538635254,227.5546875,234881024
+37,mlp.down_proj,2,32,8,128,67.51783752441406,70.21875,234881024
+37,mlp.down_proj,2,64,8,128,78.58902740478516,63.109375,234881024
+37,mlp.down_proj,2,128,8,128,88.88556671142578,59.5546875,234881024
+37,mlp.down_proj,3,32,8,128,28.888568878173828,98.21875,234881024
+37,mlp.down_proj,3,64,8,128,33.659263610839844,91.109375,234881024
+37,mlp.down_proj,3,128,8,128,37.91593551635742,87.5546875,234881024
+37,mlp.down_proj,4,32,8,128,13.453252792358398,126.21875,234881024
+37,mlp.down_proj,4,64,8,128,15.755415916442871,119.109375,234881024
+37,mlp.down_proj,4,128,8,128,17.718156814575195,115.5546875,234881024
+37,mlp.down_proj,8,32,8,128,0.8214923143386841,238.21875,234881024
+37,mlp.down_proj,8,64,8,128,0.9609149694442749,231.109375,234881024
+37,mlp.down_proj,8,128,8,128,1.0778824090957642,227.5546875,234881024
+38,self_attn.q_proj,2,32,8,128,48.81117248535156,20.0625,67108864
+38,self_attn.q_proj,2,64,8,128,58.154991149902344,18.03125,67108864
+38,self_attn.q_proj,2,128,8,128,67.70924377441406,17.015625,67108864
+38,self_attn.q_proj,3,32,8,128,20.941280364990234,28.0625,67108864
+38,self_attn.q_proj,3,64,8,128,25.039390563964844,26.03125,67108864
+38,self_attn.q_proj,3,128,8,128,29.400402069091797,25.015625,67108864
+38,self_attn.q_proj,4,32,8,128,9.765689849853516,36.0625,67108864
+38,self_attn.q_proj,4,64,8,128,11.688702583312988,34.03125,67108864
+38,self_attn.q_proj,4,128,8,128,13.716032028198242,33.015625,67108864
+38,self_attn.q_proj,8,32,8,128,0.5933027267456055,68.0625,67108864
+38,self_attn.q_proj,8,64,8,128,0.7102562189102173,66.03125,67108864
+38,self_attn.q_proj,8,128,8,128,0.8306422829627991,65.015625,67108864
+38,self_attn.v_proj,2,32,8,128,10.370458602905273,2.5078125,8388608
+38,self_attn.v_proj,2,64,8,128,12.127815246582031,2.25390625,8388608
+38,self_attn.v_proj,2,128,8,128,13.726225852966309,2.126953125,8388608
+38,self_attn.v_proj,3,32,8,128,4.433779239654541,3.5078125,8388608
+38,self_attn.v_proj,3,64,8,128,5.1891608238220215,3.25390625,8388608
+38,self_attn.v_proj,3,128,8,128,5.867914199829102,3.126953125,8388608
+38,self_attn.v_proj,4,32,8,128,2.067638635635376,4.5078125,8388608
+38,self_attn.v_proj,4,64,8,128,2.4203245639801025,4.25390625,8388608
+38,self_attn.v_proj,4,128,8,128,2.7387607097625732,4.126953125,8388608
+38,self_attn.v_proj,8,32,8,128,0.1252339631319046,8.5078125,8388608
+38,self_attn.v_proj,8,64,8,128,0.1464403122663498,8.25390625,8388608
+38,self_attn.v_proj,8,128,8,128,0.1650484800338745,8.126953125,8388608
+38,self_attn.k_proj,2,32,8,128,35.750953674316406,2.5078125,8388608
+38,self_attn.k_proj,2,64,8,128,42.777976989746094,2.25390625,8388608
+38,self_attn.k_proj,2,128,8,128,48.83155822753906,2.126953125,8388608
+38,self_attn.k_proj,3,32,8,128,15.401459693908691,3.5078125,8388608
+38,self_attn.k_proj,3,64,8,128,18.546892166137695,3.25390625,8388608
+38,self_attn.k_proj,3,128,8,128,21.522655487060547,3.126953125,8388608
+38,self_attn.k_proj,4,32,8,128,7.187498092651367,4.5078125,8388608
+38,self_attn.k_proj,4,64,8,128,8.658435821533203,4.25390625,8388608
+38,self_attn.k_proj,4,128,8,128,10.052827835083008,4.126953125,8388608
+38,self_attn.k_proj,8,32,8,128,0.4347831904888153,8.5078125,8388608
+38,self_attn.k_proj,8,64,8,128,0.5235644578933716,8.25390625,8388608
+38,self_attn.k_proj,8,128,8,128,0.6057740449905396,8.126953125,8388608
+38,self_attn.o_proj,2,32,8,128,33.50827407836914,20.0625,67108864
+38,self_attn.o_proj,2,64,8,128,39.03700256347656,18.03125,67108864
+38,self_attn.o_proj,2,128,8,128,44.07758331298828,17.015625,67108864
+38,self_attn.o_proj,3,32,8,128,14.326871871948242,28.0625,67108864
+38,self_attn.o_proj,3,64,8,128,16.716943740844727,26.03125,67108864
+38,self_attn.o_proj,3,128,8,128,18.88189125061035,25.015625,67108864
+38,self_attn.o_proj,4,32,8,128,6.683267116546631,36.0625,67108864
+38,self_attn.o_proj,4,64,8,128,7.80408239364624,34.03125,67108864
+38,self_attn.o_proj,4,128,8,128,8.812667846679688,33.015625,67108864
+38,self_attn.o_proj,8,32,8,128,0.40626096725463867,68.0625,67108864
+38,self_attn.o_proj,8,64,8,128,0.47360309958457947,66.03125,67108864
+38,self_attn.o_proj,8,128,8,128,0.5332428216934204,65.015625,67108864
+38,mlp.gate_proj,2,32,8,128,72.40433502197266,70.21875,234881024
+38,mlp.gate_proj,2,64,8,128,85.53372192382812,63.109375,234881024
+38,mlp.gate_proj,2,128,8,128,97.51160430908203,59.5546875,234881024
+38,mlp.gate_proj,3,32,8,128,31.141399383544922,98.21875,234881024
+38,mlp.gate_proj,3,64,8,128,36.56984329223633,91.109375,234881024
+38,mlp.gate_proj,3,128,8,128,42.04135513305664,87.5546875,234881024
+38,mlp.gate_proj,4,32,8,128,14.490718841552734,126.21875,234881024
+38,mlp.gate_proj,4,64,8,128,17.078691482543945,119.109375,234881024
+38,mlp.gate_proj,4,128,8,128,19.60403823852539,115.5546875,234881024
+38,mlp.gate_proj,8,32,8,128,0.8843744993209839,238.21875,234881024
+38,mlp.gate_proj,8,64,8,128,1.0436081886291504,231.109375,234881024
+38,mlp.gate_proj,8,128,8,128,1.1913179159164429,227.5546875,234881024
+38,mlp.up_proj,2,32,8,128,67.00103759765625,70.21875,234881024
+38,mlp.up_proj,2,64,8,128,77.94048309326172,63.109375,234881024
+38,mlp.up_proj,2,128,8,128,88.08341217041016,59.5546875,234881024
+38,mlp.up_proj,3,32,8,128,28.64519691467285,98.21875,234881024
+38,mlp.up_proj,3,64,8,128,33.397705078125,91.109375,234881024
+38,mlp.up_proj,3,128,8,128,37.59461212158203,87.5546875,234881024
+38,mlp.up_proj,4,32,8,128,13.341904640197754,126.21875,234881024
+38,mlp.up_proj,4,64,8,128,15.621837615966797,119.109375,234881024
+38,mlp.up_proj,4,128,8,128,17.567628860473633,115.5546875,234881024
+38,mlp.up_proj,8,32,8,128,0.8147733211517334,238.21875,234881024
+38,mlp.up_proj,8,64,8,128,0.9525849223136902,231.109375,234881024
+38,mlp.up_proj,8,128,8,128,1.0689860582351685,227.5546875,234881024
+38,mlp.down_proj,2,32,8,128,67.43077087402344,70.21875,234881024
+38,mlp.down_proj,2,64,8,128,78.4882583618164,63.109375,234881024
+38,mlp.down_proj,2,128,8,128,88.74310302734375,59.5546875,234881024
+38,mlp.down_proj,3,32,8,128,28.850187301635742,98.21875,234881024
+38,mlp.down_proj,3,64,8,128,33.61622619628906,91.109375,234881024
+38,mlp.down_proj,3,128,8,128,37.86039733886719,87.5546875,234881024
+38,mlp.down_proj,4,32,8,128,13.43558120727539,126.21875,234881024
+38,mlp.down_proj,4,64,8,128,15.737957000732422,119.109375,234881024
+38,mlp.down_proj,4,128,8,128,17.6971378326416,115.5546875,234881024
+38,mlp.down_proj,8,32,8,128,0.8205083608627319,238.21875,234881024
+38,mlp.down_proj,8,64,8,128,0.9598961472511292,231.109375,234881024
+38,mlp.down_proj,8,128,8,128,1.0771143436431885,227.5546875,234881024
+39,self_attn.q_proj,2,32,8,128,47.702579498291016,20.0625,67108864
+39,self_attn.q_proj,2,64,8,128,56.50425720214844,18.03125,67108864
+39,self_attn.q_proj,2,128,8,128,65.59249877929688,17.015625,67108864
+39,self_attn.q_proj,3,32,8,128,20.44588851928711,28.0625,67108864
+39,self_attn.q_proj,3,64,8,128,24.306482315063477,26.03125,67108864
+39,self_attn.q_proj,3,128,8,128,28.369590759277344,25.015625,67108864
+39,self_attn.q_proj,4,32,8,128,9.53781795501709,36.0625,67108864
+39,self_attn.q_proj,4,64,8,128,11.351512908935547,34.03125,67108864
+39,self_attn.q_proj,4,128,8,128,13.236985206604004,33.015625,67108864
+39,self_attn.q_proj,8,32,8,128,0.5795016288757324,68.0625,67108864
+39,self_attn.q_proj,8,64,8,128,0.6895589828491211,66.03125,67108864
+39,self_attn.q_proj,8,128,8,128,0.801480770111084,65.015625,67108864
+39,self_attn.v_proj,2,32,8,128,10.729595184326172,2.5078125,8388608
+39,self_attn.v_proj,2,64,8,128,12.568916320800781,2.25390625,8388608
+39,self_attn.v_proj,2,128,8,128,14.241296768188477,2.126953125,8388608
+39,self_attn.v_proj,3,32,8,128,4.58883810043335,3.5078125,8388608
+39,self_attn.v_proj,3,64,8,128,5.376890659332275,3.25390625,8388608
+39,self_attn.v_proj,3,128,8,128,6.090038776397705,3.126953125,8388608
+39,self_attn.v_proj,4,32,8,128,2.1410772800445557,4.5078125,8388608
+39,self_attn.v_proj,4,64,8,128,2.508882999420166,4.25390625,8388608
+39,self_attn.v_proj,4,128,8,128,2.8430514335632324,4.126953125,8388608
+39,self_attn.v_proj,8,32,8,128,0.12961198389530182,8.5078125,8388608
+39,self_attn.v_proj,8,64,8,128,0.1517712026834488,8.25390625,8388608
+39,self_attn.v_proj,8,128,8,128,0.17131583392620087,8.126953125,8388608
+39,self_attn.k_proj,2,32,8,128,36.67458724975586,2.5078125,8388608
+39,self_attn.k_proj,2,64,8,128,44.04214096069336,2.25390625,8388608
+39,self_attn.k_proj,2,128,8,128,50.33544921875,2.126953125,8388608
+39,self_attn.k_proj,3,32,8,128,15.855488777160645,3.5078125,8388608
+39,self_attn.k_proj,3,64,8,128,19.240999221801758,3.25390625,8388608
+39,self_attn.k_proj,3,128,8,128,22.492477416992188,3.126953125,8388608
+39,self_attn.k_proj,4,32,8,128,7.399962425231934,4.5078125,8388608
+39,self_attn.k_proj,4,64,8,128,8.980509757995605,4.25390625,8388608
+39,self_attn.k_proj,4,128,8,128,10.505121231079102,4.126953125,8388608
+39,self_attn.k_proj,8,32,8,128,0.4476538598537445,8.5078125,8388608
+39,self_attn.k_proj,8,64,8,128,0.542992889881134,8.25390625,8388608
+39,self_attn.k_proj,8,128,8,128,0.6329089403152466,8.126953125,8388608
+39,self_attn.o_proj,2,32,8,128,33.81058883666992,20.0625,67108864
+39,self_attn.o_proj,2,64,8,128,39.370182037353516,18.03125,67108864
+39,self_attn.o_proj,2,128,8,128,44.4337158203125,17.015625,67108864
+39,self_attn.o_proj,3,32,8,128,14.485944747924805,28.0625,67108864
+39,self_attn.o_proj,3,64,8,128,16.897361755371094,26.03125,67108864
+39,self_attn.o_proj,3,128,8,128,19.096046447753906,25.015625,67108864
+39,self_attn.o_proj,4,32,8,128,6.773406028747559,36.0625,67108864
+39,self_attn.o_proj,4,64,8,128,7.919699668884277,34.03125,67108864
+39,self_attn.o_proj,4,128,8,128,8.966453552246094,33.015625,67108864
+39,self_attn.o_proj,8,32,8,128,0.4118209183216095,68.0625,67108864
+39,self_attn.o_proj,8,64,8,128,0.4808553457260132,66.03125,67108864
+39,self_attn.o_proj,8,128,8,128,0.543539822101593,65.015625,67108864
+39,mlp.gate_proj,2,32,8,128,72.48025512695312,70.21875,234881024
+39,mlp.gate_proj,2,64,8,128,85.40560150146484,63.109375,234881024
+39,mlp.gate_proj,2,128,8,128,97.12654876708984,59.5546875,234881024
+39,mlp.gate_proj,3,32,8,128,31.164159774780273,98.21875,234881024
+39,mlp.gate_proj,3,64,8,128,36.486900329589844,91.109375,234881024
+39,mlp.gate_proj,3,128,8,128,41.73063659667969,87.5546875,234881024
+39,mlp.gate_proj,4,32,8,128,14.502073287963867,126.21875,234881024
+39,mlp.gate_proj,4,64,8,128,17.040225982666016,119.109375,234881024
+39,mlp.gate_proj,4,128,8,128,19.458717346191406,115.5546875,234881024
+39,mlp.gate_proj,8,32,8,128,0.8851938247680664,238.21875,234881024
+39,mlp.gate_proj,8,64,8,128,1.041657567024231,231.109375,234881024
+39,mlp.gate_proj,8,128,8,128,1.1834651231765747,227.5546875,234881024
+39,mlp.up_proj,2,32,8,128,66.92317199707031,70.21875,234881024
+39,mlp.up_proj,2,64,8,128,77.84870147705078,63.109375,234881024
+39,mlp.up_proj,2,128,8,128,87.98621368408203,59.5546875,234881024
+39,mlp.up_proj,3,32,8,128,28.609296798706055,98.21875,234881024
+39,mlp.up_proj,3,64,8,128,33.352821350097656,91.109375,234881024
+39,mlp.up_proj,3,128,8,128,37.53313064575195,87.5546875,234881024
+39,mlp.up_proj,4,32,8,128,13.32458782196045,126.21875,234881024
+39,mlp.up_proj,4,64,8,128,15.598309516906738,119.109375,234881024
+39,mlp.up_proj,4,128,8,128,17.536075592041016,115.5546875,234881024
+39,mlp.up_proj,8,32,8,128,0.8136534094810486,238.21875,234881024
+39,mlp.up_proj,8,64,8,128,0.9511791467666626,231.109375,234881024
+39,mlp.up_proj,8,128,8,128,1.0675238370895386,227.5546875,234881024
+39,mlp.down_proj,2,32,8,128,67.4476318359375,70.21875,234881024
+39,mlp.down_proj,2,64,8,128,78.4978256225586,63.109375,234881024
+39,mlp.down_proj,2,128,8,128,88.7271499633789,59.5546875,234881024
+39,mlp.down_proj,3,32,8,128,28.85817527770996,98.21875,234881024
+39,mlp.down_proj,3,64,8,128,33.618865966796875,91.109375,234881024
+39,mlp.down_proj,3,128,8,128,37.84467315673828,87.5546875,234881024
+39,mlp.down_proj,4,32,8,128,13.437882423400879,126.21875,234881024
+39,mlp.down_proj,4,64,8,128,15.736634254455566,119.109375,234881024
+39,mlp.down_proj,4,128,8,128,17.67955780029297,115.5546875,234881024
+39,mlp.down_proj,8,32,8,128,0.8206799626350403,238.21875,234881024
+39,mlp.down_proj,8,64,8,128,0.9597925543785095,231.109375,234881024
+39,mlp.down_proj,8,128,8,128,1.076420783996582,227.5546875,234881024
+40,self_attn.q_proj,2,32,8,128,46.208160400390625,20.0625,67108864
+40,self_attn.q_proj,2,64,8,128,54.75465393066406,18.03125,67108864
+40,self_attn.q_proj,2,128,8,128,63.644996643066406,17.015625,67108864
+40,self_attn.q_proj,3,32,8,128,19.823131561279297,28.0625,67108864
+40,self_attn.q_proj,3,64,8,128,23.634363174438477,26.03125,67108864
+40,self_attn.q_proj,3,128,8,128,27.65677833557129,25.015625,67108864
+40,self_attn.q_proj,4,32,8,128,9.247557640075684,36.0625,67108864
+40,self_attn.q_proj,4,64,8,128,11.036786079406738,34.03125,67108864
+40,self_attn.q_proj,4,128,8,128,12.9072847366333,33.015625,67108864
+40,self_attn.q_proj,8,32,8,128,0.5617331862449646,68.0625,67108864
+40,self_attn.q_proj,8,64,8,128,0.6701843738555908,66.03125,67108864
+40,self_attn.q_proj,8,128,8,128,0.7814801931381226,65.015625,67108864
+40,self_attn.v_proj,2,32,8,128,11.688714981079102,2.5078125,8388608
+40,self_attn.v_proj,2,64,8,128,13.652423858642578,2.25390625,8388608
+40,self_attn.v_proj,2,128,8,128,15.434215545654297,2.126953125,8388608
+40,self_attn.v_proj,3,32,8,128,4.99672269821167,3.5078125,8388608
+40,self_attn.v_proj,3,64,8,128,5.83676290512085,3.25390625,8388608
+40,self_attn.v_proj,3,128,8,128,6.594868183135986,3.126953125,8388608
+40,self_attn.v_proj,4,32,8,128,2.331019163131714,4.5078125,8388608
+40,self_attn.v_proj,4,64,8,128,2.7243618965148926,4.25390625,8388608
+40,self_attn.v_proj,4,128,8,128,3.078658103942871,4.126953125,8388608
+40,self_attn.v_proj,8,32,8,128,0.14116792380809784,8.5078125,8388608
+40,self_attn.v_proj,8,64,8,128,0.16471853852272034,8.25390625,8388608
+40,self_attn.v_proj,8,128,8,128,0.18558289110660553,8.126953125,8388608
+40,self_attn.k_proj,2,32,8,128,32.4831428527832,2.5078125,8388608
+40,self_attn.k_proj,2,64,8,128,38.797176361083984,2.25390625,8388608
+40,self_attn.k_proj,2,128,8,128,44.19651412963867,2.126953125,8388608
+40,self_attn.k_proj,3,32,8,128,13.992293357849121,3.5078125,8388608
+40,self_attn.k_proj,3,64,8,128,16.8132381439209,3.25390625,8388608
+40,self_attn.k_proj,3,128,8,128,19.463300704956055,3.126953125,8388608
+40,self_attn.k_proj,4,32,8,128,6.5291876792907715,4.5078125,8388608
+40,self_attn.k_proj,4,64,8,128,7.847931385040283,4.25390625,8388608
+40,self_attn.k_proj,4,128,8,128,9.088887214660645,4.126953125,8388608
+40,self_attn.k_proj,8,32,8,128,0.39515626430511475,8.5078125,8388608
+40,self_attn.k_proj,8,64,8,128,0.47452935576438904,8.25390625,8388608
+40,self_attn.k_proj,8,128,8,128,0.5475208759307861,8.126953125,8388608
+40,self_attn.o_proj,2,32,8,128,34.38907241821289,20.0625,67108864
+40,self_attn.o_proj,2,64,8,128,40.08850860595703,18.03125,67108864
+40,self_attn.o_proj,2,128,8,128,45.26538848876953,17.015625,67108864
+40,self_attn.o_proj,3,32,8,128,14.714380264282227,28.0625,67108864
+40,self_attn.o_proj,3,64,8,128,17.168743133544922,26.03125,67108864
+40,self_attn.o_proj,3,128,8,128,19.40982437133789,25.015625,67108864
+40,self_attn.o_proj,4,32,8,128,6.8638482093811035,36.0625,67108864
+40,self_attn.o_proj,4,64,8,128,8.018959999084473,34.03125,67108864
+40,self_attn.o_proj,4,128,8,128,9.058540344238281,33.015625,67108864
+40,self_attn.o_proj,8,32,8,128,0.41717758774757385,68.0625,67108864
+40,self_attn.o_proj,8,64,8,128,0.4864044189453125,66.03125,67108864
+40,self_attn.o_proj,8,128,8,128,0.5481789708137512,65.015625,67108864
+40,mlp.gate_proj,2,32,8,128,72.86029815673828,70.21875,234881024
+40,mlp.gate_proj,2,64,8,128,85.86495208740234,63.109375,234881024
+40,mlp.gate_proj,2,128,8,128,97.54362487792969,59.5546875,234881024
+40,mlp.gate_proj,3,32,8,128,31.33269500732422,98.21875,234881024
+40,mlp.gate_proj,3,64,8,128,36.662940979003906,91.109375,234881024
+40,mlp.gate_proj,3,128,8,128,41.92482376098633,87.5546875,234881024
+40,mlp.gate_proj,4,32,8,128,14.57835865020752,126.21875,234881024
+40,mlp.gate_proj,4,64,8,128,17.120357513427734,119.109375,234881024
+40,mlp.gate_proj,4,128,8,128,19.54804801940918,115.5546875,234881024
+40,mlp.gate_proj,8,32,8,128,0.889820396900177,238.21875,234881024
+40,mlp.gate_proj,8,64,8,128,1.0464510917663574,231.109375,234881024
+40,mlp.gate_proj,8,128,8,128,1.1885584592819214,227.5546875,234881024
+40,mlp.up_proj,2,32,8,128,66.9774398803711,70.21875,234881024
+40,mlp.up_proj,2,64,8,128,77.9189224243164,63.109375,234881024
+40,mlp.up_proj,2,128,8,128,88.03173828125,59.5546875,234881024
+40,mlp.up_proj,3,32,8,128,28.638511657714844,98.21875,234881024
+40,mlp.up_proj,3,64,8,128,33.38407897949219,91.109375,234881024
+40,mlp.up_proj,3,128,8,128,37.567588806152344,87.5546875,234881024
+40,mlp.up_proj,4,32,8,128,13.337662696838379,126.21875,234881024
+40,mlp.up_proj,4,64,8,128,15.614280700683594,119.109375,234881024
+40,mlp.up_proj,4,128,8,128,17.55264663696289,115.5546875,234881024
+40,mlp.up_proj,8,32,8,128,0.8146010041236877,238.21875,234881024
+40,mlp.up_proj,8,64,8,128,0.9522619247436523,231.109375,234881024
+40,mlp.up_proj,8,128,8,128,1.0685402154922485,227.5546875,234881024
+40,mlp.down_proj,2,32,8,128,67.45592498779297,70.21875,234881024
+40,mlp.down_proj,2,64,8,128,78.50223541259766,63.109375,234881024
+40,mlp.down_proj,2,128,8,128,88.74594116210938,59.5546875,234881024
+40,mlp.down_proj,3,32,8,128,28.86054229736328,98.21875,234881024
+40,mlp.down_proj,3,64,8,128,33.6246337890625,91.109375,234881024
+40,mlp.down_proj,3,128,8,128,37.86384582519531,87.5546875,234881024
+40,mlp.down_proj,4,32,8,128,13.43843936920166,126.21875,234881024
+40,mlp.down_proj,4,64,8,128,15.739042282104492,119.109375,234881024
+40,mlp.down_proj,4,128,8,128,17.69170379638672,115.5546875,234881024
+40,mlp.down_proj,8,32,8,128,0.8206921219825745,238.21875,234881024
+40,mlp.down_proj,8,64,8,128,0.9599099159240723,231.109375,234881024
+40,mlp.down_proj,8,128,8,128,1.076674461364746,227.5546875,234881024
+41,self_attn.q_proj,2,32,8,128,47.144996643066406,20.0625,67108864
+41,self_attn.q_proj,2,64,8,128,55.56779479980469,18.03125,67108864
+41,self_attn.q_proj,2,128,8,128,64.27327728271484,17.015625,67108864
+41,self_attn.q_proj,3,32,8,128,20.191753387451172,28.0625,67108864
+41,self_attn.q_proj,3,64,8,128,23.86530303955078,26.03125,67108864
+41,self_attn.q_proj,3,128,8,128,27.667194366455078,25.015625,67108864
+41,self_attn.q_proj,4,32,8,128,9.417218208312988,36.0625,67108864
+41,self_attn.q_proj,4,64,8,128,11.140992164611816,34.03125,67108864
+41,self_attn.q_proj,4,128,8,128,12.913605690002441,33.015625,67108864
+41,self_attn.q_proj,8,32,8,128,0.5720597505569458,68.0625,67108864
+41,self_attn.q_proj,8,64,8,128,0.6766618490219116,66.03125,67108864
+41,self_attn.q_proj,8,128,8,128,0.7819150686264038,65.015625,67108864
+41,self_attn.v_proj,2,32,8,128,10.822153091430664,2.5078125,8388608
+41,self_attn.v_proj,2,64,8,128,12.622425079345703,2.25390625,8388608
+41,self_attn.v_proj,2,128,8,128,14.253490447998047,2.126953125,8388608
+41,self_attn.v_proj,3,32,8,128,4.625828266143799,3.5078125,8388608
+41,self_attn.v_proj,3,64,8,128,5.397848129272461,3.25390625,8388608
+41,self_attn.v_proj,3,128,8,128,6.096958160400391,3.126953125,8388608
+41,self_attn.v_proj,4,32,8,128,2.159303903579712,4.5078125,8388608
+41,self_attn.v_proj,4,64,8,128,2.5206658840179443,4.25390625,8388608
+41,self_attn.v_proj,4,128,8,128,2.8453831672668457,4.126953125,8388608
+41,self_attn.v_proj,8,32,8,128,0.13070832192897797,8.5078125,8388608
+41,self_attn.v_proj,8,64,8,128,0.1523590385913849,8.25390625,8388608
+41,self_attn.v_proj,8,128,8,128,0.17144542932510376,8.126953125,8388608
+41,self_attn.k_proj,2,32,8,128,36.50194549560547,2.5078125,8388608
+41,self_attn.k_proj,2,64,8,128,43.63142013549805,2.25390625,8388608
+41,self_attn.k_proj,2,128,8,128,49.69706344604492,2.126953125,8388608
+41,self_attn.k_proj,3,32,8,128,15.729552268981934,3.5078125,8388608
+41,self_attn.k_proj,3,64,8,128,18.93177032470703,3.25390625,8388608
+41,self_attn.k_proj,3,128,8,128,21.933258056640625,3.126953125,8388608
+41,self_attn.k_proj,4,32,8,128,7.341017246246338,4.5078125,8388608
+41,self_attn.k_proj,4,64,8,128,8.837186813354492,4.25390625,8388608
+41,self_attn.k_proj,4,128,8,128,10.23975658416748,4.126953125,8388608
+41,self_attn.k_proj,8,32,8,128,0.44426271319389343,8.5078125,8388608
+41,self_attn.k_proj,8,64,8,128,0.5342881679534912,8.25390625,8388608
+41,self_attn.k_proj,8,128,8,128,0.6171364784240723,8.126953125,8388608
+41,self_attn.o_proj,2,32,8,128,33.85589599609375,20.0625,67108864
+41,self_attn.o_proj,2,64,8,128,39.39896774291992,18.03125,67108864
+41,self_attn.o_proj,2,128,8,128,44.39799118041992,17.015625,67108864
+41,self_attn.o_proj,3,32,8,128,14.476641654968262,28.0625,67108864
+41,self_attn.o_proj,3,64,8,128,16.861488342285156,26.03125,67108864
+41,self_attn.o_proj,3,128,8,128,18.997379302978516,25.015625,67108864
+41,self_attn.o_proj,4,32,8,128,6.75196647644043,36.0625,67108864
+41,self_attn.o_proj,4,64,8,128,7.871068477630615,34.03125,67108864
+41,self_attn.o_proj,4,128,8,128,8.866022109985352,33.015625,67108864
+41,self_attn.o_proj,8,32,8,128,0.4104026257991791,68.0625,67108864
+41,self_attn.o_proj,8,64,8,128,0.4775233864784241,66.03125,67108864
+41,self_attn.o_proj,8,128,8,128,0.5363680720329285,65.015625,67108864
+41,mlp.gate_proj,2,32,8,128,72.76805114746094,70.21875,234881024
+41,mlp.gate_proj,2,64,8,128,85.67806243896484,63.109375,234881024
+41,mlp.gate_proj,2,128,8,128,97.2476577758789,59.5546875,234881024
+41,mlp.gate_proj,3,32,8,128,31.287689208984375,98.21875,234881024
+41,mlp.gate_proj,3,64,8,128,36.57448959350586,91.109375,234881024
+41,mlp.gate_proj,3,128,8,128,41.774906158447266,87.5546875,234881024
+41,mlp.gate_proj,4,32,8,128,14.558725357055664,126.21875,234881024
+41,mlp.gate_proj,4,64,8,128,17.08295440673828,119.109375,234881024
+41,mlp.gate_proj,4,128,8,128,19.468364715576172,115.5546875,234881024
+41,mlp.gate_proj,8,32,8,128,0.8886621594429016,238.21875,234881024
+41,mlp.gate_proj,8,64,8,128,1.044183373451233,231.109375,234881024
+41,mlp.gate_proj,8,128,8,128,1.1841503381729126,227.5546875,234881024
+41,mlp.up_proj,2,32,8,128,66.98749542236328,70.21875,234881024
+41,mlp.up_proj,2,64,8,128,77.9141845703125,63.109375,234881024
+41,mlp.up_proj,2,128,8,128,88.0218734741211,59.5546875,234881024
+41,mlp.up_proj,3,32,8,128,28.636873245239258,98.21875,234881024
+41,mlp.up_proj,3,64,8,128,33.379913330078125,91.109375,234881024
+41,mlp.up_proj,3,128,8,128,37.56983947753906,87.5546875,234881024
+41,mlp.up_proj,4,32,8,128,13.335919380187988,126.21875,234881024
+41,mlp.up_proj,4,64,8,128,15.6109619140625,119.109375,234881024
+41,mlp.up_proj,4,128,8,128,17.54947853088379,115.5546875,234881024
+41,mlp.up_proj,8,32,8,128,0.814592719078064,238.21875,234881024
+41,mlp.up_proj,8,64,8,128,0.9520053863525391,231.109375,234881024
+41,mlp.up_proj,8,128,8,128,1.0680066347122192,227.5546875,234881024
+41,mlp.down_proj,2,32,8,128,67.4456558227539,70.21875,234881024
+41,mlp.down_proj,2,64,8,128,78.47855377197266,63.109375,234881024
+41,mlp.down_proj,2,128,8,128,88.67603302001953,59.5546875,234881024
+41,mlp.down_proj,3,32,8,128,28.851882934570312,98.21875,234881024
+41,mlp.down_proj,3,64,8,128,33.60783004760742,91.109375,234881024
+41,mlp.down_proj,3,128,8,128,37.82402801513672,87.5546875,234881024
+41,mlp.down_proj,4,32,8,128,13.436448097229004,126.21875,234881024
+41,mlp.down_proj,4,64,8,128,15.729095458984375,119.109375,234881024
+41,mlp.down_proj,4,128,8,128,17.667362213134766,115.5546875,234881024
+41,mlp.down_proj,8,32,8,128,0.8205360174179077,238.21875,234881024
+41,mlp.down_proj,8,64,8,128,0.9593890905380249,231.109375,234881024
+41,mlp.down_proj,8,128,8,128,1.0757725238800049,227.5546875,234881024
+42,self_attn.q_proj,2,32,8,128,43.36602783203125,20.0625,67108864
+42,self_attn.q_proj,2,64,8,128,51.36442184448242,18.03125,67108864
+42,self_attn.q_proj,2,128,8,128,59.52371597290039,17.015625,67108864
+42,self_attn.q_proj,3,32,8,128,18.606870651245117,28.0625,67108864
+42,self_attn.q_proj,3,64,8,128,22.19917106628418,26.03125,67108864
+42,self_attn.q_proj,3,128,8,128,25.900707244873047,25.015625,67108864
+42,self_attn.q_proj,4,32,8,128,8.682658195495605,36.0625,67108864
+42,self_attn.q_proj,4,64,8,128,10.355265617370605,34.03125,67108864
+42,self_attn.q_proj,4,128,8,128,12.097424507141113,33.015625,67108864
+42,self_attn.q_proj,8,32,8,128,0.5273026823997498,68.0625,67108864
+42,self_attn.q_proj,8,64,8,128,0.6285464763641357,66.03125,67108864
+42,self_attn.q_proj,8,128,8,128,0.732229471206665,65.015625,67108864
+42,self_attn.v_proj,2,32,8,128,11.65095329284668,2.5078125,8388608
+42,self_attn.v_proj,2,64,8,128,13.566142082214355,2.25390625,8388608
+42,self_attn.v_proj,2,128,8,128,15.285043716430664,2.126953125,8388608
+42,self_attn.v_proj,3,32,8,128,4.981750965118408,3.5078125,8388608
+42,self_attn.v_proj,3,64,8,128,5.8029890060424805,3.25390625,8388608
+42,self_attn.v_proj,3,128,8,128,6.5364532470703125,3.126953125,8388608
+42,self_attn.v_proj,4,32,8,128,2.323693037033081,4.5078125,8388608
+42,self_attn.v_proj,4,64,8,128,2.7071378231048584,4.25390625,8388608
+42,self_attn.v_proj,4,128,8,128,3.049984931945801,4.126953125,8388608
+42,self_attn.v_proj,8,32,8,128,0.14074617624282837,8.5078125,8388608
+42,self_attn.v_proj,8,64,8,128,0.1638219654560089,8.25390625,8388608
+42,self_attn.v_proj,8,128,8,128,0.1839367151260376,8.126953125,8388608
+42,self_attn.k_proj,2,32,8,128,33.81072235107422,2.5078125,8388608
+42,self_attn.k_proj,2,64,8,128,40.54521179199219,2.25390625,8388608
+42,self_attn.k_proj,2,128,8,128,46.06313705444336,2.126953125,8388608
+42,self_attn.k_proj,3,32,8,128,14.76662540435791,3.5078125,8388608
+42,self_attn.k_proj,3,64,8,128,17.99306869506836,3.25390625,8388608
+42,self_attn.k_proj,3,128,8,128,21.08199691772461,3.126953125,8388608
+42,self_attn.k_proj,4,32,8,128,6.8902387619018555,4.5078125,8388608
+42,self_attn.k_proj,4,64,8,128,8.405324935913086,4.25390625,8388608
+42,self_attn.k_proj,4,128,8,128,9.850715637207031,4.126953125,8388608
+42,self_attn.k_proj,8,32,8,128,0.4167911410331726,8.5078125,8388608
+42,self_attn.k_proj,8,64,8,128,0.5080803036689758,8.25390625,8388608
+42,self_attn.k_proj,8,128,8,128,0.5938116312026978,8.126953125,8388608
+42,self_attn.o_proj,2,32,8,128,34.28117370605469,20.0625,67108864
+42,self_attn.o_proj,2,64,8,128,39.89688491821289,18.03125,67108864
+42,self_attn.o_proj,2,128,8,128,44.96490478515625,17.015625,67108864
+42,self_attn.o_proj,3,32,8,128,14.662089347839355,28.0625,67108864
+42,self_attn.o_proj,3,64,8,128,17.073745727539062,26.03125,67108864
+42,self_attn.o_proj,3,128,8,128,19.2498836517334,25.015625,67108864
+42,self_attn.o_proj,4,32,8,128,6.838352203369141,36.0625,67108864
+42,self_attn.o_proj,4,64,8,128,7.972696304321289,34.03125,67108864
+42,self_attn.o_proj,4,128,8,128,8.977917671203613,33.015625,67108864
+42,self_attn.o_proj,8,32,8,128,0.4155738651752472,68.0625,67108864
+42,self_attn.o_proj,8,64,8,128,0.4835386276245117,66.03125,67108864
+42,self_attn.o_proj,8,128,8,128,0.5431820750236511,65.015625,67108864
+42,mlp.gate_proj,2,32,8,128,73.07642364501953,70.21875,234881024
+42,mlp.gate_proj,2,64,8,128,86.11751556396484,63.109375,234881024
+42,mlp.gate_proj,2,128,8,128,97.81194305419922,59.5546875,234881024
+42,mlp.gate_proj,3,32,8,128,31.4307918548584,98.21875,234881024
+42,mlp.gate_proj,3,64,8,128,36.76255798339844,91.109375,234881024
+42,mlp.gate_proj,3,128,8,128,42.02373123168945,87.5546875,234881024
+42,mlp.gate_proj,4,32,8,128,14.623417854309082,126.21875,234881024
+42,mlp.gate_proj,4,64,8,128,17.166091918945312,119.109375,234881024
+42,mlp.gate_proj,4,128,8,128,19.601470947265625,115.5546875,234881024
+42,mlp.gate_proj,8,32,8,128,0.8925521373748779,238.21875,234881024
+42,mlp.gate_proj,8,64,8,128,1.0491890907287598,231.109375,234881024
+42,mlp.gate_proj,8,128,8,128,1.191459059715271,227.5546875,234881024
+42,mlp.up_proj,2,32,8,128,66.89308166503906,70.21875,234881024
+42,mlp.up_proj,2,64,8,128,77.79241180419922,63.109375,234881024
+42,mlp.up_proj,2,128,8,128,87.87067413330078,59.5546875,234881024
+42,mlp.up_proj,3,32,8,128,28.594133377075195,98.21875,234881024
+42,mlp.up_proj,3,64,8,128,33.33050537109375,91.109375,234881024
+42,mlp.up_proj,3,128,8,128,37.50752258300781,87.5546875,234881024
+42,mlp.up_proj,4,32,8,128,13.316927909851074,126.21875,234881024
+42,mlp.up_proj,4,64,8,128,15.585668563842773,119.109375,234881024
+42,mlp.up_proj,4,128,8,128,17.51656723022461,115.5546875,234881024
+42,mlp.up_proj,8,32,8,128,0.8134232759475708,238.21875,234881024
+42,mlp.up_proj,8,64,8,128,0.9505223035812378,231.109375,234881024
+42,mlp.up_proj,8,128,8,128,1.0663670301437378,227.5546875,234881024
+42,mlp.down_proj,2,32,8,128,67.3494873046875,70.21875,234881024
+42,mlp.down_proj,2,64,8,128,78.35686492919922,63.109375,234881024
+42,mlp.down_proj,2,128,8,128,88.52523803710938,59.5546875,234881024
+42,mlp.down_proj,3,32,8,128,28.810386657714844,98.21875,234881024
+42,mlp.down_proj,3,64,8,128,33.556854248046875,91.109375,234881024
+42,mlp.down_proj,3,128,8,128,37.77118682861328,87.5546875,234881024
+42,mlp.down_proj,4,32,8,128,13.416614532470703,126.21875,234881024
+42,mlp.down_proj,4,64,8,128,15.70386028289795,119.109375,234881024
+42,mlp.down_proj,4,128,8,128,17.63862419128418,115.5546875,234881024
+42,mlp.down_proj,8,32,8,128,0.8192898631095886,238.21875,234881024
+42,mlp.down_proj,8,64,8,128,0.9577595591545105,231.109375,234881024
+42,mlp.down_proj,8,128,8,128,1.0738871097564697,227.5546875,234881024
+43,self_attn.q_proj,2,32,8,128,44.389923095703125,20.0625,67108864
+43,self_attn.q_proj,2,64,8,128,52.41548156738281,18.03125,67108864
+43,self_attn.q_proj,2,128,8,128,60.34274673461914,17.015625,67108864
+43,self_attn.q_proj,3,32,8,128,19.00701332092285,28.0625,67108864
+43,self_attn.q_proj,3,64,8,128,22.555706024169922,26.03125,67108864
+43,self_attn.q_proj,3,128,8,128,26.060203552246094,25.015625,67108864
+43,self_attn.q_proj,4,32,8,128,8.868762016296387,36.0625,67108864
+43,self_attn.q_proj,4,64,8,128,10.522554397583008,34.03125,67108864
+43,self_attn.q_proj,4,128,8,128,12.168608665466309,33.015625,67108864
+43,self_attn.q_proj,8,32,8,128,0.5388153195381165,68.0625,67108864
+43,self_attn.q_proj,8,64,8,128,0.6391758322715759,66.03125,67108864
+43,self_attn.q_proj,8,128,8,128,0.7370218634605408,65.015625,67108864
+43,self_attn.v_proj,2,32,8,128,11.442712783813477,2.5078125,8388608
+43,self_attn.v_proj,2,64,8,128,13.32003402709961,2.25390625,8388608
+43,self_attn.v_proj,2,128,8,128,15.011310577392578,2.126953125,8388608
+43,self_attn.v_proj,3,32,8,128,4.892576694488525,3.5078125,8388608
+43,self_attn.v_proj,3,64,8,128,5.701953887939453,3.25390625,8388608
+43,self_attn.v_proj,3,128,8,128,6.4251861572265625,3.126953125,8388608
+43,self_attn.v_proj,4,32,8,128,2.282567262649536,4.5078125,8388608
+43,self_attn.v_proj,4,64,8,128,2.660261392593384,4.25390625,8388608
+43,self_attn.v_proj,4,128,8,128,2.999386787414551,4.126953125,8388608
+43,self_attn.v_proj,8,32,8,128,0.13819672167301178,8.5078125,8388608
+43,self_attn.v_proj,8,64,8,128,0.16084444522857666,8.25390625,8388608
+43,self_attn.v_proj,8,128,8,128,0.1807282269001007,8.126953125,8388608
+43,self_attn.k_proj,2,32,8,128,33.31045150756836,2.5078125,8388608
+43,self_attn.k_proj,2,64,8,128,39.763484954833984,2.25390625,8388608
+43,self_attn.k_proj,2,128,8,128,45.24285125732422,2.126953125,8388608
+43,self_attn.k_proj,3,32,8,128,14.421406745910645,3.5078125,8388608
+43,self_attn.k_proj,3,64,8,128,17.399892807006836,3.25390625,8388608
+43,self_attn.k_proj,3,128,8,128,20.227066040039062,3.126953125,8388608
+43,self_attn.k_proj,4,32,8,128,6.7294182777404785,4.5078125,8388608
+43,self_attn.k_proj,4,64,8,128,8.128619194030762,4.25390625,8388608
+43,self_attn.k_proj,4,128,8,128,9.454068183898926,4.126953125,8388608
+43,self_attn.k_proj,8,32,8,128,0.4075189232826233,8.5078125,8388608
+43,self_attn.k_proj,8,64,8,128,0.4914335012435913,8.25390625,8388608
+43,self_attn.k_proj,8,128,8,128,0.5701614618301392,8.126953125,8388608
+43,self_attn.o_proj,2,32,8,128,34.05000686645508,20.0625,67108864
+43,self_attn.o_proj,2,64,8,128,39.66487121582031,18.03125,67108864
+43,self_attn.o_proj,2,128,8,128,44.74393844604492,17.015625,67108864
+43,self_attn.o_proj,3,32,8,128,14.561798095703125,28.0625,67108864
+43,self_attn.o_proj,3,64,8,128,16.974653244018555,26.03125,67108864
+43,self_attn.o_proj,3,128,8,128,19.170684814453125,25.015625,67108864
+43,self_attn.o_proj,4,32,8,128,6.792040824890137,36.0625,67108864
+43,self_attn.o_proj,4,64,8,128,7.9286346435546875,34.03125,67108864
+43,self_attn.o_proj,4,128,8,128,8.945514678955078,33.015625,67108864
+43,self_attn.o_proj,8,32,8,128,0.41293349862098694,68.0625,67108864
+43,self_attn.o_proj,8,64,8,128,0.48096227645874023,66.03125,67108864
+43,self_attn.o_proj,8,128,8,128,0.5411837697029114,65.015625,67108864
+43,mlp.gate_proj,2,32,8,128,73.29714965820312,70.21875,234881024
+43,mlp.gate_proj,2,64,8,128,86.33870697021484,63.109375,234881024
+43,mlp.gate_proj,2,128,8,128,98.01742553710938,59.5546875,234881024
+43,mlp.gate_proj,3,32,8,128,31.52519416809082,98.21875,234881024
+43,mlp.gate_proj,3,64,8,128,36.86177444458008,91.109375,234881024
+43,mlp.gate_proj,3,128,8,128,42.10293960571289,87.5546875,234881024
+43,mlp.gate_proj,4,32,8,128,14.668590545654297,126.21875,234881024
+43,mlp.gate_proj,4,64,8,128,17.213346481323242,119.109375,234881024
+43,mlp.gate_proj,4,128,8,128,19.628021240234375,115.5546875,234881024
+43,mlp.gate_proj,8,32,8,128,0.8954508304595947,238.21875,234881024
+43,mlp.gate_proj,8,64,8,128,1.0518462657928467,231.109375,234881024
+43,mlp.gate_proj,8,128,8,128,1.1938315629959106,227.5546875,234881024
+43,mlp.up_proj,2,32,8,128,66.93367767333984,70.21875,234881024
+43,mlp.up_proj,2,64,8,128,77.84380340576172,63.109375,234881024
+43,mlp.up_proj,2,128,8,128,87.9275131225586,59.5546875,234881024
+43,mlp.up_proj,3,32,8,128,28.611665725708008,98.21875,234881024
+43,mlp.up_proj,3,64,8,128,33.352088928222656,91.109375,234881024
+43,mlp.up_proj,3,128,8,128,37.52348709106445,87.5546875,234881024
+43,mlp.up_proj,4,32,8,128,13.32532024383545,126.21875,234881024
+43,mlp.up_proj,4,64,8,128,15.597128868103027,119.109375,234881024
+43,mlp.up_proj,4,128,8,128,17.534337997436523,115.5546875,234881024
+43,mlp.up_proj,8,32,8,128,0.8138563632965088,238.21875,234881024
+43,mlp.up_proj,8,64,8,128,0.9510639905929565,231.109375,234881024
+43,mlp.up_proj,8,128,8,128,1.0669806003570557,227.5546875,234881024
+43,mlp.down_proj,2,32,8,128,67.3672103881836,70.21875,234881024
+43,mlp.down_proj,2,64,8,128,78.37300109863281,63.109375,234881024
+43,mlp.down_proj,2,128,8,128,88.57797241210938,59.5546875,234881024
+43,mlp.down_proj,3,32,8,128,28.815099716186523,98.21875,234881024
+43,mlp.down_proj,3,64,8,128,33.56431198120117,91.109375,234881024
+43,mlp.down_proj,3,128,8,128,37.77107620239258,87.5546875,234881024
+43,mlp.down_proj,4,32,8,128,13.418885231018066,126.21875,234881024
+43,mlp.down_proj,4,64,8,128,15.712289810180664,119.109375,234881024
+43,mlp.down_proj,4,128,8,128,17.659759521484375,115.5546875,234881024
+43,mlp.down_proj,8,32,8,128,0.8194864988327026,238.21875,234881024
+43,mlp.down_proj,8,64,8,128,0.9580633640289307,231.109375,234881024
+43,mlp.down_proj,8,128,8,128,1.0742955207824707,227.5546875,234881024
+44,self_attn.q_proj,2,32,8,128,46.533748626708984,20.0625,67108864
+44,self_attn.q_proj,2,64,8,128,55.049041748046875,18.03125,67108864
+44,self_attn.q_proj,2,128,8,128,63.62704086303711,17.015625,67108864
+44,self_attn.q_proj,3,32,8,128,19.910192489624023,28.0625,67108864
+44,self_attn.q_proj,3,64,8,128,23.61671257019043,26.03125,67108864
+44,self_attn.q_proj,3,128,8,128,27.340620040893555,25.015625,67108864
+44,self_attn.q_proj,4,32,8,128,9.28849983215332,36.0625,67108864
+44,self_attn.q_proj,4,64,8,128,11.025700569152832,34.03125,67108864
+44,self_attn.q_proj,4,128,8,128,12.763594627380371,33.015625,67108864
+44,self_attn.q_proj,8,32,8,128,0.5643801093101501,68.0625,67108864
+44,self_attn.q_proj,8,64,8,128,0.6697564125061035,66.03125,67108864
+44,self_attn.q_proj,8,128,8,128,0.7728790044784546,65.015625,67108864
+44,self_attn.v_proj,2,32,8,128,11.000033378601074,2.5078125,8388608
+44,self_attn.v_proj,2,64,8,128,12.811030387878418,2.25390625,8388608
+44,self_attn.v_proj,2,128,8,128,14.453746795654297,2.126953125,8388608
+44,self_attn.v_proj,3,32,8,128,4.702386856079102,3.5078125,8388608
+44,self_attn.v_proj,3,64,8,128,5.483011722564697,3.25390625,8388608
+44,self_attn.v_proj,3,128,8,128,6.182624340057373,3.126953125,8388608
+44,self_attn.v_proj,4,32,8,128,2.1933188438415527,4.5078125,8388608
+44,self_attn.v_proj,4,64,8,128,2.558455228805542,4.25390625,8388608
+44,self_attn.v_proj,4,128,8,128,2.88651442527771,4.126953125,8388608
+44,self_attn.v_proj,8,32,8,128,0.13283023238182068,8.5078125,8388608
+44,self_attn.v_proj,8,64,8,128,0.1547144502401352,8.25390625,8388608
+44,self_attn.v_proj,8,128,8,128,0.1739635318517685,8.126953125,8388608
+44,self_attn.k_proj,2,32,8,128,33.64914321899414,2.5078125,8388608
+44,self_attn.k_proj,2,64,8,128,40.32521057128906,2.25390625,8388608
+44,self_attn.k_proj,2,128,8,128,46.245906829833984,2.126953125,8388608
+44,self_attn.k_proj,3,32,8,128,14.443167686462402,3.5078125,8388608
+44,self_attn.k_proj,3,64,8,128,17.36426544189453,3.25390625,8388608
+44,self_attn.k_proj,3,128,8,128,20.12685203552246,3.126953125,8388608
+44,self_attn.k_proj,4,32,8,128,6.739865779876709,4.5078125,8388608
+44,self_attn.k_proj,4,64,8,128,8.105449676513672,4.25390625,8388608
+44,self_attn.k_proj,4,128,8,128,9.3982572555542,4.126953125,8388608
+44,self_attn.k_proj,8,32,8,128,0.40774375200271606,8.5078125,8388608
+44,self_attn.k_proj,8,64,8,128,0.4898754358291626,8.25390625,8388608
+44,self_attn.k_proj,8,128,8,128,0.5663259625434875,8.126953125,8388608
+44,self_attn.o_proj,2,32,8,128,32.980777740478516,20.0625,67108864
+44,self_attn.o_proj,2,64,8,128,38.41195297241211,18.03125,67108864
+44,self_attn.o_proj,2,128,8,128,43.35028076171875,17.015625,67108864
+44,self_attn.o_proj,3,32,8,128,14.097983360290527,28.0625,67108864
+44,self_attn.o_proj,3,64,8,128,16.449453353881836,26.03125,67108864
+44,self_attn.o_proj,3,128,8,128,18.567670822143555,25.015625,67108864
+44,self_attn.o_proj,4,32,8,128,6.575790882110596,36.0625,67108864
+44,self_attn.o_proj,4,64,8,128,7.677188873291016,34.03125,67108864
+44,self_attn.o_proj,4,128,8,128,8.66549301147461,33.015625,67108864
+44,self_attn.o_proj,8,32,8,128,0.3997792899608612,68.0625,67108864
+44,self_attn.o_proj,8,64,8,128,0.465817391872406,66.03125,67108864
+44,self_attn.o_proj,8,128,8,128,0.5242487192153931,65.015625,67108864
+44,mlp.gate_proj,2,32,8,128,73.69697570800781,70.21875,234881024
+44,mlp.gate_proj,2,64,8,128,86.92499542236328,63.109375,234881024
+44,mlp.gate_proj,2,128,8,128,98.74552154541016,59.5546875,234881024
+44,mlp.gate_proj,3,32,8,128,31.709993362426758,98.21875,234881024
+44,mlp.gate_proj,3,64,8,128,37.11006546020508,91.109375,234881024
+44,mlp.gate_proj,3,128,8,128,42.473907470703125,87.5546875,234881024
+44,mlp.gate_proj,4,32,8,128,14.753701210021973,126.21875,234881024
+44,mlp.gate_proj,4,64,8,128,17.32762336730957,119.109375,234881024
+44,mlp.gate_proj,4,128,8,128,19.803489685058594,115.5546875,234881024
+44,mlp.gate_proj,8,32,8,128,0.9004591107368469,238.21875,234881024
+44,mlp.gate_proj,8,64,8,128,1.0585983991622925,231.109375,234881024
+44,mlp.gate_proj,8,128,8,128,1.2041525840759277,227.5546875,234881024
+44,mlp.up_proj,2,32,8,128,66.9874038696289,70.21875,234881024
+44,mlp.up_proj,2,64,8,128,77.92176818847656,63.109375,234881024
+44,mlp.up_proj,2,128,8,128,88.02734375,59.5546875,234881024
+44,mlp.up_proj,3,32,8,128,28.642507553100586,98.21875,234881024
+44,mlp.up_proj,3,64,8,128,33.38420486450195,91.109375,234881024
+44,mlp.up_proj,3,128,8,128,37.564083099365234,87.5546875,234881024
+44,mlp.up_proj,4,32,8,128,13.33820629119873,126.21875,234881024
+44,mlp.up_proj,4,64,8,128,15.614079475402832,119.109375,234881024
+44,mlp.up_proj,4,128,8,128,17.547273635864258,115.5546875,234881024
+44,mlp.up_proj,8,32,8,128,0.8146205544471741,238.21875,234881024
+44,mlp.up_proj,8,64,8,128,0.9520849585533142,231.109375,234881024
+44,mlp.up_proj,8,128,8,128,1.0682586431503296,227.5546875,234881024
+44,mlp.down_proj,2,32,8,128,67.36439514160156,70.21875,234881024
+44,mlp.down_proj,2,64,8,128,78.3900146484375,63.109375,234881024
+44,mlp.down_proj,2,128,8,128,88.57388305664062,59.5546875,234881024
+44,mlp.down_proj,3,32,8,128,28.815444946289062,98.21875,234881024
+44,mlp.down_proj,3,64,8,128,33.5680046081543,91.109375,234881024
+44,mlp.down_proj,3,128,8,128,37.783199310302734,87.5546875,234881024
+44,mlp.down_proj,4,32,8,128,13.419448852539062,126.21875,234881024
+44,mlp.down_proj,4,64,8,128,15.71078109741211,119.109375,234881024
+44,mlp.down_proj,4,128,8,128,17.660531997680664,115.5546875,234881024
+44,mlp.down_proj,8,32,8,128,0.8194013237953186,238.21875,234881024
+44,mlp.down_proj,8,64,8,128,0.9580613970756531,231.109375,234881024
+44,mlp.down_proj,8,128,8,128,1.0743517875671387,227.5546875,234881024
+45,self_attn.q_proj,2,32,8,128,44.66831588745117,20.0625,67108864
+45,self_attn.q_proj,2,64,8,128,53.035526275634766,18.03125,67108864
+45,self_attn.q_proj,2,128,8,128,61.35528564453125,17.015625,67108864
+45,self_attn.q_proj,3,32,8,128,19.14750862121582,28.0625,67108864
+45,self_attn.q_proj,3,64,8,128,22.879669189453125,26.03125,67108864
+45,self_attn.q_proj,3,128,8,128,26.64273452758789,25.015625,67108864
+45,self_attn.q_proj,4,32,8,128,8.932506561279297,36.0625,67108864
+45,self_attn.q_proj,4,64,8,128,10.674449920654297,34.03125,67108864
+45,self_attn.q_proj,4,128,8,128,12.437538146972656,33.015625,67108864
+45,self_attn.q_proj,8,32,8,128,0.5426734089851379,68.0625,67108864
+45,self_attn.q_proj,8,64,8,128,0.6484254002571106,66.03125,67108864
+45,self_attn.q_proj,8,128,8,128,0.753384530544281,65.015625,67108864
+45,self_attn.v_proj,2,32,8,128,11.732988357543945,2.5078125,8388608
+45,self_attn.v_proj,2,64,8,128,13.668310165405273,2.25390625,8388608
+45,self_attn.v_proj,2,128,8,128,15.403094291687012,2.126953125,8388608
+45,self_attn.v_proj,3,32,8,128,5.018603801727295,3.5078125,8388608
+45,self_attn.v_proj,3,64,8,128,5.845300197601318,3.25390625,8388608
+45,self_attn.v_proj,3,128,8,128,6.590423583984375,3.126953125,8388608
+45,self_attn.v_proj,4,32,8,128,2.341033458709717,4.5078125,8388608
+45,self_attn.v_proj,4,64,8,128,2.7280471324920654,4.25390625,8388608
+45,self_attn.v_proj,4,128,8,128,3.076289653778076,4.126953125,8388608
+45,self_attn.v_proj,8,32,8,128,0.14170370995998383,8.5078125,8388608
+45,self_attn.v_proj,8,64,8,128,0.16497930884361267,8.25390625,8388608
+45,self_attn.v_proj,8,128,8,128,0.18533429503440857,8.126953125,8388608
+45,self_attn.k_proj,2,32,8,128,33.761539459228516,2.5078125,8388608
+45,self_attn.k_proj,2,64,8,128,40.37125778198242,2.25390625,8388608
+45,self_attn.k_proj,2,128,8,128,45.83311080932617,2.126953125,8388608
+45,self_attn.k_proj,3,32,8,128,14.588956832885742,3.5078125,8388608
+45,self_attn.k_proj,3,64,8,128,17.65705108642578,3.25390625,8388608
+45,self_attn.k_proj,3,128,8,128,20.491701126098633,3.126953125,8388608
+45,self_attn.k_proj,4,32,8,128,6.814208030700684,4.5078125,8388608
+45,self_attn.k_proj,4,64,8,128,8.246665000915527,4.25390625,8388608
+45,self_attn.k_proj,4,128,8,128,9.582051277160645,4.126953125,8388608
+45,self_attn.k_proj,8,32,8,128,0.4122065603733063,8.5078125,8388608
+45,self_attn.k_proj,8,64,8,128,0.4984695017337799,8.25390625,8388608
+45,self_attn.k_proj,8,128,8,128,0.5772191882133484,8.126953125,8388608
+45,self_attn.o_proj,2,32,8,128,33.83187484741211,20.0625,67108864
+45,self_attn.o_proj,2,64,8,128,39.41790008544922,18.03125,67108864
+45,self_attn.o_proj,2,128,8,128,44.50209426879883,17.015625,67108864
+45,self_attn.o_proj,3,32,8,128,14.469254493713379,28.0625,67108864
+45,self_attn.o_proj,3,64,8,128,16.87953758239746,26.03125,67108864
+45,self_attn.o_proj,3,128,8,128,19.07027816772461,25.015625,67108864
+45,self_attn.o_proj,4,32,8,128,6.748599529266357,36.0625,67108864
+45,self_attn.o_proj,4,64,8,128,7.879608631134033,34.03125,67108864
+45,self_attn.o_proj,4,128,8,128,8.898579597473145,33.015625,67108864
+45,self_attn.o_proj,8,32,8,128,0.41023609042167664,68.0625,67108864
+45,self_attn.o_proj,8,64,8,128,0.4779943823814392,66.03125,67108864
+45,self_attn.o_proj,8,128,8,128,0.5382140278816223,65.015625,67108864
+45,mlp.gate_proj,2,32,8,128,73.69805145263672,70.21875,234881024
+45,mlp.gate_proj,2,64,8,128,86.89090728759766,63.109375,234881024
+45,mlp.gate_proj,2,128,8,128,98.69658660888672,59.5546875,234881024
+45,mlp.gate_proj,3,32,8,128,31.714853286743164,98.21875,234881024
+45,mlp.gate_proj,3,64,8,128,37.111595153808594,91.109375,234881024
+45,mlp.gate_proj,3,128,8,128,42.45512390136719,87.5546875,234881024
+45,mlp.gate_proj,4,32,8,128,14.755873680114746,126.21875,234881024
+45,mlp.gate_proj,4,64,8,128,17.324247360229492,119.109375,234881024
+45,mlp.gate_proj,4,128,8,128,19.801288604736328,115.5546875,234881024
+45,mlp.gate_proj,8,32,8,128,0.9007071852684021,238.21875,234881024
+45,mlp.gate_proj,8,64,8,128,1.0585224628448486,231.109375,234881024
+45,mlp.gate_proj,8,128,8,128,1.2036244869232178,227.5546875,234881024
+45,mlp.up_proj,2,32,8,128,67.0142822265625,70.21875,234881024
+45,mlp.up_proj,2,64,8,128,77.92594909667969,63.109375,234881024
+45,mlp.up_proj,2,128,8,128,88.0631332397461,59.5546875,234881024
+45,mlp.up_proj,3,32,8,128,28.651765823364258,98.21875,234881024
+45,mlp.up_proj,3,64,8,128,33.39378356933594,91.109375,234881024
+45,mlp.up_proj,3,128,8,128,37.567649841308594,87.5546875,234881024
+45,mlp.up_proj,4,32,8,128,13.342472076416016,126.21875,234881024
+45,mlp.up_proj,4,64,8,128,15.61843490600586,119.109375,234881024
+45,mlp.up_proj,4,128,8,128,17.551843643188477,115.5546875,234881024
+45,mlp.up_proj,8,32,8,128,0.8148605227470398,238.21875,234881024
+45,mlp.up_proj,8,64,8,128,0.9523614048957825,231.109375,234881024
+45,mlp.up_proj,8,128,8,128,1.0684646368026733,227.5546875,234881024
+45,mlp.down_proj,2,32,8,128,67.35442352294922,70.21875,234881024
+45,mlp.down_proj,2,64,8,128,78.36260986328125,63.109375,234881024
+45,mlp.down_proj,2,128,8,128,88.56124877929688,59.5546875,234881024
+45,mlp.down_proj,3,32,8,128,28.811695098876953,98.21875,234881024
+45,mlp.down_proj,3,64,8,128,33.56398391723633,91.109375,234881024
+45,mlp.down_proj,3,128,8,128,37.773651123046875,87.5546875,234881024
+45,mlp.down_proj,4,32,8,128,13.416972160339355,126.21875,234881024
+45,mlp.down_proj,4,64,8,128,15.708373069763184,119.109375,234881024
+45,mlp.down_proj,4,128,8,128,17.660991668701172,115.5546875,234881024
+45,mlp.down_proj,8,32,8,128,0.8194159269332886,238.21875,234881024
+45,mlp.down_proj,8,64,8,128,0.9581830501556396,231.109375,234881024
+45,mlp.down_proj,8,128,8,128,1.074641466140747,227.5546875,234881024
+46,self_attn.q_proj,2,32,8,128,33.07374954223633,20.0625,67108864
+46,self_attn.q_proj,2,64,8,128,39.14200973510742,18.03125,67108864
+46,self_attn.q_proj,2,128,8,128,45.38581085205078,17.015625,67108864
+46,self_attn.q_proj,3,32,8,128,14.179903030395508,28.0625,67108864
+46,self_attn.q_proj,3,64,8,128,16.847990036010742,26.03125,67108864
+46,self_attn.q_proj,3,128,8,128,19.65625,25.015625,67108864
+46,self_attn.q_proj,4,32,8,128,6.613727569580078,36.0625,67108864
+46,self_attn.q_proj,4,64,8,128,7.869314193725586,34.03125,67108864
+46,self_attn.q_proj,4,128,8,128,9.175289154052734,33.015625,67108864
+46,self_attn.q_proj,8,32,8,128,0.40200963616371155,68.0625,67108864
+46,self_attn.q_proj,8,64,8,128,0.4775310158729553,66.03125,67108864
+46,self_attn.q_proj,8,128,8,128,0.5552332401275635,65.015625,67108864
+46,self_attn.v_proj,2,32,8,128,13.90028190612793,2.5078125,8388608
+46,self_attn.v_proj,2,64,8,128,16.179035186767578,2.25390625,8388608
+46,self_attn.v_proj,2,128,8,128,18.216018676757812,2.126953125,8388608
+46,self_attn.v_proj,3,32,8,128,5.942164897918701,3.5078125,8388608
+46,self_attn.v_proj,3,64,8,128,6.919670581817627,3.25390625,8388608
+46,self_attn.v_proj,3,128,8,128,7.800212383270264,3.126953125,8388608
+46,self_attn.v_proj,4,32,8,128,2.77248215675354,4.5078125,8388608
+46,self_attn.v_proj,4,64,8,128,3.229130983352661,4.25390625,8388608
+46,self_attn.v_proj,4,128,8,128,3.6386756896972656,4.126953125,8388608
+46,self_attn.v_proj,8,32,8,128,0.16790629923343658,8.5078125,8388608
+46,self_attn.v_proj,8,64,8,128,0.19523286819458008,8.25390625,8388608
+46,self_attn.v_proj,8,128,8,128,0.21922948956489563,8.126953125,8388608
+46,self_attn.k_proj,2,32,8,128,23.356643676757812,2.5078125,8388608
+46,self_attn.k_proj,2,64,8,128,28.308414459228516,2.25390625,8388608
+46,self_attn.k_proj,2,128,8,128,32.54825210571289,2.126953125,8388608
+46,self_attn.k_proj,3,32,8,128,10.134140968322754,3.5078125,8388608
+46,self_attn.k_proj,3,64,8,128,12.441202163696289,3.25390625,8388608
+46,self_attn.k_proj,3,128,8,128,14.607378005981445,3.126953125,8388608
+46,self_attn.k_proj,4,32,8,128,4.739351272583008,4.5078125,8388608
+46,self_attn.k_proj,4,64,8,128,5.816651344299316,4.25390625,8388608
+46,self_attn.k_proj,4,128,8,128,6.8414530754089355,4.126953125,8388608
+46,self_attn.k_proj,8,32,8,128,0.2869332730770111,8.5078125,8388608
+46,self_attn.k_proj,8,64,8,128,0.35206112265586853,8.25390625,8388608
+46,self_attn.k_proj,8,128,8,128,0.4130723178386688,8.126953125,8388608
+46,self_attn.o_proj,2,32,8,128,35.068294525146484,20.0625,67108864
+46,self_attn.o_proj,2,64,8,128,40.82563400268555,18.03125,67108864
+46,self_attn.o_proj,2,128,8,128,45.99352264404297,17.015625,67108864
+46,self_attn.o_proj,3,32,8,128,14.999524116516113,28.0625,67108864
+46,self_attn.o_proj,3,64,8,128,17.456119537353516,26.03125,67108864
+46,self_attn.o_proj,3,128,8,128,19.683950424194336,25.015625,67108864
+46,self_attn.o_proj,4,32,8,128,6.993298530578613,36.0625,67108864
+46,self_attn.o_proj,4,64,8,128,8.150209426879883,34.03125,67108864
+46,self_attn.o_proj,4,128,8,128,9.174812316894531,33.015625,67108864
+46,self_attn.o_proj,8,32,8,128,0.42504003643989563,68.0625,67108864
+46,self_attn.o_proj,8,64,8,128,0.4943331778049469,66.03125,67108864
+46,self_attn.o_proj,8,128,8,128,0.5551255941390991,65.015625,67108864
+46,mlp.gate_proj,2,32,8,128,73.75445556640625,70.21875,234881024
+46,mlp.gate_proj,2,64,8,128,86.97998809814453,63.109375,234881024
+46,mlp.gate_proj,2,128,8,128,98.85993957519531,59.5546875,234881024
+46,mlp.gate_proj,3,32,8,128,31.7373046875,98.21875,234881024
+46,mlp.gate_proj,3,64,8,128,37.13303756713867,91.109375,234881024
+46,mlp.gate_proj,3,128,8,128,42.50054931640625,87.5546875,234881024
+46,mlp.gate_proj,4,32,8,128,14.766219139099121,126.21875,234881024
+46,mlp.gate_proj,4,64,8,128,17.339611053466797,119.109375,234881024
+46,mlp.gate_proj,4,128,8,128,19.81884002685547,115.5546875,234881024
+46,mlp.gate_proj,8,32,8,128,0.9013376832008362,238.21875,234881024
+46,mlp.gate_proj,8,64,8,128,1.0592693090438843,231.109375,234881024
+46,mlp.gate_proj,8,128,8,128,1.20497727394104,227.5546875,234881024
+46,mlp.up_proj,2,32,8,128,67.00633239746094,70.21875,234881024
+46,mlp.up_proj,2,64,8,128,77.924560546875,63.109375,234881024
+46,mlp.up_proj,2,128,8,128,88.0217056274414,59.5546875,234881024
+46,mlp.up_proj,3,32,8,128,28.647987365722656,98.21875,234881024
+46,mlp.up_proj,3,64,8,128,33.3841438293457,91.109375,234881024
+46,mlp.up_proj,3,128,8,128,37.56757736206055,87.5546875,234881024
+46,mlp.up_proj,4,32,8,128,13.341965675354004,126.21875,234881024
+46,mlp.up_proj,4,64,8,128,15.613386154174805,119.109375,234881024
+46,mlp.up_proj,4,128,8,128,17.549440383911133,115.5546875,234881024
+46,mlp.up_proj,8,32,8,128,0.814782440662384,238.21875,234881024
+46,mlp.up_proj,8,64,8,128,0.9521632194519043,231.109375,234881024
+46,mlp.up_proj,8,128,8,128,1.0680440664291382,227.5546875,234881024
+46,mlp.down_proj,2,32,8,128,67.34281921386719,70.21875,234881024
+46,mlp.down_proj,2,64,8,128,78.34907531738281,63.109375,234881024
+46,mlp.down_proj,2,128,8,128,88.51546478271484,59.5546875,234881024
+46,mlp.down_proj,3,32,8,128,28.804889678955078,98.21875,234881024
+46,mlp.down_proj,3,64,8,128,33.5521125793457,91.109375,234881024
+46,mlp.down_proj,3,128,8,128,37.75220489501953,87.5546875,234881024
+46,mlp.down_proj,4,32,8,128,13.4133939743042,126.21875,234881024
+46,mlp.down_proj,4,64,8,128,15.70014762878418,119.109375,234881024
+46,mlp.down_proj,4,128,8,128,17.63566017150879,115.5546875,234881024
+46,mlp.down_proj,8,32,8,128,0.8190996050834656,238.21875,234881024
+46,mlp.down_proj,8,64,8,128,0.9574612975120544,231.109375,234881024
+46,mlp.down_proj,8,128,8,128,1.0735527276992798,227.5546875,234881024
+47,self_attn.q_proj,2,32,8,128,41.88862991333008,20.0625,67108864
+47,self_attn.q_proj,2,64,8,128,49.606136322021484,18.03125,67108864
+47,self_attn.q_proj,2,128,8,128,57.41631317138672,17.015625,67108864
+47,self_attn.q_proj,3,32,8,128,17.993478775024414,28.0625,67108864
+47,self_attn.q_proj,3,64,8,128,21.468549728393555,26.03125,67108864
+47,self_attn.q_proj,3,128,8,128,25.081783294677734,25.015625,67108864
+47,self_attn.q_proj,4,32,8,128,8.398050308227539,36.0625,67108864
+47,self_attn.q_proj,4,64,8,128,10.013509750366211,34.03125,67108864
+47,self_attn.q_proj,4,128,8,128,11.715579986572266,33.015625,67108864
+47,self_attn.q_proj,8,32,8,128,0.5099574327468872,68.0625,67108864
+47,self_attn.q_proj,8,64,8,128,0.6082870960235596,66.03125,67108864
+47,self_attn.q_proj,8,128,8,128,0.7095174789428711,65.015625,67108864
+47,self_attn.v_proj,2,32,8,128,12.18216609954834,2.5078125,8388608
+47,self_attn.v_proj,2,64,8,128,14.310644149780273,2.25390625,8388608
+47,self_attn.v_proj,2,128,8,128,16.287038803100586,2.126953125,8388608
+47,self_attn.v_proj,3,32,8,128,5.212124824523926,3.5078125,8388608
+47,self_attn.v_proj,3,64,8,128,6.12705659866333,3.25390625,8388608
+47,self_attn.v_proj,3,128,8,128,6.978005409240723,3.126953125,8388608
+47,self_attn.v_proj,4,32,8,128,2.431384563446045,4.5078125,8388608
+47,self_attn.v_proj,4,64,8,128,2.8579838275909424,4.25390625,8388608
+47,self_attn.v_proj,4,128,8,128,3.2546772956848145,4.126953125,8388608
+47,self_attn.v_proj,8,32,8,128,0.14716185629367828,8.5078125,8388608
+47,self_attn.v_proj,8,64,8,128,0.17287017405033112,8.25390625,8388608
+47,self_attn.v_proj,8,128,8,128,0.19603194296360016,8.126953125,8388608
+47,self_attn.k_proj,2,32,8,128,32.41608428955078,2.5078125,8388608
+47,self_attn.k_proj,2,64,8,128,38.827999114990234,2.25390625,8388608
+47,self_attn.k_proj,2,128,8,128,44.14604568481445,2.126953125,8388608
+47,self_attn.k_proj,3,32,8,128,14.139856338500977,3.5078125,8388608
+47,self_attn.k_proj,3,64,8,128,17.224653244018555,3.25390625,8388608
+47,self_attn.k_proj,3,128,8,128,20.154512405395508,3.126953125,8388608
+47,self_attn.k_proj,4,32,8,128,6.601849555969238,4.5078125,8388608
+47,self_attn.k_proj,4,64,8,128,8.050243377685547,4.25390625,8388608
+47,self_attn.k_proj,4,128,8,128,9.430561065673828,4.126953125,8388608
+47,self_attn.k_proj,8,32,8,128,0.3997699022293091,8.5078125,8388608
+47,self_attn.k_proj,8,64,8,128,0.4869077801704407,8.25390625,8388608
+47,self_attn.k_proj,8,128,8,128,0.5685588121414185,8.126953125,8388608
+47,self_attn.o_proj,2,32,8,128,34.144100189208984,20.0625,67108864
+47,self_attn.o_proj,2,64,8,128,39.7586555480957,18.03125,67108864
+47,self_attn.o_proj,2,128,8,128,44.83449172973633,17.015625,67108864
+47,self_attn.o_proj,3,32,8,128,14.600237846374512,28.0625,67108864
+47,self_attn.o_proj,3,64,8,128,17.009122848510742,26.03125,67108864
+47,self_attn.o_proj,3,128,8,128,19.18636131286621,25.015625,67108864
+47,self_attn.o_proj,4,32,8,128,6.809091567993164,36.0625,67108864
+47,self_attn.o_proj,4,64,8,128,7.940968036651611,34.03125,67108864
+47,self_attn.o_proj,4,128,8,128,8.947844505310059,33.015625,67108864
+47,self_attn.o_proj,8,32,8,128,0.4139258861541748,68.0625,67108864
+47,self_attn.o_proj,8,64,8,128,0.4817003905773163,66.03125,67108864
+47,self_attn.o_proj,8,128,8,128,0.5413659811019897,65.015625,67108864
+47,mlp.gate_proj,2,32,8,128,73.70323181152344,70.21875,234881024
+47,mlp.gate_proj,2,64,8,128,86.81108093261719,63.109375,234881024
+47,mlp.gate_proj,2,128,8,128,98.52267456054688,59.5546875,234881024
+47,mlp.gate_proj,3,32,8,128,31.71010971069336,98.21875,234881024
+47,mlp.gate_proj,3,64,8,128,37.05392837524414,91.109375,234881024
+47,mlp.gate_proj,3,128,8,128,42.32307434082031,87.5546875,234881024
+47,mlp.gate_proj,4,32,8,128,14.752424240112305,126.21875,234881024
+47,mlp.gate_proj,4,64,8,128,17.30134391784668,119.109375,234881024
+47,mlp.gate_proj,4,128,8,128,19.734224319458008,115.5546875,234881024
+47,mlp.gate_proj,8,32,8,128,0.9004930257797241,238.21875,234881024
+47,mlp.gate_proj,8,64,8,128,1.0573163032531738,231.109375,234881024
+47,mlp.gate_proj,8,128,8,128,1.200161337852478,227.5546875,234881024
+47,mlp.up_proj,2,32,8,128,67.0693359375,70.21875,234881024
+47,mlp.up_proj,2,64,8,128,77.99166107177734,63.109375,234881024
+47,mlp.up_proj,2,128,8,128,88.13277435302734,59.5546875,234881024
+47,mlp.up_proj,3,32,8,128,28.67729377746582,98.21875,234881024
+47,mlp.up_proj,3,64,8,128,33.41237258911133,91.109375,234881024
+47,mlp.up_proj,3,128,8,128,37.602169036865234,87.5546875,234881024
+47,mlp.up_proj,4,32,8,128,13.355624198913574,126.21875,234881024
+47,mlp.up_proj,4,64,8,128,15.629155158996582,119.109375,234881024
+47,mlp.up_proj,4,128,8,128,17.564594268798828,115.5546875,234881024
+47,mlp.up_proj,8,32,8,128,0.8156527280807495,238.21875,234881024
+47,mlp.up_proj,8,64,8,128,0.9530927538871765,231.109375,234881024
+47,mlp.up_proj,8,128,8,128,1.068934440612793,227.5546875,234881024
+47,mlp.down_proj,2,32,8,128,67.39747619628906,70.21875,234881024
+47,mlp.down_proj,2,64,8,128,78.41231536865234,63.109375,234881024
+47,mlp.down_proj,2,128,8,128,88.59102630615234,59.5546875,234881024
+47,mlp.down_proj,3,32,8,128,28.824203491210938,98.21875,234881024
+47,mlp.down_proj,3,64,8,128,33.57784652709961,91.109375,234881024
+47,mlp.down_proj,3,128,8,128,37.784671783447266,87.5546875,234881024
+47,mlp.down_proj,4,32,8,128,13.4232816696167,126.21875,234881024
+47,mlp.down_proj,4,64,8,128,15.71212387084961,119.109375,234881024
+47,mlp.down_proj,4,128,8,128,17.647144317626953,115.5546875,234881024
+47,mlp.down_proj,8,32,8,128,0.8196788430213928,238.21875,234881024
+47,mlp.down_proj,8,64,8,128,0.9582773447036743,231.109375,234881024
+47,mlp.down_proj,8,128,8,128,1.0744510889053345,227.5546875,234881024
+48,self_attn.q_proj,2,32,8,128,42.75474166870117,20.0625,67108864
+48,self_attn.q_proj,2,64,8,128,50.5051383972168,18.03125,67108864
+48,self_attn.q_proj,2,128,8,128,58.23652648925781,17.015625,67108864
+48,self_attn.q_proj,3,32,8,128,18.329456329345703,28.0625,67108864
+48,self_attn.q_proj,3,64,8,128,21.782621383666992,26.03125,67108864
+48,self_attn.q_proj,3,128,8,128,25.252910614013672,25.015625,67108864
+48,self_attn.q_proj,4,32,8,128,8.555161476135254,36.0625,67108864
+48,self_attn.q_proj,4,64,8,128,10.159960746765137,34.03125,67108864
+48,self_attn.q_proj,4,128,8,128,11.795351028442383,33.015625,67108864
+48,self_attn.q_proj,8,32,8,128,0.5194847583770752,68.0625,67108864
+48,self_attn.q_proj,8,64,8,128,0.6170913577079773,66.03125,67108864
+48,self_attn.q_proj,8,128,8,128,0.7144477367401123,65.015625,67108864
+48,self_attn.v_proj,2,32,8,128,12.551719665527344,2.5078125,8388608
+48,self_attn.v_proj,2,64,8,128,14.746847152709961,2.25390625,8388608
+48,self_attn.v_proj,2,128,8,128,16.78072166442871,2.126953125,8388608
+48,self_attn.v_proj,3,32,8,128,5.367072582244873,3.5078125,8388608
+48,self_attn.v_proj,3,64,8,128,6.309472560882568,3.25390625,8388608
+48,self_attn.v_proj,3,128,8,128,7.185246467590332,3.126953125,8388608
+48,self_attn.v_proj,4,32,8,128,2.505286931991577,4.5078125,8388608
+48,self_attn.v_proj,4,64,8,128,2.9443154335021973,4.25390625,8388608
+48,self_attn.v_proj,4,128,8,128,3.353142261505127,4.126953125,8388608
+48,self_attn.v_proj,8,32,8,128,0.15161526203155518,8.5078125,8388608
+48,self_attn.v_proj,8,64,8,128,0.17802679538726807,8.25390625,8388608
+48,self_attn.v_proj,8,128,8,128,0.20199286937713623,8.126953125,8388608
+48,self_attn.k_proj,2,32,8,128,33.20490646362305,2.5078125,8388608
+48,self_attn.k_proj,2,64,8,128,39.70198440551758,2.25390625,8388608
+48,self_attn.k_proj,2,128,8,128,45.190467834472656,2.126953125,8388608
+48,self_attn.k_proj,3,32,8,128,14.39284610748291,3.5078125,8388608
+48,self_attn.k_proj,3,64,8,128,17.426782608032227,3.25390625,8388608
+48,self_attn.k_proj,3,128,8,128,20.302202224731445,3.126953125,8388608
+48,self_attn.k_proj,4,32,8,128,6.718226432800293,4.5078125,8388608
+48,self_attn.k_proj,4,64,8,128,8.139481544494629,4.25390625,8388608
+48,self_attn.k_proj,4,128,8,128,9.486781120300293,4.126953125,8388608
+48,self_attn.k_proj,8,32,8,128,0.4064233899116516,8.5078125,8388608
+48,self_attn.k_proj,8,64,8,128,0.4920719861984253,8.25390625,8388608
+48,self_attn.k_proj,8,128,8,128,0.5720940232276917,8.126953125,8388608
+48,self_attn.o_proj,2,32,8,128,34.18947982788086,20.0625,67108864
+48,self_attn.o_proj,2,64,8,128,39.827823638916016,18.03125,67108864
+48,self_attn.o_proj,2,128,8,128,44.92872619628906,17.015625,67108864
+48,self_attn.o_proj,3,32,8,128,14.626492500305176,28.0625,67108864
+48,self_attn.o_proj,3,64,8,128,17.04945945739746,26.03125,67108864
+48,self_attn.o_proj,3,128,8,128,19.256776809692383,25.015625,67108864
+48,self_attn.o_proj,4,32,8,128,6.819639205932617,36.0625,67108864
+48,self_attn.o_proj,4,64,8,128,7.960466384887695,34.03125,67108864
+48,self_attn.o_proj,4,128,8,128,8.983887672424316,33.015625,67108864
+48,self_attn.o_proj,8,32,8,128,0.4146237075328827,68.0625,67108864
+48,self_attn.o_proj,8,64,8,128,0.48291632533073425,66.03125,67108864
+48,self_attn.o_proj,8,128,8,128,0.5435115098953247,65.015625,67108864
+48,mlp.gate_proj,2,32,8,128,73.93285369873047,70.21875,234881024
+48,mlp.gate_proj,2,64,8,128,87.125244140625,63.109375,234881024
+48,mlp.gate_proj,2,128,8,128,98.8986587524414,59.5546875,234881024
+48,mlp.gate_proj,3,32,8,128,31.81260871887207,98.21875,234881024
+48,mlp.gate_proj,3,64,8,128,37.18643569946289,91.109375,234881024
+48,mlp.gate_proj,3,128,8,128,42.51704788208008,87.5546875,234881024
+48,mlp.gate_proj,4,32,8,128,14.800883293151855,126.21875,234881024
+48,mlp.gate_proj,4,64,8,128,17.362627029418945,119.109375,234881024
+48,mlp.gate_proj,4,128,8,128,19.81534767150879,115.5546875,234881024
+48,mlp.gate_proj,8,32,8,128,0.9033291339874268,238.21875,234881024
+48,mlp.gate_proj,8,64,8,128,1.0608112812042236,231.109375,234881024
+48,mlp.gate_proj,8,128,8,128,1.2049850225448608,227.5546875,234881024
+48,mlp.up_proj,2,32,8,128,67.12738037109375,70.21875,234881024
+48,mlp.up_proj,2,64,8,128,78.07495880126953,63.109375,234881024
+48,mlp.up_proj,2,128,8,128,88.18572998046875,59.5546875,234881024
+48,mlp.up_proj,3,32,8,128,28.703763961791992,98.21875,234881024
+48,mlp.up_proj,3,64,8,128,33.44729232788086,91.109375,234881024
+48,mlp.up_proj,3,128,8,128,37.62908935546875,87.5546875,234881024
+48,mlp.up_proj,4,32,8,128,13.367666244506836,126.21875,234881024
+48,mlp.up_proj,4,64,8,128,15.64653205871582,119.109375,234881024
+48,mlp.up_proj,4,128,8,128,17.578989028930664,115.5546875,234881024
+48,mlp.up_proj,8,32,8,128,0.8163517713546753,238.21875,234881024
+48,mlp.up_proj,8,64,8,128,0.9542070031166077,231.109375,234881024
+48,mlp.up_proj,8,128,8,128,1.0702835321426392,227.5546875,234881024
+48,mlp.down_proj,2,32,8,128,67.41004943847656,70.21875,234881024
+48,mlp.down_proj,2,64,8,128,78.42401885986328,63.109375,234881024
+48,mlp.down_proj,2,128,8,128,88.62925720214844,59.5546875,234881024
+48,mlp.down_proj,3,32,8,128,28.83295440673828,98.21875,234881024
+48,mlp.down_proj,3,64,8,128,33.58579635620117,91.109375,234881024
+48,mlp.down_proj,3,128,8,128,37.80756759643555,87.5546875,234881024
+48,mlp.down_proj,4,32,8,128,13.426764488220215,126.21875,234881024
+48,mlp.down_proj,4,64,8,128,15.718056678771973,119.109375,234881024
+48,mlp.down_proj,4,128,8,128,17.655363082885742,115.5546875,234881024
+48,mlp.down_proj,8,32,8,128,0.8199697136878967,238.21875,234881024
+48,mlp.down_proj,8,64,8,128,0.9586257338523865,231.109375,234881024
+48,mlp.down_proj,8,128,8,128,1.0749019384384155,227.5546875,234881024
+49,self_attn.q_proj,2,32,8,128,41.55817413330078,20.0625,67108864
+49,self_attn.q_proj,2,64,8,128,49.04204559326172,18.03125,67108864
+49,self_attn.q_proj,2,128,8,128,56.773921966552734,17.015625,67108864
+49,self_attn.q_proj,3,32,8,128,17.827648162841797,28.0625,67108864
+49,self_attn.q_proj,3,64,8,128,21.154726028442383,26.03125,67108864
+49,self_attn.q_proj,3,128,8,128,24.679542541503906,25.015625,67108864
+49,self_attn.q_proj,4,32,8,128,8.320907592773438,36.0625,67108864
+49,self_attn.q_proj,4,64,8,128,9.869521141052246,34.03125,67108864
+49,self_attn.q_proj,4,128,8,128,11.52386474609375,33.015625,67108864
+49,self_attn.q_proj,8,32,8,128,0.5054245591163635,68.0625,67108864
+49,self_attn.q_proj,8,64,8,128,0.5993776321411133,66.03125,67108864
+49,self_attn.q_proj,8,128,8,128,0.6981526613235474,65.015625,67108864
+49,self_attn.v_proj,2,32,8,128,11.691324234008789,2.5078125,8388608
+49,self_attn.v_proj,2,64,8,128,13.625853538513184,2.25390625,8388608
+49,self_attn.v_proj,2,128,8,128,15.363326072692871,2.126953125,8388608
+49,self_attn.v_proj,3,32,8,128,4.997766494750977,3.5078125,8388608
+49,self_attn.v_proj,3,64,8,128,5.829087257385254,3.25390625,8388608
+49,self_attn.v_proj,3,128,8,128,6.574118137359619,3.126953125,8388608
+49,self_attn.v_proj,4,32,8,128,2.331937074661255,4.5078125,8388608
+49,self_attn.v_proj,4,64,8,128,2.720003366470337,4.25390625,8388608
+49,self_attn.v_proj,4,128,8,128,3.0680782794952393,4.126953125,8388608
+49,self_attn.v_proj,8,32,8,128,0.1412230134010315,8.5078125,8388608
+49,self_attn.v_proj,8,64,8,128,0.1645384132862091,8.25390625,8388608
+49,self_attn.v_proj,8,128,8,128,0.18491972982883453,8.126953125,8388608
+49,self_attn.k_proj,2,32,8,128,32.791831970214844,2.5078125,8388608
+49,self_attn.k_proj,2,64,8,128,39.39252853393555,2.25390625,8388608
+49,self_attn.k_proj,2,128,8,128,44.95381164550781,2.126953125,8388608
+49,self_attn.k_proj,3,32,8,128,14.303227424621582,3.5078125,8388608
+49,self_attn.k_proj,3,64,8,128,17.482088088989258,3.25390625,8388608
+49,self_attn.k_proj,3,128,8,128,20.543956756591797,3.126953125,8388608
+49,self_attn.k_proj,4,32,8,128,6.678488731384277,4.5078125,8388608
+49,self_attn.k_proj,4,64,8,128,8.168581008911133,4.25390625,8388608
+49,self_attn.k_proj,4,128,8,128,9.614110946655273,4.126953125,8388608
+49,self_attn.k_proj,8,32,8,128,0.4041021764278412,8.5078125,8388608
+49,self_attn.k_proj,8,64,8,128,0.4938357472419739,8.25390625,8388608
+49,self_attn.k_proj,8,128,8,128,0.5795847177505493,8.126953125,8388608
+49,self_attn.o_proj,2,32,8,128,33.715797424316406,20.0625,67108864
+49,self_attn.o_proj,2,64,8,128,39.252784729003906,18.03125,67108864
+49,self_attn.o_proj,2,128,8,128,44.26153564453125,17.015625,67108864
+49,self_attn.o_proj,3,32,8,128,14.415535926818848,28.0625,67108864
+49,self_attn.o_proj,3,64,8,128,16.799867630004883,26.03125,67108864
+49,self_attn.o_proj,3,128,8,128,18.944719314575195,25.015625,67108864
+49,self_attn.o_proj,4,32,8,128,6.723541259765625,36.0625,67108864
+49,self_attn.o_proj,4,64,8,128,7.841063499450684,34.03125,67108864
+49,self_attn.o_proj,4,128,8,128,8.835956573486328,33.015625,67108864
+49,self_attn.o_proj,8,32,8,128,0.4087204337120056,68.0625,67108864
+49,self_attn.o_proj,8,64,8,128,0.4757342040538788,66.03125,67108864
+49,self_attn.o_proj,8,128,8,128,0.5345524549484253,65.015625,67108864
+49,mlp.gate_proj,2,32,8,128,74.00435638427734,70.21875,234881024
+49,mlp.gate_proj,2,64,8,128,87.25935363769531,63.109375,234881024
+49,mlp.gate_proj,2,128,8,128,99.1235580444336,59.5546875,234881024
+49,mlp.gate_proj,3,32,8,128,31.84816551208496,98.21875,234881024
+49,mlp.gate_proj,3,64,8,128,37.25536346435547,91.109375,234881024
+49,mlp.gate_proj,3,128,8,128,42.62614822387695,87.5546875,234881024
+49,mlp.gate_proj,4,32,8,128,14.818818092346191,126.21875,234881024
+49,mlp.gate_proj,4,64,8,128,17.392108917236328,119.109375,234881024
+49,mlp.gate_proj,4,128,8,128,19.883676528930664,115.5546875,234881024
+49,mlp.gate_proj,8,32,8,128,0.9044886827468872,238.21875,234881024
+49,mlp.gate_proj,8,64,8,128,1.062693476676941,231.109375,234881024
+49,mlp.gate_proj,8,128,8,128,1.208604335784912,227.5546875,234881024
+49,mlp.up_proj,2,32,8,128,67.15093231201172,70.21875,234881024
+49,mlp.up_proj,2,64,8,128,78.09907531738281,63.109375,234881024
+49,mlp.up_proj,2,128,8,128,88.24002075195312,59.5546875,234881024
+49,mlp.up_proj,3,32,8,128,28.714569091796875,98.21875,234881024
+49,mlp.up_proj,3,64,8,128,33.460636138916016,91.109375,234881024
+49,mlp.up_proj,3,128,8,128,37.65470504760742,87.5546875,234881024
+49,mlp.up_proj,4,32,8,128,13.373547554016113,126.21875,234881024
+49,mlp.up_proj,4,64,8,128,15.655231475830078,119.109375,234881024
+49,mlp.up_proj,4,128,8,128,17.59136390686035,115.5546875,234881024
+49,mlp.up_proj,8,32,8,128,0.8167153000831604,238.21875,234881024
+49,mlp.up_proj,8,64,8,128,0.9546181559562683,231.109375,234881024
+49,mlp.up_proj,8,128,8,128,1.070786476135254,227.5546875,234881024
+49,mlp.down_proj,2,32,8,128,67.37598419189453,70.21875,234881024
+49,mlp.down_proj,2,64,8,128,78.3998794555664,63.109375,234881024
+49,mlp.down_proj,2,128,8,128,88.58515930175781,59.5546875,234881024
+49,mlp.down_proj,3,32,8,128,28.818889617919922,98.21875,234881024
+49,mlp.down_proj,3,64,8,128,33.57279586791992,91.109375,234881024
+49,mlp.down_proj,3,128,8,128,37.782588958740234,87.5546875,234881024
+49,mlp.down_proj,4,32,8,128,13.420001983642578,126.21875,234881024
+49,mlp.down_proj,4,64,8,128,15.70971393585205,119.109375,234881024
+49,mlp.down_proj,4,128,8,128,17.645538330078125,115.5546875,234881024
+49,mlp.down_proj,8,32,8,128,0.8196076154708862,238.21875,234881024
+49,mlp.down_proj,8,64,8,128,0.9583225250244141,231.109375,234881024
+49,mlp.down_proj,8,128,8,128,1.074610710144043,227.5546875,234881024
+50,self_attn.q_proj,2,32,8,128,38.69044876098633,20.0625,67108864
+50,self_attn.q_proj,2,64,8,128,45.70438003540039,18.03125,67108864
+50,self_attn.q_proj,2,128,8,128,52.91432571411133,17.015625,67108864
+50,self_attn.q_proj,3,32,8,128,16.625898361206055,28.0625,67108864
+50,self_attn.q_proj,3,64,8,128,19.746244430541992,26.03125,67108864
+50,self_attn.q_proj,3,128,8,128,23.194814682006836,25.015625,67108864
+50,self_attn.q_proj,4,32,8,128,7.76073694229126,36.0625,67108864
+50,self_attn.q_proj,4,64,8,128,9.212562561035156,34.03125,67108864
+50,self_attn.q_proj,4,128,8,128,10.827637672424316,33.015625,67108864
+50,self_attn.q_proj,8,32,8,128,0.4710680842399597,68.0625,67108864
+50,self_attn.q_proj,8,64,8,128,0.5593442320823669,66.03125,67108864
+50,self_attn.q_proj,8,128,8,128,0.6555463671684265,65.015625,67108864
+50,self_attn.v_proj,2,32,8,128,12.961758613586426,2.5078125,8388608
+50,self_attn.v_proj,2,64,8,128,15.143828392028809,2.25390625,8388608
+50,self_attn.v_proj,2,128,8,128,17.131256103515625,2.126953125,8388608
+50,self_attn.v_proj,3,32,8,128,5.540149688720703,3.5078125,8388608
+50,self_attn.v_proj,3,64,8,128,6.47633171081543,3.25390625,8388608
+50,self_attn.v_proj,3,128,8,128,7.329452991485596,3.126953125,8388608
+50,self_attn.v_proj,4,32,8,128,2.5854461193084717,4.5078125,8388608
+50,self_attn.v_proj,4,64,8,128,3.0223300457000732,4.25390625,8388608
+50,self_attn.v_proj,4,128,8,128,3.421661138534546,4.126953125,8388608
+50,self_attn.v_proj,8,32,8,128,0.15652543306350708,8.5078125,8388608
+50,self_attn.v_proj,8,64,8,128,0.182789608836174,8.25390625,8388608
+50,self_attn.v_proj,8,128,8,128,0.20612362027168274,8.126953125,8388608
+50,self_attn.k_proj,2,32,8,128,29.69169044494629,2.5078125,8388608
+50,self_attn.k_proj,2,64,8,128,35.920310974121094,2.25390625,8388608
+50,self_attn.k_proj,2,128,8,128,41.2724609375,2.126953125,8388608
+50,self_attn.k_proj,3,32,8,128,13.120783805847168,3.5078125,8388608
+50,self_attn.k_proj,3,64,8,128,16.316011428833008,3.25390625,8388608
+50,self_attn.k_proj,3,128,8,128,19.529123306274414,3.126953125,8388608
+50,self_attn.k_proj,4,32,8,128,6.128735065460205,4.5078125,8388608
+50,self_attn.k_proj,4,64,8,128,7.630778789520264,4.25390625,8388608
+50,self_attn.k_proj,4,128,8,128,9.16757869720459,4.126953125,8388608
+50,self_attn.k_proj,8,32,8,128,0.3707149028778076,8.5078125,8388608
+50,self_attn.k_proj,8,64,8,128,0.46149206161499023,8.25390625,8388608
+50,self_attn.k_proj,8,128,8,128,0.5526161193847656,8.126953125,8388608
+50,self_attn.o_proj,2,32,8,128,34.409969329833984,20.0625,67108864
+50,self_attn.o_proj,2,64,8,128,40.04841613769531,18.03125,67108864
+50,self_attn.o_proj,2,128,8,128,45.12452697753906,17.015625,67108864
+50,self_attn.o_proj,3,32,8,128,14.714488983154297,28.0625,67108864
+50,self_attn.o_proj,3,64,8,128,17.128976821899414,26.03125,67108864
+50,self_attn.o_proj,3,128,8,128,19.296262741088867,25.015625,67108864
+50,self_attn.o_proj,4,32,8,128,6.86084508895874,36.0625,67108864
+50,self_attn.o_proj,4,64,8,128,7.998870372772217,34.03125,67108864
+50,self_attn.o_proj,4,128,8,128,9.002357482910156,33.015625,67108864
+50,self_attn.o_proj,8,32,8,128,0.41705721616744995,68.0625,67108864
+50,self_attn.o_proj,8,64,8,128,0.4851283133029938,66.03125,67108864
+50,self_attn.o_proj,8,128,8,128,0.5445810556411743,65.015625,67108864
+50,mlp.gate_proj,2,32,8,128,74.14884185791016,70.21875,234881024
+50,mlp.gate_proj,2,64,8,128,87.54200744628906,63.109375,234881024
+50,mlp.gate_proj,2,128,8,128,99.57586669921875,59.5546875,234881024
+50,mlp.gate_proj,3,32,8,128,31.91617774963379,98.21875,234881024
+50,mlp.gate_proj,3,64,8,128,37.37941360473633,91.109375,234881024
+50,mlp.gate_proj,3,128,8,128,42.86003112792969,87.5546875,234881024
+50,mlp.gate_proj,4,32,8,128,14.850088119506836,126.21875,234881024
+50,mlp.gate_proj,4,64,8,128,17.453786849975586,119.109375,234881024
+50,mlp.gate_proj,4,128,8,128,19.977313995361328,115.5546875,234881024
+50,mlp.gate_proj,8,32,8,128,0.9063097238540649,238.21875,234881024
+50,mlp.gate_proj,8,64,8,128,1.065877914428711,231.109375,234881024
+50,mlp.gate_proj,8,128,8,128,1.2145878076553345,227.5546875,234881024
+50,mlp.up_proj,2,32,8,128,67.1590347290039,70.21875,234881024
+50,mlp.up_proj,2,64,8,128,78.1212158203125,63.109375,234881024
+50,mlp.up_proj,2,128,8,128,88.2654800415039,59.5546875,234881024
+50,mlp.up_proj,3,32,8,128,28.72030258178711,98.21875,234881024
+50,mlp.up_proj,3,64,8,128,33.46231460571289,91.109375,234881024
+50,mlp.up_proj,3,128,8,128,37.65021896362305,87.5546875,234881024
+50,mlp.up_proj,4,32,8,128,13.374326705932617,126.21875,234881024
+50,mlp.up_proj,4,64,8,128,15.656843185424805,119.109375,234881024
+50,mlp.up_proj,4,128,8,128,17.58805274963379,115.5546875,234881024
+50,mlp.up_proj,8,32,8,128,0.8167883157730103,238.21875,234881024
+50,mlp.up_proj,8,64,8,128,0.9546166062355042,231.109375,234881024
+50,mlp.up_proj,8,128,8,128,1.0706071853637695,227.5546875,234881024
+50,mlp.down_proj,2,32,8,128,67.37797546386719,70.21875,234881024
+50,mlp.down_proj,2,64,8,128,78.39753723144531,63.109375,234881024
+50,mlp.down_proj,2,128,8,128,88.60389709472656,59.5546875,234881024
+50,mlp.down_proj,3,32,8,128,28.818302154541016,98.21875,234881024
+50,mlp.down_proj,3,64,8,128,33.56850051879883,91.109375,234881024
+50,mlp.down_proj,3,128,8,128,37.77184295654297,87.5546875,234881024
+50,mlp.down_proj,4,32,8,128,13.421090126037598,126.21875,234881024
+50,mlp.down_proj,4,64,8,128,15.706878662109375,119.109375,234881024
+50,mlp.down_proj,4,128,8,128,17.653772354125977,115.5546875,234881024
+50,mlp.down_proj,8,32,8,128,0.8195101022720337,238.21875,234881024
+50,mlp.down_proj,8,64,8,128,0.9579325914382935,231.109375,234881024
+50,mlp.down_proj,8,128,8,128,1.0738887786865234,227.5546875,234881024
+51,self_attn.q_proj,2,32,8,128,42.04945755004883,20.0625,67108864
+51,self_attn.q_proj,2,64,8,128,49.62029266357422,18.03125,67108864
+51,self_attn.q_proj,2,128,8,128,57.11650848388672,17.015625,67108864
+51,self_attn.q_proj,3,32,8,128,18.010051727294922,28.0625,67108864
+51,self_attn.q_proj,3,64,8,128,21.346834182739258,26.03125,67108864
+51,self_attn.q_proj,3,128,8,128,24.700363159179688,25.015625,67108864
+51,self_attn.q_proj,4,32,8,128,8.40643310546875,36.0625,67108864
+51,self_attn.q_proj,4,64,8,128,9.95595932006836,34.03125,67108864
+51,self_attn.q_proj,4,128,8,128,11.538736343383789,33.015625,67108864
+51,self_attn.q_proj,8,32,8,128,0.5105367302894592,68.0625,67108864
+51,self_attn.q_proj,8,64,8,128,0.6046388745307922,66.03125,67108864
+51,self_attn.q_proj,8,128,8,128,0.6986956000328064,65.015625,67108864
+51,self_attn.v_proj,2,32,8,128,12.71059799194336,2.5078125,8388608
+51,self_attn.v_proj,2,64,8,128,14.888504981994629,2.25390625,8388608
+51,self_attn.v_proj,2,128,8,128,16.877052307128906,2.126953125,8388608
+51,self_attn.v_proj,3,32,8,128,5.434027194976807,3.5078125,8388608
+51,self_attn.v_proj,3,64,8,128,6.368951797485352,3.25390625,8388608
+51,self_attn.v_proj,3,128,8,128,7.229095935821533,3.126953125,8388608
+51,self_attn.v_proj,4,32,8,128,2.535001516342163,4.5078125,8388608
+51,self_attn.v_proj,4,64,8,128,2.9729888439178467,4.25390625,8388608
+51,self_attn.v_proj,4,128,8,128,3.3733410835266113,4.126953125,8388608
+51,self_attn.v_proj,8,32,8,128,0.15354080498218536,8.5078125,8388608
+51,self_attn.v_proj,8,64,8,128,0.17971046268939972,8.25390625,8388608
+51,self_attn.v_proj,8,128,8,128,0.20317232608795166,8.126953125,8388608
+51,self_attn.k_proj,2,32,8,128,32.2258186340332,2.5078125,8388608
+51,self_attn.k_proj,2,64,8,128,38.65294647216797,2.25390625,8388608
+51,self_attn.k_proj,2,128,8,128,44.08636474609375,2.126953125,8388608
+51,self_attn.k_proj,3,32,8,128,13.95390510559082,3.5078125,8388608
+51,self_attn.k_proj,3,64,8,128,16.92561912536621,3.25390625,8388608
+51,self_attn.k_proj,3,128,8,128,19.775678634643555,3.126953125,8388608
+51,self_attn.k_proj,4,32,8,128,6.511787414550781,4.5078125,8388608
+51,self_attn.k_proj,4,64,8,128,7.9032745361328125,4.25390625,8388608
+51,self_attn.k_proj,4,128,8,128,9.23332691192627,4.126953125,8388608
+51,self_attn.k_proj,8,32,8,128,0.39420580863952637,8.5078125,8388608
+51,self_attn.k_proj,8,64,8,128,0.47800225019454956,8.25390625,8388608
+51,self_attn.k_proj,8,128,8,128,0.5566169023513794,8.126953125,8388608
+51,self_attn.o_proj,2,32,8,128,34.109867095947266,20.0625,67108864
+51,self_attn.o_proj,2,64,8,128,39.72843933105469,18.03125,67108864
+51,self_attn.o_proj,2,128,8,128,44.82476806640625,17.015625,67108864
+51,self_attn.o_proj,3,32,8,128,14.584275245666504,28.0625,67108864
+51,self_attn.o_proj,3,64,8,128,17.000051498413086,26.03125,67108864
+51,self_attn.o_proj,3,128,8,128,19.189542770385742,25.015625,67108864
+51,self_attn.o_proj,4,32,8,128,6.802095413208008,36.0625,67108864
+51,self_attn.o_proj,4,64,8,128,7.9367499351501465,34.03125,67108864
+51,self_attn.o_proj,4,128,8,128,8.952648162841797,33.015625,67108864
+51,self_attn.o_proj,8,32,8,128,0.41348740458488464,68.0625,67108864
+51,self_attn.o_proj,8,64,8,128,0.48144781589508057,66.03125,67108864
+51,self_attn.o_proj,8,128,8,128,0.541509747505188,65.015625,67108864
+51,mlp.gate_proj,2,32,8,128,74.0965576171875,70.21875,234881024
+51,mlp.gate_proj,2,64,8,128,87.43866729736328,63.109375,234881024
+51,mlp.gate_proj,2,128,8,128,99.36560821533203,59.5546875,234881024
+51,mlp.gate_proj,3,32,8,128,31.89478302001953,98.21875,234881024
+51,mlp.gate_proj,3,64,8,128,37.3293571472168,91.109375,234881024
+51,mlp.gate_proj,3,128,8,128,42.77140426635742,87.5546875,234881024
+51,mlp.gate_proj,4,32,8,128,14.841056823730469,126.21875,234881024
+51,mlp.gate_proj,4,64,8,128,17.43130874633789,119.109375,234881024
+51,mlp.gate_proj,4,128,8,128,19.946102142333984,115.5546875,234881024
+51,mlp.gate_proj,8,32,8,128,0.9058587551116943,238.21875,234881024
+51,mlp.gate_proj,8,64,8,128,1.0648937225341797,231.109375,234881024
+51,mlp.gate_proj,8,128,8,128,1.2128052711486816,227.5546875,234881024
+51,mlp.up_proj,2,32,8,128,67.24855041503906,70.21875,234881024
+51,mlp.up_proj,2,64,8,128,78.220703125,63.109375,234881024
+51,mlp.up_proj,2,128,8,128,88.4031982421875,59.5546875,234881024
+51,mlp.up_proj,3,32,8,128,28.7609806060791,98.21875,234881024
+51,mlp.up_proj,3,64,8,128,33.511295318603516,91.109375,234881024
+51,mlp.up_proj,3,128,8,128,37.71316909790039,87.5546875,234881024
+51,mlp.up_proj,4,32,8,128,13.39443302154541,126.21875,234881024
+51,mlp.up_proj,4,64,8,128,15.6790189743042,119.109375,234881024
+51,mlp.up_proj,4,128,8,128,17.61656379699707,115.5546875,234881024
+51,mlp.up_proj,8,32,8,128,0.8179135918617249,238.21875,234881024
+51,mlp.up_proj,8,64,8,128,0.9561282396316528,231.109375,234881024
+51,mlp.up_proj,8,128,8,128,1.0723077058792114,227.5546875,234881024
+51,mlp.down_proj,2,32,8,128,67.44241333007812,70.21875,234881024
+51,mlp.down_proj,2,64,8,128,78.47358703613281,63.109375,234881024
+51,mlp.down_proj,2,128,8,128,88.66727447509766,59.5546875,234881024
+51,mlp.down_proj,3,32,8,128,28.85005760192871,98.21875,234881024
+51,mlp.down_proj,3,64,8,128,33.60512161254883,91.109375,234881024
+51,mlp.down_proj,3,128,8,128,37.81795883178711,87.5546875,234881024
+51,mlp.down_proj,4,32,8,128,13.434735298156738,126.21875,234881024
+51,mlp.down_proj,4,64,8,128,15.726635932922363,119.109375,234881024
+51,mlp.down_proj,4,128,8,128,17.666240692138672,115.5546875,234881024
+51,mlp.down_proj,8,32,8,128,0.8204967975616455,238.21875,234881024
+51,mlp.down_proj,8,64,8,128,0.9592645764350891,231.109375,234881024
+51,mlp.down_proj,8,128,8,128,1.075515866279602,227.5546875,234881024
+52,self_attn.q_proj,2,32,8,128,46.2410774230957,20.0625,67108864
+52,self_attn.q_proj,2,64,8,128,54.697574615478516,18.03125,67108864
+52,self_attn.q_proj,2,128,8,128,63.34103775024414,17.015625,67108864
+52,self_attn.q_proj,3,32,8,128,19.795639038085938,28.0625,67108864
+52,self_attn.q_proj,3,64,8,128,23.497615814208984,26.03125,67108864
+52,self_attn.q_proj,3,128,8,128,27.28960418701172,25.015625,67108864
+52,self_attn.q_proj,4,32,8,128,9.233983993530273,36.0625,67108864
+52,self_attn.q_proj,4,64,8,128,10.96748161315918,34.03125,67108864
+52,self_attn.q_proj,4,128,8,128,12.73377513885498,33.015625,67108864
+52,self_attn.q_proj,8,32,8,128,0.5608720779418945,68.0625,67108864
+52,self_attn.q_proj,8,64,8,128,0.6660153865814209,66.03125,67108864
+52,self_attn.q_proj,8,128,8,128,0.7709869146347046,65.015625,67108864
+52,self_attn.v_proj,2,32,8,128,11.060070991516113,2.5078125,8388608
+52,self_attn.v_proj,2,64,8,128,12.939879417419434,2.25390625,8388608
+52,self_attn.v_proj,2,128,8,128,14.645830154418945,2.126953125,8388608
+52,self_attn.v_proj,3,32,8,128,4.7302727699279785,3.5078125,8388608
+52,self_attn.v_proj,3,64,8,128,5.534723281860352,3.25390625,8388608
+52,self_attn.v_proj,3,128,8,128,6.266499042510986,3.126953125,8388608
+52,self_attn.v_proj,4,32,8,128,2.206082820892334,4.5078125,8388608
+52,self_attn.v_proj,4,64,8,128,2.582369565963745,4.25390625,8388608
+52,self_attn.v_proj,4,128,8,128,2.926506996154785,4.126953125,8388608
+52,self_attn.v_proj,8,32,8,128,0.13356243073940277,8.5078125,8388608
+52,self_attn.v_proj,8,64,8,128,0.15621532499790192,8.25390625,8388608
+52,self_attn.v_proj,8,128,8,128,0.17624631524085999,8.126953125,8388608
+52,self_attn.k_proj,2,32,8,128,33.55439376831055,2.5078125,8388608
+52,self_attn.k_proj,2,64,8,128,40.30548095703125,2.25390625,8388608
+52,self_attn.k_proj,2,128,8,128,46.3602294921875,2.126953125,8388608
+52,self_attn.k_proj,3,32,8,128,14.432565689086914,3.5078125,8388608
+52,self_attn.k_proj,3,64,8,128,17.45225715637207,3.25390625,8388608
+52,self_attn.k_proj,3,128,8,128,20.367019653320312,3.126953125,8388608
+52,self_attn.k_proj,4,32,8,128,6.735119342803955,4.5078125,8388608
+52,self_attn.k_proj,4,64,8,128,8.146063804626465,4.25390625,8388608
+52,self_attn.k_proj,4,128,8,128,9.509270668029785,4.126953125,8388608
+52,self_attn.k_proj,8,32,8,128,0.40766850113868713,8.5078125,8388608
+52,self_attn.k_proj,8,64,8,128,0.4922788441181183,8.25390625,8388608
+52,self_attn.k_proj,8,128,8,128,0.5725817084312439,8.126953125,8388608
+52,self_attn.o_proj,2,32,8,128,32.742374420166016,20.0625,67108864
+52,self_attn.o_proj,2,64,8,128,38.14075469970703,18.03125,67108864
+52,self_attn.o_proj,2,128,8,128,43.0750732421875,17.015625,67108864
+52,self_attn.o_proj,3,32,8,128,14.004478454589844,28.0625,67108864
+52,self_attn.o_proj,3,64,8,128,16.359081268310547,26.03125,67108864
+52,self_attn.o_proj,3,128,8,128,18.499805450439453,25.015625,67108864
+52,self_attn.o_proj,4,32,8,128,6.5332489013671875,36.0625,67108864
+52,self_attn.o_proj,4,64,8,128,7.63424825668335,34.03125,67108864
+52,self_attn.o_proj,4,128,8,128,8.641610145568848,33.015625,67108864
+52,self_attn.o_proj,8,32,8,128,0.39715057611465454,68.0625,67108864
+52,self_attn.o_proj,8,64,8,128,0.4633426070213318,66.03125,67108864
+52,self_attn.o_proj,8,128,8,128,0.5225140452384949,65.015625,67108864
+52,mlp.gate_proj,2,32,8,128,74.26083374023438,70.21875,234881024
+52,mlp.gate_proj,2,64,8,128,87.5726089477539,63.109375,234881024
+52,mlp.gate_proj,2,128,8,128,99.38154602050781,59.5546875,234881024
+52,mlp.gate_proj,3,32,8,128,31.95894432067871,98.21875,234881024
+52,mlp.gate_proj,3,64,8,128,37.36513137817383,91.109375,234881024
+52,mlp.gate_proj,3,128,8,128,42.73399353027344,87.5546875,234881024
+52,mlp.gate_proj,4,32,8,128,14.869640350341797,126.21875,234881024
+52,mlp.gate_proj,4,64,8,128,17.44605255126953,119.109375,234881024
+52,mlp.gate_proj,4,128,8,128,19.92057228088379,115.5546875,234881024
+52,mlp.gate_proj,8,32,8,128,0.9076813459396362,238.21875,234881024
+52,mlp.gate_proj,8,64,8,128,1.0658798217773438,231.109375,234881024
+52,mlp.gate_proj,8,128,8,128,1.2114204168319702,227.5546875,234881024
+52,mlp.up_proj,2,32,8,128,67.2550277709961,70.21875,234881024
+52,mlp.up_proj,2,64,8,128,78.232421875,63.109375,234881024
+52,mlp.up_proj,2,128,8,128,88.3899917602539,59.5546875,234881024
+52,mlp.up_proj,3,32,8,128,28.76525115966797,98.21875,234881024
+52,mlp.up_proj,3,64,8,128,33.51512145996094,91.109375,234881024
+52,mlp.up_proj,3,128,8,128,37.72269821166992,87.5546875,234881024
+52,mlp.up_proj,4,32,8,128,13.395151138305664,126.21875,234881024
+52,mlp.up_proj,4,64,8,128,15.684372901916504,119.109375,234881024
+52,mlp.up_proj,4,128,8,128,17.62211036682129,115.5546875,234881024
+52,mlp.up_proj,8,32,8,128,0.8179901838302612,238.21875,234881024
+52,mlp.up_proj,8,64,8,128,0.9563564658164978,231.109375,234881024
+52,mlp.up_proj,8,128,8,128,1.0727356672286987,227.5546875,234881024
+52,mlp.down_proj,2,32,8,128,67.43844604492188,70.21875,234881024
+52,mlp.down_proj,2,64,8,128,78.46015930175781,63.109375,234881024
+52,mlp.down_proj,2,128,8,128,88.69992065429688,59.5546875,234881024
+52,mlp.down_proj,3,32,8,128,28.849721908569336,98.21875,234881024
+52,mlp.down_proj,3,64,8,128,33.60414123535156,91.109375,234881024
+52,mlp.down_proj,3,128,8,128,37.826148986816406,87.5546875,234881024
+52,mlp.down_proj,4,32,8,128,13.434651374816895,126.21875,234881024
+52,mlp.down_proj,4,64,8,128,15.727668762207031,119.109375,234881024
+52,mlp.down_proj,4,128,8,128,17.66876220703125,115.5546875,234881024
+52,mlp.down_proj,8,32,8,128,0.8204707503318787,238.21875,234881024
+52,mlp.down_proj,8,64,8,128,0.9592309594154358,231.109375,234881024
+52,mlp.down_proj,8,128,8,128,1.0756560564041138,227.5546875,234881024
+53,self_attn.q_proj,2,32,8,128,41.569862365722656,20.0625,67108864
+53,self_attn.q_proj,2,64,8,128,48.95644760131836,18.03125,67108864
+53,self_attn.q_proj,2,128,8,128,56.63172149658203,17.015625,67108864
+53,self_attn.q_proj,3,32,8,128,17.806421279907227,28.0625,67108864
+53,self_attn.q_proj,3,64,8,128,21.054595947265625,26.03125,67108864
+53,self_attn.q_proj,3,128,8,128,24.484636306762695,25.015625,67108864
+53,self_attn.q_proj,4,32,8,128,8.311920166015625,36.0625,67108864
+53,self_attn.q_proj,4,64,8,128,9.825268745422363,34.03125,67108864
+53,self_attn.q_proj,4,128,8,128,11.434738159179688,33.015625,67108864
+53,self_attn.q_proj,8,32,8,128,0.5047361850738525,68.0625,67108864
+53,self_attn.q_proj,8,64,8,128,0.5964109897613525,66.03125,67108864
+53,self_attn.q_proj,8,128,8,128,0.6924493908882141,65.015625,67108864
+53,self_attn.v_proj,2,32,8,128,11.981741905212402,2.5078125,8388608
+53,self_attn.v_proj,2,64,8,128,13.952746391296387,2.25390625,8388608
+53,self_attn.v_proj,2,128,8,128,15.72281265258789,2.126953125,8388608
+53,self_attn.v_proj,3,32,8,128,5.123473167419434,3.5078125,8388608
+53,self_attn.v_proj,3,64,8,128,5.968129634857178,3.25390625,8388608
+53,self_attn.v_proj,3,128,8,128,6.733577251434326,3.126953125,8388608
+53,self_attn.v_proj,4,32,8,128,2.3896803855895996,4.5078125,8388608
+53,self_attn.v_proj,4,64,8,128,2.784593105316162,4.25390625,8388608
+53,self_attn.v_proj,4,128,8,128,3.1410255432128906,4.126953125,8388608
+53,self_attn.v_proj,8,32,8,128,0.14476831257343292,8.5078125,8388608
+53,self_attn.v_proj,8,64,8,128,0.1684647649526596,8.25390625,8388608
+53,self_attn.v_proj,8,128,8,128,0.1892404705286026,8.126953125,8388608
+53,self_attn.k_proj,2,32,8,128,33.03573226928711,2.5078125,8388608
+53,self_attn.k_proj,2,64,8,128,39.739566802978516,2.25390625,8388608
+53,self_attn.k_proj,2,128,8,128,45.372371673583984,2.126953125,8388608
+53,self_attn.k_proj,3,32,8,128,14.39354133605957,3.5078125,8388608
+53,self_attn.k_proj,3,64,8,128,17.60256004333496,3.25390625,8388608
+53,self_attn.k_proj,3,128,8,128,20.689294815063477,3.126953125,8388608
+53,self_attn.k_proj,4,32,8,128,6.72144079208374,4.5078125,8388608
+53,self_attn.k_proj,4,64,8,128,8.22337532043457,4.25390625,8388608
+53,self_attn.k_proj,4,128,8,128,9.672806739807129,4.126953125,8388608
+53,self_attn.k_proj,8,32,8,128,0.4066151976585388,8.5078125,8388608
+53,self_attn.k_proj,8,64,8,128,0.49724721908569336,8.25390625,8388608
+53,self_attn.k_proj,8,128,8,128,0.583095371723175,8.126953125,8388608
+53,self_attn.o_proj,2,32,8,128,33.888675689697266,20.0625,67108864
+53,self_attn.o_proj,2,64,8,128,39.448150634765625,18.03125,67108864
+53,self_attn.o_proj,2,128,8,128,44.453983306884766,17.015625,67108864
+53,self_attn.o_proj,3,32,8,128,14.488508224487305,28.0625,67108864
+53,self_attn.o_proj,3,64,8,128,16.875808715820312,26.03125,67108864
+53,self_attn.o_proj,3,128,8,128,19.01791763305664,25.015625,67108864
+53,self_attn.o_proj,4,32,8,128,6.755908966064453,36.0625,67108864
+53,self_attn.o_proj,4,64,8,128,7.876598834991455,34.03125,67108864
+53,self_attn.o_proj,4,128,8,128,8.870698928833008,33.015625,67108864
+53,self_attn.o_proj,8,32,8,128,0.41081702709198,68.0625,67108864
+53,self_attn.o_proj,8,64,8,128,0.4778541624546051,66.03125,67108864
+53,self_attn.o_proj,8,128,8,128,0.536673367023468,65.015625,67108864
+53,mlp.gate_proj,2,32,8,128,74.35889434814453,70.21875,234881024
+53,mlp.gate_proj,2,64,8,128,87.77432250976562,63.109375,234881024
+53,mlp.gate_proj,2,128,8,128,99.74461364746094,59.5546875,234881024
+53,mlp.gate_proj,3,32,8,128,32.0046501159668,98.21875,234881024
+53,mlp.gate_proj,3,64,8,128,37.458778381347656,91.109375,234881024
+53,mlp.gate_proj,3,128,8,128,42.92732620239258,87.5546875,234881024
+53,mlp.gate_proj,4,32,8,128,14.890552520751953,126.21875,234881024
+53,mlp.gate_proj,4,64,8,128,17.486661911010742,119.109375,234881024
+53,mlp.gate_proj,4,128,8,128,20.000524520874023,115.5546875,234881024
+53,mlp.gate_proj,8,32,8,128,0.9088312983512878,238.21875,234881024
+53,mlp.gate_proj,8,64,8,128,1.0682481527328491,231.109375,234881024
+53,mlp.gate_proj,8,128,8,128,1.2160813808441162,227.5546875,234881024
+53,mlp.up_proj,2,32,8,128,67.2683334350586,70.21875,234881024
+53,mlp.up_proj,2,64,8,128,78.25328826904297,63.109375,234881024
+53,mlp.up_proj,2,128,8,128,88.3838119506836,59.5546875,234881024
+53,mlp.up_proj,3,32,8,128,28.767242431640625,98.21875,234881024
+53,mlp.up_proj,3,64,8,128,33.51169967651367,91.109375,234881024
+53,mlp.up_proj,3,128,8,128,37.71158218383789,87.5546875,234881024
+53,mlp.up_proj,4,32,8,128,13.395927429199219,126.21875,234881024
+53,mlp.up_proj,4,64,8,128,15.680472373962402,119.109375,234881024
+53,mlp.up_proj,4,128,8,128,17.61606216430664,115.5546875,234881024
+53,mlp.up_proj,8,32,8,128,0.8181264996528625,238.21875,234881024
+53,mlp.up_proj,8,64,8,128,0.9562508463859558,231.109375,234881024
+53,mlp.up_proj,8,128,8,128,1.0722988843917847,227.5546875,234881024
+53,mlp.down_proj,2,32,8,128,67.42005157470703,70.21875,234881024
+53,mlp.down_proj,2,64,8,128,78.45710754394531,63.109375,234881024
+53,mlp.down_proj,2,128,8,128,88.63882446289062,59.5546875,234881024
+53,mlp.down_proj,3,32,8,128,28.84033203125,98.21875,234881024
+53,mlp.down_proj,3,64,8,128,33.59003829956055,91.109375,234881024
+53,mlp.down_proj,3,128,8,128,37.79269790649414,87.5546875,234881024
+53,mlp.down_proj,4,32,8,128,13.431459426879883,126.21875,234881024
+53,mlp.down_proj,4,64,8,128,15.721224784851074,119.109375,234881024
+53,mlp.down_proj,4,128,8,128,17.663915634155273,115.5546875,234881024
+53,mlp.down_proj,8,32,8,128,0.8201905488967896,238.21875,234881024
+53,mlp.down_proj,8,64,8,128,0.9588964581489563,231.109375,234881024
+53,mlp.down_proj,8,128,8,128,1.075105905532837,227.5546875,234881024
+54,self_attn.q_proj,2,32,8,128,39.249427795410156,20.0625,67108864
+54,self_attn.q_proj,2,64,8,128,46.8551139831543,18.03125,67108864
+54,self_attn.q_proj,2,128,8,128,54.43247985839844,17.015625,67108864
+54,self_attn.q_proj,3,32,8,128,16.91651725769043,28.0625,67108864
+54,self_attn.q_proj,3,64,8,128,20.383575439453125,26.03125,67108864
+54,self_attn.q_proj,3,128,8,128,24.105958938598633,25.015625,67108864
+54,self_attn.q_proj,4,32,8,128,7.896571636199951,36.0625,67108864
+54,self_attn.q_proj,4,64,8,128,9.509722709655762,34.03125,67108864
+54,self_attn.q_proj,4,128,8,128,11.266100883483887,33.015625,67108864
+54,self_attn.q_proj,8,32,8,128,0.4793313145637512,68.0625,67108864
+54,self_attn.q_proj,8,64,8,128,0.5771214365959167,66.03125,67108864
+54,self_attn.q_proj,8,128,8,128,0.6816413998603821,65.015625,67108864
+54,self_attn.v_proj,2,32,8,128,12.239532470703125,2.5078125,8388608
+54,self_attn.v_proj,2,64,8,128,14.255985260009766,2.25390625,8388608
+54,self_attn.v_proj,2,128,8,128,16.075403213500977,2.126953125,8388608
+54,self_attn.v_proj,3,32,8,128,5.233312129974365,3.5078125,8388608
+54,self_attn.v_proj,3,64,8,128,6.1003899574279785,3.25390625,8388608
+54,self_attn.v_proj,3,128,8,128,6.885213375091553,3.126953125,8388608
+54,self_attn.v_proj,4,32,8,128,2.441946029663086,4.5078125,8388608
+54,self_attn.v_proj,4,64,8,128,2.846435546875,4.25390625,8388608
+54,self_attn.v_proj,4,128,8,128,3.2121405601501465,4.126953125,8388608
+54,self_attn.v_proj,8,32,8,128,0.14790257811546326,8.5078125,8388608
+54,self_attn.v_proj,8,64,8,128,0.17221671342849731,8.25390625,8388608
+54,self_attn.v_proj,8,128,8,128,0.19357958436012268,8.126953125,8388608
+54,self_attn.k_proj,2,32,8,128,28.96541404724121,2.5078125,8388608
+54,self_attn.k_proj,2,64,8,128,34.87678146362305,2.25390625,8388608
+54,self_attn.k_proj,2,128,8,128,39.738590240478516,2.126953125,8388608
+54,self_attn.k_proj,3,32,8,128,12.742772102355957,3.5078125,8388608
+54,self_attn.k_proj,3,64,8,128,15.763087272644043,3.25390625,8388608
+54,self_attn.k_proj,3,128,8,128,18.706520080566406,3.126953125,8388608
+54,self_attn.k_proj,4,32,8,128,5.952512264251709,4.5078125,8388608
+54,self_attn.k_proj,4,64,8,128,7.368692398071289,4.25390625,8388608
+54,self_attn.k_proj,4,128,8,128,8.756891250610352,4.126953125,8388608
+54,self_attn.k_proj,8,32,8,128,0.3600020110607147,8.5078125,8388608
+54,self_attn.k_proj,8,64,8,128,0.44566887617111206,8.25390625,8388608
+54,self_attn.k_proj,8,128,8,128,0.5280682444572449,8.126953125,8388608
+54,self_attn.o_proj,2,32,8,128,33.731117248535156,20.0625,67108864
+54,self_attn.o_proj,2,64,8,128,39.30580139160156,18.03125,67108864
+54,self_attn.o_proj,2,128,8,128,44.385169982910156,17.015625,67108864
+54,self_attn.o_proj,3,32,8,128,14.425516128540039,28.0625,67108864
+54,self_attn.o_proj,3,64,8,128,16.833969116210938,26.03125,67108864
+54,self_attn.o_proj,3,128,8,128,19.023815155029297,25.015625,67108864
+54,self_attn.o_proj,4,32,8,128,6.7274322509765625,36.0625,67108864
+54,self_attn.o_proj,4,64,8,128,7.857855319976807,34.03125,67108864
+54,self_attn.o_proj,4,128,8,128,8.878423690795898,33.015625,67108864
+54,self_attn.o_proj,8,32,8,128,0.4089667499065399,68.0625,67108864
+54,self_attn.o_proj,8,64,8,128,0.47669917345046997,66.03125,67108864
+54,self_attn.o_proj,8,128,8,128,0.5370094776153564,65.015625,67108864
+54,mlp.gate_proj,2,32,8,128,74.0001220703125,70.21875,234881024
+54,mlp.gate_proj,2,64,8,128,87.23823547363281,63.109375,234881024
+54,mlp.gate_proj,2,128,8,128,99.02980041503906,59.5546875,234881024
+54,mlp.gate_proj,3,32,8,128,31.8410587310791,98.21875,234881024
+54,mlp.gate_proj,3,64,8,128,37.226009368896484,91.109375,234881024
+54,mlp.gate_proj,3,128,8,128,42.55548858642578,87.5546875,234881024
+54,mlp.gate_proj,4,32,8,128,14.814108848571777,126.21875,234881024
+54,mlp.gate_proj,4,64,8,128,17.38499641418457,119.109375,234881024
+54,mlp.gate_proj,4,128,8,128,19.846542358398438,115.5546875,234881024
+54,mlp.gate_proj,8,32,8,128,0.9041767120361328,238.21875,234881024
+54,mlp.gate_proj,8,64,8,128,1.0618702173233032,231.109375,234881024
+54,mlp.gate_proj,8,128,8,128,1.2063158750534058,227.5546875,234881024
+54,mlp.up_proj,2,32,8,128,67.40941619873047,70.21875,234881024
+54,mlp.up_proj,2,64,8,128,78.41588592529297,63.109375,234881024
+54,mlp.up_proj,2,128,8,128,88.59095001220703,59.5546875,234881024
+54,mlp.up_proj,3,32,8,128,28.833765029907227,98.21875,234881024
+54,mlp.up_proj,3,64,8,128,33.58460235595703,91.109375,234881024
+54,mlp.up_proj,3,128,8,128,37.801998138427734,87.5546875,234881024
+54,mlp.up_proj,4,32,8,128,13.427448272705078,126.21875,234881024
+54,mlp.up_proj,4,64,8,128,15.716592788696289,119.109375,234881024
+54,mlp.up_proj,4,128,8,128,17.653127670288086,115.5546875,234881024
+54,mlp.up_proj,8,32,8,128,0.819957971572876,238.21875,234881024
+54,mlp.up_proj,8,64,8,128,0.9584872722625732,231.109375,234881024
+54,mlp.up_proj,8,128,8,128,1.0745919942855835,227.5546875,234881024
+54,mlp.down_proj,2,32,8,128,67.5570297241211,70.21875,234881024
+54,mlp.down_proj,2,64,8,128,78.59541320800781,63.109375,234881024
+54,mlp.down_proj,2,128,8,128,88.80152893066406,59.5546875,234881024
+54,mlp.down_proj,3,32,8,128,28.899572372436523,98.21875,234881024
+54,mlp.down_proj,3,64,8,128,33.65366744995117,91.109375,234881024
+54,mlp.down_proj,3,128,8,128,37.871402740478516,87.5546875,234881024
+54,mlp.down_proj,4,32,8,128,13.457481384277344,126.21875,234881024
+54,mlp.down_proj,4,64,8,128,15.751625061035156,119.109375,234881024
+54,mlp.down_proj,4,128,8,128,17.692672729492188,115.5546875,234881024
+54,mlp.down_proj,8,32,8,128,0.8218164443969727,238.21875,234881024
+54,mlp.down_proj,8,64,8,128,0.9607595205307007,231.109375,234881024
+54,mlp.down_proj,8,128,8,128,1.076769232749939,227.5546875,234881024
+55,self_attn.q_proj,2,32,8,128,40.56990432739258,20.0625,67108864
+55,self_attn.q_proj,2,64,8,128,48.058746337890625,18.03125,67108864
+55,self_attn.q_proj,2,128,8,128,55.568695068359375,17.015625,67108864
+55,self_attn.q_proj,3,32,8,128,17.41628646850586,28.0625,67108864
+55,self_attn.q_proj,3,64,8,128,20.752994537353516,26.03125,67108864
+55,self_attn.q_proj,3,128,8,128,24.237327575683594,25.015625,67108864
+55,self_attn.q_proj,4,32,8,128,8.132290840148926,36.0625,67108864
+55,self_attn.q_proj,4,64,8,128,9.683880805969238,34.03125,67108864
+55,self_attn.q_proj,4,128,8,128,11.32651424407959,33.015625,67108864
+55,self_attn.q_proj,8,32,8,128,0.4937272369861603,68.0625,67108864
+55,self_attn.q_proj,8,64,8,128,0.5881300568580627,66.03125,67108864
+55,self_attn.q_proj,8,128,8,128,0.6856171488761902,65.015625,67108864
+55,self_attn.v_proj,2,32,8,128,12.25698471069336,2.5078125,8388608
+55,self_attn.v_proj,2,64,8,128,14.291569709777832,2.25390625,8388608
+55,self_attn.v_proj,2,128,8,128,16.122074127197266,2.126953125,8388608
+55,self_attn.v_proj,3,32,8,128,5.242306232452393,3.5078125,8388608
+55,self_attn.v_proj,3,64,8,128,6.115605354309082,3.25390625,8388608
+55,self_attn.v_proj,3,128,8,128,6.905040740966797,3.126953125,8388608
+55,self_attn.v_proj,4,32,8,128,2.4446990489959717,4.5078125,8388608
+55,self_attn.v_proj,4,64,8,128,2.8541407585144043,4.25390625,8388608
+55,self_attn.v_proj,4,128,8,128,3.2210497856140137,4.126953125,8388608
+55,self_attn.v_proj,8,32,8,128,0.14801062643527985,8.5078125,8388608
+55,self_attn.v_proj,8,64,8,128,0.17261441051959991,8.25390625,8388608
+55,self_attn.v_proj,8,128,8,128,0.19415798783302307,8.126953125,8388608
+55,self_attn.k_proj,2,32,8,128,29.758623123168945,2.5078125,8388608
+55,self_attn.k_proj,2,64,8,128,35.6055793762207,2.25390625,8388608
+55,self_attn.k_proj,2,128,8,128,40.53006362915039,2.126953125,8388608
+55,self_attn.k_proj,3,32,8,128,12.843358993530273,3.5078125,8388608
+55,self_attn.k_proj,3,64,8,128,15.521919250488281,3.25390625,8388608
+55,self_attn.k_proj,3,128,8,128,18.01385498046875,3.126953125,8388608
+55,self_attn.k_proj,4,32,8,128,5.992343902587891,4.5078125,8388608
+55,self_attn.k_proj,4,64,8,128,7.245814323425293,4.25390625,8388608
+55,self_attn.k_proj,4,128,8,128,8.414450645446777,4.126953125,8388608
+55,self_attn.k_proj,8,32,8,128,0.3628051280975342,8.5078125,8388608
+55,self_attn.k_proj,8,64,8,128,0.4380407929420471,8.25390625,8388608
+55,self_attn.k_proj,8,128,8,128,0.5070092082023621,8.126953125,8388608
+55,self_attn.o_proj,2,32,8,128,33.59809875488281,20.0625,67108864
+55,self_attn.o_proj,2,64,8,128,39.28446578979492,18.03125,67108864
+55,self_attn.o_proj,2,128,8,128,44.58836364746094,17.015625,67108864
+55,self_attn.o_proj,3,32,8,128,14.401481628417969,28.0625,67108864
+55,self_attn.o_proj,3,64,8,128,16.90453338623047,26.03125,67108864
+55,self_attn.o_proj,3,128,8,128,19.28582000732422,25.015625,67108864
+55,self_attn.o_proj,4,32,8,128,6.732936859130859,36.0625,67108864
+55,self_attn.o_proj,4,64,8,128,7.922384738922119,34.03125,67108864
+55,self_attn.o_proj,4,128,8,128,9.05025577545166,33.015625,67108864
+55,self_attn.o_proj,8,32,8,128,0.41167038679122925,68.0625,67108864
+55,self_attn.o_proj,8,64,8,128,0.4851287305355072,66.03125,67108864
+55,self_attn.o_proj,8,128,8,128,0.5561374425888062,65.015625,67108864
+55,mlp.gate_proj,2,32,8,128,74.17977142333984,70.21875,234881024
+55,mlp.gate_proj,2,64,8,128,87.46391296386719,63.109375,234881024
+55,mlp.gate_proj,2,128,8,128,99.31480407714844,59.5546875,234881024
+55,mlp.gate_proj,3,32,8,128,31.920177459716797,98.21875,234881024
+55,mlp.gate_proj,3,64,8,128,37.32277297973633,91.109375,234881024
+55,mlp.gate_proj,3,128,8,128,42.70172119140625,87.5546875,234881024
+55,mlp.gate_proj,4,32,8,128,14.850040435791016,126.21875,234881024
+55,mlp.gate_proj,4,64,8,128,17.423938751220703,119.109375,234881024
+55,mlp.gate_proj,4,128,8,128,19.8994083404541,115.5546875,234881024
+55,mlp.gate_proj,8,32,8,128,0.9064744710922241,238.21875,234881024
+55,mlp.gate_proj,8,64,8,128,1.0645235776901245,231.109375,234881024
+55,mlp.gate_proj,8,128,8,128,1.2100826501846313,227.5546875,234881024
+55,mlp.up_proj,2,32,8,128,67.44631958007812,70.21875,234881024
+55,mlp.up_proj,2,64,8,128,78.46819305419922,63.109375,234881024
+55,mlp.up_proj,2,128,8,128,88.6911392211914,59.5546875,234881024
+55,mlp.up_proj,3,32,8,128,28.850303649902344,98.21875,234881024
+55,mlp.up_proj,3,64,8,128,33.604095458984375,91.109375,234881024
+55,mlp.up_proj,3,128,8,128,37.83155822753906,87.5546875,234881024
+55,mlp.up_proj,4,32,8,128,13.43616008758545,126.21875,234881024
+55,mlp.up_proj,4,64,8,128,15.72747802734375,119.109375,234881024
+55,mlp.up_proj,4,128,8,128,17.6661434173584,115.5546875,234881024
+55,mlp.up_proj,8,32,8,128,0.8204723596572876,238.21875,234881024
+55,mlp.up_proj,8,64,8,128,0.95916748046875,231.109375,234881024
+55,mlp.up_proj,8,128,8,128,1.0752958059310913,227.5546875,234881024
+55,mlp.down_proj,2,32,8,128,67.58561706542969,70.21875,234881024
+55,mlp.down_proj,2,64,8,128,78.6537857055664,63.109375,234881024
+55,mlp.down_proj,2,128,8,128,88.8930435180664,59.5546875,234881024
+55,mlp.down_proj,3,32,8,128,28.914976119995117,98.21875,234881024
+55,mlp.down_proj,3,64,8,128,33.67095947265625,91.109375,234881024
+55,mlp.down_proj,3,128,8,128,37.903297424316406,87.5546875,234881024
+55,mlp.down_proj,4,32,8,128,13.464611053466797,126.21875,234881024
+55,mlp.down_proj,4,64,8,128,15.759300231933594,119.109375,234881024
+55,mlp.down_proj,4,128,8,128,17.69732093811035,115.5546875,234881024
+55,mlp.down_proj,8,32,8,128,0.8221872448921204,238.21875,234881024
+55,mlp.down_proj,8,64,8,128,0.9613041877746582,231.109375,234881024
+55,mlp.down_proj,8,128,8,128,1.077470302581787,227.5546875,234881024
+56,self_attn.q_proj,2,32,8,128,43.87620162963867,20.0625,67108864
+56,self_attn.q_proj,2,64,8,128,51.49464797973633,18.03125,67108864
+56,self_attn.q_proj,2,128,8,128,59.497013092041016,17.015625,67108864
+56,self_attn.q_proj,3,32,8,128,18.763473510742188,28.0625,67108864
+56,self_attn.q_proj,3,64,8,128,22.10165786743164,26.03125,67108864
+56,self_attn.q_proj,3,128,8,128,25.578298568725586,25.015625,67108864
+56,self_attn.q_proj,4,32,8,128,8.755913734436035,36.0625,67108864
+56,self_attn.q_proj,4,64,8,128,10.309966087341309,34.03125,67108864
+56,self_attn.q_proj,4,128,8,128,11.942545890808105,33.015625,67108864
+56,self_attn.q_proj,8,32,8,128,0.5317762494087219,68.0625,67108864
+56,self_attn.q_proj,8,64,8,128,0.6261802315711975,66.03125,67108864
+56,self_attn.q_proj,8,128,8,128,0.7233961224555969,65.015625,67108864
+56,self_attn.v_proj,2,32,8,128,11.798242568969727,2.5078125,8388608
+56,self_attn.v_proj,2,64,8,128,13.733071327209473,2.25390625,8388608
+56,self_attn.v_proj,2,128,8,128,15.473725318908691,2.126953125,8388608
+56,self_attn.v_proj,3,32,8,128,5.04462194442749,3.5078125,8388608
+56,self_attn.v_proj,3,64,8,128,5.874096870422363,3.25390625,8388608
+56,self_attn.v_proj,3,128,8,128,6.619505882263184,3.126953125,8388608
+56,self_attn.v_proj,4,32,8,128,2.353705883026123,4.5078125,8388608
+56,self_attn.v_proj,4,64,8,128,2.741083860397339,4.25390625,8388608
+56,self_attn.v_proj,4,128,8,128,3.0899264812469482,4.126953125,8388608
+56,self_attn.v_proj,8,32,8,128,0.14255090057849884,8.5078125,8388608
+56,self_attn.v_proj,8,64,8,128,0.16581346094608307,8.25390625,8388608
+56,self_attn.v_proj,8,128,8,128,0.18623343110084534,8.126953125,8388608
+56,self_attn.k_proj,2,32,8,128,31.692384719848633,2.5078125,8388608
+56,self_attn.k_proj,2,64,8,128,37.82472610473633,2.25390625,8388608
+56,self_attn.k_proj,2,128,8,128,43.2342643737793,2.126953125,8388608
+56,self_attn.k_proj,3,32,8,128,13.622679710388184,3.5078125,8388608
+56,self_attn.k_proj,3,64,8,128,16.333524703979492,3.25390625,8388608
+56,self_attn.k_proj,3,128,8,128,18.898395538330078,3.126953125,8388608
+56,self_attn.k_proj,4,32,8,128,6.356379985809326,4.5078125,8388608
+56,self_attn.k_proj,4,64,8,128,7.6229987144470215,4.25390625,8388608
+56,self_attn.k_proj,4,128,8,128,8.82217025756836,4.126953125,8388608
+56,self_attn.k_proj,8,32,8,128,0.3848208785057068,8.5078125,8388608
+56,self_attn.k_proj,8,64,8,128,0.4608488380908966,8.25390625,8388608
+56,self_attn.k_proj,8,128,8,128,0.5312046408653259,8.126953125,8388608
+56,self_attn.o_proj,2,32,8,128,33.59645080566406,20.0625,67108864
+56,self_attn.o_proj,2,64,8,128,39.10022735595703,18.03125,67108864
+56,self_attn.o_proj,2,128,8,128,44.05947494506836,17.015625,67108864
+56,self_attn.o_proj,3,32,8,128,14.362443923950195,28.0625,67108864
+56,self_attn.o_proj,3,64,8,128,16.72585678100586,26.03125,67108864
+56,self_attn.o_proj,3,128,8,128,18.85149383544922,25.015625,67108864
+56,self_attn.o_proj,4,32,8,128,6.698688983917236,36.0625,67108864
+56,self_attn.o_proj,4,64,8,128,7.808315753936768,34.03125,67108864
+56,self_attn.o_proj,4,128,8,128,8.791555404663086,33.015625,67108864
+56,self_attn.o_proj,8,32,8,128,0.40719354152679443,68.0625,67108864
+56,self_attn.o_proj,8,64,8,128,0.4736632704734802,66.03125,67108864
+56,self_attn.o_proj,8,128,8,128,0.5317232608795166,65.015625,67108864
+56,mlp.gate_proj,2,32,8,128,74.60127258300781,70.21875,234881024
+56,mlp.gate_proj,2,64,8,128,88.06282043457031,63.109375,234881024
+56,mlp.gate_proj,2,128,8,128,100.0769271850586,59.5546875,234881024
+56,mlp.gate_proj,3,32,8,128,32.09475326538086,98.21875,234881024
+56,mlp.gate_proj,3,64,8,128,37.575721740722656,91.109375,234881024
+56,mlp.gate_proj,3,128,8,128,43.06573486328125,87.5546875,234881024
+56,mlp.gate_proj,4,32,8,128,14.93848991394043,126.21875,234881024
+56,mlp.gate_proj,4,64,8,128,17.5435733795166,119.109375,234881024
+56,mlp.gate_proj,4,128,8,128,20.07000160217285,115.5546875,234881024
+56,mlp.gate_proj,8,32,8,128,0.9118632078170776,238.21875,234881024
+56,mlp.gate_proj,8,64,8,128,1.0714000463485718,231.109375,234881024
+56,mlp.gate_proj,8,128,8,128,1.2199900150299072,227.5546875,234881024
+56,mlp.up_proj,2,32,8,128,67.37001037597656,70.21875,234881024
+56,mlp.up_proj,2,64,8,128,78.38288116455078,63.109375,234881024
+56,mlp.up_proj,2,128,8,128,88.5545883178711,59.5546875,234881024
+56,mlp.up_proj,3,32,8,128,28.814105987548828,98.21875,234881024
+56,mlp.up_proj,3,64,8,128,33.56886291503906,91.109375,234881024
+56,mlp.up_proj,3,128,8,128,37.77330017089844,87.5546875,234881024
+56,mlp.up_proj,4,32,8,128,13.41838550567627,126.21875,234881024
+56,mlp.up_proj,4,64,8,128,15.70688247680664,119.109375,234881024
+56,mlp.up_proj,4,128,8,128,17.653900146484375,115.5546875,234881024
+56,mlp.up_proj,8,32,8,128,0.8194065690040588,238.21875,234881024
+56,mlp.up_proj,8,64,8,128,0.9579480886459351,231.109375,234881024
+56,mlp.up_proj,8,128,8,128,1.074022889137268,227.5546875,234881024
+56,mlp.down_proj,2,32,8,128,67.49398040771484,70.21875,234881024
+56,mlp.down_proj,2,64,8,128,78.52323150634766,63.109375,234881024
+56,mlp.down_proj,2,128,8,128,88.75371551513672,59.5546875,234881024
+56,mlp.down_proj,3,32,8,128,28.872196197509766,98.21875,234881024
+56,mlp.down_proj,3,64,8,128,33.622589111328125,91.109375,234881024
+56,mlp.down_proj,3,128,8,128,37.82685470581055,87.5546875,234881024
+56,mlp.down_proj,4,32,8,128,13.44374942779541,126.21875,234881024
+56,mlp.down_proj,4,64,8,128,15.737311363220215,119.109375,234881024
+56,mlp.down_proj,4,128,8,128,17.675691604614258,115.5546875,234881024
+56,mlp.down_proj,8,32,8,128,0.821048378944397,238.21875,234881024
+56,mlp.down_proj,8,64,8,128,0.9598550200462341,231.109375,234881024
+56,mlp.down_proj,8,128,8,128,1.0757743120193481,227.5546875,234881024
+57,self_attn.q_proj,2,32,8,128,42.85846710205078,20.0625,67108864
+57,self_attn.q_proj,2,64,8,128,50.661319732666016,18.03125,67108864
+57,self_attn.q_proj,2,128,8,128,58.45608139038086,17.015625,67108864
+57,self_attn.q_proj,3,32,8,128,18.349687576293945,28.0625,67108864
+57,self_attn.q_proj,3,64,8,128,21.79133415222168,26.03125,67108864
+57,self_attn.q_proj,3,128,8,128,25.2443790435791,25.015625,67108864
+57,self_attn.q_proj,4,32,8,128,8.564016342163086,36.0625,67108864
+57,self_attn.q_proj,4,64,8,128,10.1651029586792,34.03125,67108864
+57,self_attn.q_proj,4,128,8,128,11.79159927368164,33.015625,67108864
+57,self_attn.q_proj,8,32,8,128,0.5200287103652954,68.0625,67108864
+57,self_attn.q_proj,8,64,8,128,0.6173112988471985,66.03125,67108864
+57,self_attn.q_proj,8,128,8,128,0.7143703699111938,65.015625,67108864
+57,self_attn.v_proj,2,32,8,128,12.989782333374023,2.5078125,8388608
+57,self_attn.v_proj,2,64,8,128,15.148744583129883,2.25390625,8388608
+57,self_attn.v_proj,2,128,8,128,17.101806640625,2.126953125,8388608
+57,self_attn.v_proj,3,32,8,128,5.552268981933594,3.5078125,8388608
+57,self_attn.v_proj,3,64,8,128,6.479983329772949,3.25390625,8388608
+57,self_attn.v_proj,3,128,8,128,7.320120811462402,3.126953125,8388608
+57,self_attn.v_proj,4,32,8,128,2.590792655944824,4.5078125,8388608
+57,self_attn.v_proj,4,64,8,128,3.0236148834228516,4.25390625,8388608
+57,self_attn.v_proj,4,128,8,128,3.4170737266540527,4.126953125,8388608
+57,self_attn.v_proj,8,32,8,128,0.15686902403831482,8.5078125,8388608
+57,self_attn.v_proj,8,64,8,128,0.1828427016735077,8.25390625,8388608
+57,self_attn.v_proj,8,128,8,128,0.20573510229587555,8.126953125,8388608
+57,self_attn.k_proj,2,32,8,128,32.45452880859375,2.5078125,8388608
+57,self_attn.k_proj,2,64,8,128,38.78204345703125,2.25390625,8388608
+57,self_attn.k_proj,2,128,8,128,44.17359161376953,2.126953125,8388608
+57,self_attn.k_proj,3,32,8,128,14.004440307617188,3.5078125,8388608
+57,self_attn.k_proj,3,64,8,128,16.883586883544922,3.25390625,8388608
+57,self_attn.k_proj,3,128,8,128,19.57182502746582,3.126953125,8388608
+57,self_attn.k_proj,4,32,8,128,6.534045696258545,4.5078125,8388608
+57,self_attn.k_proj,4,64,8,128,7.88316535949707,4.25390625,8388608
+57,self_attn.k_proj,4,128,8,128,9.146933555603027,4.126953125,8388608
+57,self_attn.k_proj,8,32,8,128,0.3954992890357971,8.5078125,8388608
+57,self_attn.k_proj,8,64,8,128,0.47633540630340576,8.25390625,8388608
+57,self_attn.k_proj,8,128,8,128,0.5509874820709229,8.126953125,8388608
+57,self_attn.o_proj,2,32,8,128,34.21621322631836,20.0625,67108864
+57,self_attn.o_proj,2,64,8,128,39.82251739501953,18.03125,67108864
+57,self_attn.o_proj,2,128,8,128,44.89662170410156,17.015625,67108864
+57,self_attn.o_proj,3,32,8,128,14.631689071655273,28.0625,67108864
+57,self_attn.o_proj,3,64,8,128,17.03797721862793,26.03125,67108864
+57,self_attn.o_proj,3,128,8,128,19.20110511779785,25.015625,67108864
+57,self_attn.o_proj,4,32,8,128,6.824492454528809,36.0625,67108864
+57,self_attn.o_proj,4,64,8,128,7.955810070037842,34.03125,67108864
+57,self_attn.o_proj,4,128,8,128,8.959942817687988,33.015625,67108864
+57,self_attn.o_proj,8,32,8,128,0.4147461950778961,68.0625,67108864
+57,self_attn.o_proj,8,64,8,128,0.48245060443878174,66.03125,67108864
+57,self_attn.o_proj,8,128,8,128,0.542047917842865,65.015625,67108864
+57,mlp.gate_proj,2,32,8,128,74.74286651611328,70.21875,234881024
+57,mlp.gate_proj,2,64,8,128,88.30633544921875,63.109375,234881024
+57,mlp.gate_proj,2,128,8,128,100.43818664550781,59.5546875,234881024
+57,mlp.gate_proj,3,32,8,128,32.1463623046875,98.21875,234881024
+57,mlp.gate_proj,3,64,8,128,37.681671142578125,91.109375,234881024
+57,mlp.gate_proj,3,128,8,128,43.238258361816406,87.5546875,234881024
+57,mlp.gate_proj,4,32,8,128,14.967938423156738,126.21875,234881024
+57,mlp.gate_proj,4,64,8,128,17.5858154296875,119.109375,234881024
+57,mlp.gate_proj,4,128,8,128,20.142263412475586,115.5546875,234881024
+57,mlp.gate_proj,8,32,8,128,0.9136084318161011,238.21875,234881024
+57,mlp.gate_proj,8,64,8,128,1.074019432067871,231.109375,234881024
+57,mlp.gate_proj,8,128,8,128,1.224131464958191,227.5546875,234881024
+57,mlp.up_proj,2,32,8,128,67.38514709472656,70.21875,234881024
+57,mlp.up_proj,2,64,8,128,78.40018463134766,63.109375,234881024
+57,mlp.up_proj,2,128,8,128,88.57624816894531,59.5546875,234881024
+57,mlp.up_proj,3,32,8,128,28.825239181518555,98.21875,234881024
+57,mlp.up_proj,3,64,8,128,33.57402038574219,91.109375,234881024
+57,mlp.up_proj,3,128,8,128,37.780601501464844,87.5546875,234881024
+57,mlp.up_proj,4,32,8,128,13.4228515625,126.21875,234881024
+57,mlp.up_proj,4,64,8,128,15.711897850036621,119.109375,234881024
+57,mlp.up_proj,4,128,8,128,17.646757125854492,115.5546875,234881024
+57,mlp.up_proj,8,32,8,128,0.8195822834968567,238.21875,234881024
+57,mlp.up_proj,8,64,8,128,0.9581327438354492,231.109375,234881024
+57,mlp.up_proj,8,128,8,128,1.0740714073181152,227.5546875,234881024
+57,mlp.down_proj,2,32,8,128,67.48255920410156,70.21875,234881024
+57,mlp.down_proj,2,64,8,128,78.53009796142578,63.109375,234881024
+57,mlp.down_proj,2,128,8,128,88.72081756591797,59.5546875,234881024
+57,mlp.down_proj,3,32,8,128,28.867464065551758,98.21875,234881024
+57,mlp.down_proj,3,64,8,128,33.62115478515625,91.109375,234881024
+57,mlp.down_proj,3,128,8,128,37.82460403442383,87.5546875,234881024
+57,mlp.down_proj,4,32,8,128,13.44274616241455,126.21875,234881024
+57,mlp.down_proj,4,64,8,128,15.734841346740723,119.109375,234881024
+57,mlp.down_proj,4,128,8,128,17.67094612121582,115.5546875,234881024
+57,mlp.down_proj,8,32,8,128,0.8208925127983093,238.21875,234881024
+57,mlp.down_proj,8,64,8,128,0.9596756100654602,231.109375,234881024
+57,mlp.down_proj,8,128,8,128,1.075747013092041,227.5546875,234881024
+58,self_attn.q_proj,2,32,8,128,39.425498962402344,20.0625,67108864
+58,self_attn.q_proj,2,64,8,128,46.638526916503906,18.03125,67108864
+58,self_attn.q_proj,2,128,8,128,53.92063903808594,17.015625,67108864
+58,self_attn.q_proj,3,32,8,128,16.92674446105957,28.0625,67108864
+58,self_attn.q_proj,3,64,8,128,20.119474411010742,26.03125,67108864
+58,self_attn.q_proj,3,128,8,128,23.494348526000977,25.015625,67108864
+58,self_attn.q_proj,4,32,8,128,7.903947353363037,36.0625,67108864
+58,self_attn.q_proj,4,64,8,128,9.38468074798584,34.03125,67108864
+58,self_attn.q_proj,4,128,8,128,10.975442886352539,33.015625,67108864
+58,self_attn.q_proj,8,32,8,128,0.4798099100589752,68.0625,67108864
+58,self_attn.q_proj,8,64,8,128,0.5698453187942505,66.03125,67108864
+58,self_attn.q_proj,8,128,8,128,0.6641814708709717,65.015625,67108864
+58,self_attn.v_proj,2,32,8,128,13.700727462768555,2.5078125,8388608
+58,self_attn.v_proj,2,64,8,128,16.066253662109375,2.25390625,8388608
+58,self_attn.v_proj,2,128,8,128,18.25057601928711,2.126953125,8388608
+58,self_attn.v_proj,3,32,8,128,5.859604358673096,3.5078125,8388608
+58,self_attn.v_proj,3,64,8,128,6.876149654388428,3.25390625,8388608
+58,self_attn.v_proj,3,128,8,128,7.816343307495117,3.126953125,8388608
+58,self_attn.v_proj,4,32,8,128,2.734663486480713,4.5078125,8388608
+58,self_attn.v_proj,4,64,8,128,3.2078359127044678,4.25390625,8388608
+58,self_attn.v_proj,4,128,8,128,3.6476597785949707,4.126953125,8388608
+58,self_attn.v_proj,8,32,8,128,0.1655382513999939,8.5078125,8388608
+58,self_attn.v_proj,8,64,8,128,0.1940414160490036,8.25390625,8388608
+58,self_attn.v_proj,8,128,8,128,0.21981889009475708,8.126953125,8388608
+58,self_attn.k_proj,2,32,8,128,27.857711791992188,2.5078125,8388608
+58,self_attn.k_proj,2,64,8,128,33.545352935791016,2.25390625,8388608
+58,self_attn.k_proj,2,128,8,128,38.456058502197266,2.126953125,8388608
+58,self_attn.k_proj,3,32,8,128,12.099510192871094,3.5078125,8388608
+58,self_attn.k_proj,3,64,8,128,14.784231185913086,3.25390625,8388608
+58,self_attn.k_proj,3,128,8,128,17.411245346069336,3.126953125,8388608
+58,self_attn.k_proj,4,32,8,128,5.648931503295898,4.5078125,8388608
+58,self_attn.k_proj,4,64,8,128,6.9079155921936035,4.25390625,8388608
+58,self_attn.k_proj,4,128,8,128,8.141322135925293,4.126953125,8388608
+58,self_attn.k_proj,8,32,8,128,0.3418181240558624,8.5078125,8388608
+58,self_attn.k_proj,8,64,8,128,0.41769900918006897,8.25390625,8388608
+58,self_attn.k_proj,8,128,8,128,0.49038809537887573,8.126953125,8388608
+58,self_attn.o_proj,2,32,8,128,34.40038299560547,20.0625,67108864
+58,self_attn.o_proj,2,64,8,128,40.037254333496094,18.03125,67108864
+58,self_attn.o_proj,2,128,8,128,45.129390716552734,17.015625,67108864
+58,self_attn.o_proj,3,32,8,128,14.706823348999023,28.0625,67108864
+58,self_attn.o_proj,3,64,8,128,17.12470245361328,26.03125,67108864
+58,self_attn.o_proj,3,128,8,128,19.306352615356445,25.015625,67108864
+58,self_attn.o_proj,4,32,8,128,6.859638690948486,36.0625,67108864
+58,self_attn.o_proj,4,64,8,128,7.997882843017578,34.03125,67108864
+58,self_attn.o_proj,4,128,8,128,9.007405281066895,33.015625,67108864
+58,self_attn.o_proj,8,32,8,128,0.41692495346069336,68.0625,67108864
+58,self_attn.o_proj,8,64,8,128,0.48510682582855225,66.03125,67108864
+58,self_attn.o_proj,8,128,8,128,0.5449389219284058,65.015625,67108864
+58,mlp.gate_proj,2,32,8,128,74.34040832519531,70.21875,234881024
+58,mlp.gate_proj,2,64,8,128,87.72673034667969,63.109375,234881024
+58,mlp.gate_proj,2,128,8,128,99.71549987792969,59.5546875,234881024
+58,mlp.gate_proj,3,32,8,128,31.99078941345215,98.21875,234881024
+58,mlp.gate_proj,3,64,8,128,37.434879302978516,91.109375,234881024
+58,mlp.gate_proj,3,128,8,128,42.868526458740234,87.5546875,234881024
+58,mlp.gate_proj,4,32,8,128,14.88133716583252,126.21875,234881024
+58,mlp.gate_proj,4,64,8,128,17.4698429107666,119.109375,234881024
+58,mlp.gate_proj,4,128,8,128,19.980371475219727,115.5546875,234881024
+58,mlp.gate_proj,8,32,8,128,0.9083348512649536,238.21875,234881024
+58,mlp.gate_proj,8,64,8,128,1.0672800540924072,231.109375,234881024
+58,mlp.gate_proj,8,128,8,128,1.2145757675170898,227.5546875,234881024
+58,mlp.up_proj,2,32,8,128,67.59613037109375,70.21875,234881024
+58,mlp.up_proj,2,64,8,128,78.66363525390625,63.109375,234881024
+58,mlp.up_proj,2,128,8,128,88.86957550048828,59.5546875,234881024
+58,mlp.up_proj,3,32,8,128,28.920846939086914,98.21875,234881024
+58,mlp.up_proj,3,64,8,128,33.679744720458984,91.109375,234881024
+58,mlp.up_proj,3,128,8,128,37.902320861816406,87.5546875,234881024
+58,mlp.up_proj,4,32,8,128,13.468317031860352,126.21875,234881024
+58,mlp.up_proj,4,64,8,128,15.766176223754883,119.109375,234881024
+58,mlp.up_proj,4,128,8,128,17.70819091796875,115.5546875,234881024
+58,mlp.up_proj,8,32,8,128,0.8223936557769775,238.21875,234881024
+58,mlp.up_proj,8,64,8,128,0.9615448117256165,231.109375,234881024
+58,mlp.up_proj,8,128,8,128,1.0776385068893433,227.5546875,234881024
+58,mlp.down_proj,2,32,8,128,67.65601348876953,70.21875,234881024
+58,mlp.down_proj,2,64,8,128,78.73197937011719,63.109375,234881024
+58,mlp.down_proj,2,128,8,128,88.93867492675781,59.5546875,234881024
+58,mlp.down_proj,3,32,8,128,28.947364807128906,98.21875,234881024
+58,mlp.down_proj,3,64,8,128,33.699859619140625,91.109375,234881024
+58,mlp.down_proj,3,128,8,128,37.93180847167969,87.5546875,234881024
+58,mlp.down_proj,4,32,8,128,13.478936195373535,126.21875,234881024
+58,mlp.down_proj,4,64,8,128,15.77641487121582,119.109375,234881024
+58,mlp.down_proj,4,128,8,128,17.71137809753418,115.5546875,234881024
+58,mlp.down_proj,8,32,8,128,0.8231168985366821,238.21875,234881024
+58,mlp.down_proj,8,64,8,128,0.9622901678085327,231.109375,234881024
+58,mlp.down_proj,8,128,8,128,1.0783286094665527,227.5546875,234881024
+59,self_attn.q_proj,2,32,8,128,39.2701416015625,20.0625,67108864
+59,self_attn.q_proj,2,64,8,128,46.60835266113281,18.03125,67108864
+59,self_attn.q_proj,2,128,8,128,53.801673889160156,17.015625,67108864
+59,self_attn.q_proj,3,32,8,128,16.862812042236328,28.0625,67108864
+59,self_attn.q_proj,3,64,8,128,20.123401641845703,26.03125,67108864
+59,self_attn.q_proj,3,128,8,128,23.49970817565918,25.015625,67108864
+59,self_attn.q_proj,4,32,8,128,7.874965190887451,36.0625,67108864
+59,self_attn.q_proj,4,64,8,128,9.388465881347656,34.03125,67108864
+59,self_attn.q_proj,4,128,8,128,10.971816062927246,33.015625,67108864
+59,self_attn.q_proj,8,32,8,128,0.47788867354393005,68.0625,67108864
+59,self_attn.q_proj,8,64,8,128,0.5701588988304138,66.03125,67108864
+59,self_attn.q_proj,8,128,8,128,0.664284884929657,65.015625,67108864
+59,self_attn.v_proj,2,32,8,128,13.162670135498047,2.5078125,8388608
+59,self_attn.v_proj,2,64,8,128,15.354419708251953,2.25390625,8388608
+59,self_attn.v_proj,2,128,8,128,17.336923599243164,2.126953125,8388608
+59,self_attn.v_proj,3,32,8,128,5.626540184020996,3.5078125,8388608
+59,self_attn.v_proj,3,64,8,128,6.567514419555664,3.25390625,8388608
+59,self_attn.v_proj,3,128,8,128,7.416077136993408,3.126953125,8388608
+59,self_attn.v_proj,4,32,8,128,2.6266589164733887,4.5078125,8388608
+59,self_attn.v_proj,4,64,8,128,3.0655221939086914,4.25390625,8388608
+59,self_attn.v_proj,4,128,8,128,3.461200714111328,4.126953125,8388608
+59,self_attn.v_proj,8,32,8,128,0.15902318060398102,8.5078125,8388608
+59,self_attn.v_proj,8,64,8,128,0.18533480167388916,8.25390625,8388608
+59,self_attn.v_proj,8,128,8,128,0.20852121710777283,8.126953125,8388608
+59,self_attn.k_proj,2,32,8,128,29.401058197021484,2.5078125,8388608
+59,self_attn.k_proj,2,64,8,128,35.32689666748047,2.25390625,8388608
+59,self_attn.k_proj,2,128,8,128,40.28376388549805,2.126953125,8388608
+59,self_attn.k_proj,3,32,8,128,12.754972457885742,3.5078125,8388608
+59,self_attn.k_proj,3,64,8,128,15.559125900268555,3.25390625,8388608
+59,self_attn.k_proj,3,128,8,128,18.190214157104492,3.126953125,8388608
+59,self_attn.k_proj,4,32,8,128,5.95247220993042,4.5078125,8388608
+59,self_attn.k_proj,4,64,8,128,7.2620439529418945,4.25390625,8388608
+59,self_attn.k_proj,4,128,8,128,8.50261402130127,4.126953125,8388608
+59,self_attn.k_proj,8,32,8,128,0.36017483472824097,8.5078125,8388608
+59,self_attn.k_proj,8,64,8,128,0.43894389271736145,8.25390625,8388608
+59,self_attn.k_proj,8,128,8,128,0.5124502778053284,8.126953125,8388608
+59,self_attn.o_proj,2,32,8,128,33.934566497802734,20.0625,67108864
+59,self_attn.o_proj,2,64,8,128,39.49851989746094,18.03125,67108864
+59,self_attn.o_proj,2,128,8,128,44.503089904785156,17.015625,67108864
+59,self_attn.o_proj,3,32,8,128,14.509501457214355,28.0625,67108864
+59,self_attn.o_proj,3,64,8,128,16.895349502563477,26.03125,67108864
+59,self_attn.o_proj,3,128,8,128,19.04216194152832,25.015625,67108864
+59,self_attn.o_proj,4,32,8,128,6.766356945037842,36.0625,67108864
+59,self_attn.o_proj,4,64,8,128,7.888190746307373,34.03125,67108864
+59,self_attn.o_proj,4,128,8,128,8.885581016540527,33.015625,67108864
+59,self_attn.o_proj,8,32,8,128,0.4113270342350006,68.0625,67108864
+59,self_attn.o_proj,8,64,8,128,0.47855424880981445,66.03125,67108864
+59,self_attn.o_proj,8,128,8,128,0.5374066829681396,65.015625,67108864
+59,mlp.gate_proj,2,32,8,128,75.21849822998047,70.21875,234881024
+59,mlp.gate_proj,2,64,8,128,89.12264251708984,63.109375,234881024
+59,mlp.gate_proj,2,128,8,128,101.66844177246094,59.5546875,234881024
+59,mlp.gate_proj,3,32,8,128,32.334251403808594,98.21875,234881024
+59,mlp.gate_proj,3,64,8,128,38.05775833129883,91.109375,234881024
+59,mlp.gate_proj,3,128,8,128,43.84445571899414,87.5546875,234881024
+59,mlp.gate_proj,4,32,8,128,15.069782257080078,126.21875,234881024
+59,mlp.gate_proj,4,64,8,128,17.75200653076172,119.109375,234881024
+59,mlp.gate_proj,4,128,8,128,20.42919921875,115.5546875,234881024
+59,mlp.gate_proj,8,32,8,128,0.919658899307251,238.21875,234881024
+59,mlp.gate_proj,8,64,8,128,1.0839327573776245,231.109375,234881024
+59,mlp.gate_proj,8,128,8,128,1.2414761781692505,227.5546875,234881024
+59,mlp.up_proj,2,32,8,128,67.4483642578125,70.21875,234881024
+59,mlp.up_proj,2,64,8,128,78.48065185546875,63.109375,234881024
+59,mlp.up_proj,2,128,8,128,88.66421508789062,59.5546875,234881024
+59,mlp.up_proj,3,32,8,128,28.852825164794922,98.21875,234881024
+59,mlp.up_proj,3,64,8,128,33.60743713378906,91.109375,234881024
+59,mlp.up_proj,3,128,8,128,37.83370590209961,87.5546875,234881024
+59,mlp.up_proj,4,32,8,128,13.435503959655762,126.21875,234881024
+59,mlp.up_proj,4,64,8,128,15.728187561035156,119.109375,234881024
+59,mlp.up_proj,4,128,8,128,17.667020797729492,115.5546875,234881024
+59,mlp.up_proj,8,32,8,128,0.8205093145370483,238.21875,234881024
+59,mlp.up_proj,8,64,8,128,0.9592403769493103,231.109375,234881024
+59,mlp.up_proj,8,128,8,128,1.0754485130310059,227.5546875,234881024
+59,mlp.down_proj,2,32,8,128,67.46237182617188,70.21875,234881024
+59,mlp.down_proj,2,64,8,128,78.49662780761719,63.109375,234881024
+59,mlp.down_proj,2,128,8,128,88.70198822021484,59.5546875,234881024
+59,mlp.down_proj,3,32,8,128,28.859241485595703,98.21875,234881024
+59,mlp.down_proj,3,64,8,128,33.6164436340332,91.109375,234881024
+59,mlp.down_proj,3,128,8,128,37.833213806152344,87.5546875,234881024
+59,mlp.down_proj,4,32,8,128,13.439879417419434,126.21875,234881024
+59,mlp.down_proj,4,64,8,128,15.7325439453125,119.109375,234881024
+59,mlp.down_proj,4,128,8,128,17.66667366027832,115.5546875,234881024
+59,mlp.down_proj,8,32,8,128,0.8207367062568665,238.21875,234881024
+59,mlp.down_proj,8,64,8,128,0.9596270322799683,231.109375,234881024
+59,mlp.down_proj,8,128,8,128,1.0756505727767944,227.5546875,234881024
+60,self_attn.q_proj,2,32,8,128,43.715328216552734,20.0625,67108864
+60,self_attn.q_proj,2,64,8,128,51.39675521850586,18.03125,67108864
+60,self_attn.q_proj,2,128,8,128,59.365238189697266,17.015625,67108864
+60,self_attn.q_proj,3,32,8,128,18.701522827148438,28.0625,67108864
+60,self_attn.q_proj,3,64,8,128,22.0684871673584,26.03125,67108864
+60,self_attn.q_proj,3,128,8,128,25.546777725219727,25.015625,67108864
+60,self_attn.q_proj,4,32,8,128,8.727008819580078,36.0625,67108864
+60,self_attn.q_proj,4,64,8,128,10.298078536987305,34.03125,67108864
+60,self_attn.q_proj,4,128,8,128,11.92385196685791,33.015625,67108864
+60,self_attn.q_proj,8,32,8,128,0.5299493670463562,68.0625,67108864
+60,self_attn.q_proj,8,64,8,128,0.6253095269203186,66.03125,67108864
+60,self_attn.q_proj,8,128,8,128,0.7224632501602173,65.015625,67108864
+60,self_attn.v_proj,2,32,8,128,13.14790153503418,2.5078125,8388608
+60,self_attn.v_proj,2,64,8,128,15.346366882324219,2.25390625,8388608
+60,self_attn.v_proj,2,128,8,128,17.33112144470215,2.126953125,8388608
+60,self_attn.v_proj,3,32,8,128,5.621884346008301,3.5078125,8388608
+60,self_attn.v_proj,3,64,8,128,6.564789295196533,3.25390625,8388608
+60,self_attn.v_proj,3,128,8,128,7.423012733459473,3.126953125,8388608
+60,self_attn.v_proj,4,32,8,128,2.623168468475342,4.5078125,8388608
+60,self_attn.v_proj,4,64,8,128,3.063364267349243,4.25390625,8388608
+60,self_attn.v_proj,4,128,8,128,3.463324785232544,4.126953125,8388608
+60,self_attn.v_proj,8,32,8,128,0.15880005061626434,8.5078125,8388608
+60,self_attn.v_proj,8,64,8,128,0.18526054918766022,8.25390625,8388608
+60,self_attn.v_proj,8,128,8,128,0.2086431235074997,8.126953125,8388608
+60,self_attn.k_proj,2,32,8,128,31.66526985168457,2.5078125,8388608
+60,self_attn.k_proj,2,64,8,128,37.82787322998047,2.25390625,8388608
+60,self_attn.k_proj,2,128,8,128,43.2412109375,2.126953125,8388608
+60,self_attn.k_proj,3,32,8,128,13.603410720825195,3.5078125,8388608
+60,self_attn.k_proj,3,64,8,128,16.300561904907227,3.25390625,8388608
+60,self_attn.k_proj,3,128,8,128,18.84320831298828,3.126953125,8388608
+60,self_attn.k_proj,4,32,8,128,6.346245765686035,4.5078125,8388608
+60,self_attn.k_proj,4,64,8,128,7.609442234039307,4.25390625,8388608
+60,self_attn.k_proj,4,128,8,128,8.794570922851562,4.126953125,8388608
+60,self_attn.k_proj,8,32,8,128,0.3839884102344513,8.5078125,8388608
+60,self_attn.k_proj,8,64,8,128,0.459894061088562,8.25390625,8388608
+60,self_attn.k_proj,8,128,8,128,0.529900848865509,8.126953125,8388608
+60,self_attn.o_proj,2,32,8,128,34.073272705078125,20.0625,67108864
+60,self_attn.o_proj,2,64,8,128,39.67479705810547,18.03125,67108864
+60,self_attn.o_proj,2,128,8,128,44.74182891845703,17.015625,67108864
+60,self_attn.o_proj,3,32,8,128,14.570452690124512,28.0625,67108864
+60,self_attn.o_proj,3,64,8,128,16.978300094604492,26.03125,67108864
+60,self_attn.o_proj,3,128,8,128,19.14500617980957,25.015625,67108864
+60,self_attn.o_proj,4,32,8,128,6.795092582702637,36.0625,67108864
+60,self_attn.o_proj,4,64,8,128,7.926553726196289,34.03125,67108864
+60,self_attn.o_proj,4,128,8,128,8.936878204345703,33.015625,67108864
+60,self_attn.o_proj,8,32,8,128,0.4130799472332001,68.0625,67108864
+60,self_attn.o_proj,8,64,8,128,0.4807862341403961,66.03125,67108864
+60,self_attn.o_proj,8,128,8,128,0.5403887629508972,65.015625,67108864
+60,mlp.gate_proj,2,32,8,128,75.76948547363281,70.21875,234881024
+60,mlp.gate_proj,2,64,8,128,89.9057846069336,63.109375,234881024
+60,mlp.gate_proj,2,128,8,128,102.66978454589844,59.5546875,234881024
+60,mlp.gate_proj,3,32,8,128,32.54499816894531,98.21875,234881024
+60,mlp.gate_proj,3,64,8,128,38.37843322753906,91.109375,234881024
+60,mlp.gate_proj,3,128,8,128,44.304264068603516,87.5546875,234881024
+60,mlp.gate_proj,4,32,8,128,15.184892654418945,126.21875,234881024
+60,mlp.gate_proj,4,64,8,128,17.90265655517578,119.109375,234881024
+60,mlp.gate_proj,4,128,8,128,20.639257431030273,115.5546875,234881024
+60,mlp.gate_proj,8,32,8,128,0.9265992641448975,238.21875,234881024
+60,mlp.gate_proj,8,64,8,128,1.0927612781524658,231.109375,234881024
+60,mlp.gate_proj,8,128,8,128,1.2539386749267578,227.5546875,234881024
+60,mlp.up_proj,2,32,8,128,67.33573913574219,70.21875,234881024
+60,mlp.up_proj,2,64,8,128,78.33284759521484,63.109375,234881024
+60,mlp.up_proj,2,128,8,128,88.51492309570312,59.5546875,234881024
+60,mlp.up_proj,3,32,8,128,28.801116943359375,98.21875,234881024
+60,mlp.up_proj,3,64,8,128,33.54736328125,91.109375,234881024
+60,mlp.up_proj,3,128,8,128,37.75339126586914,87.5546875,234881024
+60,mlp.up_proj,4,32,8,128,13.412123680114746,126.21875,234881024
+60,mlp.up_proj,4,64,8,128,15.69827651977539,119.109375,234881024
+60,mlp.up_proj,4,128,8,128,17.63593864440918,115.5546875,234881024
+60,mlp.up_proj,8,32,8,128,0.8189941048622131,238.21875,234881024
+60,mlp.up_proj,8,64,8,128,0.9573838710784912,231.109375,234881024
+60,mlp.up_proj,8,128,8,128,1.073511004447937,227.5546875,234881024
+60,mlp.down_proj,2,32,8,128,67.33570861816406,70.21875,234881024
+60,mlp.down_proj,2,64,8,128,78.32613372802734,63.109375,234881024
+60,mlp.down_proj,2,128,8,128,88.51353454589844,59.5546875,234881024
+60,mlp.down_proj,3,32,8,128,28.79682159423828,98.21875,234881024
+60,mlp.down_proj,3,64,8,128,33.543949127197266,91.109375,234881024
+60,mlp.down_proj,3,128,8,128,37.744049072265625,87.5546875,234881024
+60,mlp.down_proj,4,32,8,128,13.411508560180664,126.21875,234881024
+60,mlp.down_proj,4,64,8,128,15.6987943649292,119.109375,234881024
+60,mlp.down_proj,4,128,8,128,17.628725051879883,115.5546875,234881024
+60,mlp.down_proj,8,32,8,128,0.8190664052963257,238.21875,234881024
+60,mlp.down_proj,8,64,8,128,0.9575161337852478,231.109375,234881024
+60,mlp.down_proj,8,128,8,128,1.0736315250396729,227.5546875,234881024
+61,self_attn.q_proj,2,32,8,128,38.973541259765625,20.0625,67108864
+61,self_attn.q_proj,2,64,8,128,45.845298767089844,18.03125,67108864
+61,self_attn.q_proj,2,128,8,128,52.71784591674805,17.015625,67108864
+61,self_attn.q_proj,3,32,8,128,16.72574806213379,28.0625,67108864
+61,self_attn.q_proj,3,64,8,128,19.73444366455078,26.03125,67108864
+61,self_attn.q_proj,3,128,8,128,22.908206939697266,25.015625,67108864
+61,self_attn.q_proj,4,32,8,128,7.807056427001953,36.0625,67108864
+61,self_attn.q_proj,4,64,8,128,9.20737075805664,34.03125,67108864
+61,self_attn.q_proj,4,128,8,128,10.694247245788574,33.015625,67108864
+61,self_attn.q_proj,8,32,8,128,0.47395724058151245,68.0625,67108864
+61,self_attn.q_proj,8,64,8,128,0.5590010285377502,66.03125,67108864
+61,self_attn.q_proj,8,128,8,128,0.6472654938697815,65.015625,67108864
+61,self_attn.v_proj,2,32,8,128,14.164259910583496,2.5078125,8388608
+61,self_attn.v_proj,2,64,8,128,16.701602935791016,2.25390625,8388608
+61,self_attn.v_proj,2,128,8,128,19.102109909057617,2.126953125,8388608
+61,self_attn.v_proj,3,32,8,128,6.05617094039917,3.5078125,8388608
+61,self_attn.v_proj,3,64,8,128,7.1500067710876465,3.25390625,8388608
+61,self_attn.v_proj,3,128,8,128,8.183921813964844,3.126953125,8388608
+61,self_attn.v_proj,4,32,8,128,2.8265137672424316,4.5078125,8388608
+61,self_attn.v_proj,4,64,8,128,3.3363349437713623,4.25390625,8388608
+61,self_attn.v_proj,4,128,8,128,3.821249008178711,4.126953125,8388608
+61,self_attn.v_proj,8,32,8,128,0.17118647694587708,8.5078125,8388608
+61,self_attn.v_proj,8,64,8,128,0.20183341205120087,8.25390625,8388608
+61,self_attn.v_proj,8,128,8,128,0.2302669733762741,8.126953125,8388608
+61,self_attn.k_proj,2,32,8,128,29.862363815307617,2.5078125,8388608
+61,self_attn.k_proj,2,64,8,128,35.85988235473633,2.25390625,8388608
+61,self_attn.k_proj,2,128,8,128,40.84939193725586,2.126953125,8388608
+61,self_attn.k_proj,3,32,8,128,13.01622486114502,3.5078125,8388608
+61,self_attn.k_proj,3,64,8,128,15.916674613952637,3.25390625,8388608
+61,self_attn.k_proj,3,128,8,128,18.683326721191406,3.126953125,8388608
+61,self_attn.k_proj,4,32,8,128,6.075058937072754,4.5078125,8388608
+61,self_attn.k_proj,4,64,8,128,7.4342041015625,4.25390625,8388608
+61,self_attn.k_proj,4,128,8,128,8.736970901489258,4.126953125,8388608
+61,self_attn.k_proj,8,32,8,128,0.36766231060028076,8.5078125,8388608
+61,self_attn.k_proj,8,64,8,128,0.44918307662010193,8.25390625,8388608
+61,self_attn.k_proj,8,128,8,128,0.5262781381607056,8.126953125,8388608
+61,self_attn.o_proj,2,32,8,128,34.37571334838867,20.0625,67108864
+61,self_attn.o_proj,2,64,8,128,40.00499725341797,18.03125,67108864
+61,self_attn.o_proj,2,128,8,128,45.08881378173828,17.015625,67108864
+61,self_attn.o_proj,3,32,8,128,14.698253631591797,28.0625,67108864
+61,self_attn.o_proj,3,64,8,128,17.111522674560547,26.03125,67108864
+61,self_attn.o_proj,3,128,8,128,19.27420997619629,25.015625,67108864
+61,self_attn.o_proj,4,32,8,128,6.855103969573975,36.0625,67108864
+61,self_attn.o_proj,4,64,8,128,7.990299224853516,34.03125,67108864
+61,self_attn.o_proj,4,128,8,128,8.992043495178223,33.015625,67108864
+61,self_attn.o_proj,8,32,8,128,0.41663646697998047,68.0625,67108864
+61,self_attn.o_proj,8,64,8,128,0.48454150557518005,66.03125,67108864
+61,self_attn.o_proj,8,128,8,128,0.544036328792572,65.015625,67108864
+61,mlp.gate_proj,2,32,8,128,75.743896484375,70.21875,234881024
+61,mlp.gate_proj,2,64,8,128,89.85423278808594,63.109375,234881024
+61,mlp.gate_proj,2,128,8,128,102.558349609375,59.5546875,234881024
+61,mlp.gate_proj,3,32,8,128,32.53206253051758,98.21875,234881024
+61,mlp.gate_proj,3,64,8,128,38.34189224243164,91.109375,234881024
+61,mlp.gate_proj,3,128,8,128,44.24352264404297,87.5546875,234881024
+61,mlp.gate_proj,4,32,8,128,15.177080154418945,126.21875,234881024
+61,mlp.gate_proj,4,64,8,128,17.891611099243164,119.109375,234881024
+61,mlp.gate_proj,4,128,8,128,20.610952377319336,115.5546875,234881024
+61,mlp.gate_proj,8,32,8,128,0.92622971534729,238.21875,234881024
+61,mlp.gate_proj,8,64,8,128,1.0918970108032227,231.109375,234881024
+61,mlp.gate_proj,8,128,8,128,1.2521106004714966,227.5546875,234881024
+61,mlp.up_proj,2,32,8,128,67.44733428955078,70.21875,234881024
+61,mlp.up_proj,2,64,8,128,78.46923828125,63.109375,234881024
+61,mlp.up_proj,2,128,8,128,88.6889877319336,59.5546875,234881024
+61,mlp.up_proj,3,32,8,128,28.851415634155273,98.21875,234881024
+61,mlp.up_proj,3,64,8,128,33.60603713989258,91.109375,234881024
+61,mlp.up_proj,3,128,8,128,37.83018493652344,87.5546875,234881024
+61,mlp.up_proj,4,32,8,128,13.434486389160156,126.21875,234881024
+61,mlp.up_proj,4,64,8,128,15.72931957244873,119.109375,234881024
+61,mlp.up_proj,4,128,8,128,17.664539337158203,115.5546875,234881024
+61,mlp.up_proj,8,32,8,128,0.8203868865966797,238.21875,234881024
+61,mlp.up_proj,8,64,8,128,0.9590937495231628,231.109375,234881024
+61,mlp.up_proj,8,128,8,128,1.0753191709518433,227.5546875,234881024
+61,mlp.down_proj,2,32,8,128,67.40216064453125,70.21875,234881024
+61,mlp.down_proj,2,64,8,128,78.41160583496094,63.109375,234881024
+61,mlp.down_proj,2,128,8,128,88.59757995605469,59.5546875,234881024
+61,mlp.down_proj,3,32,8,128,28.829999923706055,98.21875,234881024
+61,mlp.down_proj,3,64,8,128,33.57952117919922,91.109375,234881024
+61,mlp.down_proj,3,128,8,128,37.787315368652344,87.5546875,234881024
+61,mlp.down_proj,4,32,8,128,13.426426887512207,126.21875,234881024
+61,mlp.down_proj,4,64,8,128,15.717398643493652,119.109375,234881024
+61,mlp.down_proj,4,128,8,128,17.65062141418457,115.5546875,234881024
+61,mlp.down_proj,8,32,8,128,0.8199287056922913,238.21875,234881024
+61,mlp.down_proj,8,64,8,128,0.9585103392601013,231.109375,234881024
+61,mlp.down_proj,8,128,8,128,1.0745633840560913,227.5546875,234881024
+62,self_attn.q_proj,2,32,8,128,33.16127014160156,20.0625,67108864
+62,self_attn.q_proj,2,64,8,128,39.26675796508789,18.03125,67108864
+62,self_attn.q_proj,2,128,8,128,45.21831512451172,17.015625,67108864
+62,self_attn.q_proj,3,32,8,128,14.2050142288208,28.0625,67108864
+62,self_attn.q_proj,3,64,8,128,16.87083625793457,26.03125,67108864
+62,self_attn.q_proj,3,128,8,128,19.51991844177246,25.015625,67108864
+62,self_attn.q_proj,4,32,8,128,6.625704288482666,36.0625,67108864
+62,self_attn.q_proj,4,64,8,128,7.880497932434082,34.03125,67108864
+62,self_attn.q_proj,4,128,8,128,9.11164379119873,33.015625,67108864
+62,self_attn.q_proj,8,32,8,128,0.40274667739868164,68.0625,67108864
+62,self_attn.q_proj,8,64,8,128,0.47802072763442993,66.03125,67108864
+62,self_attn.q_proj,8,128,8,128,0.5511553287506104,65.015625,67108864
+62,self_attn.v_proj,2,32,8,128,14.29792594909668,2.5078125,8388608
+62,self_attn.v_proj,2,64,8,128,16.639434814453125,2.25390625,8388608
+62,self_attn.v_proj,2,128,8,128,18.745302200317383,2.126953125,8388608
+62,self_attn.v_proj,3,32,8,128,6.112382888793945,3.5078125,8388608
+62,self_attn.v_proj,3,64,8,128,7.117050647735596,3.25390625,8388608
+62,self_attn.v_proj,3,128,8,128,8.01989459991455,3.126953125,8388608
+62,self_attn.v_proj,4,32,8,128,2.8522708415985107,4.5078125,8388608
+62,self_attn.v_proj,4,64,8,128,3.3215179443359375,4.25390625,8388608
+62,self_attn.v_proj,4,128,8,128,3.743574857711792,4.126953125,8388608
+62,self_attn.v_proj,8,32,8,128,0.17269602417945862,8.5078125,8388608
+62,self_attn.v_proj,8,64,8,128,0.20089605450630188,8.25390625,8388608
+62,self_attn.v_proj,8,128,8,128,0.22550548613071442,8.126953125,8388608
+62,self_attn.k_proj,2,32,8,128,23.684188842773438,2.5078125,8388608
+62,self_attn.k_proj,2,64,8,128,28.831796646118164,2.25390625,8388608
+62,self_attn.k_proj,2,128,8,128,33.065914154052734,2.126953125,8388608
+62,self_attn.k_proj,3,32,8,128,10.291169166564941,3.5078125,8388608
+62,self_attn.k_proj,3,64,8,128,12.711946487426758,3.25390625,8388608
+62,self_attn.k_proj,3,128,8,128,14.934733390808105,3.126953125,8388608
+62,self_attn.k_proj,4,32,8,128,4.809582710266113,4.5078125,8388608
+62,self_attn.k_proj,4,64,8,128,5.951900959014893,4.25390625,8388608
+62,self_attn.k_proj,4,128,8,128,7.006351470947266,4.126953125,8388608
+62,self_attn.k_proj,8,32,8,128,0.2910732626914978,8.5078125,8388608
+62,self_attn.k_proj,8,64,8,128,0.359929621219635,8.25390625,8388608
+62,self_attn.k_proj,8,128,8,128,0.4220229685306549,8.126953125,8388608
+62,self_attn.o_proj,2,32,8,128,34.49824523925781,20.0625,67108864
+62,self_attn.o_proj,2,64,8,128,40.1680793762207,18.03125,67108864
+62,self_attn.o_proj,2,128,8,128,45.29377746582031,17.015625,67108864
+62,self_attn.o_proj,3,32,8,128,14.762136459350586,28.0625,67108864
+62,self_attn.o_proj,3,64,8,128,17.20389175415039,26.03125,67108864
+62,self_attn.o_proj,3,128,8,128,19.41390609741211,25.015625,67108864
+62,self_attn.o_proj,4,32,8,128,6.884294509887695,36.0625,67108864
+62,self_attn.o_proj,4,64,8,128,8.034402847290039,34.03125,67108864
+62,self_attn.o_proj,4,128,8,128,9.061327934265137,33.015625,67108864
+62,self_attn.o_proj,8,32,8,128,0.4184496998786926,68.0625,67108864
+62,self_attn.o_proj,8,64,8,128,0.4872736930847168,66.03125,67108864
+62,self_attn.o_proj,8,128,8,128,0.5482164025306702,65.015625,67108864
+62,mlp.gate_proj,2,32,8,128,75.47796630859375,70.21875,234881024
+62,mlp.gate_proj,2,64,8,128,89.52935028076172,63.109375,234881024
+62,mlp.gate_proj,2,128,8,128,102.23421478271484,59.5546875,234881024
+62,mlp.gate_proj,3,32,8,128,32.43706512451172,98.21875,234881024
+62,mlp.gate_proj,3,64,8,128,38.217708587646484,91.109375,234881024
+62,mlp.gate_proj,3,128,8,128,44.08865737915039,87.5546875,234881024
+62,mlp.gate_proj,4,32,8,128,15.12441349029541,126.21875,234881024
+62,mlp.gate_proj,4,64,8,128,17.826047897338867,119.109375,234881024
+62,mlp.gate_proj,4,128,8,128,20.547039031982422,115.5546875,234881024
+62,mlp.gate_proj,8,32,8,128,0.923049807548523,238.21875,234881024
+62,mlp.gate_proj,8,64,8,128,1.0882408618927002,231.109375,234881024
+62,mlp.gate_proj,8,128,8,128,1.2482062578201294,227.5546875,234881024
+62,mlp.up_proj,2,32,8,128,67.74840545654297,70.21875,234881024
+62,mlp.up_proj,2,64,8,128,78.86077117919922,63.109375,234881024
+62,mlp.up_proj,2,128,8,128,89.114990234375,59.5546875,234881024
+62,mlp.up_proj,3,32,8,128,28.99092674255371,98.21875,234881024
+62,mlp.up_proj,3,64,8,128,33.7600212097168,91.109375,234881024
+62,mlp.up_proj,3,128,8,128,38.00096893310547,87.5546875,234881024
+62,mlp.up_proj,4,32,8,128,13.500089645385742,126.21875,234881024
+62,mlp.up_proj,4,64,8,128,15.804059982299805,119.109375,234881024
+62,mlp.up_proj,4,128,8,128,17.745237350463867,115.5546875,234881024
+62,mlp.up_proj,8,32,8,128,0.8243322372436523,238.21875,234881024
+62,mlp.up_proj,8,64,8,128,0.9639585614204407,231.109375,234881024
+62,mlp.up_proj,8,128,8,128,1.0802463293075562,227.5546875,234881024
+62,mlp.down_proj,2,32,8,128,67.65412139892578,70.21875,234881024
+62,mlp.down_proj,2,64,8,128,78.73932647705078,63.109375,234881024
+62,mlp.down_proj,2,128,8,128,88.9809799194336,59.5546875,234881024
+62,mlp.down_proj,3,32,8,128,28.945667266845703,98.21875,234881024
+62,mlp.down_proj,3,64,8,128,33.70509719848633,91.109375,234881024
+62,mlp.down_proj,3,128,8,128,37.92435836791992,87.5546875,234881024
+62,mlp.down_proj,4,32,8,128,13.479736328125,126.21875,234881024
+62,mlp.down_proj,4,64,8,128,15.778360366821289,119.109375,234881024
+62,mlp.down_proj,4,128,8,128,17.716798782348633,115.5546875,234881024
+62,mlp.down_proj,8,32,8,128,0.8231130242347717,238.21875,234881024
+62,mlp.down_proj,8,64,8,128,0.9623863101005554,231.109375,234881024
+62,mlp.down_proj,8,128,8,128,1.0786713361740112,227.5546875,234881024
+63,self_attn.q_proj,2,32,8,128,37.58094024658203,20.0625,67108864
+63,self_attn.q_proj,2,64,8,128,44.64314270019531,18.03125,67108864
+63,self_attn.q_proj,2,128,8,128,51.32870864868164,17.015625,67108864
+63,self_attn.q_proj,3,32,8,128,16.189167022705078,28.0625,67108864
+63,self_attn.q_proj,3,64,8,128,19.378171920776367,26.03125,67108864
+63,self_attn.q_proj,3,128,8,128,22.655580520629883,25.015625,67108864
+63,self_attn.q_proj,4,32,8,128,7.552465438842773,36.0625,67108864
+63,self_attn.q_proj,4,64,8,128,9.04287052154541,34.03125,67108864
+63,self_attn.q_proj,4,128,8,128,10.570756912231445,33.015625,67108864
+63,self_attn.q_proj,8,32,8,128,0.458551287651062,68.0625,67108864
+63,self_attn.q_proj,8,64,8,128,0.5490501523017883,66.03125,67108864
+63,self_attn.q_proj,8,128,8,128,0.6400566101074219,65.015625,67108864
+63,self_attn.v_proj,2,32,8,128,12.993329048156738,2.5078125,8388608
+63,self_attn.v_proj,2,64,8,128,15.1444091796875,2.25390625,8388608
+63,self_attn.v_proj,2,128,8,128,17.067733764648438,2.126953125,8388608
+63,self_attn.v_proj,3,32,8,128,5.556256294250488,3.5078125,8388608
+63,self_attn.v_proj,3,64,8,128,6.476415157318115,3.25390625,8388608
+63,self_attn.v_proj,3,128,8,128,7.3071608543396,3.126953125,8388608
+63,self_attn.v_proj,4,32,8,128,2.5927393436431885,4.5078125,8388608
+63,self_attn.v_proj,4,64,8,128,3.022320508956909,4.25390625,8388608
+63,self_attn.v_proj,4,128,8,128,3.4110758304595947,4.126953125,8388608
+63,self_attn.v_proj,8,32,8,128,0.15688839554786682,8.5078125,8388608
+63,self_attn.v_proj,8,64,8,128,0.18275412917137146,8.25390625,8388608
+63,self_attn.v_proj,8,128,8,128,0.20543789863586426,8.126953125,8388608
+63,self_attn.k_proj,2,32,8,128,28.954383850097656,2.5078125,8388608
+63,self_attn.k_proj,2,64,8,128,34.84510803222656,2.25390625,8388608
+63,self_attn.k_proj,2,128,8,128,39.55474090576172,2.126953125,8388608
+63,self_attn.k_proj,3,32,8,128,12.72195816040039,3.5078125,8388608
+63,self_attn.k_proj,3,64,8,128,15.668394088745117,3.25390625,8388608
+63,self_attn.k_proj,3,128,8,128,18.450824737548828,3.126953125,8388608
+63,self_attn.k_proj,4,32,8,128,5.941433906555176,4.5078125,8388608
+63,self_attn.k_proj,4,64,8,128,7.3232221603393555,4.25390625,8388608
+63,self_attn.k_proj,4,128,8,128,8.637715339660645,4.126953125,8388608
+63,self_attn.k_proj,8,32,8,128,0.3594670593738556,8.5078125,8388608
+63,self_attn.k_proj,8,64,8,128,0.44262799620628357,8.25390625,8388608
+63,self_attn.k_proj,8,128,8,128,0.5211894512176514,8.126953125,8388608
+63,self_attn.o_proj,2,32,8,128,33.67514419555664,20.0625,67108864
+63,self_attn.o_proj,2,64,8,128,39.233497619628906,18.03125,67108864
+63,self_attn.o_proj,2,128,8,128,44.28489685058594,17.015625,67108864
+63,self_attn.o_proj,3,32,8,128,14.399726867675781,28.0625,67108864
+63,self_attn.o_proj,3,64,8,128,16.797008514404297,26.03125,67108864
+63,self_attn.o_proj,3,128,8,128,18.987823486328125,25.015625,67108864
+63,self_attn.o_proj,4,32,8,128,6.7164740562438965,36.0625,67108864
+63,self_attn.o_proj,4,64,8,128,7.842055320739746,34.03125,67108864
+63,self_attn.o_proj,4,128,8,128,8.855427742004395,33.015625,67108864
+63,self_attn.o_proj,8,32,8,128,0.40824148058891296,68.0625,67108864
+63,self_attn.o_proj,8,64,8,128,0.4757193326950073,66.03125,67108864
+63,self_attn.o_proj,8,128,8,128,0.5354966521263123,65.015625,67108864
+63,mlp.gate_proj,2,32,8,128,76.42575073242188,70.21875,234881024
+63,mlp.gate_proj,2,64,8,128,90.9824447631836,63.109375,234881024
+63,mlp.gate_proj,2,128,8,128,104.39635467529297,59.5546875,234881024
+63,mlp.gate_proj,3,32,8,128,32.8078498840332,98.21875,234881024
+63,mlp.gate_proj,3,64,8,128,38.88457107543945,91.109375,234881024
+63,mlp.gate_proj,3,128,8,128,45.164676666259766,87.5546875,234881024
+63,mlp.gate_proj,4,32,8,128,15.325652122497559,126.21875,234881024
+63,mlp.gate_proj,4,64,8,128,18.131166458129883,119.109375,234881024
+63,mlp.gate_proj,4,128,8,128,21.04170799255371,115.5546875,234881024
+63,mlp.gate_proj,8,32,8,128,0.9350093007087708,238.21875,234881024
+63,mlp.gate_proj,8,64,8,128,1.1062854528427124,231.109375,234881024
+63,mlp.gate_proj,8,128,8,128,1.277557611465454,227.5546875,234881024
+63,mlp.up_proj,2,32,8,128,67.56266784667969,70.21875,234881024
+63,mlp.up_proj,2,64,8,128,78.61321258544922,63.109375,234881024
+63,mlp.up_proj,2,128,8,128,88.83979797363281,59.5546875,234881024
+63,mlp.up_proj,3,32,8,128,28.90641975402832,98.21875,234881024
+63,mlp.up_proj,3,64,8,128,33.666107177734375,91.109375,234881024
+63,mlp.up_proj,3,128,8,128,37.898590087890625,87.5546875,234881024
+63,mlp.up_proj,4,32,8,128,13.46016788482666,126.21875,234881024
+63,mlp.up_proj,4,64,8,128,15.75747299194336,119.109375,234881024
+63,mlp.up_proj,4,128,8,128,17.696977615356445,115.5546875,234881024
+63,mlp.up_proj,8,32,8,128,0.8219993710517883,238.21875,234881024
+63,mlp.up_proj,8,64,8,128,0.961009681224823,231.109375,234881024
+63,mlp.up_proj,8,128,8,128,1.0773146152496338,227.5546875,234881024
+63,mlp.down_proj,2,32,8,128,67.39888000488281,70.21875,234881024
+63,mlp.down_proj,2,64,8,128,78.41475677490234,63.109375,234881024
+63,mlp.down_proj,2,128,8,128,88.59690856933594,59.5546875,234881024
+63,mlp.down_proj,3,32,8,128,28.82672691345215,98.21875,234881024
+63,mlp.down_proj,3,64,8,128,33.577213287353516,91.109375,234881024
+63,mlp.down_proj,3,128,8,128,37.78432846069336,87.5546875,234881024
+63,mlp.down_proj,4,32,8,128,13.423266410827637,126.21875,234881024
+63,mlp.down_proj,4,64,8,128,15.715346336364746,119.109375,234881024
+63,mlp.down_proj,4,128,8,128,17.647968292236328,115.5546875,234881024
+63,mlp.down_proj,8,32,8,128,0.8197218179702759,238.21875,234881024
+63,mlp.down_proj,8,64,8,128,0.9583899974822998,231.109375,234881024
+63,mlp.down_proj,8,128,8,128,1.0744006633758545,227.5546875,234881024
+64,self_attn.q_proj,2,32,8,128,42.819366455078125,20.0625,67108864
+64,self_attn.q_proj,2,64,8,128,50.21886444091797,18.03125,67108864
+64,self_attn.q_proj,2,128,8,128,58.032466888427734,17.015625,67108864
+64,self_attn.q_proj,3,32,8,128,18.317514419555664,28.0625,67108864
+64,self_attn.q_proj,3,64,8,128,21.556615829467773,26.03125,67108864
+64,self_attn.q_proj,3,128,8,128,24.975194931030273,25.015625,67108864
+64,self_attn.q_proj,4,32,8,128,8.54777717590332,36.0625,67108864
+64,self_attn.q_proj,4,64,8,128,10.055856704711914,34.03125,67108864
+64,self_attn.q_proj,4,128,8,128,11.663802146911621,33.015625,67108864
+64,self_attn.q_proj,8,32,8,128,0.5190718173980713,68.0625,67108864
+64,self_attn.q_proj,8,64,8,128,0.6106762886047363,66.03125,67108864
+64,self_attn.q_proj,8,128,8,128,0.7062397003173828,65.015625,67108864
+64,self_attn.v_proj,2,32,8,128,14.038131713867188,2.5078125,8388608
+64,self_attn.v_proj,2,64,8,128,16.469144821166992,2.25390625,8388608
+64,self_attn.v_proj,2,128,8,128,18.721162796020508,2.126953125,8388608
+64,self_attn.v_proj,3,32,8,128,6.003831386566162,3.5078125,8388608
+64,self_attn.v_proj,3,64,8,128,7.0487775802612305,3.25390625,8388608
+64,self_attn.v_proj,3,128,8,128,8.014094352722168,3.126953125,8388608
+64,self_attn.v_proj,4,32,8,128,2.800537109375,4.5078125,8388608
+64,self_attn.v_proj,4,64,8,128,3.2880258560180664,4.25390625,8388608
+64,self_attn.v_proj,4,128,8,128,3.7403016090393066,4.126953125,8388608
+64,self_attn.v_proj,8,32,8,128,0.1696086972951889,8.5078125,8388608
+64,self_attn.v_proj,8,64,8,128,0.19886595010757446,8.25390625,8388608
+64,self_attn.v_proj,8,128,8,128,0.22530344128608704,8.126953125,8388608
+64,self_attn.k_proj,2,32,8,128,32.365970611572266,2.5078125,8388608
+64,self_attn.k_proj,2,64,8,128,38.74104309082031,2.25390625,8388608
+64,self_attn.k_proj,2,128,8,128,44.19805908203125,2.126953125,8388608
+64,self_attn.k_proj,3,32,8,128,13.956207275390625,3.5078125,8388608
+64,self_attn.k_proj,3,64,8,128,16.81561279296875,3.25390625,8388608
+64,self_attn.k_proj,3,128,8,128,19.528528213500977,3.126953125,8388608
+64,self_attn.k_proj,4,32,8,128,6.509464263916016,4.5078125,8388608
+64,self_attn.k_proj,4,64,8,128,7.848601818084717,4.25390625,8388608
+64,self_attn.k_proj,4,128,8,128,9.11415958404541,4.126953125,8388608
+64,self_attn.k_proj,8,32,8,128,0.3940204381942749,8.5078125,8388608
+64,self_attn.k_proj,8,64,8,128,0.4745347201824188,8.25390625,8388608
+64,self_attn.k_proj,8,128,8,128,0.5493104457855225,8.126953125,8388608
+64,self_attn.o_proj,2,32,8,128,34.521060943603516,20.0625,67108864
+64,self_attn.o_proj,2,64,8,128,40.190311431884766,18.03125,67108864
+64,self_attn.o_proj,2,128,8,128,45.29783248901367,17.015625,67108864
+64,self_attn.o_proj,3,32,8,128,14.766611099243164,28.0625,67108864
+64,self_attn.o_proj,3,64,8,128,17.203161239624023,26.03125,67108864
+64,self_attn.o_proj,3,128,8,128,19.390905380249023,25.015625,67108864
+64,self_attn.o_proj,4,32,8,128,6.886738300323486,36.0625,67108864
+64,self_attn.o_proj,4,64,8,128,8.032516479492188,34.03125,67108864
+64,self_attn.o_proj,4,128,8,128,9.047829627990723,33.015625,67108864
+64,self_attn.o_proj,8,32,8,128,0.41853147745132446,68.0625,67108864
+64,self_attn.o_proj,8,64,8,128,0.4871586561203003,66.03125,67108864
+64,self_attn.o_proj,8,128,8,128,0.5473939180374146,65.015625,67108864
+64,mlp.gate_proj,2,32,8,128,77.4906005859375,70.21875,234881024
+64,mlp.gate_proj,2,64,8,128,92.36436462402344,63.109375,234881024
+64,mlp.gate_proj,2,128,8,128,106.58409118652344,59.5546875,234881024
+64,mlp.gate_proj,3,32,8,128,33.21943664550781,98.21875,234881024
+64,mlp.gate_proj,3,64,8,128,39.57835006713867,91.109375,234881024
+64,mlp.gate_proj,3,128,8,128,46.06911087036133,87.5546875,234881024
+64,mlp.gate_proj,4,32,8,128,15.545063018798828,126.21875,234881024
+64,mlp.gate_proj,4,64,8,128,18.44978904724121,119.109375,234881024
+64,mlp.gate_proj,4,128,8,128,21.523025512695312,115.5546875,234881024
+64,mlp.gate_proj,8,32,8,128,0.9483956098556519,238.21875,234881024
+64,mlp.gate_proj,8,64,8,128,1.1253129243850708,231.109375,234881024
+64,mlp.gate_proj,8,128,8,128,1.3062058687210083,227.5546875,234881024
+64,mlp.up_proj,2,32,8,128,67.24283599853516,70.21875,234881024
+64,mlp.up_proj,2,64,8,128,78.21695709228516,63.109375,234881024
+64,mlp.up_proj,2,128,8,128,88.39877319335938,59.5546875,234881024
+64,mlp.up_proj,3,32,8,128,28.75704574584961,98.21875,234881024
+64,mlp.up_proj,3,64,8,128,33.504093170166016,91.109375,234881024
+64,mlp.up_proj,3,128,8,128,37.68867111206055,87.5546875,234881024
+64,mlp.up_proj,4,32,8,128,13.391504287719727,126.21875,234881024
+64,mlp.up_proj,4,64,8,128,15.674460411071777,119.109375,234881024
+64,mlp.up_proj,4,128,8,128,17.614107131958008,115.5546875,234881024
+64,mlp.up_proj,8,32,8,128,0.8177882432937622,238.21875,234881024
+64,mlp.up_proj,8,64,8,128,0.9559713006019592,231.109375,234881024
+64,mlp.up_proj,8,128,8,128,1.0719205141067505,227.5546875,234881024
+64,mlp.down_proj,2,32,8,128,67.02141571044922,70.21875,234881024
+64,mlp.down_proj,2,64,8,128,77.94024658203125,63.109375,234881024
+64,mlp.down_proj,2,128,8,128,88.04471588134766,59.5546875,234881024
+64,mlp.down_proj,3,32,8,128,28.6521053314209,98.21875,234881024
+64,mlp.down_proj,3,64,8,128,33.38684844970703,91.109375,234881024
+64,mlp.down_proj,3,128,8,128,37.55854797363281,87.5546875,234881024
+64,mlp.down_proj,4,32,8,128,13.344470024108887,126.21875,234881024
+64,mlp.down_proj,4,64,8,128,15.616251945495605,119.109375,234881024
+64,mlp.down_proj,4,128,8,128,17.548776626586914,115.5546875,234881024
+64,mlp.down_proj,8,32,8,128,0.8149448037147522,238.21875,234881024
+64,mlp.down_proj,8,64,8,128,0.9523642063140869,231.109375,234881024
+64,mlp.down_proj,8,128,8,128,1.0681616067886353,227.5546875,234881024
+65,self_attn.q_proj,2,32,8,128,36.329490661621094,20.0625,67108864
+65,self_attn.q_proj,2,64,8,128,43.12436294555664,18.03125,67108864
+65,self_attn.q_proj,2,128,8,128,49.51023483276367,17.015625,67108864
+65,self_attn.q_proj,3,32,8,128,15.586019515991211,28.0625,67108864
+65,self_attn.q_proj,3,64,8,128,18.526418685913086,26.03125,67108864
+65,self_attn.q_proj,3,128,8,128,21.410940170288086,25.015625,67108864
+65,self_attn.q_proj,4,32,8,128,7.275989532470703,36.0625,67108864
+65,self_attn.q_proj,4,64,8,128,8.655049324035645,34.03125,67108864
+65,self_attn.q_proj,4,128,8,128,9.999711036682129,33.015625,67108864
+65,self_attn.q_proj,8,32,8,128,0.44221141934394836,68.0625,67108864
+65,self_attn.q_proj,8,64,8,128,0.5259088277816772,66.03125,67108864
+65,self_attn.q_proj,8,128,8,128,0.6061524748802185,65.015625,67108864
+65,self_attn.v_proj,2,32,8,128,13.861909866333008,2.5078125,8388608
+65,self_attn.v_proj,2,64,8,128,16.18387222290039,2.25390625,8388608
+65,self_attn.v_proj,2,128,8,128,18.28928565979004,2.126953125,8388608
+65,self_attn.v_proj,3,32,8,128,5.927091598510742,3.5078125,8388608
+65,self_attn.v_proj,3,64,8,128,6.923969268798828,3.25390625,8388608
+65,self_attn.v_proj,3,128,8,128,7.828524589538574,3.126953125,8388608
+65,self_attn.v_proj,4,32,8,128,2.7649950981140137,4.5078125,8388608
+65,self_attn.v_proj,4,64,8,128,3.23111629486084,4.25390625,8388608
+65,self_attn.v_proj,4,128,8,128,3.654599666595459,4.126953125,8388608
+65,self_attn.v_proj,8,32,8,128,0.1674090325832367,8.5078125,8388608
+65,self_attn.v_proj,8,64,8,128,0.19542135298252106,8.25390625,8388608
+65,self_attn.v_proj,8,128,8,128,0.22002652287483215,8.126953125,8388608
+65,self_attn.k_proj,2,32,8,128,26.813446044921875,2.5078125,8388608
+65,self_attn.k_proj,2,64,8,128,32.19272994995117,2.25390625,8388608
+65,self_attn.k_proj,2,128,8,128,36.87970733642578,2.126953125,8388608
+65,self_attn.k_proj,3,32,8,128,11.603473663330078,3.5078125,8388608
+65,self_attn.k_proj,3,64,8,128,14.102288246154785,3.25390625,8388608
+65,self_attn.k_proj,3,128,8,128,16.474794387817383,3.126953125,8388608
+65,self_attn.k_proj,4,32,8,128,5.43253231048584,4.5078125,8388608
+65,self_attn.k_proj,4,64,8,128,6.60510778427124,4.25390625,8388608
+65,self_attn.k_proj,4,128,8,128,7.7312469482421875,4.126953125,8388608
+65,self_attn.k_proj,8,32,8,128,0.32878240942955017,8.5078125,8388608
+65,self_attn.k_proj,8,64,8,128,0.3998721241950989,8.25390625,8388608
+65,self_attn.k_proj,8,128,8,128,0.4676685333251953,8.126953125,8388608
+65,self_attn.o_proj,2,32,8,128,33.4189453125,20.0625,67108864
+65,self_attn.o_proj,2,64,8,128,38.898555755615234,18.03125,67108864
+65,self_attn.o_proj,2,128,8,128,43.8623046875,17.015625,67108864
+65,self_attn.o_proj,3,32,8,128,14.294992446899414,28.0625,67108864
+65,self_attn.o_proj,3,64,8,128,16.665603637695312,26.03125,67108864
+65,self_attn.o_proj,3,128,8,128,18.801679611206055,25.015625,67108864
+65,self_attn.o_proj,4,32,8,128,6.682906150817871,36.0625,67108864
+65,self_attn.o_proj,4,64,8,128,7.803160667419434,34.03125,67108864
+65,self_attn.o_proj,4,128,8,128,8.814871788024902,33.015625,67108864
+65,self_attn.o_proj,8,32,8,128,0.4080032408237457,68.0625,67108864
+65,self_attn.o_proj,8,64,8,128,0.4771054685115814,66.03125,67108864
+65,self_attn.o_proj,8,128,8,128,0.5404240489006042,65.015625,67108864
+65,mlp.gate_proj,2,32,8,128,77.75199890136719,70.21875,234881024
+65,mlp.gate_proj,2,64,8,128,92.78266906738281,63.109375,234881024
+65,mlp.gate_proj,2,128,8,128,107.23893737792969,59.5546875,234881024
+65,mlp.gate_proj,3,32,8,128,33.326316833496094,98.21875,234881024
+65,mlp.gate_proj,3,64,8,128,39.7974967956543,91.109375,234881024
+65,mlp.gate_proj,3,128,8,128,46.36573028564453,87.5546875,234881024
+65,mlp.gate_proj,4,32,8,128,15.601041793823242,126.21875,234881024
+65,mlp.gate_proj,4,64,8,128,18.548358917236328,119.109375,234881024
+65,mlp.gate_proj,4,128,8,128,21.681699752807617,115.5546875,234881024
+65,mlp.gate_proj,8,32,8,128,0.9517542719841003,238.21875,234881024
+65,mlp.gate_proj,8,64,8,128,1.13104248046875,231.109375,234881024
+65,mlp.gate_proj,8,128,8,128,1.3156931400299072,227.5546875,234881024
+65,mlp.up_proj,2,32,8,128,67.30894470214844,70.21875,234881024
+65,mlp.up_proj,2,64,8,128,78.29441833496094,63.109375,234881024
+65,mlp.up_proj,2,128,8,128,88.46066284179688,59.5546875,234881024
+65,mlp.up_proj,3,32,8,128,28.785274505615234,98.21875,234881024
+65,mlp.up_proj,3,64,8,128,33.53602981567383,91.109375,234881024
+65,mlp.up_proj,3,128,8,128,37.729740142822266,87.5546875,234881024
+65,mlp.up_proj,4,32,8,128,13.405630111694336,126.21875,234881024
+65,mlp.up_proj,4,64,8,128,15.69153118133545,119.109375,234881024
+65,mlp.up_proj,4,128,8,128,17.638450622558594,115.5546875,234881024
+65,mlp.up_proj,8,32,8,128,0.8186416625976562,238.21875,234881024
+65,mlp.up_proj,8,64,8,128,0.9570550322532654,231.109375,234881024
+65,mlp.up_proj,8,128,8,128,1.0730164051055908,227.5546875,234881024
+65,mlp.down_proj,2,32,8,128,67.02510070800781,70.21875,234881024
+65,mlp.down_proj,2,64,8,128,77.9443130493164,63.109375,234881024
+65,mlp.down_proj,2,128,8,128,88.04894256591797,59.5546875,234881024
+65,mlp.down_proj,3,32,8,128,28.653772354125977,98.21875,234881024
+65,mlp.down_proj,3,64,8,128,33.3892936706543,91.109375,234881024
+65,mlp.down_proj,3,128,8,128,37.557395935058594,87.5546875,234881024
+65,mlp.down_proj,4,32,8,128,13.345603942871094,126.21875,234881024
+65,mlp.down_proj,4,64,8,128,15.618847846984863,119.109375,234881024
+65,mlp.down_proj,4,128,8,128,17.549768447875977,115.5546875,234881024
+65,mlp.down_proj,8,32,8,128,0.8150051236152649,238.21875,234881024
+65,mlp.down_proj,8,64,8,128,0.9523598551750183,231.109375,234881024
+65,mlp.down_proj,8,128,8,128,1.0679457187652588,227.5546875,234881024
+66,self_attn.q_proj,2,32,8,128,38.7794303894043,20.0625,67108864
+66,self_attn.q_proj,2,64,8,128,46.0344123840332,18.03125,67108864
+66,self_attn.q_proj,2,128,8,128,53.05609893798828,17.015625,67108864
+66,self_attn.q_proj,3,32,8,128,16.653215408325195,28.0625,67108864
+66,self_attn.q_proj,3,64,8,128,19.862144470214844,26.03125,67108864
+66,self_attn.q_proj,3,128,8,128,23.146211624145508,25.015625,67108864
+66,self_attn.q_proj,4,32,8,128,7.7720184326171875,36.0625,67108864
+66,self_attn.q_proj,4,64,8,128,9.264443397521973,34.03125,67108864
+66,self_attn.q_proj,4,128,8,128,10.80466365814209,33.015625,67108864
+66,self_attn.q_proj,8,32,8,128,0.47180798649787903,68.0625,67108864
+66,self_attn.q_proj,8,64,8,128,0.5622888803482056,66.03125,67108864
+66,self_attn.q_proj,8,128,8,128,0.653881311416626,65.015625,67108864
+66,self_attn.v_proj,2,32,8,128,13.974505424499512,2.5078125,8388608
+66,self_attn.v_proj,2,64,8,128,16.337491989135742,2.25390625,8388608
+66,self_attn.v_proj,2,128,8,128,18.482759475708008,2.126953125,8388608
+66,self_attn.v_proj,3,32,8,128,5.975247859954834,3.5078125,8388608
+66,self_attn.v_proj,3,64,8,128,6.993313789367676,3.25390625,8388608
+66,self_attn.v_proj,3,128,8,128,7.91671895980835,3.126953125,8388608
+66,self_attn.v_proj,4,32,8,128,2.788114309310913,4.5078125,8388608
+66,self_attn.v_proj,4,64,8,128,3.2619645595550537,4.25390625,8388608
+66,self_attn.v_proj,4,128,8,128,3.6959388256073,4.126953125,8388608
+66,self_attn.v_proj,8,32,8,128,0.16879767179489136,8.5078125,8388608
+66,self_attn.v_proj,8,64,8,128,0.19729551672935486,8.25390625,8388608
+66,self_attn.v_proj,8,128,8,128,0.22268208861351013,8.126953125,8388608
+66,self_attn.k_proj,2,32,8,128,29.672889709472656,2.5078125,8388608
+66,self_attn.k_proj,2,64,8,128,35.635929107666016,2.25390625,8388608
+66,self_attn.k_proj,2,128,8,128,40.66949462890625,2.126953125,8388608
+66,self_attn.k_proj,3,32,8,128,12.839678764343262,3.5078125,8388608
+66,self_attn.k_proj,3,64,8,128,15.61888313293457,3.25390625,8388608
+66,self_attn.k_proj,3,128,8,128,18.2392520904541,3.126953125,8388608
+66,self_attn.k_proj,4,32,8,128,5.991870880126953,4.5078125,8388608
+66,self_attn.k_proj,4,64,8,128,7.294279098510742,4.25390625,8388608
+66,self_attn.k_proj,4,128,8,128,8.524703979492188,4.126953125,8388608
+66,self_attn.k_proj,8,32,8,128,0.36271706223487854,8.5078125,8388608
+66,self_attn.k_proj,8,64,8,128,0.44099608063697815,8.25390625,8388608
+66,self_attn.k_proj,8,128,8,128,0.5136765241622925,8.126953125,8388608
+66,self_attn.o_proj,2,32,8,128,34.24116134643555,20.0625,67108864
+66,self_attn.o_proj,2,64,8,128,39.95665740966797,18.03125,67108864
+66,self_attn.o_proj,2,128,8,128,45.20613098144531,17.015625,67108864
+66,self_attn.o_proj,3,32,8,128,14.67123031616211,28.0625,67108864
+66,self_attn.o_proj,3,64,8,128,17.166067123413086,26.03125,67108864
+66,self_attn.o_proj,3,128,8,128,19.506874084472656,25.015625,67108864
+66,self_attn.o_proj,4,32,8,128,6.844421863555908,36.0625,67108864
+66,self_attn.o_proj,4,64,8,128,8.021551132202148,34.03125,67108864
+66,self_attn.o_proj,4,128,8,128,9.108867645263672,33.015625,67108864
+66,self_attn.o_proj,8,32,8,128,0.416020005941391,68.0625,67108864
+66,self_attn.o_proj,8,64,8,128,0.48655399680137634,66.03125,67108864
+66,self_attn.o_proj,8,128,8,128,0.5511406064033508,65.015625,67108864
+66,mlp.gate_proj,2,32,8,128,76.27597045898438,70.21875,234881024
+66,mlp.gate_proj,2,64,8,128,90.81880187988281,63.109375,234881024
+66,mlp.gate_proj,2,128,8,128,104.22380828857422,59.5546875,234881024
+66,mlp.gate_proj,3,32,8,128,32.744529724121094,98.21875,234881024
+66,mlp.gate_proj,3,64,8,128,38.799224853515625,91.109375,234881024
+66,mlp.gate_proj,3,128,8,128,45.08484649658203,87.5546875,234881024
+66,mlp.gate_proj,4,32,8,128,15.29305648803711,126.21875,234881024
+66,mlp.gate_proj,4,64,8,128,18.09764289855957,119.109375,234881024
+66,mlp.gate_proj,4,128,8,128,21.006444931030273,115.5546875,234881024
+66,mlp.gate_proj,8,32,8,128,0.9330369830131531,238.21875,234881024
+66,mlp.gate_proj,8,64,8,128,1.1040418148040771,231.109375,234881024
+66,mlp.gate_proj,8,128,8,128,1.2750846147537231,227.5546875,234881024
+66,mlp.up_proj,2,32,8,128,67.87973022460938,70.21875,234881024
+66,mlp.up_proj,2,64,8,128,79.010986328125,63.109375,234881024
+66,mlp.up_proj,2,128,8,128,89.31341552734375,59.5546875,234881024
+66,mlp.up_proj,3,32,8,128,29.054590225219727,98.21875,234881024
+66,mlp.up_proj,3,64,8,128,33.8223876953125,91.109375,234881024
+66,mlp.up_proj,3,128,8,128,38.074710845947266,87.5546875,234881024
+66,mlp.up_proj,4,32,8,128,13.52694034576416,126.21875,234881024
+66,mlp.up_proj,4,64,8,128,15.839090347290039,119.109375,234881024
+66,mlp.up_proj,4,128,8,128,17.78289222717285,115.5546875,234881024
+66,mlp.up_proj,8,32,8,128,0.8259626030921936,238.21875,234881024
+66,mlp.up_proj,8,64,8,128,0.9660632610321045,231.109375,234881024
+66,mlp.up_proj,8,128,8,128,1.0824954509735107,227.5546875,234881024
+66,mlp.down_proj,2,32,8,128,67.5672378540039,70.21875,234881024
+66,mlp.down_proj,2,64,8,128,78.6178970336914,63.109375,234881024
+66,mlp.down_proj,2,128,8,128,88.81820678710938,59.5546875,234881024
+66,mlp.down_proj,3,32,8,128,28.901615142822266,98.21875,234881024
+66,mlp.down_proj,3,64,8,128,33.65853500366211,91.109375,234881024
+66,mlp.down_proj,3,128,8,128,37.872039794921875,87.5546875,234881024
+66,mlp.down_proj,4,32,8,128,13.45999526977539,126.21875,234881024
+66,mlp.down_proj,4,64,8,128,15.756766319274902,119.109375,234881024
+66,mlp.down_proj,4,128,8,128,17.69424819946289,115.5546875,234881024
+66,mlp.down_proj,8,32,8,128,0.8219252824783325,238.21875,234881024
+66,mlp.down_proj,8,64,8,128,0.9608999490737915,231.109375,234881024
+66,mlp.down_proj,8,128,8,128,1.0769327878952026,227.5546875,234881024
+67,self_attn.q_proj,2,32,8,128,41.87763214111328,20.0625,67108864
+67,self_attn.q_proj,2,64,8,128,49.1953125,18.03125,67108864
+67,self_attn.q_proj,2,128,8,128,56.52548599243164,17.015625,67108864
+67,self_attn.q_proj,3,32,8,128,17.9111328125,28.0625,67108864
+67,self_attn.q_proj,3,64,8,128,21.09305763244629,26.03125,67108864
+67,self_attn.q_proj,3,128,8,128,24.280925750732422,25.015625,67108864
+67,self_attn.q_proj,4,32,8,128,8.360442161560059,36.0625,67108864
+67,self_attn.q_proj,4,64,8,128,9.838826179504395,34.03125,67108864
+67,self_attn.q_proj,4,128,8,128,11.340537071228027,33.015625,67108864
+67,self_attn.q_proj,8,32,8,128,0.5075060725212097,68.0625,67108864
+67,self_attn.q_proj,8,64,8,128,0.5976356267929077,66.03125,67108864
+67,self_attn.q_proj,8,128,8,128,0.6866402626037598,65.015625,67108864
+67,self_attn.v_proj,2,32,8,128,14.141161918640137,2.5078125,8388608
+67,self_attn.v_proj,2,64,8,128,16.50977325439453,2.25390625,8388608
+67,self_attn.v_proj,2,128,8,128,18.659975051879883,2.126953125,8388608
+67,self_attn.v_proj,3,32,8,128,6.04632568359375,3.5078125,8388608
+67,self_attn.v_proj,3,64,8,128,7.063683986663818,3.25390625,8388608
+67,self_attn.v_proj,3,128,8,128,7.989662170410156,3.126953125,8388608
+67,self_attn.v_proj,4,32,8,128,2.8217005729675293,4.5078125,8388608
+67,self_attn.v_proj,4,64,8,128,3.297024726867676,4.25390625,8388608
+67,self_attn.v_proj,4,128,8,128,3.728530168533325,4.126953125,8388608
+67,self_attn.v_proj,8,32,8,128,0.17084762454032898,8.5078125,8388608
+67,self_attn.v_proj,8,64,8,128,0.19930483400821686,8.25390625,8388608
+67,self_attn.v_proj,8,128,8,128,0.22459186613559723,8.126953125,8388608
+67,self_attn.k_proj,2,32,8,128,31.872743606567383,2.5078125,8388608
+67,self_attn.k_proj,2,64,8,128,38.07038116455078,2.25390625,8388608
+67,self_attn.k_proj,2,128,8,128,43.447418212890625,2.126953125,8388608
+67,self_attn.k_proj,3,32,8,128,13.692203521728516,3.5078125,8388608
+67,self_attn.k_proj,3,64,8,128,16.43685531616211,3.25390625,8388608
+67,self_attn.k_proj,3,128,8,128,18.980356216430664,3.126953125,8388608
+67,self_attn.k_proj,4,32,8,128,6.3920063972473145,4.5078125,8388608
+67,self_attn.k_proj,4,64,8,128,7.669287204742432,4.25390625,8388608
+67,self_attn.k_proj,4,128,8,128,8.86434268951416,4.126953125,8388608
+67,self_attn.k_proj,8,32,8,128,0.3868117034435272,8.5078125,8388608
+67,self_attn.k_proj,8,64,8,128,0.46366190910339355,8.25390625,8388608
+67,self_attn.k_proj,8,128,8,128,0.5339415073394775,8.126953125,8388608
+67,self_attn.o_proj,2,32,8,128,34.68899917602539,20.0625,67108864
+67,self_attn.o_proj,2,64,8,128,40.39021301269531,18.03125,67108864
+67,self_attn.o_proj,2,128,8,128,45.523136138916016,17.015625,67108864
+67,self_attn.o_proj,3,32,8,128,14.838661193847656,28.0625,67108864
+67,self_attn.o_proj,3,64,8,128,17.280132293701172,26.03125,67108864
+67,self_attn.o_proj,3,128,8,128,19.488264083862305,25.015625,67108864
+67,self_attn.o_proj,4,32,8,128,6.91993522644043,36.0625,67108864
+67,self_attn.o_proj,4,64,8,128,8.068282127380371,34.03125,67108864
+67,self_attn.o_proj,4,128,8,128,9.092334747314453,33.015625,67108864
+67,self_attn.o_proj,8,32,8,128,0.42054077982902527,68.0625,67108864
+67,self_attn.o_proj,8,64,8,128,0.4894316494464874,66.03125,67108864
+67,self_attn.o_proj,8,128,8,128,0.5501255989074707,65.015625,67108864
+67,mlp.gate_proj,2,32,8,128,76.54330444335938,70.21875,234881024
+67,mlp.gate_proj,2,64,8,128,91.16223907470703,63.109375,234881024
+67,mlp.gate_proj,2,128,8,128,104.71990966796875,59.5546875,234881024
+67,mlp.gate_proj,3,32,8,128,32.849613189697266,98.21875,234881024
+67,mlp.gate_proj,3,64,8,128,38.9803581237793,91.109375,234881024
+67,mlp.gate_proj,3,128,8,128,45.3189582824707,87.5546875,234881024
+67,mlp.gate_proj,4,32,8,128,15.349717140197754,126.21875,234881024
+67,mlp.gate_proj,4,64,8,128,18.176387786865234,119.109375,234881024
+67,mlp.gate_proj,4,128,8,128,21.1143798828125,115.5546875,234881024
+67,mlp.gate_proj,8,32,8,128,0.9364266395568848,238.21875,234881024
+67,mlp.gate_proj,8,64,8,128,1.108932375907898,231.109375,234881024
+67,mlp.gate_proj,8,128,8,128,1.2820361852645874,227.5546875,234881024
+67,mlp.up_proj,2,32,8,128,67.92131042480469,70.21875,234881024
+67,mlp.up_proj,2,64,8,128,79.05140686035156,63.109375,234881024
+67,mlp.up_proj,2,128,8,128,89.3563232421875,59.5546875,234881024
+67,mlp.up_proj,3,32,8,128,29.071630477905273,98.21875,234881024
+67,mlp.up_proj,3,64,8,128,33.84465408325195,91.109375,234881024
+67,mlp.up_proj,3,128,8,128,38.09027862548828,87.5546875,234881024
+67,mlp.up_proj,4,32,8,128,13.53543758392334,126.21875,234881024
+67,mlp.up_proj,4,64,8,128,15.849507331848145,119.109375,234881024
+67,mlp.up_proj,4,128,8,128,17.79135513305664,115.5546875,234881024
+67,mlp.up_proj,8,32,8,128,0.8264915347099304,238.21875,234881024
+67,mlp.up_proj,8,64,8,128,0.966688334941864,231.109375,234881024
+67,mlp.up_proj,8,128,8,128,1.0831878185272217,227.5546875,234881024
+67,mlp.down_proj,2,32,8,128,67.54988861083984,70.21875,234881024
+67,mlp.down_proj,2,64,8,128,78.59307861328125,63.109375,234881024
+67,mlp.down_proj,2,128,8,128,88.81293487548828,59.5546875,234881024
+67,mlp.down_proj,3,32,8,128,28.898155212402344,98.21875,234881024
+67,mlp.down_proj,3,64,8,128,33.65370178222656,91.109375,234881024
+67,mlp.down_proj,3,128,8,128,37.86188888549805,87.5546875,234881024
+67,mlp.down_proj,4,32,8,128,13.457674980163574,126.21875,234881024
+67,mlp.down_proj,4,64,8,128,15.751675605773926,119.109375,234881024
+67,mlp.down_proj,4,128,8,128,17.686805725097656,115.5546875,234881024
+67,mlp.down_proj,8,32,8,128,0.8217827081680298,238.21875,234881024
+67,mlp.down_proj,8,64,8,128,0.9607451558113098,231.109375,234881024
+67,mlp.down_proj,8,128,8,128,1.0767126083374023,227.5546875,234881024
+68,self_attn.q_proj,2,32,8,128,42.996917724609375,20.0625,67108864
+68,self_attn.q_proj,2,64,8,128,50.308048248291016,18.03125,67108864
+68,self_attn.q_proj,2,128,8,128,57.63694763183594,17.015625,67108864
+68,self_attn.q_proj,3,32,8,128,18.38302993774414,28.0625,67108864
+68,self_attn.q_proj,3,64,8,128,21.562314987182617,26.03125,67108864
+68,self_attn.q_proj,3,128,8,128,24.72186279296875,25.015625,67108864
+68,self_attn.q_proj,4,32,8,128,8.578042984008789,36.0625,67108864
+68,self_attn.q_proj,4,64,8,128,10.05810832977295,34.03125,67108864
+68,self_attn.q_proj,4,128,8,128,11.546043395996094,33.015625,67108864
+68,self_attn.q_proj,8,32,8,128,0.52088862657547,68.0625,67108864
+68,self_attn.q_proj,8,64,8,128,0.6109640598297119,66.03125,67108864
+68,self_attn.q_proj,8,128,8,128,0.6993457078933716,65.015625,67108864
+68,self_attn.v_proj,2,32,8,128,14.98930835723877,2.5078125,8388608
+68,self_attn.v_proj,2,64,8,128,17.529664993286133,2.25390625,8388608
+68,self_attn.v_proj,2,128,8,128,19.84973907470703,2.126953125,8388608
+68,self_attn.v_proj,3,32,8,128,6.404816627502441,3.5078125,8388608
+68,self_attn.v_proj,3,64,8,128,7.498355865478516,3.25390625,8388608
+68,self_attn.v_proj,3,128,8,128,8.496915817260742,3.126953125,8388608
+68,self_attn.v_proj,4,32,8,128,2.9906091690063477,4.5078125,8388608
+68,self_attn.v_proj,4,64,8,128,3.4994311332702637,4.25390625,8388608
+68,self_attn.v_proj,4,128,8,128,3.9646248817443848,4.126953125,8388608
+68,self_attn.v_proj,8,32,8,128,0.1810348778963089,8.5078125,8388608
+68,self_attn.v_proj,8,64,8,128,0.21159625053405762,8.25390625,8388608
+68,self_attn.v_proj,8,128,8,128,0.2388579100370407,8.126953125,8388608
+68,self_attn.k_proj,2,32,8,128,30.838842391967773,2.5078125,8388608
+68,self_attn.k_proj,2,64,8,128,36.82102584838867,2.25390625,8388608
+68,self_attn.k_proj,2,128,8,128,42.182369232177734,2.126953125,8388608
+68,self_attn.k_proj,3,32,8,128,13.226166725158691,3.5078125,8388608
+68,self_attn.k_proj,3,64,8,128,15.83836555480957,3.25390625,8388608
+68,self_attn.k_proj,3,128,8,128,18.289793014526367,3.126953125,8388608
+68,self_attn.k_proj,4,32,8,128,6.169673442840576,4.5078125,8388608
+68,self_attn.k_proj,4,64,8,128,7.3937482833862305,4.25390625,8388608
+68,self_attn.k_proj,4,128,8,128,8.535738945007324,4.126953125,8388608
+68,self_attn.k_proj,8,32,8,128,0.3734605610370636,8.5078125,8388608
+68,self_attn.k_proj,8,64,8,128,0.44660499691963196,8.25390625,8388608
+68,self_attn.k_proj,8,128,8,128,0.5144127011299133,8.126953125,8388608
+68,self_attn.o_proj,2,32,8,128,35.336387634277344,20.0625,67108864
+68,self_attn.o_proj,2,64,8,128,41.148712158203125,18.03125,67108864
+68,self_attn.o_proj,2,128,8,128,46.36989212036133,17.015625,67108864
+68,self_attn.o_proj,3,32,8,128,15.117834091186523,28.0625,67108864
+68,self_attn.o_proj,3,64,8,128,17.601490020751953,26.03125,67108864
+68,self_attn.o_proj,3,128,8,128,19.848608016967773,25.015625,67108864
+68,self_attn.o_proj,4,32,8,128,7.049229145050049,36.0625,67108864
+68,self_attn.o_proj,4,64,8,128,8.217535972595215,34.03125,67108864
+68,self_attn.o_proj,4,128,8,128,9.261478424072266,33.015625,67108864
+68,self_attn.o_proj,8,32,8,128,0.42832088470458984,68.0625,67108864
+68,self_attn.o_proj,8,64,8,128,0.4984959363937378,66.03125,67108864
+68,self_attn.o_proj,8,128,8,128,0.5604457855224609,65.015625,67108864
+68,mlp.gate_proj,2,32,8,128,77.9117431640625,70.21875,234881024
+68,mlp.gate_proj,2,64,8,128,93.0105209350586,63.109375,234881024
+68,mlp.gate_proj,2,128,8,128,107.70882415771484,59.5546875,234881024
+68,mlp.gate_proj,3,32,8,128,33.385066986083984,98.21875,234881024
+68,mlp.gate_proj,3,64,8,128,39.91365051269531,91.109375,234881024
+68,mlp.gate_proj,3,128,8,128,46.54311752319336,87.5546875,234881024
+68,mlp.gate_proj,4,32,8,128,15.630667686462402,126.21875,234881024
+68,mlp.gate_proj,4,64,8,128,18.60211753845215,119.109375,234881024
+68,mlp.gate_proj,4,128,8,128,21.782941818237305,115.5546875,234881024
+68,mlp.gate_proj,8,32,8,128,0.9535031914710999,238.21875,234881024
+68,mlp.gate_proj,8,64,8,128,1.134141206741333,231.109375,234881024
+68,mlp.gate_proj,8,128,8,128,1.3212354183197021,227.5546875,234881024
+68,mlp.up_proj,2,32,8,128,67.4381332397461,70.21875,234881024
+68,mlp.up_proj,2,64,8,128,78.47795104980469,63.109375,234881024
+68,mlp.up_proj,2,128,8,128,88.67298126220703,59.5546875,234881024
+68,mlp.up_proj,3,32,8,128,28.849563598632812,98.21875,234881024
+68,mlp.up_proj,3,64,8,128,33.609432220458984,91.109375,234881024
+68,mlp.up_proj,3,128,8,128,37.84211349487305,87.5546875,234881024
+68,mlp.up_proj,4,32,8,128,13.434784889221191,126.21875,234881024
+68,mlp.up_proj,4,64,8,128,15.730012893676758,119.109375,234881024
+68,mlp.up_proj,4,128,8,128,17.671184539794922,115.5546875,234881024
+68,mlp.up_proj,8,32,8,128,0.8204174637794495,238.21875,234881024
+68,mlp.up_proj,8,64,8,128,0.9593989253044128,231.109375,234881024
+68,mlp.up_proj,8,128,8,128,1.0757280588150024,227.5546875,234881024
+68,mlp.down_proj,2,32,8,128,66.96928405761719,70.21875,234881024
+68,mlp.down_proj,2,64,8,128,77.8817367553711,63.109375,234881024
+68,mlp.down_proj,2,128,8,128,88.00737762451172,59.5546875,234881024
+68,mlp.down_proj,3,32,8,128,28.63149070739746,98.21875,234881024
+68,mlp.down_proj,3,64,8,128,33.36552429199219,91.109375,234881024
+68,mlp.down_proj,3,128,8,128,37.53841781616211,87.5546875,234881024
+68,mlp.down_proj,4,32,8,128,13.332662582397461,126.21875,234881024
+68,mlp.down_proj,4,64,8,128,15.606319427490234,119.109375,234881024
+68,mlp.down_proj,4,128,8,128,17.542068481445312,115.5546875,234881024
+68,mlp.down_proj,8,32,8,128,0.8142945170402527,238.21875,234881024
+68,mlp.down_proj,8,64,8,128,0.9515939950942993,231.109375,234881024
+68,mlp.down_proj,8,128,8,128,1.0672000646591187,227.5546875,234881024
+69,self_attn.q_proj,2,32,8,128,42.20248031616211,20.0625,67108864
+69,self_attn.q_proj,2,64,8,128,49.8327751159668,18.03125,67108864
+69,self_attn.q_proj,2,128,8,128,57.46018600463867,17.015625,67108864
+69,self_attn.q_proj,3,32,8,128,18.056442260742188,28.0625,67108864
+69,self_attn.q_proj,3,64,8,128,21.399200439453125,26.03125,67108864
+69,self_attn.q_proj,3,128,8,128,24.73744773864746,25.015625,67108864
+69,self_attn.q_proj,4,32,8,128,8.42861557006836,36.0625,67108864
+69,self_attn.q_proj,4,64,8,128,9.980915069580078,34.03125,67108864
+69,self_attn.q_proj,4,128,8,128,11.552515983581543,33.015625,67108864
+69,self_attn.q_proj,8,32,8,128,0.5116274356842041,68.0625,67108864
+69,self_attn.q_proj,8,64,8,128,0.6062206625938416,66.03125,67108864
+69,self_attn.q_proj,8,128,8,128,0.699862003326416,65.015625,67108864
+69,self_attn.v_proj,2,32,8,128,14.842782020568848,2.5078125,8388608
+69,self_attn.v_proj,2,64,8,128,17.356426239013672,2.25390625,8388608
+69,self_attn.v_proj,2,128,8,128,19.63593101501465,2.126953125,8388608
+69,self_attn.v_proj,3,32,8,128,6.346794605255127,3.5078125,8388608
+69,self_attn.v_proj,3,64,8,128,7.42772912979126,3.25390625,8388608
+69,self_attn.v_proj,3,128,8,128,8.4048490524292,3.126953125,8388608
+69,self_attn.v_proj,4,32,8,128,2.9614622592926025,4.5078125,8388608
+69,self_attn.v_proj,4,64,8,128,3.4658656120300293,4.25390625,8388608
+69,self_attn.v_proj,4,128,8,128,3.9225854873657227,4.126953125,8388608
+69,self_attn.v_proj,8,32,8,128,0.17930050194263458,8.5078125,8388608
+69,self_attn.v_proj,8,64,8,128,0.20954196155071259,8.25390625,8388608
+69,self_attn.v_proj,8,128,8,128,0.23633894324302673,8.126953125,8388608
+69,self_attn.k_proj,2,32,8,128,31.405559539794922,2.5078125,8388608
+69,self_attn.k_proj,2,64,8,128,37.61735916137695,2.25390625,8388608
+69,self_attn.k_proj,2,128,8,128,43.15043258666992,2.126953125,8388608
+69,self_attn.k_proj,3,32,8,128,13.499991416931152,3.5078125,8388608
+69,self_attn.k_proj,3,64,8,128,16.250951766967773,3.25390625,8388608
+69,self_attn.k_proj,3,128,8,128,18.875097274780273,3.126953125,8388608
+69,self_attn.k_proj,4,32,8,128,6.295987129211426,4.5078125,8388608
+69,self_attn.k_proj,4,64,8,128,7.58663272857666,4.25390625,8388608
+69,self_attn.k_proj,4,128,8,128,8.809903144836426,4.126953125,8388608
+69,self_attn.k_proj,8,32,8,128,0.3811444044113159,8.5078125,8388608
+69,self_attn.k_proj,8,64,8,128,0.4585740864276886,8.25390625,8388608
+69,self_attn.k_proj,8,128,8,128,0.5306520462036133,8.126953125,8388608
+69,self_attn.o_proj,2,32,8,128,35.38931655883789,20.0625,67108864
+69,self_attn.o_proj,2,64,8,128,41.22873306274414,18.03125,67108864
+69,self_attn.o_proj,2,128,8,128,46.49451446533203,17.015625,67108864
+69,self_attn.o_proj,3,32,8,128,15.141733169555664,28.0625,67108864
+69,self_attn.o_proj,3,64,8,128,17.641101837158203,26.03125,67108864
+69,self_attn.o_proj,3,128,8,128,19.916149139404297,25.015625,67108864
+69,self_attn.o_proj,4,32,8,128,7.061334609985352,36.0625,67108864
+69,self_attn.o_proj,4,64,8,128,8.234591484069824,34.03125,67108864
+69,self_attn.o_proj,4,128,8,128,9.29265308380127,33.015625,67108864
+69,self_attn.o_proj,8,32,8,128,0.4290388524532318,68.0625,67108864
+69,self_attn.o_proj,8,64,8,128,0.49960705637931824,66.03125,67108864
+69,self_attn.o_proj,8,128,8,128,0.5623963475227356,65.015625,67108864
+69,mlp.gate_proj,2,32,8,128,77.63627624511719,70.21875,234881024
+69,mlp.gate_proj,2,64,8,128,92.6241455078125,63.109375,234881024
+69,mlp.gate_proj,2,128,8,128,107.11105346679688,59.5546875,234881024
+69,mlp.gate_proj,3,32,8,128,33.277278900146484,98.21875,234881024
+69,mlp.gate_proj,3,64,8,128,39.71714401245117,91.109375,234881024
+69,mlp.gate_proj,3,128,8,128,46.28458786010742,87.5546875,234881024
+69,mlp.gate_proj,4,32,8,128,15.573941230773926,126.21875,234881024
+69,mlp.gate_proj,4,64,8,128,18.5102481842041,119.109375,234881024
+69,mlp.gate_proj,4,128,8,128,21.630252838134766,115.5546875,234881024
+69,mlp.gate_proj,8,32,8,128,0.9500958919525146,238.21875,234881024
+69,mlp.gate_proj,8,64,8,128,1.1287362575531006,231.109375,234881024
+69,mlp.gate_proj,8,128,8,128,1.3128478527069092,227.5546875,234881024
+69,mlp.up_proj,2,32,8,128,67.73368072509766,70.21875,234881024
+69,mlp.up_proj,2,64,8,128,78.83189392089844,63.109375,234881024
+69,mlp.up_proj,2,128,8,128,89.09113311767578,59.5546875,234881024
+69,mlp.up_proj,3,32,8,128,28.983076095581055,98.21875,234881024
+69,mlp.up_proj,3,64,8,128,33.750640869140625,91.109375,234881024
+69,mlp.up_proj,3,128,8,128,37.988224029541016,87.5546875,234881024
+69,mlp.up_proj,4,32,8,128,13.496142387390137,126.21875,234881024
+69,mlp.up_proj,4,64,8,128,15.803689956665039,119.109375,234881024
+69,mlp.up_proj,4,128,8,128,17.7470703125,115.5546875,234881024
+69,mlp.up_proj,8,32,8,128,0.824127733707428,238.21875,234881024
+69,mlp.up_proj,8,64,8,128,0.9638040065765381,231.109375,234881024
+69,mlp.up_proj,8,128,8,128,1.0802112817764282,227.5546875,234881024
+69,mlp.down_proj,2,32,8,128,67.24958038330078,70.21875,234881024
+69,mlp.down_proj,2,64,8,128,78.22705841064453,63.109375,234881024
+69,mlp.down_proj,2,128,8,128,88.4117660522461,59.5546875,234881024
+69,mlp.down_proj,3,32,8,128,28.76068115234375,98.21875,234881024
+69,mlp.down_proj,3,64,8,128,33.50086975097656,91.109375,234881024
+69,mlp.down_proj,3,128,8,128,37.703311920166016,87.5546875,234881024
+69,mlp.down_proj,4,32,8,128,13.39233684539795,126.21875,234881024
+69,mlp.down_proj,4,64,8,128,15.674565315246582,119.109375,234881024
+69,mlp.down_proj,4,128,8,128,17.605365753173828,115.5546875,234881024
+69,mlp.down_proj,8,32,8,128,0.8178876042366028,238.21875,234881024
+69,mlp.down_proj,8,64,8,128,0.9558658599853516,231.109375,234881024
+69,mlp.down_proj,8,128,8,128,1.071731686592102,227.5546875,234881024
+70,self_attn.q_proj,2,32,8,128,43.85414505004883,20.0625,67108864
+70,self_attn.q_proj,2,64,8,128,51.76820373535156,18.03125,67108864
+70,self_attn.q_proj,2,128,8,128,59.50196838378906,17.015625,67108864
+70,self_attn.q_proj,3,32,8,128,18.76361083984375,28.0625,67108864
+70,self_attn.q_proj,3,64,8,128,22.22780418395996,26.03125,67108864
+70,self_attn.q_proj,3,128,8,128,25.6020565032959,25.015625,67108864
+70,self_attn.q_proj,4,32,8,128,8.75325870513916,36.0625,67108864
+70,self_attn.q_proj,4,64,8,128,10.367703437805176,34.03125,67108864
+70,self_attn.q_proj,4,128,8,128,11.9556884765625,33.015625,67108864
+70,self_attn.q_proj,8,32,8,128,0.5317333936691284,68.0625,67108864
+70,self_attn.q_proj,8,64,8,128,0.6297582387924194,66.03125,67108864
+70,self_attn.q_proj,8,128,8,128,0.7244271636009216,65.015625,67108864
+70,self_attn.v_proj,2,32,8,128,14.010323524475098,2.5078125,8388608
+70,self_attn.v_proj,2,64,8,128,16.384572982788086,2.25390625,8388608
+70,self_attn.v_proj,2,128,8,128,18.55001449584961,2.126953125,8388608
+70,self_attn.v_proj,3,32,8,128,5.989038467407227,3.5078125,8388608
+70,self_attn.v_proj,3,64,8,128,7.014025688171387,3.25390625,8388608
+70,self_attn.v_proj,3,128,8,128,7.93963098526001,3.126953125,8388608
+70,self_attn.v_proj,4,32,8,128,2.7954518795013428,4.5078125,8388608
+70,self_attn.v_proj,4,64,8,128,3.2717807292938232,4.25390625,8388608
+70,self_attn.v_proj,4,128,8,128,3.7052507400512695,4.126953125,8388608
+70,self_attn.v_proj,8,32,8,128,0.16924989223480225,8.5078125,8388608
+70,self_attn.v_proj,8,64,8,128,0.19784781336784363,8.25390625,8388608
+70,self_attn.v_proj,8,128,8,128,0.22331149876117706,8.126953125,8388608
+70,self_attn.k_proj,2,32,8,128,30.274782180786133,2.5078125,8388608
+70,self_attn.k_proj,2,64,8,128,35.994300842285156,2.25390625,8388608
+70,self_attn.k_proj,2,128,8,128,41.0926513671875,2.126953125,8388608
+70,self_attn.k_proj,3,32,8,128,12.975082397460938,3.5078125,8388608
+70,self_attn.k_proj,3,64,8,128,15.467161178588867,3.25390625,8388608
+70,self_attn.k_proj,3,128,8,128,17.776256561279297,3.126953125,8388608
+70,self_attn.k_proj,4,32,8,128,6.05685567855835,4.5078125,8388608
+70,self_attn.k_proj,4,64,8,128,7.222330570220947,4.25390625,8388608
+70,self_attn.k_proj,4,128,8,128,8.301847457885742,4.126953125,8388608
+70,self_attn.k_proj,8,32,8,128,0.3664975166320801,8.5078125,8388608
+70,self_attn.k_proj,8,64,8,128,0.43637561798095703,8.25390625,8388608
+70,self_attn.k_proj,8,128,8,128,0.4999316334724426,8.126953125,8388608
+70,self_attn.o_proj,2,32,8,128,34.425960540771484,20.0625,67108864
+70,self_attn.o_proj,2,64,8,128,40.147743225097656,18.03125,67108864
+70,self_attn.o_proj,2,128,8,128,45.34928894042969,17.015625,67108864
+70,self_attn.o_proj,3,32,8,128,14.741358757019043,28.0625,67108864
+70,self_attn.o_proj,3,64,8,128,17.2127628326416,26.03125,67108864
+70,self_attn.o_proj,3,128,8,128,19.487916946411133,25.015625,67108864
+70,self_attn.o_proj,4,32,8,128,6.880132675170898,36.0625,67108864
+70,self_attn.o_proj,4,64,8,128,8.047780990600586,34.03125,67108864
+70,self_attn.o_proj,4,128,8,128,9.111714363098145,33.015625,67108864
+70,self_attn.o_proj,8,32,8,128,0.4180791974067688,68.0625,67108864
+70,self_attn.o_proj,8,64,8,128,0.4883270859718323,66.03125,67108864
+70,self_attn.o_proj,8,128,8,128,0.5515293478965759,65.015625,67108864
+70,mlp.gate_proj,2,32,8,128,76.47567749023438,70.21875,234881024
+70,mlp.gate_proj,2,64,8,128,91.01229858398438,63.109375,234881024
+70,mlp.gate_proj,2,128,8,128,104.45394897460938,59.5546875,234881024
+70,mlp.gate_proj,3,32,8,128,32.81861114501953,98.21875,234881024
+70,mlp.gate_proj,3,64,8,128,38.8919792175293,91.109375,234881024
+70,mlp.gate_proj,3,128,8,128,45.174774169921875,87.5546875,234881024
+70,mlp.gate_proj,4,32,8,128,15.3292818069458,126.21875,234881024
+70,mlp.gate_proj,4,64,8,128,18.13567352294922,119.109375,234881024
+70,mlp.gate_proj,4,128,8,128,21.045066833496094,115.5546875,234881024
+70,mlp.gate_proj,8,32,8,128,0.9353014230728149,238.21875,234881024
+70,mlp.gate_proj,8,64,8,128,1.1065781116485596,231.109375,234881024
+70,mlp.gate_proj,8,128,8,128,1.277785062789917,227.5546875,234881024
+70,mlp.up_proj,2,32,8,128,68.46273803710938,70.21875,234881024
+70,mlp.up_proj,2,64,8,128,79.75574493408203,63.109375,234881024
+70,mlp.up_proj,2,128,8,128,90.17842864990234,59.5546875,234881024
+70,mlp.up_proj,3,32,8,128,29.322778701782227,98.21875,234881024
+70,mlp.up_proj,3,64,8,128,34.12318801879883,91.109375,234881024
+70,mlp.up_proj,3,128,8,128,38.43038558959961,87.5546875,234881024
+70,mlp.up_proj,4,32,8,128,13.65205192565918,126.21875,234881024
+70,mlp.up_proj,4,64,8,128,15.991703987121582,119.109375,234881024
+70,mlp.up_proj,4,128,8,128,17.9500789642334,115.5546875,234881024
+70,mlp.up_proj,8,32,8,128,0.8335393667221069,238.21875,234881024
+70,mlp.up_proj,8,64,8,128,0.9754669070243835,231.109375,234881024
+70,mlp.up_proj,8,128,8,128,1.092573642730713,227.5546875,234881024
+70,mlp.down_proj,2,32,8,128,68.0191421508789,70.21875,234881024
+70,mlp.down_proj,2,64,8,128,79.18224334716797,63.109375,234881024
+70,mlp.down_proj,2,128,8,128,89.53915405273438,59.5546875,234881024
+70,mlp.down_proj,3,32,8,128,29.11890411376953,98.21875,234881024
+70,mlp.down_proj,3,64,8,128,33.89120864868164,91.109375,234881024
+70,mlp.down_proj,3,128,8,128,38.15570068359375,87.5546875,234881024
+70,mlp.down_proj,4,32,8,128,13.556910514831543,126.21875,234881024
+70,mlp.down_proj,4,64,8,128,15.87414264678955,119.109375,234881024
+70,mlp.down_proj,4,128,8,128,17.820886611938477,115.5546875,234881024
+70,mlp.down_proj,8,32,8,128,0.8277688026428223,238.21875,234881024
+70,mlp.down_proj,8,64,8,128,0.9682811498641968,231.109375,234881024
+70,mlp.down_proj,8,128,8,128,1.0848784446716309,227.5546875,234881024
+71,self_attn.q_proj,2,32,8,128,46.20585250854492,20.0625,67108864
+71,self_attn.q_proj,2,64,8,128,54.372806549072266,18.03125,67108864
+71,self_attn.q_proj,2,128,8,128,62.60104751586914,17.015625,67108864
+71,self_attn.q_proj,3,32,8,128,19.76317596435547,28.0625,67108864
+71,self_attn.q_proj,3,64,8,128,23.315311431884766,26.03125,67108864
+71,self_attn.q_proj,3,128,8,128,26.864534378051758,25.015625,67108864
+71,self_attn.q_proj,4,32,8,128,9.217568397521973,36.0625,67108864
+71,self_attn.q_proj,4,64,8,128,10.879681587219238,34.03125,67108864
+71,self_attn.q_proj,4,128,8,128,12.535223007202148,33.015625,67108864
+71,self_attn.q_proj,8,32,8,128,0.560032308101654,68.0625,67108864
+71,self_attn.q_proj,8,64,8,128,0.6610955595970154,66.03125,67108864
+71,self_attn.q_proj,8,128,8,128,0.7593925595283508,65.015625,67108864
+71,self_attn.v_proj,2,32,8,128,14.65397834777832,2.5078125,8388608
+71,self_attn.v_proj,2,64,8,128,17.1112117767334,2.25390625,8388608
+71,self_attn.v_proj,2,128,8,128,19.353404998779297,2.126953125,8388608
+71,self_attn.v_proj,3,32,8,128,6.26489782333374,3.5078125,8388608
+71,self_attn.v_proj,3,64,8,128,7.322195529937744,3.25390625,8388608
+71,self_attn.v_proj,3,128,8,128,8.283472061157227,3.126953125,8388608
+71,self_attn.v_proj,4,32,8,128,2.9224939346313477,4.5078125,8388608
+71,self_attn.v_proj,4,64,8,128,3.417940378189087,4.25390625,8388608
+71,self_attn.v_proj,4,128,8,128,3.8664910793304443,4.126953125,8388608
+71,self_attn.v_proj,8,32,8,128,0.1769656091928482,8.5078125,8388608
+71,self_attn.v_proj,8,64,8,128,0.20656238496303558,8.25390625,8388608
+71,self_attn.v_proj,8,128,8,128,0.2329462617635727,8.126953125,8388608
+71,self_attn.k_proj,2,32,8,128,30.891511917114258,2.5078125,8388608
+71,self_attn.k_proj,2,64,8,128,36.71682357788086,2.25390625,8388608
+71,self_attn.k_proj,2,128,8,128,42.04000473022461,2.126953125,8388608
+71,self_attn.k_proj,3,32,8,128,13.229418754577637,3.5078125,8388608
+71,self_attn.k_proj,3,64,8,128,15.742876052856445,3.25390625,8388608
+71,self_attn.k_proj,3,128,8,128,18.095964431762695,3.126953125,8388608
+71,self_attn.k_proj,4,32,8,128,6.171253681182861,4.5078125,8388608
+71,self_attn.k_proj,4,64,8,128,7.3462347984313965,4.25390625,8388608
+71,self_attn.k_proj,4,128,8,128,8.448288917541504,4.126953125,8388608
+71,self_attn.k_proj,8,32,8,128,0.3734912574291229,8.5078125,8388608
+71,self_attn.k_proj,8,64,8,128,0.4442631006240845,8.25390625,8388608
+71,self_attn.k_proj,8,128,8,128,0.5089350342750549,8.126953125,8388608
+71,self_attn.o_proj,2,32,8,128,35.44900131225586,20.0625,67108864
+71,self_attn.o_proj,2,64,8,128,41.30754852294922,18.03125,67108864
+71,self_attn.o_proj,2,128,8,128,46.6004753112793,17.015625,67108864
+71,self_attn.o_proj,3,32,8,128,15.171170234680176,28.0625,67108864
+71,self_attn.o_proj,3,64,8,128,17.684022903442383,26.03125,67108864
+71,self_attn.o_proj,3,128,8,128,19.98017120361328,25.015625,67108864
+71,self_attn.o_proj,4,32,8,128,7.074581146240234,36.0625,67108864
+71,self_attn.o_proj,4,64,8,128,8.256081581115723,34.03125,67108864
+71,self_attn.o_proj,4,128,8,128,9.322015762329102,33.015625,67108864
+71,self_attn.o_proj,8,32,8,128,0.4298321604728699,68.0625,67108864
+71,self_attn.o_proj,8,64,8,128,0.5008589029312134,66.03125,67108864
+71,self_attn.o_proj,8,128,8,128,0.5641148686408997,65.015625,67108864
+71,mlp.gate_proj,2,32,8,128,76.27523040771484,70.21875,234881024
+71,mlp.gate_proj,2,64,8,128,90.81085968017578,63.109375,234881024
+71,mlp.gate_proj,2,128,8,128,104.19676971435547,59.5546875,234881024
+71,mlp.gate_proj,3,32,8,128,32.7415657043457,98.21875,234881024
+71,mlp.gate_proj,3,64,8,128,38.786720275878906,91.109375,234881024
+71,mlp.gate_proj,3,128,8,128,45.06076431274414,87.5546875,234881024
+71,mlp.gate_proj,4,32,8,128,15.290729522705078,126.21875,234881024
+71,mlp.gate_proj,4,64,8,128,18.08896827697754,119.109375,234881024
+71,mlp.gate_proj,4,128,8,128,21.000080108642578,115.5546875,234881024
+71,mlp.gate_proj,8,32,8,128,0.9329917430877686,238.21875,234881024
+71,mlp.gate_proj,8,64,8,128,1.103611946105957,231.109375,234881024
+71,mlp.gate_proj,8,128,8,128,1.2746168375015259,227.5546875,234881024
+71,mlp.up_proj,2,32,8,128,68.39393615722656,70.21875,234881024
+71,mlp.up_proj,2,64,8,128,79.66326141357422,63.109375,234881024
+71,mlp.up_proj,2,128,8,128,90.10916137695312,59.5546875,234881024
+71,mlp.up_proj,3,32,8,128,29.291790008544922,98.21875,234881024
+71,mlp.up_proj,3,64,8,128,34.089786529541016,91.109375,234881024
+71,mlp.up_proj,3,128,8,128,38.39808654785156,87.5546875,234881024
+71,mlp.up_proj,4,32,8,128,13.637269020080566,126.21875,234881024
+71,mlp.up_proj,4,64,8,128,15.975635528564453,119.109375,234881024
+71,mlp.up_proj,4,128,8,128,17.940425872802734,115.5546875,234881024
+71,mlp.up_proj,8,32,8,128,0.8325954079627991,238.21875,234881024
+71,mlp.up_proj,8,64,8,128,0.9744822978973389,231.109375,234881024
+71,mlp.up_proj,8,128,8,128,1.091763973236084,227.5546875,234881024
+71,mlp.down_proj,2,32,8,128,67.9198989868164,70.21875,234881024
+71,mlp.down_proj,2,64,8,128,79.0812759399414,63.109375,234881024
+71,mlp.down_proj,2,128,8,128,89.38114929199219,59.5546875,234881024
+71,mlp.down_proj,3,32,8,128,29.074405670166016,98.21875,234881024
+71,mlp.down_proj,3,64,8,128,33.847232818603516,91.109375,234881024
+71,mlp.down_proj,3,128,8,128,38.09489059448242,87.5546875,234881024
+71,mlp.down_proj,4,32,8,128,13.53721809387207,126.21875,234881024
+71,mlp.down_proj,4,64,8,128,15.851770401000977,119.109375,234881024
+71,mlp.down_proj,4,128,8,128,17.793489456176758,115.5546875,234881024
+71,mlp.down_proj,8,32,8,128,0.8265733122825623,238.21875,234881024
+71,mlp.down_proj,8,64,8,128,0.966854453086853,231.109375,234881024
+71,mlp.down_proj,8,128,8,128,1.0833534002304077,227.5546875,234881024
+72,self_attn.q_proj,2,32,8,128,45.87093734741211,20.0625,67108864
+72,self_attn.q_proj,2,64,8,128,53.970970153808594,18.03125,67108864
+72,self_attn.q_proj,2,128,8,128,62.01472854614258,17.015625,67108864
+72,self_attn.q_proj,3,32,8,128,19.610729217529297,28.0625,67108864
+72,self_attn.q_proj,3,64,8,128,23.13291358947754,26.03125,67108864
+72,self_attn.q_proj,3,128,8,128,26.58592987060547,25.015625,67108864
+72,self_attn.q_proj,4,32,8,128,9.146471977233887,36.0625,67108864
+72,self_attn.q_proj,4,64,8,128,10.795269012451172,34.03125,67108864
+72,self_attn.q_proj,4,128,8,128,12.411182403564453,33.015625,67108864
+72,self_attn.q_proj,8,32,8,128,0.5556230545043945,68.0625,67108864
+72,self_attn.q_proj,8,64,8,128,0.6556134819984436,66.03125,67108864
+72,self_attn.q_proj,8,128,8,128,0.7518096566200256,65.015625,67108864
+72,self_attn.v_proj,2,32,8,128,14.785261154174805,2.5078125,8388608
+72,self_attn.v_proj,2,64,8,128,17.289037704467773,2.25390625,8388608
+72,self_attn.v_proj,2,128,8,128,19.557937622070312,2.126953125,8388608
+72,self_attn.v_proj,3,32,8,128,6.317192554473877,3.5078125,8388608
+72,self_attn.v_proj,3,64,8,128,7.394779682159424,3.25390625,8388608
+72,self_attn.v_proj,3,128,8,128,8.373653411865234,3.126953125,8388608
+72,self_attn.v_proj,4,32,8,128,2.9483704566955566,4.5078125,8388608
+72,self_attn.v_proj,4,64,8,128,3.450660467147827,4.25390625,8388608
+72,self_attn.v_proj,4,128,8,128,3.9086735248565674,4.126953125,8388608
+72,self_attn.v_proj,8,32,8,128,0.17855603992938995,8.5078125,8388608
+72,self_attn.v_proj,8,64,8,128,0.20865142345428467,8.25390625,8388608
+72,self_attn.v_proj,8,128,8,128,0.23534859716892242,8.126953125,8388608
+72,self_attn.k_proj,2,32,8,128,30.582002639770508,2.5078125,8388608
+72,self_attn.k_proj,2,64,8,128,36.35914993286133,2.25390625,8388608
+72,self_attn.k_proj,2,128,8,128,41.652748107910156,2.126953125,8388608
+72,self_attn.k_proj,3,32,8,128,13.090524673461914,3.5078125,8388608
+72,self_attn.k_proj,3,64,8,128,15.600629806518555,3.25390625,8388608
+72,self_attn.k_proj,3,128,8,128,17.957857131958008,3.126953125,8388608
+72,self_attn.k_proj,4,32,8,128,6.112730979919434,4.5078125,8388608
+72,self_attn.k_proj,4,64,8,128,7.283754825592041,4.25390625,8388608
+72,self_attn.k_proj,4,128,8,128,8.388301849365234,4.126953125,8388608
+72,self_attn.k_proj,8,32,8,128,0.3699299395084381,8.5078125,8388608
+72,self_attn.k_proj,8,64,8,128,0.44029298424720764,8.25390625,8388608
+72,self_attn.k_proj,8,128,8,128,0.5049984455108643,8.126953125,8388608
+72,self_attn.o_proj,2,32,8,128,35.15370178222656,20.0625,67108864
+72,self_attn.o_proj,2,64,8,128,40.97298812866211,18.03125,67108864
+72,self_attn.o_proj,2,128,8,128,46.20988082885742,17.015625,67108864
+72,self_attn.o_proj,3,32,8,128,15.052817344665527,28.0625,67108864
+72,self_attn.o_proj,3,64,8,128,17.553760528564453,26.03125,67108864
+72,self_attn.o_proj,3,128,8,128,19.842687606811523,25.015625,67108864
+72,self_attn.o_proj,4,32,8,128,7.036149978637695,36.0625,67108864
+72,self_attn.o_proj,4,64,8,128,8.218515396118164,34.03125,67108864
+72,self_attn.o_proj,4,128,8,128,9.296625137329102,33.015625,67108864
+72,self_attn.o_proj,8,32,8,128,0.4278172552585602,68.0625,67108864
+72,self_attn.o_proj,8,64,8,128,0.4997723698616028,66.03125,67108864
+72,self_attn.o_proj,8,128,8,128,0.5650157332420349,65.015625,67108864
+72,mlp.gate_proj,2,32,8,128,77.87389373779297,70.21875,234881024
+72,mlp.gate_proj,2,64,8,128,92.91389465332031,63.109375,234881024
+72,mlp.gate_proj,2,128,8,128,107.4586410522461,59.5546875,234881024
+72,mlp.gate_proj,3,32,8,128,33.36417007446289,98.21875,234881024
+72,mlp.gate_proj,3,64,8,128,39.85036849975586,91.109375,234881024
+72,mlp.gate_proj,3,128,8,128,46.42362976074219,87.5546875,234881024
+72,mlp.gate_proj,4,32,8,128,15.619500160217285,126.21875,234881024
+72,mlp.gate_proj,4,64,8,128,18.571430206298828,119.109375,234881024
+72,mlp.gate_proj,4,128,8,128,21.714038848876953,115.5546875,234881024
+72,mlp.gate_proj,8,32,8,128,0.9530026316642761,238.21875,234881024
+72,mlp.gate_proj,8,64,8,128,1.1323366165161133,231.109375,234881024
+72,mlp.gate_proj,8,128,8,128,1.3170850276947021,227.5546875,234881024
+72,mlp.up_proj,2,32,8,128,67.97492218017578,70.21875,234881024
+72,mlp.up_proj,2,64,8,128,79.14541625976562,63.109375,234881024
+72,mlp.up_proj,2,128,8,128,89.50260925292969,59.5546875,234881024
+72,mlp.up_proj,3,32,8,128,29.099220275878906,98.21875,234881024
+72,mlp.up_proj,3,64,8,128,33.882076263427734,91.109375,234881024
+72,mlp.up_proj,3,128,8,128,38.17097854614258,87.5546875,234881024
+72,mlp.up_proj,4,32,8,128,13.549552917480469,126.21875,234881024
+72,mlp.up_proj,4,64,8,128,15.869789123535156,119.109375,234881024
+72,mlp.up_proj,4,128,8,128,17.827640533447266,115.5546875,234881024
+72,mlp.up_proj,8,32,8,128,0.8273308277130127,238.21875,234881024
+72,mlp.up_proj,8,64,8,128,0.9679552912712097,231.109375,234881024
+72,mlp.up_proj,8,128,8,128,1.0849127769470215,227.5546875,234881024
+72,mlp.down_proj,2,32,8,128,67.4945068359375,70.21875,234881024
+72,mlp.down_proj,2,64,8,128,78.55665588378906,63.109375,234881024
+72,mlp.down_proj,2,128,8,128,88.77367401123047,59.5546875,234881024
+72,mlp.down_proj,3,32,8,128,28.87398338317871,98.21875,234881024
+72,mlp.down_proj,3,64,8,128,33.63195037841797,91.109375,234881024
+72,mlp.down_proj,3,128,8,128,37.859073638916016,87.5546875,234881024
+72,mlp.down_proj,4,32,8,128,13.445395469665527,126.21875,234881024
+72,mlp.down_proj,4,64,8,128,15.742737770080566,119.109375,234881024
+72,mlp.down_proj,4,128,8,128,17.694564819335938,115.5546875,234881024
+72,mlp.down_proj,8,32,8,128,0.8212090730667114,238.21875,234881024
+72,mlp.down_proj,8,64,8,128,0.9602127075195312,231.109375,234881024
+72,mlp.down_proj,8,128,8,128,1.076486587524414,227.5546875,234881024
+73,self_attn.q_proj,2,32,8,128,44.908748626708984,20.0625,67108864
+73,self_attn.q_proj,2,64,8,128,52.642093658447266,18.03125,67108864
+73,self_attn.q_proj,2,128,8,128,60.68070602416992,17.015625,67108864
+73,self_attn.q_proj,3,32,8,128,19.197282791137695,28.0625,67108864
+73,self_attn.q_proj,3,64,8,128,22.587039947509766,26.03125,67108864
+73,self_attn.q_proj,3,128,8,128,26.0538387298584,25.015625,67108864
+73,self_attn.q_proj,4,32,8,128,8.956133842468262,36.0625,67108864
+73,self_attn.q_proj,4,64,8,128,10.534462928771973,34.03125,67108864
+73,self_attn.q_proj,4,128,8,128,12.157283782958984,33.015625,67108864
+73,self_attn.q_proj,8,32,8,128,0.5440316200256348,68.0625,67108864
+73,self_attn.q_proj,8,64,8,128,0.6400073170661926,66.03125,67108864
+73,self_attn.q_proj,8,128,8,128,0.7366273999214172,65.015625,67108864
+73,self_attn.v_proj,2,32,8,128,15.360084533691406,2.5078125,8388608
+73,self_attn.v_proj,2,64,8,128,18.05657196044922,2.25390625,8388608
+73,self_attn.v_proj,2,128,8,128,20.545289993286133,2.126953125,8388608
+73,self_attn.v_proj,3,32,8,128,6.567314624786377,3.5078125,8388608
+73,self_attn.v_proj,3,64,8,128,7.72932767868042,3.25390625,8388608
+73,self_attn.v_proj,3,128,8,128,8.795120239257812,3.126953125,8388608
+73,self_attn.v_proj,4,32,8,128,3.064174175262451,4.5078125,8388608
+73,self_attn.v_proj,4,64,8,128,3.604372501373291,4.25390625,8388608
+73,self_attn.v_proj,4,128,8,128,4.104313373565674,4.126953125,8388608
+73,self_attn.v_proj,8,32,8,128,0.18550726771354675,8.5078125,8388608
+73,self_attn.v_proj,8,64,8,128,0.218012735247612,8.25390625,8388608
+73,self_attn.v_proj,8,128,8,128,0.2472943663597107,8.126953125,8388608
+73,self_attn.k_proj,2,32,8,128,30.556140899658203,2.5078125,8388608
+73,self_attn.k_proj,2,64,8,128,36.41156768798828,2.25390625,8388608
+73,self_attn.k_proj,2,128,8,128,41.68701934814453,2.126953125,8388608
+73,self_attn.k_proj,3,32,8,128,13.097455024719238,3.5078125,8388608
+73,self_attn.k_proj,3,64,8,128,15.64253044128418,3.25390625,8388608
+73,self_attn.k_proj,3,128,8,128,18.028018951416016,3.126953125,8388608
+73,self_attn.k_proj,4,32,8,128,6.111292839050293,4.5078125,8388608
+73,self_attn.k_proj,4,64,8,128,7.3028106689453125,4.25390625,8388608
+73,self_attn.k_proj,4,128,8,128,8.41177749633789,4.126953125,8388608
+73,self_attn.k_proj,8,32,8,128,0.3698863685131073,8.5078125,8388608
+73,self_attn.k_proj,8,64,8,128,0.4413740336894989,8.25390625,8388608
+73,self_attn.k_proj,8,128,8,128,0.5064254403114319,8.126953125,8388608
+73,self_attn.o_proj,2,32,8,128,35.71365737915039,20.0625,67108864
+73,self_attn.o_proj,2,64,8,128,41.674896240234375,18.03125,67108864
+73,self_attn.o_proj,2,128,8,128,47.094444274902344,17.015625,67108864
+73,self_attn.o_proj,3,32,8,128,15.31737232208252,28.0625,67108864
+73,self_attn.o_proj,3,64,8,128,17.896757125854492,26.03125,67108864
+73,self_attn.o_proj,3,128,8,128,20.30257225036621,25.015625,67108864
+73,self_attn.o_proj,4,32,8,128,7.154277801513672,36.0625,67108864
+73,self_attn.o_proj,4,64,8,128,8.378737449645996,34.03125,67108864
+73,self_attn.o_proj,4,128,8,128,9.524287223815918,33.015625,67108864
+73,self_attn.o_proj,8,32,8,128,0.4346255362033844,68.0625,67108864
+73,self_attn.o_proj,8,64,8,128,0.5085970759391785,66.03125,67108864
+73,self_attn.o_proj,8,128,8,128,0.576569139957428,65.015625,67108864
+73,mlp.gate_proj,2,32,8,128,77.71849822998047,70.21875,234881024
+73,mlp.gate_proj,2,64,8,128,92.646728515625,63.109375,234881024
+73,mlp.gate_proj,2,128,8,128,106.94808959960938,59.5546875,234881024
+73,mlp.gate_proj,3,32,8,128,33.30611038208008,98.21875,234881024
+73,mlp.gate_proj,3,64,8,128,39.703346252441406,91.109375,234881024
+73,mlp.gate_proj,3,128,8,128,46.205020904541016,87.5546875,234881024
+73,mlp.gate_proj,4,32,8,128,15.589932441711426,126.21875,234881024
+73,mlp.gate_proj,4,64,8,128,18.504695892333984,119.109375,234881024
+73,mlp.gate_proj,4,128,8,128,21.589311599731445,115.5546875,234881024
+73,mlp.gate_proj,8,32,8,128,0.95123291015625,238.21875,234881024
+73,mlp.gate_proj,8,64,8,128,1.1287516355514526,231.109375,234881024
+73,mlp.gate_proj,8,128,8,128,1.3102949857711792,227.5546875,234881024
+73,mlp.up_proj,2,32,8,128,68.20610809326172,70.21875,234881024
+73,mlp.up_proj,2,64,8,128,79.46021270751953,63.109375,234881024
+73,mlp.up_proj,2,128,8,128,89.91656494140625,59.5546875,234881024
+73,mlp.up_proj,3,32,8,128,29.209928512573242,98.21875,234881024
+73,mlp.up_proj,3,64,8,128,34.01777267456055,91.109375,234881024
+73,mlp.up_proj,3,128,8,128,38.34321975708008,87.5546875,234881024
+73,mlp.up_proj,4,32,8,128,13.601468086242676,126.21875,234881024
+73,mlp.up_proj,4,64,8,128,15.93851089477539,119.109375,234881024
+73,mlp.up_proj,4,128,8,128,17.909387588500977,115.5546875,234881024
+73,mlp.up_proj,8,32,8,128,0.8303664326667786,238.21875,234881024
+73,mlp.up_proj,8,64,8,128,0.972183883190155,231.109375,234881024
+73,mlp.up_proj,8,128,8,128,1.0900572538375854,227.5546875,234881024
+73,mlp.down_proj,2,32,8,128,67.70048522949219,70.21875,234881024
+73,mlp.down_proj,2,64,8,128,78.8084945678711,63.109375,234881024
+73,mlp.down_proj,2,128,8,128,89.08326721191406,59.5546875,234881024
+73,mlp.down_proj,3,32,8,128,28.971731185913086,98.21875,234881024
+73,mlp.down_proj,3,64,8,128,33.740264892578125,91.109375,234881024
+73,mlp.down_proj,3,128,8,128,37.98360824584961,87.5546875,234881024
+73,mlp.down_proj,4,32,8,128,13.491297721862793,126.21875,234881024
+73,mlp.down_proj,4,64,8,128,15.798612594604492,119.109375,234881024
+73,mlp.down_proj,4,128,8,128,17.74402618408203,115.5546875,234881024
+73,mlp.down_proj,8,32,8,128,0.8238307237625122,238.21875,234881024
+73,mlp.down_proj,8,64,8,128,0.9637379050254822,231.109375,234881024
+73,mlp.down_proj,8,128,8,128,1.0803669691085815,227.5546875,234881024
+74,self_attn.q_proj,2,32,8,128,45.908111572265625,20.0625,67108864
+74,self_attn.q_proj,2,64,8,128,54.356529235839844,18.03125,67108864
+74,self_attn.q_proj,2,128,8,128,62.826969146728516,17.015625,67108864
+74,self_attn.q_proj,3,32,8,128,19.638975143432617,28.0625,67108864
+74,self_attn.q_proj,3,64,8,128,23.32144546508789,26.03125,67108864
+74,self_attn.q_proj,3,128,8,128,27.010229110717773,25.015625,67108864
+74,self_attn.q_proj,4,32,8,128,9.161828994750977,36.0625,67108864
+74,self_attn.q_proj,4,64,8,128,10.882911682128906,34.03125,67108864
+74,self_attn.q_proj,4,128,8,128,12.603344917297363,33.015625,67108864
+74,self_attn.q_proj,8,32,8,128,0.5565611124038696,68.0625,67108864
+74,self_attn.q_proj,8,64,8,128,0.6612533330917358,66.03125,67108864
+74,self_attn.q_proj,8,128,8,128,0.7637158632278442,65.015625,67108864
+74,self_attn.v_proj,2,32,8,128,15.615221977233887,2.5078125,8388608
+74,self_attn.v_proj,2,64,8,128,18.50690460205078,2.25390625,8388608
+74,self_attn.v_proj,2,128,8,128,21.25008773803711,2.126953125,8388608
+74,self_attn.v_proj,3,32,8,128,6.678697109222412,3.5078125,8388608
+74,self_attn.v_proj,3,64,8,128,7.928497314453125,3.25390625,8388608
+74,self_attn.v_proj,3,128,8,128,9.116799354553223,3.126953125,8388608
+74,self_attn.v_proj,4,32,8,128,3.1170694828033447,4.5078125,8388608
+74,self_attn.v_proj,4,64,8,128,3.6992039680480957,4.25390625,8388608
+74,self_attn.v_proj,4,128,8,128,4.254657745361328,4.126953125,8388608
+74,self_attn.v_proj,8,32,8,128,0.18863582611083984,8.5078125,8388608
+74,self_attn.v_proj,8,64,8,128,0.22366201877593994,8.25390625,8388608
+74,self_attn.v_proj,8,128,8,128,0.25627195835113525,8.126953125,8388608
+74,self_attn.k_proj,2,32,8,128,29.896936416625977,2.5078125,8388608
+74,self_attn.k_proj,2,64,8,128,35.65000915527344,2.25390625,8388608
+74,self_attn.k_proj,2,128,8,128,41.01694107055664,2.126953125,8388608
+74,self_attn.k_proj,3,32,8,128,12.806458473205566,3.5078125,8388608
+74,self_attn.k_proj,3,64,8,128,15.308561325073242,3.25390625,8388608
+74,self_attn.k_proj,3,128,8,128,17.69248390197754,3.126953125,8388608
+74,self_attn.k_proj,4,32,8,128,5.978397369384766,4.5078125,8388608
+74,self_attn.k_proj,4,64,8,128,7.14662504196167,4.25390625,8388608
+74,self_attn.k_proj,4,128,8,128,8.266164779663086,4.126953125,8388608
+74,self_attn.k_proj,8,32,8,128,0.36178767681121826,8.5078125,8388608
+74,self_attn.k_proj,8,64,8,128,0.4319888949394226,8.25390625,8388608
+74,self_attn.k_proj,8,128,8,128,0.4978649616241455,8.126953125,8388608
+74,self_attn.o_proj,2,32,8,128,35.20986557006836,20.0625,67108864
+74,self_attn.o_proj,2,64,8,128,41.076419830322266,18.03125,67108864
+74,self_attn.o_proj,2,128,8,128,46.38601303100586,17.015625,67108864
+74,self_attn.o_proj,3,32,8,128,15.085688591003418,28.0625,67108864
+74,self_attn.o_proj,3,64,8,128,17.618566513061523,26.03125,67108864
+74,self_attn.o_proj,3,128,8,128,19.96009063720703,25.015625,67108864
+74,self_attn.o_proj,4,32,8,128,7.044442653656006,36.0625,67108864
+74,self_attn.o_proj,4,64,8,128,8.24317741394043,34.03125,67108864
+74,self_attn.o_proj,4,128,8,128,9.345968246459961,33.015625,67108864
+74,self_attn.o_proj,8,32,8,128,0.4280918836593628,68.0625,67108864
+74,self_attn.o_proj,8,64,8,128,0.5004184246063232,66.03125,67108864
+74,self_attn.o_proj,8,128,8,128,0.5661728978157043,65.015625,67108864
+74,mlp.gate_proj,2,32,8,128,77.5662612915039,70.21875,234881024
+74,mlp.gate_proj,2,64,8,128,92.41680145263672,63.109375,234881024
+74,mlp.gate_proj,2,128,8,128,106.57329559326172,59.5546875,234881024
+74,mlp.gate_proj,3,32,8,128,33.243263244628906,98.21875,234881024
+74,mlp.gate_proj,3,64,8,128,39.58908462524414,91.109375,234881024
+74,mlp.gate_proj,3,128,8,128,46.035465240478516,87.5546875,234881024
+74,mlp.gate_proj,4,32,8,128,15.555849075317383,126.21875,234881024
+74,mlp.gate_proj,4,64,8,128,18.453081130981445,119.109375,234881024
+74,mlp.gate_proj,4,128,8,128,21.49773597717285,115.5546875,234881024
+74,mlp.gate_proj,8,32,8,128,0.9491239786148071,238.21875,234881024
+74,mlp.gate_proj,8,64,8,128,1.1254314184188843,231.109375,234881024
+74,mlp.gate_proj,8,128,8,128,1.3049715757369995,227.5546875,234881024
+74,mlp.up_proj,2,32,8,128,68.49195098876953,70.21875,234881024
+74,mlp.up_proj,2,64,8,128,79.7950210571289,63.109375,234881024
+74,mlp.up_proj,2,128,8,128,90.26329040527344,59.5546875,234881024
+74,mlp.up_proj,3,32,8,128,29.334569931030273,98.21875,234881024
+74,mlp.up_proj,3,64,8,128,34.14316177368164,91.109375,234881024
+74,mlp.up_proj,3,128,8,128,38.466957092285156,87.5546875,234881024
+74,mlp.up_proj,4,32,8,128,13.65725040435791,126.21875,234881024
+74,mlp.up_proj,4,64,8,128,16.001522064208984,119.109375,234881024
+74,mlp.up_proj,4,128,8,128,17.966344833374023,115.5546875,234881024
+74,mlp.up_proj,8,32,8,128,0.8338944315910339,238.21875,234881024
+74,mlp.up_proj,8,64,8,128,0.9761962890625,231.109375,234881024
+74,mlp.up_proj,8,128,8,128,1.0936640501022339,227.5546875,234881024
+74,mlp.down_proj,2,32,8,128,68.01387023925781,70.21875,234881024
+74,mlp.down_proj,2,64,8,128,79.19795989990234,63.109375,234881024
+74,mlp.down_proj,2,128,8,128,89.59196472167969,59.5546875,234881024
+74,mlp.down_proj,3,32,8,128,29.11899757385254,98.21875,234881024
+74,mlp.down_proj,3,64,8,128,33.90629196166992,91.109375,234881024
+74,mlp.down_proj,3,128,8,128,38.18498229980469,87.5546875,234881024
+74,mlp.down_proj,4,32,8,128,13.558594703674316,126.21875,234881024
+74,mlp.down_proj,4,64,8,128,15.881946563720703,119.109375,234881024
+74,mlp.down_proj,4,128,8,128,17.842344284057617,115.5546875,234881024
+74,mlp.down_proj,8,32,8,128,0.8278955221176147,238.21875,234881024
+74,mlp.down_proj,8,64,8,128,0.9688405990600586,231.109375,234881024
+74,mlp.down_proj,8,128,8,128,1.085961103439331,227.5546875,234881024
+75,self_attn.q_proj,2,32,8,128,46.86625289916992,20.0625,67108864
+75,self_attn.q_proj,2,64,8,128,55.34519577026367,18.03125,67108864
+75,self_attn.q_proj,2,128,8,128,63.81245803833008,17.015625,67108864
+75,self_attn.q_proj,3,32,8,128,20.05324363708496,28.0625,67108864
+75,self_attn.q_proj,3,64,8,128,23.729326248168945,26.03125,67108864
+75,self_attn.q_proj,3,128,8,128,27.397777557373047,25.015625,67108864
+75,self_attn.q_proj,4,32,8,128,9.352805137634277,36.0625,67108864
+75,self_attn.q_proj,4,64,8,128,11.078514099121094,34.03125,67108864
+75,self_attn.q_proj,4,128,8,128,12.789298057556152,33.015625,67108864
+75,self_attn.q_proj,8,32,8,128,0.568340003490448,68.0625,67108864
+75,self_attn.q_proj,8,64,8,128,0.6730284094810486,66.03125,67108864
+75,self_attn.q_proj,8,128,8,128,0.7745082378387451,65.015625,67108864
+75,self_attn.v_proj,2,32,8,128,16.094350814819336,2.5078125,8388608
+75,self_attn.v_proj,2,64,8,128,18.82871437072754,2.25390625,8388608
+75,self_attn.v_proj,2,128,8,128,21.317386627197266,2.126953125,8388608
+75,self_attn.v_proj,3,32,8,128,6.881319046020508,3.5078125,8388608
+75,self_attn.v_proj,3,64,8,128,8.055254936218262,3.25390625,8388608
+75,self_attn.v_proj,3,128,8,128,9.12129020690918,3.126953125,8388608
+75,self_attn.v_proj,4,32,8,128,3.2102701663970947,4.5078125,8388608
+75,self_attn.v_proj,4,64,8,128,3.758492946624756,4.25390625,8388608
+75,self_attn.v_proj,4,128,8,128,4.2574357986450195,4.126953125,8388608
+75,self_attn.v_proj,8,32,8,128,0.19439326226711273,8.5078125,8388608
+75,self_attn.v_proj,8,64,8,128,0.22729763388633728,8.25390625,8388608
+75,self_attn.v_proj,8,128,8,128,0.2565464675426483,8.126953125,8388608
+75,self_attn.k_proj,2,32,8,128,28.6390323638916,2.5078125,8388608
+75,self_attn.k_proj,2,64,8,128,34.00221633911133,2.25390625,8388608
+75,self_attn.k_proj,2,128,8,128,38.90826416015625,2.126953125,8388608
+75,self_attn.k_proj,3,32,8,128,12.268054008483887,3.5078125,8388608
+75,self_attn.k_proj,3,64,8,128,14.609052658081055,3.25390625,8388608
+75,self_attn.k_proj,3,128,8,128,16.77063751220703,3.126953125,8388608
+75,self_attn.k_proj,4,32,8,128,5.727261066436768,4.5078125,8388608
+75,self_attn.k_proj,4,64,8,128,6.819714069366455,4.25390625,8388608
+75,self_attn.k_proj,4,128,8,128,7.83676290512085,4.126953125,8388608
+75,self_attn.k_proj,8,32,8,128,0.34680312871932983,8.5078125,8388608
+75,self_attn.k_proj,8,64,8,128,0.4125382900238037,8.25390625,8388608
+75,self_attn.k_proj,8,128,8,128,0.4722334146499634,8.126953125,8388608
+75,self_attn.o_proj,2,32,8,128,37.050636291503906,20.0625,67108864
+75,self_attn.o_proj,2,64,8,128,43.26202392578125,18.03125,67108864
+75,self_attn.o_proj,2,128,8,128,48.8711051940918,17.015625,67108864
+75,self_attn.o_proj,3,32,8,128,15.870832443237305,28.0625,67108864
+75,self_attn.o_proj,3,64,8,128,18.53050994873047,26.03125,67108864
+75,self_attn.o_proj,3,128,8,128,21.014774322509766,25.015625,67108864
+75,self_attn.o_proj,4,32,8,128,7.400918483734131,36.0625,67108864
+75,self_attn.o_proj,4,64,8,128,8.651154518127441,34.03125,67108864
+75,self_attn.o_proj,4,128,8,128,9.813300132751465,33.015625,67108864
+75,self_attn.o_proj,8,32,8,128,0.4493739902973175,68.0625,67108864
+75,self_attn.o_proj,8,64,8,128,0.5249952673912048,66.03125,67108864
+75,self_attn.o_proj,8,128,8,128,0.5936371684074402,65.015625,67108864
+75,mlp.gate_proj,2,32,8,128,79.02632141113281,70.21875,234881024
+75,mlp.gate_proj,2,64,8,128,94.2256088256836,63.109375,234881024
+75,mlp.gate_proj,2,128,8,128,109.20569610595703,59.5546875,234881024
+75,mlp.gate_proj,3,32,8,128,33.81875228881836,98.21875,234881024
+75,mlp.gate_proj,3,64,8,128,40.47719955444336,91.109375,234881024
+75,mlp.gate_proj,3,128,8,128,47.135257720947266,87.5546875,234881024
+75,mlp.gate_proj,4,32,8,128,15.856170654296875,126.21875,234881024
+75,mlp.gate_proj,4,64,8,128,18.8575496673584,119.109375,234881024
+75,mlp.gate_proj,4,128,8,128,22.07779312133789,115.5546875,234881024
+75,mlp.gate_proj,8,32,8,128,0.9674955606460571,238.21875,234881024
+75,mlp.gate_proj,8,64,8,128,1.1498275995254517,231.109375,234881024
+75,mlp.gate_proj,8,128,8,128,1.3389769792556763,227.5546875,234881024
+75,mlp.up_proj,2,32,8,128,68.4249267578125,70.21875,234881024
+75,mlp.up_proj,2,64,8,128,79.72909545898438,63.109375,234881024
+75,mlp.up_proj,2,128,8,128,90.20531463623047,59.5546875,234881024
+75,mlp.up_proj,3,32,8,128,29.307039260864258,98.21875,234881024
+75,mlp.up_proj,3,64,8,128,34.11951446533203,91.109375,234881024
+75,mlp.up_proj,3,128,8,128,38.444766998291016,87.5546875,234881024
+75,mlp.up_proj,4,32,8,128,13.644506454467773,126.21875,234881024
+75,mlp.up_proj,4,64,8,128,15.989699363708496,119.109375,234881024
+75,mlp.up_proj,4,128,8,128,17.957698822021484,115.5546875,234881024
+75,mlp.up_proj,8,32,8,128,0.8330489993095398,238.21875,234881024
+75,mlp.up_proj,8,64,8,128,0.9753963351249695,231.109375,234881024
+75,mlp.up_proj,8,128,8,128,1.0929673910140991,227.5546875,234881024
+75,mlp.down_proj,2,32,8,128,67.86714935302734,70.21875,234881024
+75,mlp.down_proj,2,64,8,128,79.02397155761719,63.109375,234881024
+75,mlp.down_proj,2,128,8,128,89.38908386230469,59.5546875,234881024
+75,mlp.down_proj,3,32,8,128,29.054061889648438,98.21875,234881024
+75,mlp.down_proj,3,64,8,128,33.83174133300781,91.109375,234881024
+75,mlp.down_proj,3,128,8,128,38.10285949707031,87.5546875,234881024
+75,mlp.down_proj,4,32,8,128,13.527694702148438,126.21875,234881024
+75,mlp.down_proj,4,64,8,128,15.84890079498291,119.109375,234881024
+75,mlp.down_proj,4,128,8,128,17.798532485961914,115.5546875,234881024
+75,mlp.down_proj,8,32,8,128,0.8260056972503662,238.21875,234881024
+75,mlp.down_proj,8,64,8,128,0.9668017625808716,231.109375,234881024
+75,mlp.down_proj,8,128,8,128,1.0836479663848877,227.5546875,234881024
+76,self_attn.q_proj,2,32,8,128,44.96638488769531,20.0625,67108864
+76,self_attn.q_proj,2,64,8,128,52.79098892211914,18.03125,67108864
+76,self_attn.q_proj,2,128,8,128,60.85258483886719,17.015625,67108864
+76,self_attn.q_proj,3,32,8,128,19.22578239440918,28.0625,67108864
+76,self_attn.q_proj,3,64,8,128,22.652141571044922,26.03125,67108864
+76,self_attn.q_proj,3,128,8,128,26.12206268310547,25.015625,67108864
+76,self_attn.q_proj,4,32,8,128,8.96791934967041,36.0625,67108864
+76,self_attn.q_proj,4,64,8,128,10.5626802444458,34.03125,67108864
+76,self_attn.q_proj,4,128,8,128,12.19727897644043,33.015625,67108864
+76,self_attn.q_proj,8,32,8,128,0.5446987748146057,68.0625,67108864
+76,self_attn.q_proj,8,64,8,128,0.6416283845901489,66.03125,67108864
+76,self_attn.q_proj,8,128,8,128,0.7388189435005188,65.015625,67108864
+76,self_attn.v_proj,2,32,8,128,16.06108283996582,2.5078125,8388608
+76,self_attn.v_proj,2,64,8,128,18.849332809448242,2.25390625,8388608
+76,self_attn.v_proj,2,128,8,128,21.393030166625977,2.126953125,8388608
+76,self_attn.v_proj,3,32,8,128,6.866220474243164,3.5078125,8388608
+76,self_attn.v_proj,3,64,8,128,8.062259674072266,3.25390625,8388608
+76,self_attn.v_proj,3,128,8,128,9.153508186340332,3.126953125,8388608
+76,self_attn.v_proj,4,32,8,128,3.203481435775757,4.5078125,8388608
+76,self_attn.v_proj,4,64,8,128,3.762338161468506,4.25390625,8388608
+76,self_attn.v_proj,4,128,8,128,4.273663520812988,4.126953125,8388608
+76,self_attn.v_proj,8,32,8,128,0.1939712017774582,8.5078125,8388608
+76,self_attn.v_proj,8,64,8,128,0.22737886011600494,8.25390625,8388608
+76,self_attn.v_proj,8,128,8,128,0.25735941529273987,8.126953125,8388608
+76,self_attn.k_proj,2,32,8,128,28.182710647583008,2.5078125,8388608
+76,self_attn.k_proj,2,64,8,128,33.58625411987305,2.25390625,8388608
+76,self_attn.k_proj,2,128,8,128,38.56458282470703,2.126953125,8388608
+76,self_attn.k_proj,3,32,8,128,12.065102577209473,3.5078125,8388608
+76,self_attn.k_proj,3,64,8,128,14.421436309814453,3.25390625,8388608
+76,self_attn.k_proj,3,128,8,128,16.632572174072266,3.126953125,8388608
+76,self_attn.k_proj,4,32,8,128,5.631641864776611,4.5078125,8388608
+76,self_attn.k_proj,4,64,8,128,6.726741790771484,4.25390625,8388608
+76,self_attn.k_proj,4,128,8,128,7.761914253234863,4.126953125,8388608
+76,self_attn.k_proj,8,32,8,128,0.34074726700782776,8.5078125,8388608
+76,self_attn.k_proj,8,64,8,128,0.4068009555339813,8.25390625,8388608
+76,self_attn.k_proj,8,128,8,128,0.4677245318889618,8.126953125,8388608
+76,self_attn.o_proj,2,32,8,128,36.240142822265625,20.0625,67108864
+76,self_attn.o_proj,2,64,8,128,42.31157302856445,18.03125,67108864
+76,self_attn.o_proj,2,128,8,128,47.78789520263672,17.015625,67108864
+76,self_attn.o_proj,3,32,8,128,15.539855003356934,28.0625,67108864
+76,self_attn.o_proj,3,64,8,128,18.154129028320312,26.03125,67108864
+76,self_attn.o_proj,3,128,8,128,20.59672737121582,25.015625,67108864
+76,self_attn.o_proj,4,32,8,128,7.251073360443115,36.0625,67108864
+76,self_attn.o_proj,4,64,8,128,8.487772941589355,34.03125,67108864
+76,self_attn.o_proj,4,128,8,128,9.639594078063965,33.015625,67108864
+76,self_attn.o_proj,8,32,8,128,0.4404081404209137,68.0625,67108864
+76,self_attn.o_proj,8,64,8,128,0.5149508714675903,66.03125,67108864
+76,self_attn.o_proj,8,128,8,128,0.5831468105316162,65.015625,67108864
+76,mlp.gate_proj,2,32,8,128,87.50687408447266,70.21875,234881024
+76,mlp.gate_proj,2,64,8,128,105.46006774902344,63.109375,234881024
+76,mlp.gate_proj,2,128,8,128,124.30660247802734,59.5546875,234881024
+76,mlp.gate_proj,3,32,8,128,37.322357177734375,98.21875,234881024
+76,mlp.gate_proj,3,64,8,128,45.688045501708984,91.109375,234881024
+76,mlp.gate_proj,3,128,8,128,53.897525787353516,87.5546875,234881024
+76,mlp.gate_proj,4,32,8,128,17.41168212890625,126.21875,234881024
+76,mlp.gate_proj,4,64,8,128,21.303808212280273,119.109375,234881024
+76,mlp.gate_proj,4,128,8,128,25.12656021118164,115.5546875,234881024
+76,mlp.gate_proj,8,32,8,128,1.0630604028701782,238.21875,234881024
+76,mlp.gate_proj,8,64,8,128,1.2963154315948486,231.109375,234881024
+76,mlp.gate_proj,8,128,8,128,1.5235844850540161,227.5546875,234881024
+76,mlp.up_proj,2,32,8,128,65.95216369628906,70.21875,234881024
+76,mlp.up_proj,2,64,8,128,76.72928619384766,63.109375,234881024
+76,mlp.up_proj,2,128,8,128,86.83141326904297,59.5546875,234881024
+76,mlp.up_proj,3,32,8,128,28.163097381591797,98.21875,234881024
+76,mlp.up_proj,3,64,8,128,32.925941467285156,91.109375,234881024
+76,mlp.up_proj,3,128,8,128,37.1202278137207,87.5546875,234881024
+76,mlp.up_proj,4,32,8,128,13.120469093322754,126.21875,234881024
+76,mlp.up_proj,4,64,8,128,15.379888534545898,119.109375,234881024
+76,mlp.up_proj,4,128,8,128,17.347118377685547,115.5546875,234881024
+76,mlp.up_proj,8,32,8,128,0.8013165593147278,238.21875,234881024
+76,mlp.up_proj,8,64,8,128,0.9375361204147339,231.109375,234881024
+76,mlp.up_proj,8,128,8,128,1.0552701950073242,227.5546875,234881024
+76,mlp.down_proj,2,32,8,128,64.67002868652344,70.21875,234881024
+76,mlp.down_proj,2,64,8,128,74.96887969970703,63.109375,234881024
+76,mlp.down_proj,2,128,8,128,84.75555419921875,59.5546875,234881024
+76,mlp.down_proj,3,32,8,128,27.537456512451172,98.21875,234881024
+76,mlp.down_proj,3,64,8,128,32.24800491333008,91.109375,234881024
+76,mlp.down_proj,3,128,8,128,36.22049331665039,87.5546875,234881024
+76,mlp.down_proj,4,32,8,128,12.83462905883789,126.21875,234881024
+76,mlp.down_proj,4,64,8,128,15.025320053100586,119.109375,234881024
+76,mlp.down_proj,4,128,8,128,16.934677124023438,115.5546875,234881024
+76,mlp.down_proj,8,32,8,128,0.7842637300491333,238.21875,234881024
+76,mlp.down_proj,8,64,8,128,0.9156776070594788,231.109375,234881024
+76,mlp.down_proj,8,128,8,128,1.0316241979599,227.5546875,234881024
+77,self_attn.q_proj,2,32,8,128,45.46040344238281,20.0625,67108864
+77,self_attn.q_proj,2,64,8,128,53.22178649902344,18.03125,67108864
+77,self_attn.q_proj,2,128,8,128,61.40945816040039,17.015625,67108864
+77,self_attn.q_proj,3,32,8,128,19.437721252441406,28.0625,67108864
+77,self_attn.q_proj,3,64,8,128,22.830425262451172,26.03125,67108864
+77,self_attn.q_proj,3,128,8,128,26.38439178466797,25.015625,67108864
+77,self_attn.q_proj,4,32,8,128,9.067130088806152,36.0625,67108864
+77,self_attn.q_proj,4,64,8,128,10.651561737060547,34.03125,67108864
+77,self_attn.q_proj,4,128,8,128,12.311981201171875,33.015625,67108864
+77,self_attn.q_proj,8,32,8,128,0.5506958365440369,68.0625,67108864
+77,self_attn.q_proj,8,64,8,128,0.6469172239303589,66.03125,67108864
+77,self_attn.q_proj,8,128,8,128,0.7459492683410645,65.015625,67108864
+77,self_attn.v_proj,2,32,8,128,17.348421096801758,2.5078125,8388608
+77,self_attn.v_proj,2,64,8,128,20.481374740600586,2.25390625,8388608
+77,self_attn.v_proj,2,128,8,128,23.39963150024414,2.126953125,8388608
+77,self_attn.v_proj,3,32,8,128,7.425415992736816,3.5078125,8388608
+77,self_attn.v_proj,3,64,8,128,8.771838188171387,3.25390625,8388608
+77,self_attn.v_proj,3,128,8,128,10.021161079406738,3.126953125,8388608
+77,self_attn.v_proj,4,32,8,128,3.463970184326172,4.5078125,8388608
+77,self_attn.v_proj,4,64,8,128,4.092836380004883,4.25390625,8388608
+77,self_attn.v_proj,4,128,8,128,4.677574157714844,4.126953125,8388608
+77,self_attn.v_proj,8,32,8,128,0.20955178141593933,8.5078125,8388608
+77,self_attn.v_proj,8,64,8,128,0.24733172357082367,8.25390625,8388608
+77,self_attn.v_proj,8,128,8,128,0.28185445070266724,8.126953125,8388608
+77,self_attn.k_proj,2,32,8,128,28.821823120117188,2.5078125,8388608
+77,self_attn.k_proj,2,64,8,128,34.22096252441406,2.25390625,8388608
+77,self_attn.k_proj,2,128,8,128,39.14206314086914,2.126953125,8388608
+77,self_attn.k_proj,3,32,8,128,12.344230651855469,3.5078125,8388608
+77,self_attn.k_proj,3,64,8,128,14.675520896911621,3.25390625,8388608
+77,self_attn.k_proj,3,128,8,128,16.855392456054688,3.126953125,8388608
+77,self_attn.k_proj,4,32,8,128,5.757424831390381,4.5078125,8388608
+77,self_attn.k_proj,4,64,8,128,6.850033760070801,4.25390625,8388608
+77,self_attn.k_proj,4,128,8,128,7.867064476013184,4.126953125,8388608
+77,self_attn.k_proj,8,32,8,128,0.34863340854644775,8.5078125,8388608
+77,self_attn.k_proj,8,64,8,128,0.41413548588752747,8.25390625,8388608
+77,self_attn.k_proj,8,128,8,128,0.4738069474697113,8.126953125,8388608
+77,self_attn.o_proj,2,32,8,128,37.59469985961914,20.0625,67108864
+77,self_attn.o_proj,2,64,8,128,43.857444763183594,18.03125,67108864
+77,self_attn.o_proj,2,128,8,128,49.43571853637695,17.015625,67108864
+77,self_attn.o_proj,3,32,8,128,16.095687866210938,28.0625,67108864
+77,self_attn.o_proj,3,64,8,128,18.76519203186035,26.03125,67108864
+77,self_attn.o_proj,3,128,8,128,21.22408103942871,25.015625,67108864
+77,self_attn.o_proj,4,32,8,128,7.507500648498535,36.0625,67108864
+77,self_attn.o_proj,4,64,8,128,8.760346412658691,34.03125,67108864
+77,self_attn.o_proj,4,128,8,128,9.90742015838623,33.015625,67108864
+77,self_attn.o_proj,8,32,8,128,0.4558795988559723,68.0625,67108864
+77,self_attn.o_proj,8,64,8,128,0.531673014163971,66.03125,67108864
+77,self_attn.o_proj,8,128,8,128,0.5995996594429016,65.015625,67108864
+77,mlp.gate_proj,2,32,8,128,80.48353576660156,70.21875,234881024
+77,mlp.gate_proj,2,64,8,128,95.82299041748047,63.109375,234881024
+77,mlp.gate_proj,2,128,8,128,111.09007263183594,59.5546875,234881024
+77,mlp.gate_proj,3,32,8,128,34.40098190307617,98.21875,234881024
+77,mlp.gate_proj,3,64,8,128,41.193660736083984,91.109375,234881024
+77,mlp.gate_proj,3,128,8,128,47.882408142089844,87.5546875,234881024
+77,mlp.gate_proj,4,32,8,128,16.127378463745117,126.21875,234881024
+77,mlp.gate_proj,4,64,8,128,19.192602157592773,119.109375,234881024
+77,mlp.gate_proj,4,128,8,128,22.432064056396484,115.5546875,234881024
+77,mlp.gate_proj,8,32,8,128,0.9857689142227173,238.21875,234881024
+77,mlp.gate_proj,8,64,8,128,1.1700977087020874,231.109375,234881024
+77,mlp.gate_proj,8,128,8,128,1.3611618280410767,227.5546875,234881024
+77,mlp.up_proj,2,32,8,128,69.1682357788086,70.21875,234881024
+77,mlp.up_proj,2,64,8,128,80.74910736083984,63.109375,234881024
+77,mlp.up_proj,2,128,8,128,91.39458465576172,59.5546875,234881024
+77,mlp.up_proj,3,32,8,128,29.66246795654297,98.21875,234881024
+77,mlp.up_proj,3,64,8,128,34.562889099121094,91.109375,234881024
+77,mlp.up_proj,3,128,8,128,39.06763458251953,87.5546875,234881024
+77,mlp.up_proj,4,32,8,128,13.807538986206055,126.21875,234881024
+77,mlp.up_proj,4,64,8,128,16.183992385864258,119.109375,234881024
+77,mlp.up_proj,4,128,8,128,18.244230270385742,115.5546875,234881024
+77,mlp.up_proj,8,32,8,128,0.8428636193275452,238.21875,234881024
+77,mlp.up_proj,8,64,8,128,0.9890768527984619,231.109375,234881024
+77,mlp.up_proj,8,128,8,128,1.1099514961242676,227.5546875,234881024
+77,mlp.down_proj,2,32,8,128,68.31208038330078,70.21875,234881024
+77,mlp.down_proj,2,64,8,128,79.65937042236328,63.109375,234881024
+77,mlp.down_proj,2,128,8,128,90.23479461669922,59.5546875,234881024
+77,mlp.down_proj,3,32,8,128,29.271326065063477,98.21875,234881024
+77,mlp.down_proj,3,64,8,128,34.11908721923828,91.109375,234881024
+77,mlp.down_proj,3,128,8,128,38.534461975097656,87.5546875,234881024
+77,mlp.down_proj,4,32,8,128,13.628515243530273,126.21875,234881024
+77,mlp.down_proj,4,64,8,128,15.996020317077637,119.109375,234881024
+77,mlp.down_proj,4,128,8,128,17.99416160583496,115.5546875,234881024
+77,mlp.down_proj,8,32,8,128,0.8319529891014099,238.21875,234881024
+77,mlp.down_proj,8,64,8,128,0.9758437275886536,231.109375,234881024
+77,mlp.down_proj,8,128,8,128,1.0952818393707275,227.5546875,234881024
+78,self_attn.q_proj,2,32,8,128,45.151554107666016,20.0625,67108864
+78,self_attn.q_proj,2,64,8,128,53.060829162597656,18.03125,67108864
+78,self_attn.q_proj,2,128,8,128,61.05052947998047,17.015625,67108864
+78,self_attn.q_proj,3,32,8,128,19.301328659057617,28.0625,67108864
+78,self_attn.q_proj,3,64,8,128,22.754592895507812,26.03125,67108864
+78,self_attn.q_proj,3,128,8,128,26.20524024963379,25.015625,67108864
+78,self_attn.q_proj,4,32,8,128,9.003338813781738,36.0625,67108864
+78,self_attn.q_proj,4,64,8,128,10.614086151123047,34.03125,67108864
+78,self_attn.q_proj,4,128,8,128,12.227331161499023,33.015625,67108864
+78,self_attn.q_proj,8,32,8,128,0.5470260381698608,68.0625,67108864
+78,self_attn.q_proj,8,64,8,128,0.6449123620986938,66.03125,67108864
+78,self_attn.q_proj,8,128,8,128,0.7407514452934265,65.015625,67108864
+78,self_attn.v_proj,2,32,8,128,15.987213134765625,2.5078125,8388608
+78,self_attn.v_proj,2,64,8,128,18.836885452270508,2.25390625,8388608
+78,self_attn.v_proj,2,128,8,128,21.503177642822266,2.126953125,8388608
+78,self_attn.v_proj,3,32,8,128,6.840173721313477,3.5078125,8388608
+78,self_attn.v_proj,3,64,8,128,8.066368103027344,3.25390625,8388608
+78,self_attn.v_proj,3,128,8,128,9.21609115600586,3.126953125,8388608
+78,self_attn.v_proj,4,32,8,128,3.189753293991089,4.5078125,8388608
+78,self_attn.v_proj,4,64,8,128,3.7629733085632324,4.25390625,8388608
+78,self_attn.v_proj,4,128,8,128,4.301644802093506,4.126953125,8388608
+78,self_attn.v_proj,8,32,8,128,0.19318023324012756,8.5078125,8388608
+78,self_attn.v_proj,8,64,8,128,0.22753232717514038,8.25390625,8388608
+78,self_attn.v_proj,8,128,8,128,0.2589726150035858,8.126953125,8388608
+78,self_attn.k_proj,2,32,8,128,29.57811737060547,2.5078125,8388608
+78,self_attn.k_proj,2,64,8,128,35.0291748046875,2.25390625,8388608
+78,self_attn.k_proj,2,128,8,128,39.96261215209961,2.126953125,8388608
+78,self_attn.k_proj,3,32,8,128,12.657563209533691,3.5078125,8388608
+78,self_attn.k_proj,3,64,8,128,15.013450622558594,3.25390625,8388608
+78,self_attn.k_proj,3,128,8,128,17.174427032470703,3.126953125,8388608
+78,self_attn.k_proj,4,32,8,128,5.906965732574463,4.5078125,8388608
+78,self_attn.k_proj,4,64,8,128,7.003911018371582,4.25390625,8388608
+78,self_attn.k_proj,4,128,8,128,8.01267147064209,4.126953125,8388608
+78,self_attn.k_proj,8,32,8,128,0.357599675655365,8.5078125,8388608
+78,self_attn.k_proj,8,64,8,128,0.42360854148864746,8.25390625,8388608
+78,self_attn.k_proj,8,128,8,128,0.4830447733402252,8.126953125,8388608
+78,self_attn.o_proj,2,32,8,128,36.59818649291992,20.0625,67108864
+78,self_attn.o_proj,2,64,8,128,42.73003387451172,18.03125,67108864
+78,self_attn.o_proj,2,128,8,128,48.2586784362793,17.015625,67108864
+78,self_attn.o_proj,3,32,8,128,15.668342590332031,28.0625,67108864
+78,self_attn.o_proj,3,64,8,128,18.287233352661133,26.03125,67108864
+78,self_attn.o_proj,3,128,8,128,20.704593658447266,25.015625,67108864
+78,self_attn.o_proj,4,32,8,128,7.3055620193481445,36.0625,67108864
+78,self_attn.o_proj,4,64,8,128,8.534924507141113,34.03125,67108864
+78,self_attn.o_proj,4,128,8,128,9.662675857543945,33.015625,67108864
+78,self_attn.o_proj,8,32,8,128,0.44371291995048523,68.0625,67108864
+78,self_attn.o_proj,8,64,8,128,0.5178605318069458,66.03125,67108864
+78,self_attn.o_proj,8,128,8,128,0.5846696496009827,65.015625,67108864
+78,mlp.gate_proj,2,32,8,128,81.67349243164062,70.21875,234881024
+78,mlp.gate_proj,2,64,8,128,97.14733123779297,63.109375,234881024
+78,mlp.gate_proj,2,128,8,128,112.55061340332031,59.5546875,234881024
+78,mlp.gate_proj,3,32,8,128,34.88621520996094,98.21875,234881024
+78,mlp.gate_proj,3,64,8,128,41.81679153442383,91.109375,234881024
+78,mlp.gate_proj,3,128,8,128,48.517799377441406,87.5546875,234881024
+78,mlp.gate_proj,4,32,8,128,16.337440490722656,126.21875,234881024
+78,mlp.gate_proj,4,64,8,128,19.47792625427246,119.109375,234881024
+78,mlp.gate_proj,4,128,8,128,22.740617752075195,115.5546875,234881024
+78,mlp.gate_proj,8,32,8,128,1.000754952430725,238.21875,234881024
+78,mlp.gate_proj,8,64,8,128,1.1875969171524048,231.109375,234881024
+78,mlp.gate_proj,8,128,8,128,1.3811817169189453,227.5546875,234881024
+78,mlp.up_proj,2,32,8,128,71.0717544555664,70.21875,234881024
+78,mlp.up_proj,2,64,8,128,83.31240844726562,63.109375,234881024
+78,mlp.up_proj,2,128,8,128,94.36228942871094,59.5546875,234881024
+78,mlp.up_proj,3,32,8,128,30.54636573791504,98.21875,234881024
+78,mlp.up_proj,3,64,8,128,35.66396713256836,91.109375,234881024
+78,mlp.up_proj,3,128,8,128,40.5689582824707,87.5546875,234881024
+78,mlp.up_proj,4,32,8,128,14.214624404907227,126.21875,234881024
+78,mlp.up_proj,4,64,8,128,16.670373916625977,119.109375,234881024
+78,mlp.up_proj,4,128,8,128,18.923973083496094,115.5546875,234881024
+78,mlp.up_proj,8,32,8,128,0.8674629330635071,238.21875,234881024
+78,mlp.up_proj,8,64,8,128,1.0193957090377808,231.109375,234881024
+78,mlp.up_proj,8,128,8,128,1.151291012763977,227.5546875,234881024
+78,mlp.down_proj,2,32,8,128,69.10182189941406,70.21875,234881024
+78,mlp.down_proj,2,64,8,128,80.88084411621094,63.109375,234881024
+78,mlp.down_proj,2,128,8,128,91.85265350341797,59.5546875,234881024
+78,mlp.down_proj,3,32,8,128,29.672954559326172,98.21875,234881024
+78,mlp.down_proj,3,64,8,128,34.702232360839844,91.109375,234881024
+78,mlp.down_proj,3,128,8,128,39.46205520629883,87.5546875,234881024
+78,mlp.down_proj,4,32,8,128,13.812378883361816,126.21875,234881024
+78,mlp.down_proj,4,64,8,128,16.25324249267578,119.109375,234881024
+78,mlp.down_proj,4,128,8,128,18.43561363220215,115.5546875,234881024
+78,mlp.down_proj,8,32,8,128,0.8430150151252747,238.21875,234881024
+78,mlp.down_proj,8,64,8,128,0.9940943121910095,231.109375,234881024
+78,mlp.down_proj,8,128,8,128,1.1208730936050415,227.5546875,234881024
+79,self_attn.q_proj,2,32,8,128,40.157230377197266,20.0625,67108864
+79,self_attn.q_proj,2,64,8,128,47.52407455444336,18.03125,67108864
+79,self_attn.q_proj,2,128,8,128,54.6970100402832,17.015625,67108864
+79,self_attn.q_proj,3,32,8,128,17.193233489990234,28.0625,67108864
+79,self_attn.q_proj,3,64,8,128,20.411218643188477,26.03125,67108864
+79,self_attn.q_proj,3,128,8,128,23.57327651977539,25.015625,67108864
+79,self_attn.q_proj,4,32,8,128,8.028159141540527,36.0625,67108864
+79,self_attn.q_proj,4,64,8,128,9.518832206726074,34.03125,67108864
+79,self_attn.q_proj,4,128,8,128,11.003891944885254,33.015625,67108864
+79,self_attn.q_proj,8,32,8,128,0.48727157711982727,68.0625,67108864
+79,self_attn.q_proj,8,64,8,128,0.5779823660850525,66.03125,67108864
+79,self_attn.q_proj,8,128,8,128,0.6662895083427429,65.015625,67108864
+79,self_attn.v_proj,2,32,8,128,17.05845832824707,2.5078125,8388608
+79,self_attn.v_proj,2,64,8,128,20.21656036376953,2.25390625,8388608
+79,self_attn.v_proj,2,128,8,128,23.132108688354492,2.126953125,8388608
+79,self_attn.v_proj,3,32,8,128,7.302116870880127,3.5078125,8388608
+79,self_attn.v_proj,3,64,8,128,8.657798767089844,3.25390625,8388608
+79,self_attn.v_proj,3,128,8,128,9.92711353302002,3.126953125,8388608
+79,self_attn.v_proj,4,32,8,128,3.406709909439087,4.5078125,8388608
+79,self_attn.v_proj,4,64,8,128,4.042599201202393,4.25390625,8388608
+79,self_attn.v_proj,4,128,8,128,4.633411884307861,4.126953125,8388608
+79,self_attn.v_proj,8,32,8,128,0.2061319500207901,8.5078125,8388608
+79,self_attn.v_proj,8,64,8,128,0.24422593414783478,8.25390625,8388608
+79,self_attn.v_proj,8,128,8,128,0.2792515456676483,8.126953125,8388608
+79,self_attn.k_proj,2,32,8,128,28.82709312438965,2.5078125,8388608
+79,self_attn.k_proj,2,64,8,128,34.383056640625,2.25390625,8388608
+79,self_attn.k_proj,2,128,8,128,39.45365905761719,2.126953125,8388608
+79,self_attn.k_proj,3,32,8,128,12.362560272216797,3.5078125,8388608
+79,self_attn.k_proj,3,64,8,128,14.790192604064941,3.25390625,8388608
+79,self_attn.k_proj,3,128,8,128,17.0894775390625,3.126953125,8388608
+79,self_attn.k_proj,4,32,8,128,5.76865816116333,4.5078125,8388608
+79,self_attn.k_proj,4,64,8,128,6.9022908210754395,4.25390625,8388608
+79,self_attn.k_proj,4,128,8,128,7.978939056396484,4.126953125,8388608
+79,self_attn.k_proj,8,32,8,128,0.34923863410949707,8.5078125,8388608
+79,self_attn.k_proj,8,64,8,128,0.4175252616405487,8.25390625,8388608
+79,self_attn.k_proj,8,128,8,128,0.48076361417770386,8.126953125,8388608
+79,self_attn.o_proj,2,32,8,128,35.67342758178711,20.0625,67108864
+79,self_attn.o_proj,2,64,8,128,41.87866973876953,18.03125,67108864
+79,self_attn.o_proj,2,128,8,128,47.686546325683594,17.015625,67108864
+79,self_attn.o_proj,3,32,8,128,15.32429313659668,28.0625,67108864
+79,self_attn.o_proj,3,64,8,128,18.0543212890625,26.03125,67108864
+79,self_attn.o_proj,3,128,8,128,20.748706817626953,25.015625,67108864
+79,self_attn.o_proj,4,32,8,128,7.150461196899414,36.0625,67108864
+79,self_attn.o_proj,4,64,8,128,8.441455841064453,34.03125,67108864
+79,self_attn.o_proj,4,128,8,128,9.710885047912598,33.015625,67108864
+79,self_attn.o_proj,8,32,8,128,0.43441101908683777,68.0625,67108864
+79,self_attn.o_proj,8,64,8,128,0.5123983025550842,66.03125,67108864
+79,self_attn.o_proj,8,128,8,128,0.5877714157104492,65.015625,67108864
+79,mlp.gate_proj,2,32,8,128,84.88920593261719,70.21875,234881024
+79,mlp.gate_proj,2,64,8,128,100.19083404541016,63.109375,234881024
+79,mlp.gate_proj,2,128,8,128,115.42305755615234,59.5546875,234881024
+79,mlp.gate_proj,3,32,8,128,36.19486618041992,98.21875,234881024
+79,mlp.gate_proj,3,64,8,128,43.135135650634766,91.109375,234881024
+79,mlp.gate_proj,3,128,8,128,49.62994384765625,87.5546875,234881024
+79,mlp.gate_proj,4,32,8,128,16.910303115844727,126.21875,234881024
+79,mlp.gate_proj,4,64,8,128,20.094186782836914,119.109375,234881024
+79,mlp.gate_proj,4,128,8,128,23.213022232055664,115.5546875,234881024
+79,mlp.gate_proj,8,32,8,128,1.0349289178848267,238.21875,234881024
+79,mlp.gate_proj,8,64,8,128,1.2254587411880493,231.109375,234881024
+79,mlp.gate_proj,8,128,8,128,1.4139842987060547,227.5546875,234881024
+79,mlp.up_proj,2,32,8,128,77.04585266113281,70.21875,234881024
+79,mlp.up_proj,2,64,8,128,90.84526062011719,63.109375,234881024
+79,mlp.up_proj,2,128,8,128,102.86045837402344,59.5546875,234881024
+79,mlp.up_proj,3,32,8,128,33.02766418457031,98.21875,234881024
+79,mlp.up_proj,3,64,8,128,38.74699783325195,91.109375,234881024
+79,mlp.up_proj,3,128,8,128,44.289451599121094,87.5546875,234881024
+79,mlp.up_proj,4,32,8,128,15.440093040466309,126.21875,234881024
+79,mlp.up_proj,4,64,8,128,18.075700759887695,119.109375,234881024
+79,mlp.up_proj,4,128,8,128,20.630352020263672,115.5546875,234881024
+79,mlp.up_proj,8,32,8,128,0.9425826668739319,238.21875,234881024
+79,mlp.up_proj,8,64,8,128,1.1037466526031494,231.109375,234881024
+79,mlp.up_proj,8,128,8,128,1.2537542581558228,227.5546875,234881024
+79,mlp.down_proj,2,32,8,128,69.82134246826172,70.21875,234881024
+79,mlp.down_proj,2,64,8,128,81.92239379882812,63.109375,234881024
+79,mlp.down_proj,2,128,8,128,93.10791778564453,59.5546875,234881024
+79,mlp.down_proj,3,32,8,128,30.042299270629883,98.21875,234881024
+79,mlp.down_proj,3,64,8,128,35.20534133911133,91.109375,234881024
+79,mlp.down_proj,3,128,8,128,40.28462600708008,87.5546875,234881024
+79,mlp.down_proj,4,32,8,128,13.98576831817627,126.21875,234881024
+79,mlp.down_proj,4,64,8,128,16.49049949645996,119.109375,234881024
+79,mlp.down_proj,4,128,8,128,18.81648063659668,115.5546875,234881024
+79,mlp.down_proj,8,32,8,128,0.8534584045410156,238.21875,234881024
+79,mlp.down_proj,8,64,8,128,1.0090715885162354,231.109375,234881024
+79,mlp.down_proj,8,128,8,128,1.1439918279647827,227.5546875,234881024
diff --git a/lm-quant-toolkit/src/data/fnorm-Meta-Llama-3-8B.csv b/lm-quant-toolkit/src/data/fnorm-Meta-Llama-3-8B.csv
new file mode 100644
index 0000000000000000000000000000000000000000..7a1a7dbafacc9b68d366d789e84cd734ee26c2e9
--- /dev/null
+++ b/lm-quant-toolkit/src/data/fnorm-Meta-Llama-3-8B.csv
@@ -0,0 +1,2689 @@
+layer,module,nbit1,gsize1,nbit2,gsize2,fnorm,memmb,params,kurtosis,cost_sensi,sensitivity
+0,self_attn.q_proj,2,32,8,128,29.71926498413086,5.015625,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,2,64,8,128,36.01884460449219,4.5078125,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,2,128,8,128,41.75963973999024,4.25390625,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,3,32,8,128,12.962769508361816,7.015625,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,3,64,8,128,16.056236267089844,6.5078125,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,3,128,8,128,19.37236785888672,6.25390625,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,4,32,8,128,6.056663036346436,9.015625,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,4,64,8,128,7.511712551116943,8.5078125,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,4,128,8,128,9.0902862548828125,8.25390625,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,8,32,8,128,0.3672587275505066,17.015625,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,8,64,8,128,0.4547851383686065,16.5078125,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.q_proj,8,128,8,128,0.5482342839241028,16.25390625,16777216,23.477997808080044,NA,2.350435579501209e-7
+0,self_attn.v_proj,2,32,8,128,5.1504902839660645,1.25390625,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,2,64,8,128,6.072350025177002,1.126953125,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,2,128,8,128,6.909933567047119,1.0634765625,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,3,32,8,128,2.2044689655303955,1.75390625,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,3,64,8,128,2.6054704189300537,1.626953125,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,3,128,8,128,2.9787304401397705,1.5634765625,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,4,32,8,128,1.0286246538162231,2.25390625,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,4,64,8,128,1.2157593965530396,2.126953125,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,4,128,8,128,1.390173077583313,2.0634765625,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,8,32,8,128,0.0632590651512146,4.25390625,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,8,64,8,128,0.074266493320465,4.126953125,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.v_proj,8,128,8,128,0.0843224003911018,4.0634765625,4194304,5.243808587901302,NA,2.350435579501209e-7
+0,self_attn.k_proj,2,32,8,128,21.851238250732425,1.25390625,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,2,64,8,128,26.34428596496582,1.126953125,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,2,128,8,128,30.26717185974121,1.0634765625,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,3,32,8,128,9.60117244720459,1.75390625,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,3,64,8,128,11.86343479156494,1.626953125,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,3,128,8,128,14.062352180480955,1.5634765625,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,4,32,8,128,4.484797477722168,2.25390625,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,4,64,8,128,5.547629356384277,2.126953125,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,4,128,8,128,6.592917919158936,2.0634765625,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,8,32,8,128,0.2718508839607239,4.25390625,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,8,64,8,128,0.3354998826980591,4.126953125,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,self_attn.k_proj,8,128,8,128,0.3979018926620483,4.0634765625,4194304,19.91079330638038,NA,2.350435579501209e-7
+0,mlp.gate_proj,2,32,8,128,33.444522857666016,17.5546875,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,2,64,8,128,39.36700820922852,15.77734375,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,2,128,8,128,44.94590759277344,14.888671875,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,3,32,8,128,14.31833267211914,24.5546875,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,3,64,8,128,16.907611846923828,22.77734375,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,3,128,8,128,19.3914794921875,21.888671875,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,4,32,8,128,6.679895401000977,31.5546875,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,4,64,8,128,7.8970112800598145,29.77734375,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,4,128,8,128,9.0552396774292,28.888671875,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,8,32,8,128,0.40571129322052,59.5546875,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,8,64,8,128,0.4786460995674133,57.77734375,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.gate_proj,8,128,8,128,0.5475429892539978,56.888671875,58720256,4.25397183529409,NA,3.8732471807634283e-7
+0,mlp.up_proj,2,32,8,128,30.09317207336425,17.5546875,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,2,64,8,128,35.18780517578125,15.77734375,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,2,128,8,128,39.822261810302734,14.888671875,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,3,32,8,128,12.867423057556152,24.5546875,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,3,64,8,128,15.071465492248535,22.77734375,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,3,128,8,128,17.074993133544922,21.888671875,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,4,32,8,128,6.00413179397583,31.5546875,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,4,64,8,128,7.029482841491699,29.77734375,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,4,128,8,128,7.972204685211182,28.888671875,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,8,32,8,128,0.3648918867111206,59.5546875,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,8,64,8,128,0.4266665875911712,57.77734375,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.up_proj,8,128,8,128,0.4817230999469757,56.888671875,58720256,3.196049634916446,NA,3.8732471807634283e-7
+0,mlp.down_proj,2,32,8,128,30.121797561645508,17.5546875,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,2,64,8,128,35.37422180175781,15.77734375,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,2,128,8,128,40.296199798583984,14.888671875,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,3,32,8,128,12.907261848449709,24.5546875,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,3,64,8,128,15.229618072509766,22.77734375,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,3,128,8,128,17.438817977905273,21.888671875,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,4,32,8,128,6.028163909912109,31.5546875,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,4,64,8,128,7.115785121917725,29.77734375,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,4,128,8,128,8.164895057678223,28.888671875,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,8,32,8,128,0.3665659725666046,59.5546875,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,8,64,8,128,0.4323022961616516,57.77734375,58720256,6.078099806674725,NA,1.022435526465415e-6
+0,mlp.down_proj,8,128,8,128,0.4949232637882232,56.888671875,58720256,6.078099806674725,NA,1.022435526465415e-6
+1,self_attn.q_proj,2,32,8,128,30.526927947998047,5.015625,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,2,64,8,128,36.505821228027344,4.5078125,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,2,128,8,128,42.13641357421875,4.25390625,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,3,32,8,128,13.311883926391602,7.015625,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,3,64,8,128,16.21119499206543,6.5078125,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,3,128,8,128,19.35423851013184,6.25390625,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,4,32,8,128,6.221262454986572,9.015625,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,4,64,8,128,7.588432788848877,8.5078125,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,4,128,8,128,9.07388687133789,8.25390625,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,8,32,8,128,0.3765309751033783,17.015625,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,8,64,8,128,0.4591035544872284,16.5078125,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.q_proj,8,128,8,128,0.5472977161407471,16.25390625,16777216,6.518398179144177,NA,3.939759949389554e-7
+1,self_attn.v_proj,2,32,8,128,6.054126739501953,1.25390625,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,2,64,8,128,7.120873928070068,1.126953125,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,2,128,8,128,8.102333068847656,1.0634765625,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,3,32,8,128,2.589040279388428,1.75390625,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,3,64,8,128,3.047174453735352,1.626953125,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,3,128,8,128,3.4693777561187744,1.5634765625,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,4,32,8,128,1.207473874092102,2.25390625,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,4,64,8,128,1.4214165210723877,2.126953125,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,4,128,8,128,1.6196296215057373,2.0634765625,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,8,32,8,128,0.0731801688671112,4.25390625,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,8,64,8,128,0.086038313806056907,4.126953125,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.v_proj,8,128,8,128,0.0976540595293045,4.0634765625,4194304,3.511953896727226,NA,3.939759949389554e-7
+1,self_attn.k_proj,2,32,8,128,22.087326049804688,1.25390625,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,2,64,8,128,26.22841453552246,1.126953125,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,2,128,8,128,29.570714950561523,1.0634765625,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,3,32,8,128,9.688775062561035,1.75390625,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,3,64,8,128,11.815810203552246,1.626953125,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,3,128,8,128,13.7361421585083,1.5634765625,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,4,32,8,128,4.534592151641846,2.25390625,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,4,64,8,128,5.555020809173584,2.126953125,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,4,128,8,128,6.490731239318848,2.0634765625,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,8,32,8,128,0.2743088901042938,4.25390625,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,8,64,8,128,0.3359671831130981,4.126953125,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,self_attn.k_proj,8,128,8,128,0.3915407955646515,4.0634765625,4194304,6.762759900412172,NA,3.939759949389554e-7
+1,mlp.gate_proj,2,32,8,128,33.596656799316406,17.5546875,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,2,64,8,128,39.39640045166016,15.77734375,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,2,128,8,128,44.80052947998047,14.888671875,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,3,32,8,128,14.38129425048828,24.5546875,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,3,64,8,128,16.90346336364746,22.77734375,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,3,128,8,128,19.27972412109375,21.888671875,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,4,32,8,128,6.708982944488525,31.5546875,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,4,64,8,128,7.894912719726562,29.77734375,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,4,128,8,128,9.00425910949707,28.888671875,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,8,32,8,128,0.407469242811203,59.5546875,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,8,64,8,128,0.4784776568412781,57.77734375,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.gate_proj,8,128,8,128,0.544376790523529,56.888671875,58720256,3.981607530644241,NA,1.5860730400163447e-6
+1,mlp.up_proj,2,32,8,128,30.42632484436035,17.5546875,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,2,64,8,128,35.52070617675781,15.77734375,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,2,128,8,128,40.1715087890625,14.888671875,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,3,32,8,128,13.010882377624512,24.5546875,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,3,64,8,128,15.230850219726562,22.77734375,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,3,128,8,128,17.25382423400879,21.888671875,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,4,32,8,128,6.071969032287598,31.5546875,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,4,64,8,128,7.105710029602051,29.77734375,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,4,128,8,128,8.0638580322265625,28.888671875,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,8,32,8,128,0.368994653224945,59.5546875,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,8,64,8,128,0.4312596619129181,57.77734375,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.up_proj,8,128,8,128,0.4871818125247955,56.888671875,58720256,3.865750191365785,NA,1.5860730400163447e-6
+1,mlp.down_proj,2,32,8,128,30.320220947265625,17.5546875,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,2,64,8,128,35.40483474731445,15.77734375,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,2,128,8,128,39.99724960327149,14.888671875,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,3,32,8,128,12.976301193237305,24.5546875,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,3,64,8,128,15.205132484436035,22.77734375,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,3,128,8,128,17.241455078125,21.888671875,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,4,32,8,128,6.056885719299316,31.5546875,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,4,64,8,128,7.096342086791992,29.77734375,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,4,128,8,128,8.062268257141113,28.888671875,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,8,32,8,128,0.3681756258010864,59.5546875,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,8,64,8,128,0.4309409260749817,57.77734375,58720256,4.705602229051014,NA,2.590278563729953e-5
+1,mlp.down_proj,8,128,8,128,0.4879005849361419,56.888671875,58720256,4.705602229051014,NA,2.590278563729953e-5
+2,self_attn.q_proj,2,32,8,128,28.29009246826172,5.015625,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,2,64,8,128,33.812381744384766,4.5078125,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,2,128,8,128,39.00408935546875,4.25390625,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,3,32,8,128,12.197437286376951,7.015625,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,3,64,8,128,14.689261436462402,6.5078125,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,3,128,8,128,17.301185607910156,6.25390625,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,4,32,8,128,5.6953887939453125,9.015625,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,4,64,8,128,6.862927913665772,8.5078125,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,4,128,8,128,8.08891773223877,8.25390625,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,8,32,8,128,0.3447518348693847,17.015625,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,8,64,8,128,0.4150609374046325,16.5078125,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.q_proj,8,128,8,128,0.4876388013362884,16.25390625,16777216,4.870928188359852,NA,9.326020631306164e-7
+2,self_attn.v_proj,2,32,8,128,5.159294128417969,1.25390625,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,2,64,8,128,6.081952095031738,1.126953125,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,2,128,8,128,6.926016807556152,1.0634765625,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,3,32,8,128,2.206528902053833,1.75390625,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,3,64,8,128,2.6030614376068115,1.626953125,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,3,128,8,128,2.9701642990112305,1.5634765625,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,4,32,8,128,1.028939962387085,2.25390625,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,4,64,8,128,1.2139078378677368,2.126953125,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,4,128,8,128,1.3857122659683228,2.0634765625,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,8,32,8,128,0.0623427294194698,4.25390625,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,8,64,8,128,0.0734634175896644,4.126953125,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.v_proj,8,128,8,128,0.0835602283477783,4.0634765625,4194304,3.581136549491772,NA,9.326020631306164e-7
+2,self_attn.k_proj,2,32,8,128,22.59001541137696,1.25390625,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,2,64,8,128,26.932594299316406,1.126953125,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,2,128,8,128,30.45580291748047,1.0634765625,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,3,32,8,128,9.799921035766602,1.75390625,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,3,64,8,128,11.85670566558838,1.626953125,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,3,128,8,128,13.769078254699709,1.5634765625,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,4,32,8,128,4.577732563018799,2.25390625,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,4,64,8,128,5.542038440704346,2.126953125,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,4,128,8,128,6.441653728485107,2.0634765625,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,8,32,8,128,0.276971161365509,4.25390625,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,8,64,8,128,0.3351361751556396,4.126953125,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,self_attn.k_proj,8,128,8,128,0.3882458806037903,4.0634765625,4194304,5.358498552438096,NA,9.326020631306164e-7
+2,mlp.gate_proj,2,32,8,128,34.53382873535156,17.5546875,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,2,64,8,128,40.52213287353516,15.77734375,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,2,128,8,128,46.107120513916016,14.888671875,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,3,32,8,128,14.783581733703612,24.5546875,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,3,64,8,128,17.375429153442383,22.77734375,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,3,128,8,128,19.83789253234864,21.888671875,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,4,32,8,128,6.895365238189697,31.5546875,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,4,64,8,128,8.111905097961426,29.77734375,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,4,128,8,128,9.259538650512695,28.888671875,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,8,32,8,128,0.4186978340148926,59.5546875,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,8,64,8,128,0.4917322695255279,57.77734375,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.gate_proj,8,128,8,128,0.5599081516265869,56.888671875,58720256,3.4928651366638097,NA,1.9166438960382948e-6
+2,mlp.up_proj,2,32,8,128,30.22202110290528,17.5546875,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,2,64,8,128,35.24127197265625,15.77734375,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,2,128,8,128,39.81227493286133,14.888671875,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,3,32,8,128,12.916703224182127,24.5546875,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,3,64,8,128,15.095866203308104,22.77734375,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,3,128,8,128,17.067392349243164,21.888671875,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,4,32,8,128,6.027431488037109,31.5546875,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,4,64,8,128,7.041491508483887,29.77734375,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,4,128,8,128,7.969945907592773,28.888671875,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,8,32,8,128,0.3662784397602081,59.5546875,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,8,64,8,128,0.4273211061954498,57.77734375,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.up_proj,8,128,8,128,0.481521874666214,56.888671875,58720256,3.198673872490125,NA,1.9166438960382948e-6
+2,mlp.down_proj,2,32,8,128,30.312685012817383,17.5546875,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,2,64,8,128,35.36705780029297,15.77734375,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,2,128,8,128,39.947853088378906,14.888671875,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,3,32,8,128,12.969926834106444,24.5546875,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,3,64,8,128,15.17494010925293,22.77734375,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,3,128,8,128,17.175376892089844,21.888671875,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,4,32,8,128,6.054165363311768,31.5546875,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,4,64,8,128,7.08501672744751,29.77734375,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,4,128,8,128,8.038103103637695,28.888671875,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,8,32,8,128,0.36796528100967407,59.5546875,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,8,64,8,128,0.4299937486648559,57.77734375,58720256,3.6681520245954378,NA,1.924652906382107e-6
+2,mlp.down_proj,8,128,8,128,0.4857692122459411,56.888671875,58720256,3.6681520245954378,NA,1.924652906382107e-6
+3,self_attn.q_proj,2,32,8,128,28.09393882751465,5.015625,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,2,64,8,128,33.566856384277344,4.5078125,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,2,128,8,128,39.03070831298828,4.25390625,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,3,32,8,128,12.062705039978027,7.015625,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,3,64,8,128,14.456032752990724,6.5078125,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,3,128,8,128,16.985721588134766,6.25390625,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,4,32,8,128,5.631606578826904,9.015625,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,4,64,8,128,6.747907638549805,8.5078125,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,4,128,8,128,7.93041467666626,8.25390625,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,8,32,8,128,0.3408825695514679,17.015625,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,8,64,8,128,0.4081771075725555,16.5078125,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.q_proj,8,128,8,128,0.4779151082038879,16.25390625,16777216,4.191170036530165,NA,1.5665997352698469e-6
+3,self_attn.v_proj,2,32,8,128,6.054136276245117,1.25390625,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,2,64,8,128,7.143193244934082,1.126953125,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,2,128,8,128,8.157306671142578,1.0634765625,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,3,32,8,128,2.591524362564087,1.75390625,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,3,64,8,128,3.0607292652130127,1.626953125,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,3,128,8,128,3.500352144241333,1.5634765625,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,4,32,8,128,1.2083368301391602,2.25390625,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,4,64,8,128,1.4283732175827026,2.126953125,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,4,128,8,128,1.6336627006530762,2.0634765625,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,8,32,8,128,0.0732447206974029,4.25390625,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,8,64,8,128,0.0864056944847107,4.126953125,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.v_proj,8,128,8,128,0.0984654575586319,4.0634765625,4194304,3.498782785877829,NA,1.5665997352698469e-6
+3,self_attn.k_proj,2,32,8,128,22.24028968811035,1.25390625,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,2,64,8,128,26.65947151184082,1.126953125,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,2,128,8,128,30.53901100158692,1.0634765625,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,3,32,8,128,9.575121879577637,1.75390625,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,3,64,8,128,11.551334381103516,1.626953125,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,3,128,8,128,13.431496620178224,1.5634765625,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,4,32,8,128,4.46926212310791,2.25390625,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,4,64,8,128,5.3901801109313965,2.126953125,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,4,128,8,128,6.272538185119629,2.0634765625,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,8,32,8,128,0.2705325782299042,4.25390625,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,8,64,8,128,0.3260603249073028,4.126953125,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,self_attn.k_proj,8,128,8,128,0.3779822885990143,4.0634765625,4194304,4.45015065699875,NA,1.5665997352698469e-6
+3,mlp.gate_proj,2,32,8,128,37.02172470092773,17.5546875,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,2,64,8,128,43.68269729614258,15.77734375,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,2,128,8,128,49.92171859741211,14.888671875,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,3,32,8,128,15.858477592468262,24.5546875,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,3,64,8,128,18.73029518127441,22.77734375,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,3,128,8,128,21.516189575195312,21.888671875,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,4,32,8,128,7.397336006164551,31.5546875,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,4,64,8,128,8.7405548095703125,29.77734375,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,4,128,8,128,10.045246124267578,28.888671875,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,8,32,8,128,0.448901891708374,59.5546875,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,8,64,8,128,0.5302746891975403,57.77734375,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.gate_proj,8,128,8,128,0.6077975630760193,56.888671875,58720256,3.750199208201579,NA,4.331189757067477e-6
+3,mlp.up_proj,2,32,8,128,29.67856216430664,17.5546875,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,2,64,8,128,34.637977600097656,15.77734375,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,2,128,8,128,39.1311149597168,14.888671875,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,3,32,8,128,12.684122085571287,24.5546875,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,3,64,8,128,14.836777687072754,22.77734375,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,3,128,8,128,16.790557861328125,21.888671875,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,4,32,8,128,5.920221328735352,31.5546875,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,4,64,8,128,6.921647548675537,29.77734375,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,4,128,8,128,7.839502334594727,28.888671875,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,8,32,8,128,0.3598023951053619,59.5546875,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,8,64,8,128,0.4200921356678009,57.77734375,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.up_proj,8,128,8,128,0.4737437069416046,56.888671875,58720256,3.20137190077044,NA,4.331189757067477e-6
+3,mlp.down_proj,2,32,8,128,29.65584182739257,17.5546875,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,2,64,8,128,34.60835647583008,15.77734375,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,2,128,8,128,39.098388671875,14.888671875,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,3,32,8,128,12.679429054260254,24.5546875,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,3,64,8,128,14.82733917236328,22.77734375,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,3,128,8,128,16.778474807739258,21.888671875,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,4,32,8,128,5.918444633483887,31.5546875,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,4,64,8,128,6.918429374694824,29.77734375,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,4,128,8,128,7.836794376373291,28.888671875,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,8,32,8,128,0.3596154749393463,59.5546875,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,8,64,8,128,0.4199441969394684,57.77734375,58720256,3.5819436370920057,NA,3.619242534114164e-6
+3,mlp.down_proj,8,128,8,128,0.4735199809074402,56.888671875,58720256,3.5819436370920057,NA,3.619242534114164e-6
+4,self_attn.q_proj,2,32,8,128,28.03318214416504,5.015625,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,2,64,8,128,33.52735900878906,4.5078125,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,2,128,8,128,39.10169219970703,4.25390625,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,3,32,8,128,12.06218719482422,7.015625,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,3,64,8,128,14.514254570007324,6.5078125,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,3,128,8,128,17.2142276763916,6.25390625,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,4,32,8,128,5.631128311157227,9.015625,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,4,64,8,128,6.779860973358154,8.5078125,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,4,128,8,128,8.049135208129883,8.25390625,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,8,32,8,128,0.3407697677612304,17.015625,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,8,64,8,128,0.4098406732082367,16.5078125,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.q_proj,8,128,8,128,0.4849676191806793,16.25390625,16777216,4.874343182801558,NA,3.732759296326549e-6
+4,self_attn.v_proj,2,32,8,128,6.615170001983643,1.25390625,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,2,64,8,128,7.8174052238464355,1.126953125,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,2,128,8,128,8.9237060546875,1.0634765625,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,3,32,8,128,2.828190565109253,1.75390625,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,3,64,8,128,3.345294237136841,1.626953125,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,3,128,8,128,3.823539733886719,1.5634765625,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,4,32,8,128,1.3191213607788086,2.25390625,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,4,64,8,128,1.560590147972107,2.126953125,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,4,128,8,128,1.7844257354736328,2.0634765625,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,8,32,8,128,0.079975239932537,4.25390625,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,8,64,8,128,0.0944518074393272,4.126953125,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.v_proj,8,128,8,128,0.1075403913855552,4.0634765625,4194304,3.5834757689034893,NA,3.732759296326549e-6
+4,self_attn.k_proj,2,32,8,128,21.67329216003418,1.25390625,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,2,64,8,128,25.89285469055176,1.126953125,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,2,128,8,128,29.59549331665039,1.0634765625,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,3,32,8,128,9.311653137207031,1.75390625,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,3,64,8,128,11.197571754455566,1.626953125,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,3,128,8,128,12.988943099975586,1.5634765625,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,4,32,8,128,4.346807479858398,2.25390625,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,4,64,8,128,5.22780179977417,2.126953125,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,4,128,8,128,6.067966938018799,2.0634765625,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,8,32,8,128,0.2630780041217804,4.25390625,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,8,64,8,128,0.3159684240818023,4.126953125,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,self_attn.k_proj,8,128,8,128,0.365718811750412,4.0634765625,4194304,4.655811376588373,NA,3.732759296326549e-6
+4,mlp.gate_proj,2,32,8,128,39.353912353515625,17.5546875,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,2,64,8,128,46.50521469116211,15.77734375,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,2,128,8,128,53.20646667480469,14.888671875,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,3,32,8,128,16.850141525268555,24.5546875,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,3,64,8,128,19.94430923461914,22.77734375,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,3,128,8,128,22.93217086791992,21.888671875,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,4,32,8,128,7.865922927856445,31.5546875,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,4,64,8,128,9.306957244873049,29.77734375,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,4,128,8,128,10.701563835144045,28.888671875,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,8,32,8,128,0.4769289195537567,59.5546875,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,8,64,8,128,0.5647693276405334,57.77734375,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.gate_proj,8,128,8,128,0.6474993228912354,56.888671875,58720256,3.700297021380253,NA,7.5180246312811505e-6
+4,mlp.up_proj,2,32,8,128,29.0410213470459,17.5546875,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,2,64,8,128,33.95344543457031,15.77734375,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,2,128,8,128,38.41080093383789,14.888671875,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,3,32,8,128,12.422975540161133,24.5546875,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,3,64,8,128,14.561521530151367,22.77734375,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,3,128,8,128,16.53119659423828,21.888671875,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,4,32,8,128,5.799415588378906,31.5546875,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,4,64,8,128,6.79381275177002,29.77734375,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,4,128,8,128,7.719454765319824,28.888671875,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,8,32,8,128,0.3524595499038696,59.5546875,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,8,64,8,128,0.4124566912651062,57.77734375,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.up_proj,8,128,8,128,0.4665692746639251,56.888671875,58720256,3.3816166778637653,NA,7.5180246312811505e-6
+4,mlp.down_proj,2,32,8,128,28.969560623168945,17.5546875,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,2,64,8,128,33.845821380615234,15.77734375,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,2,128,8,128,38.26799011230469,14.888671875,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,3,32,8,128,12.39323902130127,24.5546875,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,3,64,8,128,14.50938892364502,22.77734375,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,3,128,8,128,16.448408126831055,21.888671875,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,4,32,8,128,5.784383296966553,31.5546875,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,4,64,8,128,6.770299911499023,29.77734375,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,4,128,8,128,7.685372829437256,28.888671875,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,8,32,8,128,0.351535826921463,59.5546875,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,8,64,8,128,0.4111148416996002,57.77734375,58720256,4.007126729903491,NA,6.131288046162808e-6
+4,mlp.down_proj,8,128,8,128,0.464570552110672,56.888671875,58720256,4.007126729903491,NA,6.131288046162808e-6
+5,self_attn.q_proj,2,32,8,128,27.14842796325684,5.015625,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,2,64,8,128,32.56990051269531,4.5078125,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,2,128,8,128,38.27643585205078,4.25390625,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,3,32,8,128,11.68593692779541,7.015625,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,3,64,8,128,14.071290969848633,6.5078125,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,3,128,8,128,16.836549758911133,6.25390625,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,4,32,8,128,5.460404872894287,9.015625,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,4,64,8,128,6.574841499328613,8.5078125,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,4,128,8,128,7.871114253997803,8.25390625,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,8,32,8,128,0.3304733335971832,17.015625,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,8,64,8,128,0.3976515531539917,16.5078125,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.q_proj,8,128,8,128,0.4741759598255157,16.25390625,16777216,4.788145663734035,NA,5.492578566190787e-6
+5,self_attn.v_proj,2,32,8,128,5.379310131072998,1.25390625,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,2,64,8,128,6.315425872802734,1.126953125,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,2,128,8,128,7.180819988250732,1.0634765625,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,3,32,8,128,2.30245590209961,1.75390625,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,3,64,8,128,2.706099510192871,1.626953125,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,3,128,8,128,3.081350803375244,1.5634765625,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,4,32,8,128,1.074188232421875,2.25390625,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,4,64,8,128,1.2629473209381104,2.126953125,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,4,128,8,128,1.4385771751403809,2.0634765625,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,8,32,8,128,0.0650263726711273,4.25390625,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,8,64,8,128,0.0763984844088554,4.126953125,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.v_proj,8,128,8,128,0.0866768211126327,4.0634765625,4194304,3.4852569288097603,NA,5.492578566190787e-6
+5,self_attn.k_proj,2,32,8,128,21.81302261352539,1.25390625,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,2,64,8,128,26.0170955657959,1.126953125,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,2,128,8,128,29.667430877685547,1.0634765625,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,3,32,8,128,9.45390796661377,1.75390625,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,3,64,8,128,11.400733947753906,1.626953125,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,3,128,8,128,13.265999794006348,1.5634765625,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,4,32,8,128,4.411849498748779,2.25390625,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,4,64,8,128,5.332928657531738,2.126953125,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,4,128,8,128,6.2343430519104,2.0634765625,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,8,32,8,128,0.2672629654407501,4.25390625,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,8,64,8,128,0.3226409554481506,4.126953125,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,self_attn.k_proj,8,128,8,128,0.3757884502410888,4.0634765625,4194304,6.048360546833415,NA,5.492578566190787e-6
+5,mlp.gate_proj,2,32,8,128,39.130516052246094,17.5546875,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,2,64,8,128,46.13655471801758,15.77734375,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,2,128,8,128,52.64202499389648,14.888671875,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,3,32,8,128,16.74797821044922,24.5546875,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,3,64,8,128,19.76881980895996,22.77734375,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,3,128,8,128,22.633087158203125,21.888671875,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,4,32,8,128,7.815927505493164,31.5546875,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,4,64,8,128,9.223048210144045,29.77734375,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,4,128,8,128,10.564090728759766,28.888671875,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,8,32,8,128,0.4739591479301452,59.5546875,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,8,64,8,128,0.5594997406005859,57.77734375,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.gate_proj,8,128,8,128,0.6392660140991211,56.888671875,58720256,3.535033662344325,NA,1.0729807399911806e-5
+5,mlp.up_proj,2,32,8,128,29.15910530090332,17.5546875,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,2,64,8,128,34.057796478271484,15.77734375,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,2,128,8,128,38.49320602416992,14.888671875,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,3,32,8,128,12.469191551208496,24.5546875,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,3,64,8,128,14.595012664794922,22.77734375,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,3,128,8,128,16.53561782836914,21.888671875,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,4,32,8,128,5.82036018371582,31.5546875,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,4,64,8,128,6.808206081390381,29.77734375,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,4,128,8,128,7.719542503356934,28.888671875,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,8,32,8,128,0.353719025850296,59.5546875,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,8,64,8,128,0.4134189784526825,57.77734375,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.up_proj,8,128,8,128,0.4666213691234588,56.888671875,58720256,3.2940036584734083,NA,1.0729807399911806e-5
+5,mlp.down_proj,2,32,8,128,29.086917877197266,17.5546875,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,2,64,8,128,33.97077560424805,15.77734375,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,2,128,8,128,38.39598846435547,14.888671875,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,3,32,8,128,12.442087173461914,24.5546875,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,3,64,8,128,14.556851387023926,22.77734375,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,3,128,8,128,16.480491638183594,21.888671875,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,4,32,8,128,5.809384822845459,31.5546875,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,4,64,8,128,6.79417085647583,29.77734375,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,4,128,8,128,7.705014228820801,28.888671875,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,8,32,8,128,0.3530448377132416,59.5546875,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,8,64,8,128,0.412678062915802,57.77734375,58720256,4.094218898332173,NA,9.16959106689319e-6
+5,mlp.down_proj,8,128,8,128,0.4659603834152221,56.888671875,58720256,4.094218898332173,NA,9.16959106689319e-6
+6,self_attn.q_proj,2,32,8,128,28.26160430908203,5.015625,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,2,64,8,128,33.862728118896484,4.5078125,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,2,128,8,128,39.84922409057617,4.25390625,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,3,32,8,128,12.133743286132812,7.015625,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,3,64,8,128,14.60372543334961,6.5078125,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,3,128,8,128,17.438888549804688,6.25390625,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,4,32,8,128,5.66416072845459,9.015625,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,4,64,8,128,6.8173508644104,8.5078125,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,4,128,8,128,8.145289421081543,8.25390625,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,8,32,8,128,0.3429482877254486,17.015625,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,8,64,8,128,0.4121284186840057,16.5078125,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.q_proj,8,128,8,128,0.4907255172729492,16.25390625,16777216,4.466482714952487,NA,6.826936896686675e-6
+6,self_attn.v_proj,2,32,8,128,5.851216793060303,1.25390625,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,2,64,8,128,6.8759050369262695,1.126953125,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,2,128,8,128,7.827797889709473,1.0634765625,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,3,32,8,128,2.5037496089935303,1.75390625,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,3,64,8,128,2.9412851333618164,1.626953125,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,3,128,8,128,3.351409435272217,1.5634765625,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,4,32,8,128,1.1683902740478516,2.25390625,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,4,64,8,128,1.3734513521194458,2.126953125,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,4,128,8,128,1.5643017292022705,2.0634765625,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,8,32,8,128,0.0707335993647575,4.25390625,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,8,64,8,128,0.0831061825156211,4.126953125,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.v_proj,8,128,8,128,0.0942878797650337,4.0634765625,4194304,3.376907655091106,NA,6.826936896686675e-6
+6,self_attn.k_proj,2,32,8,128,21.420391082763672,1.25390625,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,2,64,8,128,25.570756912231445,1.126953125,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,2,128,8,128,29.383703231811523,1.0634765625,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,3,32,8,128,9.189220428466797,1.75390625,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,3,64,8,128,10.99506378173828,1.626953125,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,3,128,8,128,12.71745777130127,1.5634765625,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,4,32,8,128,4.287354946136475,2.25390625,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,4,64,8,128,5.131492137908936,2.126953125,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,4,128,8,128,5.935476303100586,2.0634765625,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,8,32,8,128,0.2594151198863983,4.25390625,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,8,64,8,128,0.3101383447647095,4.126953125,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,self_attn.k_proj,8,128,8,128,0.3574665784835815,4.0634765625,4194304,4.107301027533515,NA,6.826936896686675e-6
+6,mlp.gate_proj,2,32,8,128,39.572059631347656,17.5546875,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,2,64,8,128,46.76994323730469,15.77734375,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,2,128,8,128,53.50589370727539,14.888671875,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,3,32,8,128,16.936744689941406,24.5546875,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,3,64,8,128,20.04867935180664,22.77734375,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,3,128,8,128,23.027267456054688,21.888671875,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,4,32,8,128,7.904539108276367,31.5546875,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,4,64,8,128,9.356647491455078,29.77734375,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,4,128,8,128,10.750324249267578,28.888671875,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,8,32,8,128,0.4793758690357208,59.5546875,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,8,64,8,128,0.5675809383392334,57.77734375,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.gate_proj,8,128,8,128,0.650355339050293,56.888671875,58720256,3.661961989142892,NA,1.3640113138535526e-5
+6,mlp.up_proj,2,32,8,128,29.13815116882324,17.5546875,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,2,64,8,128,34.03101348876953,15.77734375,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,2,128,8,128,38.463661193847656,14.888671875,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,3,32,8,128,12.457345008850098,24.5546875,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,3,64,8,128,14.575265884399414,22.77734375,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,3,128,8,128,16.508161544799805,21.888671875,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,4,32,8,128,5.814871788024902,31.5546875,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,4,64,8,128,6.801267147064209,29.77734375,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,4,128,8,128,7.705705165863037,28.888671875,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,8,32,8,128,0.3534092903137207,59.5546875,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,8,64,8,128,0.4128694534301758,57.77734375,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.up_proj,8,128,8,128,0.4657552242279053,56.888671875,58720256,3.199743077856602,NA,1.3640113138535526e-5
+6,mlp.down_proj,2,32,8,128,29.1191635131836,17.5546875,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,2,64,8,128,34.0229606628418,15.77734375,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,2,128,8,128,38.480712890625,14.888671875,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,3,32,8,128,12.451114654541016,24.5546875,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,3,64,8,128,14.576369285583496,22.77734375,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,3,128,8,128,16.521066665649414,21.888671875,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,4,32,8,128,5.813693523406982,31.5546875,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,4,64,8,128,6.803946971893311,29.77734375,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,4,128,8,128,7.718326091766357,28.888671875,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,8,32,8,128,0.353316456079483,59.5546875,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,8,64,8,128,0.4130769073963165,57.77734375,58720256,4.219834606995466,NA,1.119308035413269e-5
+6,mlp.down_proj,8,128,8,128,0.4666117131710052,56.888671875,58720256,4.219834606995466,NA,1.119308035413269e-5
+7,self_attn.q_proj,2,32,8,128,26.976608276367188,5.015625,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,2,64,8,128,32.303985595703125,4.5078125,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,2,128,8,128,37.8601188659668,4.25390625,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,3,32,8,128,11.569588661193848,7.015625,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,3,64,8,128,13.894417762756348,6.5078125,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,3,128,8,128,16.53858757019043,6.25390625,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,4,32,8,128,5.400040626525879,9.015625,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,4,64,8,128,6.484575271606445,8.5078125,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,4,128,8,128,7.722560405731201,8.25390625,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,8,32,8,128,0.327002465724945,17.015625,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,8,64,8,128,0.3922163546085357,16.5078125,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.q_proj,8,128,8,128,0.4653173089027405,16.25390625,16777216,4.290774013551546,NA,8.94385993888136e-6
+7,self_attn.v_proj,2,32,8,128,5.812392711639404,1.25390625,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,2,64,8,128,6.83394193649292,1.126953125,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,2,128,8,128,7.764544486999512,1.0634765625,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,3,32,8,128,2.4866905212402344,1.75390625,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,3,64,8,128,2.92472505569458,1.626953125,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,3,128,8,128,3.3279316425323486,1.5634765625,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,4,32,8,128,1.160157561302185,2.25390625,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,4,64,8,128,1.364603877067566,2.126953125,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,4,128,8,128,1.5533496141433716,2.0634765625,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,8,32,8,128,0.0702754035592079,4.25390625,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,8,64,8,128,0.0825935527682304,4.126953125,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.v_proj,8,128,8,128,0.0936173573136329,4.0634765625,4194304,3.364970783290237,NA,8.94385993888136e-6
+7,self_attn.k_proj,2,32,8,128,21.49588966369629,1.25390625,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,2,64,8,128,25.60579299926757,1.126953125,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,2,128,8,128,29.29781913757324,1.0634765625,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,3,32,8,128,9.206317901611328,1.75390625,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,3,64,8,128,10.975910186767578,1.626953125,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,3,128,8,128,12.625019073486328,1.5634765625,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,4,32,8,128,4.297920227050781,2.25390625,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,4,64,8,128,5.124042987823486,2.126953125,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,4,128,8,128,5.891855716705322,2.0634765625,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,8,32,8,128,0.2601531744003296,4.25390625,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,8,64,8,128,0.3099227845668793,4.126953125,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,self_attn.k_proj,8,128,8,128,0.3550952076911926,4.0634765625,4194304,3.850992348386023,NA,8.94385993888136e-6
+7,mlp.gate_proj,2,32,8,128,39.03369522094727,17.5546875,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,2,64,8,128,46.20256423950195,15.77734375,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,2,128,8,128,52.90453720092773,14.888671875,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,3,32,8,128,16.709978103637695,24.5546875,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,3,64,8,128,19.80340576171875,22.77734375,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,3,128,8,128,22.763891220092773,21.888671875,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,4,32,8,128,7.798851013183594,31.5546875,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,4,64,8,128,9.24035930633545,29.77734375,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,4,128,8,128,10.62183952331543,28.888671875,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,8,32,8,128,0.4729699194431305,59.5546875,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,8,64,8,128,0.5606297850608826,57.77734375,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.gate_proj,8,128,8,128,0.6429332494735718,56.888671875,58720256,3.681095042996137,NA,1.5917512428131886e-5
+7,mlp.up_proj,2,32,8,128,29.60834503173828,17.5546875,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,2,64,8,128,34.610897064208984,15.77734375,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,2,128,8,128,39.16851806640625,14.888671875,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,3,32,8,128,12.663779258728027,24.5546875,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,3,64,8,128,14.840734481811523,22.77734375,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,3,128,8,128,16.8330020904541,21.888671875,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,4,32,8,128,5.910112857818604,31.5546875,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,4,64,8,128,6.922947883605957,29.77734375,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,4,128,8,128,7.86088228225708,28.888671875,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,8,32,8,128,0.3591862618923187,59.5546875,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,8,64,8,128,0.4202658236026764,57.77734375,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.up_proj,8,128,8,128,0.4751189947128296,56.888671875,58720256,3.2845267502290625,NA,1.5917512428131886e-5
+7,mlp.down_proj,2,32,8,128,29.65262031555176,17.5546875,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,2,64,8,128,34.703914642333984,15.77734375,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,2,128,8,128,39.31455993652344,14.888671875,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,3,32,8,128,12.686455726623535,24.5546875,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,3,64,8,128,14.885343551635742,22.77734375,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,3,128,8,128,16.9116268157959,21.888671875,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,4,32,8,128,5.921884536743164,31.5546875,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,4,64,8,128,6.947753429412842,29.77734375,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,4,128,8,128,7.906002998352051,28.888671875,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,8,32,8,128,0.3598684668540954,59.5546875,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,8,64,8,128,0.4216315448284149,57.77734375,58720256,4.666204098636681,NA,1.246558895218186e-5
+7,mlp.down_proj,8,128,8,128,0.4777296781539917,56.888671875,58720256,4.666204098636681,NA,1.246558895218186e-5
+8,self_attn.q_proj,2,32,8,128,27.061317443847656,5.015625,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,2,64,8,128,32.52377700805664,4.5078125,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,2,128,8,128,38.69818115234375,4.25390625,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,3,32,8,128,11.656363487243652,7.015625,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,3,64,8,128,14.124595642089844,6.5078125,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,3,128,8,128,17.215373992919922,6.25390625,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,4,32,8,128,5.444422245025635,9.015625,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,4,64,8,128,6.6116204261779785,8.5078125,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,4,128,8,128,8.074461936950684,8.25390625,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,8,32,8,128,0.3296016752719879,17.015625,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,8,64,8,128,0.3997628390789032,16.5078125,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.q_proj,8,128,8,128,0.4867331981658935,16.25390625,16777216,6.092433555599315,NA,1.259053260582732e-5
+8,self_attn.v_proj,2,32,8,128,6.304591178894043,1.25390625,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,2,64,8,128,7.507236957550049,1.126953125,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,2,128,8,128,8.639371871948242,1.0634765625,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,3,32,8,128,2.69922947883606,1.75390625,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,3,64,8,128,3.2172539234161377,1.626953125,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,3,128,8,128,3.715339422225952,1.5634765625,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,4,32,8,128,1.2599207162857056,2.25390625,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,4,64,8,128,1.501517653465271,2.126953125,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,4,128,8,128,1.7340031862258911,2.0634765625,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,8,32,8,128,0.0762663707137107,4.25390625,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,8,64,8,128,0.09086574614048,4.126953125,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.v_proj,8,128,8,128,0.1045241877436637,4.0634765625,4194304,3.9236208646076234,NA,1.259053260582732e-5
+8,self_attn.k_proj,2,32,8,128,22.023910522460938,1.25390625,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,2,64,8,128,26.45915031433105,1.126953125,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,2,128,8,128,30.406747817993164,1.0634765625,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,3,32,8,128,9.50877857208252,1.75390625,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,3,64,8,128,11.526528358459473,1.626953125,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,3,128,8,128,13.486717224121094,1.5634765625,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,4,32,8,128,4.436545372009277,2.25390625,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,4,64,8,128,5.3773512840271,2.126953125,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,4,128,8,128,6.301447868347168,2.0634765625,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,8,32,8,128,0.2684425115585327,4.25390625,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,8,64,8,128,0.3253226578235626,4.126953125,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,self_attn.k_proj,8,128,8,128,0.3797986507415771,4.0634765625,4194304,5.208863520397107,NA,1.259053260582732e-5
+8,mlp.gate_proj,2,32,8,128,39.39292144775391,17.5546875,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,2,64,8,128,46.70141220092773,15.77734375,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,2,128,8,128,53.55084228515625,14.888671875,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,3,32,8,128,16.86261749267578,24.5546875,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,3,64,8,128,20.022985458374023,22.77734375,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,3,128,8,128,23.060375213623047,21.888671875,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,4,32,8,128,7.87108850479126,31.5546875,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,4,64,8,128,9.34132194519043,29.77734375,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,4,128,8,128,10.763914108276367,28.888671875,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,8,32,8,128,0.4773235917091369,59.5546875,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,8,64,8,128,0.5668628811836243,57.77734375,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.gate_proj,8,128,8,128,0.6513233780860901,56.888671875,58720256,3.737918325227297,NA,1.736560807330534e-5
+8,mlp.up_proj,2,32,8,128,29.44819641113281,17.5546875,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,2,64,8,128,34.424102783203125,15.77734375,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,2,128,8,128,38.94149017333984,14.888671875,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,3,32,8,128,12.594765663146973,24.5546875,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,3,64,8,128,14.757983207702637,22.77734375,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,3,128,8,128,16.737844467163086,21.888671875,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,4,32,8,128,5.8780717849731445,31.5546875,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,4,64,8,128,6.88339376449585,29.77734375,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,4,128,8,128,7.817975044250488,28.888671875,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,8,32,8,128,0.3572321534156799,59.5546875,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,8,64,8,128,0.4178987145423889,57.77734375,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.up_proj,8,128,8,128,0.4723220765590668,56.888671875,58720256,3.292690892929808,NA,1.736560807330534e-5
+8,mlp.down_proj,2,32,8,128,29.567283630371097,17.5546875,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,2,64,8,128,34.5979118347168,15.77734375,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,2,128,8,128,39.1610221862793,14.888671875,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,3,32,8,128,12.648090362548828,24.5546875,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,3,64,8,128,14.835460662841797,22.77734375,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,3,128,8,128,16.83810043334961,21.888671875,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,4,32,8,128,5.904628753662109,31.5546875,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,4,64,8,128,6.924561023712158,29.77734375,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,4,128,8,128,7.871026992797852,28.888671875,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,8,32,8,128,0.3588110208511352,59.5546875,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,8,64,8,128,0.4203183948993683,57.77734375,58720256,5.024922583387458,NA,1.3630980902235024e-5
+8,mlp.down_proj,8,128,8,128,0.4758267402648926,56.888671875,58720256,5.024922583387458,NA,1.3630980902235024e-5
+9,self_attn.q_proj,2,32,8,128,26.9770565032959,5.015625,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,2,64,8,128,32.41333770751953,4.5078125,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,2,128,8,128,38.673099517822266,4.25390625,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,3,32,8,128,11.569629669189451,7.015625,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,3,64,8,128,13.95190143585205,6.5078125,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,3,128,8,128,16.984012603759766,6.25390625,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,4,32,8,128,5.3989787101745605,9.015625,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,4,64,8,128,6.50968074798584,8.5078125,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,4,128,8,128,7.928092002868652,8.25390625,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,8,32,8,128,0.3267835080623626,17.015625,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,8,64,8,128,0.3938423097133636,16.5078125,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.q_proj,8,128,8,128,0.4779276549816131,16.25390625,16777216,4.688940744089759,NA,1.545005761727225e-5
+9,self_attn.v_proj,2,32,8,128,7.524624824523926,1.25390625,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,2,64,8,128,9.065919876098633,1.126953125,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,2,128,8,128,10.531023979187012,1.0634765625,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,3,32,8,128,3.2294089794158936,1.75390625,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,3,64,8,128,3.907343626022339,1.626953125,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,3,128,8,128,4.580047607421875,1.5634765625,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,4,32,8,128,1.5074481964111328,2.25390625,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,4,64,8,128,1.824461340904236,2.126953125,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,4,128,8,128,2.1387925148010254,2.0634765625,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,8,32,8,128,0.0912665277719497,4.25390625,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,8,64,8,128,0.1103613376617431,4.126953125,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.v_proj,8,128,8,128,0.128845676779747,4.0634765625,4194304,4.470621777443523,NA,1.545005761727225e-5
+9,self_attn.k_proj,2,32,8,128,21.76741981506348,1.25390625,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,2,64,8,128,26.06185340881348,1.126953125,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,2,128,8,128,29.87959861755371,1.0634765625,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,3,32,8,128,9.348188400268556,1.75390625,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,3,64,8,128,11.24947452545166,1.626953125,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,3,128,8,128,13.032777786254885,1.5634765625,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,4,32,8,128,4.36350679397583,2.25390625,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,4,64,8,128,5.2510175704956055,2.126953125,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,4,128,8,128,6.081087112426758,2.0634765625,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,8,32,8,128,0.264181911945343,4.25390625,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,8,64,8,128,0.3175808787345886,4.126953125,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,self_attn.k_proj,8,128,8,128,0.3666176497936249,4.0634765625,4194304,4.353797933169363,NA,1.545005761727225e-5
+9,mlp.gate_proj,2,32,8,128,40.1043701171875,17.5546875,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,2,64,8,128,47.65932083129883,15.77734375,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,2,128,8,128,54.80207824707031,14.888671875,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,3,32,8,128,17.172544479370117,24.5546875,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,3,64,8,128,20.46648406982422,22.77734375,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,3,128,8,128,23.66988945007324,21.888671875,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,4,32,8,128,8.018026351928711,31.5546875,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,4,64,8,128,9.550697326660156,29.77734375,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,4,128,8,128,11.0488920211792,28.888671875,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,8,32,8,128,0.4861500263214111,59.5546875,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,8,64,8,128,0.5795626640319824,57.77734375,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.gate_proj,8,128,8,128,0.6688878536224365,56.888671875,58720256,3.960293831588499,NA,1.9692888599820435e-5
+9,mlp.up_proj,2,32,8,128,29.79354286193848,17.5546875,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,2,64,8,128,34.850040435791016,15.77734375,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,2,128,8,128,39.46410369873047,14.888671875,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,3,32,8,128,12.743324279785156,24.5546875,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,3,64,8,128,14.948670387268066,22.77734375,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,3,128,8,128,16.972240447998047,21.888671875,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,4,32,8,128,5.946471691131592,31.5546875,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,4,64,8,128,6.972580909729004,29.77734375,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,4,128,8,128,7.927926540374756,28.888671875,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,8,32,8,128,0.3614477813243866,59.5546875,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,8,64,8,128,0.4233103394508362,57.77734375,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.up_proj,8,128,8,128,0.4790314733982086,56.888671875,58720256,3.31830523961292,NA,1.9692888599820435e-5
+9,mlp.down_proj,2,32,8,128,29.70336151123047,17.5546875,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,2,64,8,128,34.76505279541016,15.77734375,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,2,128,8,128,39.38078689575195,14.888671875,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,3,32,8,128,12.70130729675293,24.5546875,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,3,64,8,128,14.899825096130373,22.77734375,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,3,128,8,128,16.9156436920166,21.888671875,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,4,32,8,128,5.929380416870117,31.5546875,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,4,64,8,128,6.953718662261963,29.77734375,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,4,128,8,128,7.905187606811523,28.888671875,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,8,32,8,128,0.3603951930999756,59.5546875,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,8,64,8,128,0.4221662580966949,57.77734375,58720256,4.790787365284185,NA,1.5630836060154252e-5
+9,mlp.down_proj,8,128,8,128,0.4778989255428314,56.888671875,58720256,4.790787365284185,NA,1.5630836060154252e-5
+10,self_attn.q_proj,2,32,8,128,27.73469352722168,5.015625,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,2,64,8,128,33.35016632080078,4.5078125,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,2,128,8,128,39.38727951049805,4.25390625,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,3,32,8,128,11.962119102478027,7.015625,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,3,64,8,128,14.490964889526367,6.5078125,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,3,128,8,128,17.632457733154297,6.25390625,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,4,32,8,128,5.58771562576294,9.015625,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,4,64,8,128,6.779984474182129,8.5078125,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,4,128,8,128,8.269221305847168,8.25390625,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,8,32,8,128,0.3381999135017395,17.015625,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,8,64,8,128,0.4100817441940307,16.5078125,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.q_proj,8,128,8,128,0.4986184537410736,16.25390625,16777216,5.492602703038164,NA,1.5661613360862248e-5
+10,self_attn.v_proj,2,32,8,128,6.229307651519775,1.25390625,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,2,64,8,128,7.42263126373291,1.126953125,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,2,128,8,128,8.537671089172363,1.0634765625,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,3,32,8,128,2.6669013500213623,1.75390625,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,3,64,8,128,3.182350873947144,1.626953125,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,3,128,8,128,3.6750028133392334,1.5634765625,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,4,32,8,128,1.2443753480911257,2.25390625,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,4,64,8,128,1.4845033884048462,2.126953125,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,4,128,8,128,1.7152011394500732,2.0634765625,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,8,32,8,128,0.0753431543707847,4.25390625,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,8,64,8,128,0.0898873060941696,4.126953125,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.v_proj,8,128,8,128,0.1033799722790718,4.0634765625,4194304,3.926895145081388,NA,1.5661613360862248e-5
+10,self_attn.k_proj,2,32,8,128,22.137977600097656,1.25390625,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,2,64,8,128,26.48815155029297,1.126953125,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,2,128,8,128,30.33242416381836,1.0634765625,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,3,32,8,128,9.56244659423828,1.75390625,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,3,64,8,128,11.564157485961914,1.626953125,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,3,128,8,128,13.468461990356444,1.5634765625,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,4,32,8,128,4.46476936340332,2.25390625,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,4,64,8,128,5.40137243270874,2.126953125,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,4,128,8,128,6.312881946563721,2.0634765625,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,8,32,8,128,0.2701395153999328,4.25390625,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,8,64,8,128,0.3267165720462799,4.126953125,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,self_attn.k_proj,8,128,8,128,0.3802925944328308,4.0634765625,4194304,5.13435499245659,NA,1.5661613360862248e-5
+10,mlp.gate_proj,2,32,8,128,38.944854736328125,17.5546875,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,2,64,8,128,46.25450897216797,15.77734375,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,2,128,8,128,53.101993560791016,14.888671875,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,3,32,8,128,16.674837112426758,24.5546875,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,3,64,8,128,19.83896255493164,22.77734375,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,3,128,8,128,22.89117431640625,21.888671875,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,4,32,8,128,7.783293724060059,31.5546875,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,4,64,8,128,9.25632667541504,29.77734375,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,4,128,8,128,10.690305709838867,28.888671875,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,8,32,8,128,0.4720402657985687,59.5546875,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,8,64,8,128,0.5616547465324402,57.77734375,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.gate_proj,8,128,8,128,0.6467053294181824,56.888671875,58720256,3.81115932602854,NA,2.0056813809787855e-5
+10,mlp.up_proj,2,32,8,128,30.180099487304688,17.5546875,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,2,64,8,128,35.26427459716797,15.77734375,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,2,128,8,128,39.9040412902832,14.888671875,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,3,32,8,128,12.90137767791748,24.5546875,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,3,64,8,128,15.10992431640625,22.77734375,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,3,128,8,128,17.116329193115234,21.888671875,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,4,32,8,128,6.021190643310547,31.5546875,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,4,64,8,128,7.0478057861328125,29.77734375,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,4,128,8,128,7.995498657226562,28.888671875,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,8,32,8,128,0.36592498421669,59.5546875,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,8,64,8,128,0.4277883470058441,57.77734375,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.up_proj,8,128,8,128,0.4828921258449554,56.888671875,58720256,3.205204743649893,NA,2.0056813809787855e-5
+10,mlp.down_proj,2,32,8,128,30.11432456970215,17.5546875,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,2,64,8,128,35.22907257080078,15.77734375,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,2,128,8,128,39.86897659301758,14.888671875,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,3,32,8,128,12.874347686767578,24.5546875,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,3,64,8,128,15.087624549865724,22.77734375,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,3,128,8,128,17.10296630859375,21.888671875,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,4,32,8,128,6.009179592132568,31.5546875,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,4,64,8,128,7.041721343994141,29.77734375,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,4,128,8,128,7.99387788772583,28.888671875,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,8,32,8,128,0.3652148842811584,59.5546875,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,8,64,8,128,0.4273668527603149,57.77734375,58720256,4.387803767505477,NA,1.642370261834003e-5
+10,mlp.down_proj,8,128,8,128,0.482973575592041,56.888671875,58720256,4.387803767505477,NA,1.642370261834003e-5
+11,self_attn.q_proj,2,32,8,128,26.12057113647461,5.015625,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,2,64,8,128,31.403221130371097,4.5078125,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,2,128,8,128,37.03877639770508,4.25390625,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,3,32,8,128,11.208246231079102,7.015625,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,3,64,8,128,13.515785217285156,6.5078125,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,3,128,8,128,16.240028381347656,6.25390625,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,4,32,8,128,5.228873252868652,9.015625,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,4,64,8,128,6.310160636901856,8.5078125,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,4,128,8,128,7.584232330322266,8.25390625,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,8,32,8,128,0.3165485560894012,17.015625,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,8,64,8,128,0.3815952241420746,16.5078125,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.q_proj,8,128,8,128,0.4569851756095886,16.25390625,16777216,4.42901483877614,NA,1.598008384462446e-5
+11,self_attn.v_proj,2,32,8,128,5.915624618530273,1.25390625,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,2,64,8,128,6.933813095092773,1.126953125,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,2,128,8,128,7.871473789215088,1.0634765625,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,3,32,8,128,2.531064510345459,1.75390625,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,3,64,8,128,2.9708845615386963,1.626953125,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,3,128,8,128,3.3750290870666504,1.5634765625,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,4,32,8,128,1.1811060905456543,2.25390625,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,4,64,8,128,1.3862783908843994,2.126953125,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,4,128,8,128,1.576763391494751,2.0634765625,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,8,32,8,128,0.0715062394738197,4.25390625,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,8,64,8,128,0.0838671326637268,4.126953125,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.v_proj,8,128,8,128,0.0949677526950836,4.0634765625,4194304,3.2780002635517027,NA,1.598008384462446e-5
+11,self_attn.k_proj,2,32,8,128,21.53693389892578,1.25390625,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,2,64,8,128,25.738405227661133,1.126953125,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,2,128,8,128,29.473670959472656,1.0634765625,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,3,32,8,128,9.25912094116211,1.75390625,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,3,64,8,128,11.105602264404297,1.626953125,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,3,128,8,128,12.856239318847656,1.5634765625,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,4,32,8,128,4.320185661315918,2.25390625,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,4,64,8,128,5.183795928955078,2.126953125,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,4,128,8,128,6.003633499145508,2.0634765625,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,8,32,8,128,0.2614936530590057,4.25390625,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,8,64,8,128,0.3133919239044189,4.126953125,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,self_attn.k_proj,8,128,8,128,0.361666202545166,4.0634765625,4194304,4.226251359421093,NA,1.598008384462446e-5
+11,mlp.gate_proj,2,32,8,128,38.63543701171875,17.5546875,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,2,64,8,128,45.97889709472656,15.77734375,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,2,128,8,128,52.91830062866211,14.888671875,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,3,32,8,128,16.557937622070312,24.5546875,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,3,64,8,128,19.7586612701416,22.77734375,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,3,128,8,128,22.90111541748047,21.888671875,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,4,32,8,128,7.72847318649292,31.5546875,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,4,64,8,128,9.2195281982421875,29.77734375,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,4,128,8,128,10.691786766052246,28.888671875,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,8,32,8,128,0.4687098860740661,59.5546875,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,8,64,8,128,0.5594026446342468,57.77734375,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.gate_proj,8,128,8,128,0.6470131874084473,56.888671875,58720256,4.070416904880738,NA,2.1369023670558818e-5
+11,mlp.up_proj,2,32,8,128,30.451913833618164,17.5546875,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,2,64,8,128,35.6109504699707,15.77734375,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,2,128,8,128,40.33466339111328,14.888671875,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,3,32,8,128,13.019288063049316,24.5546875,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,3,64,8,128,15.262990951538086,22.77734375,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,3,128,8,128,17.307527542114258,21.888671875,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,4,32,8,128,6.074724197387695,31.5546875,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,4,64,8,128,7.120927333831787,29.77734375,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,4,128,8,128,8.082134246826172,28.888671875,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,8,32,8,128,0.3691640496253967,59.5546875,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,8,64,8,128,0.4320685863494873,57.77734375,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.up_proj,8,128,8,128,0.4883591830730438,56.888671875,58720256,3.2452976744640387,NA,2.1369023670558818e-5
+11,mlp.down_proj,2,32,8,128,30.37547492980957,17.5546875,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,2,64,8,128,35.56100845336914,15.77734375,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,2,128,8,128,40.31380081176758,14.888671875,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,3,32,8,128,12.995153427124023,24.5546875,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,3,64,8,128,15.258899688720703,22.77734375,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,3,128,8,128,17.339706420898438,21.888671875,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,4,32,8,128,6.06576681137085,31.5546875,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,4,64,8,128,7.121227741241455,29.77734375,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,4,128,8,128,8.104607582092285,28.888671875,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,8,32,8,128,0.3685401678085327,59.5546875,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,8,64,8,128,0.4321089386940002,57.77734375,58720256,4.4330458558097146,NA,1.702754343568813e-5
+11,mlp.down_proj,8,128,8,128,0.4897418320178985,56.888671875,58720256,4.4330458558097146,NA,1.702754343568813e-5
+12,self_attn.q_proj,2,32,8,128,25.861541748046875,5.015625,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,2,64,8,128,30.882308959960938,4.5078125,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,2,128,8,128,36.07247924804688,4.25390625,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,3,32,8,128,11.073997497558594,7.015625,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,3,64,8,128,13.24992847442627,6.5078125,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,3,128,8,128,15.60166072845459,6.25390625,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,4,32,8,128,5.16636323928833,9.015625,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,4,64,8,128,6.183772087097168,8.5078125,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,4,128,8,128,7.281061172485352,8.25390625,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,8,32,8,128,0.3127245008945465,17.015625,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,8,64,8,128,0.3739984035491943,16.5078125,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.q_proj,8,128,8,128,0.4387511610984802,16.25390625,16777216,3.918630145163863,NA,1.6118803614517674e-5
+12,self_attn.v_proj,2,32,8,128,7.040297985076904,1.25390625,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,2,64,8,128,8.279337882995605,1.126953125,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,2,128,8,128,9.429340362548828,1.0634765625,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,3,32,8,128,3.011828899383545,1.75390625,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,3,64,8,128,3.543575525283813,1.626953125,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,3,128,8,128,4.041069984436035,1.5634765625,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,4,32,8,128,1.406035542488098,2.25390625,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,4,64,8,128,1.6543664932250977,2.126953125,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,4,128,8,128,1.8858826160430908,2.0634765625,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,8,32,8,128,0.0851678997278213,4.25390625,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,8,64,8,128,0.1000915542244911,4.126953125,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.v_proj,8,128,8,128,0.1137010008096695,4.0634765625,4194304,3.3663311858528515,NA,1.6118803614517674e-5
+12,self_attn.k_proj,2,32,8,128,20.48916244506836,1.25390625,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,2,64,8,128,24.324567794799805,1.126953125,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,2,128,8,128,27.846071243286133,1.0634765625,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,3,32,8,128,8.775659561157227,1.75390625,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,3,64,8,128,10.430098533630373,1.626953125,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,3,128,8,128,11.960859298706056,1.5634765625,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,4,32,8,128,4.094570159912109,2.25390625,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,4,64,8,128,4.867015838623047,2.126953125,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,4,128,8,128,5.583600997924805,2.0634765625,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,8,32,8,128,0.2478196173906326,4.25390625,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,8,64,8,128,0.2942624986171722,4.126953125,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,self_attn.k_proj,8,128,8,128,0.3365941643714905,4.0634765625,4194304,3.674778457813903,NA,1.6118803614517674e-5
+12,mlp.gate_proj,2,32,8,128,38.39822769165039,17.5546875,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,2,64,8,128,45.75447082519531,15.77734375,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,2,128,8,128,52.67757415771485,14.888671875,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,3,32,8,128,16.485942840576172,24.5546875,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,3,64,8,128,19.719900131225582,22.77734375,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,3,128,8,128,22.919330596923828,21.888671875,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,4,32,8,128,7.694408416748047,31.5546875,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,4,64,8,128,9.2075653076171875,29.77734375,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,4,128,8,128,10.714046478271484,28.888671875,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,8,32,8,128,0.4667559862136841,59.5546875,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,8,64,8,128,0.5586689114570618,57.77734375,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.gate_proj,8,128,8,128,0.6483145952224731,56.888671875,58720256,4.633241015879171,NA,2.492406201781705e-5
+12,mlp.up_proj,2,32,8,128,31.15839958190918,17.5546875,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,2,64,8,128,36.47867584228516,15.77734375,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,2,128,8,128,41.35436630249024,14.888671875,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,3,32,8,128,13.327926635742188,24.5546875,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,3,64,8,128,15.65520191192627,22.77734375,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,3,128,8,128,17.792837142944336,21.888671875,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,4,32,8,128,6.218276500701904,31.5546875,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,4,64,8,128,7.304477214813232,29.77734375,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,4,128,8,128,8.313840866088867,28.888671875,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,8,32,8,128,0.3778644204139709,59.5546875,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,8,64,8,128,0.4431090354919433,57.77734375,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.up_proj,8,128,8,128,0.5022327303886414,56.888671875,58720256,3.3846647247334403,NA,2.492406201781705e-5
+12,mlp.down_proj,2,32,8,128,30.80364799499512,17.5546875,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,2,64,8,128,36.04652404785156,15.77734375,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,2,128,8,128,40.88020706176758,14.888671875,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,3,32,8,128,13.178635597229004,24.5546875,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,3,64,8,128,15.473698616027832,22.77734375,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,3,128,8,128,17.591928482055664,21.888671875,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,4,32,8,128,6.151588439941406,31.5546875,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,4,64,8,128,7.222397804260254,29.77734375,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,4,128,8,128,8.224127769470215,28.888671875,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,8,32,8,128,0.3738064467906952,59.5546875,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,8,64,8,128,0.4383414089679718,57.77734375,58720256,5.494694433320656,NA,1.8670300050871447e-5
+12,mlp.down_proj,8,128,8,128,0.4972971975803375,56.888671875,58720256,5.494694433320656,NA,1.8670300050871447e-5
+13,self_attn.q_proj,2,32,8,128,26.5046501159668,5.015625,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,2,64,8,128,31.847789764404297,4.5078125,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,2,128,8,128,37.72823715209961,4.25390625,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,3,32,8,128,11.362935066223145,7.015625,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,3,64,8,128,13.68783950805664,6.5078125,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,3,128,8,128,16.56133270263672,6.25390625,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,4,32,8,128,5.300324440002441,9.015625,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,4,64,8,128,6.38405704498291,8.5078125,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,4,128,8,128,7.732006549835205,8.25390625,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,8,32,8,128,0.3209334313869476,17.015625,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,8,64,8,128,0.3862396776676178,16.5078125,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.q_proj,8,128,8,128,0.4659466743469238,16.25390625,16777216,4.476935602713404,NA,2.202295399911236e-5
+13,self_attn.v_proj,2,32,8,128,6.478818416595459,1.25390625,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,2,64,8,128,7.596816062927246,1.126953125,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,2,128,8,128,8.60972785949707,1.0634765625,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,3,32,8,128,2.772458791732788,1.75390625,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,3,64,8,128,3.2537105083465576,1.626953125,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,3,128,8,128,3.693915843963623,1.5634765625,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,4,32,8,128,1.293779730796814,2.25390625,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,4,64,8,128,1.5194296836853027,2.126953125,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,4,128,8,128,1.724428415298462,2.0634765625,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,8,32,8,128,0.0783509612083435,4.25390625,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,8,64,8,128,0.0918719694018364,4.126953125,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.v_proj,8,128,8,128,0.1039059609174728,4.0634765625,4194304,3.2982611604421312,NA,2.202295399911236e-5
+13,self_attn.k_proj,2,32,8,128,21.89759635925293,1.25390625,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,2,64,8,128,26.193098068237305,1.126953125,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,2,128,8,128,29.959367752075195,1.0634765625,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,3,32,8,128,9.432870864868164,1.75390625,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,3,64,8,128,11.339022636413574,1.626953125,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,3,128,8,128,13.113142013549805,1.5634765625,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,4,32,8,128,4.4028754234313965,2.25390625,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,4,64,8,128,5.293444633483887,2.126953125,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,4,128,8,128,6.125287055969238,2.0634765625,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,8,32,8,128,0.2664675414562225,4.25390625,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,8,64,8,128,0.3202824890613556,4.126953125,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,self_attn.k_proj,8,128,8,128,0.3691720366477966,4.0634765625,4194304,4.501807253157103,NA,2.202295399911236e-5
+13,mlp.gate_proj,2,32,8,128,38.32114410400391,17.5546875,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,2,64,8,128,45.63823699951172,15.77734375,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,2,128,8,128,52.51383590698242,14.888671875,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,3,32,8,128,16.46413803100586,24.5546875,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,3,64,8,128,19.693836212158203,22.77734375,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,3,128,8,128,22.910493850708008,21.888671875,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,4,32,8,128,7.686895847320557,31.5546875,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,4,64,8,128,9.201266288757324,29.77734375,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,4,128,8,128,10.720735549926758,28.888671875,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,8,32,8,128,0.4662733376026153,59.5546875,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,8,64,8,128,0.5582589507102966,57.77734375,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.gate_proj,8,128,8,128,0.6486979722976685,56.888671875,58720256,5.069358742018072,NA,2.8637865398195572e-5
+13,mlp.up_proj,2,32,8,128,31.239431381225582,17.5546875,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,2,64,8,128,36.55526351928711,15.77734375,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,2,128,8,128,41.43852996826172,14.888671875,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,3,32,8,128,13.356857299804688,24.5546875,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,3,64,8,128,15.68277359008789,22.77734375,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,3,128,8,128,17.81464958190918,21.888671875,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,4,32,8,128,6.231928825378418,31.5546875,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,4,64,8,128,7.315294742584228,29.77734375,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,4,128,8,128,8.315678596496582,28.888671875,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,8,32,8,128,0.3787055909633636,59.5546875,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,8,64,8,128,0.4438227415084839,57.77734375,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.up_proj,8,128,8,128,0.50248783826828,56.888671875,58720256,3.3167280454758004,NA,2.8637865398195572e-5
+13,mlp.down_proj,2,32,8,128,30.7618350982666,17.5546875,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,2,64,8,128,35.97294616699219,15.77734375,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,2,128,8,128,40.74884414672852,14.888671875,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,3,32,8,128,13.150542259216309,24.5546875,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,3,64,8,128,15.41523551940918,22.77734375,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,3,128,8,128,17.47993278503418,21.888671875,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,4,32,8,128,6.136257171630859,31.5546875,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,4,64,8,128,7.191862106323242,29.77734375,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,4,128,8,128,8.164878845214844,28.888671875,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,8,32,8,128,0.3729257583618164,59.5546875,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,8,64,8,128,0.4364360272884369,57.77734375,58720256,4.430955122881719,NA,2.266871160827577e-5
+13,mlp.down_proj,8,128,8,128,0.4934674501419067,56.888671875,58720256,4.430955122881719,NA,2.266871160827577e-5
+14,self_attn.q_proj,2,32,8,128,25.61806869506836,5.015625,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,2,64,8,128,30.653871536254883,4.5078125,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,2,128,8,128,36.28984451293945,4.25390625,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,3,32,8,128,11.005311965942385,7.015625,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,3,64,8,128,13.233725547790527,6.5078125,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,3,128,8,128,15.956876754760742,6.25390625,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,4,32,8,128,5.137575149536133,9.015625,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,4,64,8,128,6.189054489135742,8.5078125,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,4,128,8,128,7.473674297332764,8.25390625,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,8,32,8,128,0.3108876943588257,17.015625,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,8,64,8,128,0.3742972612380981,16.5078125,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.q_proj,8,128,8,128,0.4504112601280212,16.25390625,16777216,5.042591826298629,NA,2.678215787454974e-5
+14,self_attn.v_proj,2,32,8,128,6.5562052726745605,1.25390625,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,2,64,8,128,7.741369724273682,1.126953125,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,2,128,8,128,8.832467079162598,1.0634765625,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,3,32,8,128,2.809292316436768,1.75390625,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,3,64,8,128,3.325889587402344,1.626953125,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,3,128,8,128,3.810516119003296,1.5634765625,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,4,32,8,128,1.31143057346344,2.25390625,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,4,64,8,128,1.5532509088516235,2.126953125,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,4,128,8,128,1.7792961597442627,2.0634765625,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,8,32,8,128,0.0793827325105667,4.25390625,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,8,64,8,128,0.0939697101712226,4.126953125,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.v_proj,8,128,8,128,0.1072141602635383,4.0634765625,4194304,3.625143634568319,NA,2.678215787454974e-5
+14,self_attn.k_proj,2,32,8,128,21.997194290161133,1.25390625,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,2,64,8,128,26.364633560180664,1.126953125,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,2,128,8,128,30.29219055175781,1.0634765625,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,3,32,8,128,9.51581573486328,1.75390625,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,3,64,8,128,11.549477577209473,1.626953125,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,3,128,8,128,13.514533996582031,1.5634765625,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,4,32,8,128,4.443383693695068,2.25390625,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,4,64,8,128,5.3974151611328125,2.126953125,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,4,128,8,128,6.329847812652588,2.0634765625,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,8,32,8,128,0.2690210342407226,4.25390625,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,8,64,8,128,0.3263379633426666,4.126953125,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,self_attn.k_proj,8,128,8,128,0.3813648819923401,4.0634765625,4194304,5.444123532860034,NA,2.678215787454974e-5
+14,mlp.gate_proj,2,32,8,128,39.41016387939453,17.5546875,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,2,64,8,128,46.97098922729492,15.77734375,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,2,128,8,128,54.20206451416016,14.888671875,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,3,32,8,128,16.946481704711914,24.5546875,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,3,64,8,128,20.342782974243164,22.77734375,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,3,128,8,128,23.739776611328125,21.888671875,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,4,32,8,128,7.914947509765625,31.5546875,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,4,64,8,128,9.501712799072266,29.77734375,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,4,128,8,128,11.119906425476074,28.888671875,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,8,32,8,128,0.4799664616584778,59.5546875,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,8,64,8,128,0.5766294598579407,57.77734375,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.gate_proj,8,128,8,128,0.6727944016456604,56.888671875,58720256,5.187594197970972,NA,3.4405660699121654e-5
+14,mlp.up_proj,2,32,8,128,31.120824813842773,17.5546875,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,2,64,8,128,36.41718673706055,15.77734375,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,2,128,8,128,41.25801849365234,14.888671875,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,3,32,8,128,13.307998657226562,24.5546875,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,3,64,8,128,15.620094299316406,22.77734375,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,3,128,8,128,17.724742889404297,21.888671875,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,4,32,8,128,6.208417892456055,31.5546875,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,4,64,8,128,7.287253856658935,29.77734375,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,4,128,8,128,8.278099060058594,28.888671875,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,8,32,8,128,0.377309650182724,59.5546875,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,8,64,8,128,0.442139208316803,57.77734375,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.up_proj,8,128,8,128,0.5002653002738953,56.888671875,58720256,3.3868219432315603,NA,3.4405660699121654e-5
+14,mlp.down_proj,2,32,8,128,30.64486122131348,17.5546875,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,2,64,8,128,35.82240676879883,15.77734375,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,2,128,8,128,40.55413055419922,14.888671875,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,3,32,8,128,13.101051330566406,24.5546875,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,3,64,8,128,15.34364128112793,22.77734375,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,3,128,8,128,17.384769439697266,21.888671875,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,4,32,8,128,6.1127753257751465,31.5546875,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,4,64,8,128,7.158343315124512,29.77734375,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,4,128,8,128,8.121088981628418,28.888671875,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,8,32,8,128,0.371524840593338,59.5546875,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,8,64,8,128,0.4344290196895599,57.77734375,58720256,3.966699917269564,NA,2.545565803302452e-5
+14,mlp.down_proj,8,128,8,128,0.4907621443271637,56.888671875,58720256,3.966699917269564,NA,2.545565803302452e-5
+15,self_attn.q_proj,2,32,8,128,29.381113052368164,5.015625,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,2,64,8,128,35.35570526123047,4.5078125,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,2,128,8,128,41.67421722412109,4.25390625,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,3,32,8,128,12.663312911987305,7.015625,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,3,64,8,128,15.373734474182127,6.5078125,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,3,128,8,128,18.746606826782227,6.25390625,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,4,32,8,128,5.9120683670043945,9.015625,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,4,64,8,128,7.184608936309815,8.5078125,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,4,128,8,128,8.775914192199707,8.25390625,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,8,32,8,128,0.3578397035598755,17.015625,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,8,64,8,128,0.4344322681427002,16.5078125,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.q_proj,8,128,8,128,0.5290461778640747,16.25390625,16777216,5.6402210746320485,NA,2.328735718037933e-5
+15,self_attn.v_proj,2,32,8,128,7.238279819488525,1.25390625,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,2,64,8,128,8.587432861328125,1.126953125,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,2,128,8,128,9.83323097229004,1.0634765625,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,3,32,8,128,3.1004669666290283,1.75390625,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,3,64,8,128,3.6899452209472656,1.626953125,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,3,128,8,128,4.236072540283203,1.5634765625,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,4,32,8,128,1.447650671005249,2.25390625,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,4,64,8,128,1.7214912176132202,2.126953125,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,4,128,8,128,1.9767045974731443,2.0634765625,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,8,32,8,128,0.087690532207489,4.25390625,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,8,64,8,128,0.1041000336408615,4.126953125,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.v_proj,8,128,8,128,0.1191994249820709,4.0634765625,4194304,3.8880310041683983,NA,2.328735718037933e-5
+15,self_attn.k_proj,2,32,8,128,21.04084205627441,1.25390625,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,2,64,8,128,25.06307411193848,1.126953125,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,2,128,8,128,28.77065658569336,1.0634765625,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,3,32,8,128,9.0236663818359375,1.75390625,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,3,64,8,128,10.787558555603027,1.626953125,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,3,128,8,128,12.454113006591797,1.5634765625,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,4,32,8,128,4.210366249084473,2.25390625,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,4,64,8,128,5.031194686889648,2.126953125,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,4,128,8,128,5.814916133880615,2.0634765625,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,8,32,8,128,0.2549245059490204,4.25390625,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,8,64,8,128,0.3044169843196869,4.126953125,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,self_attn.k_proj,8,128,8,128,0.3503837883472442,4.0634765625,4194304,4.194555335325779,NA,2.328735718037933e-5
+15,mlp.gate_proj,2,32,8,128,40.27034378051758,17.5546875,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,2,64,8,128,48.0027961730957,15.77734375,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,2,128,8,128,55.40787887573242,14.888671875,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,3,32,8,128,17.291048049926758,24.5546875,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,3,64,8,128,20.744365692138672,22.77734375,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,3,128,8,128,24.19746971130371,21.888671875,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,4,32,8,128,8.074305534362793,31.5546875,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,4,64,8,128,9.683999061584473,29.77734375,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,4,128,8,128,11.320701599121094,28.888671875,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,8,32,8,128,0.4895641803741455,59.5546875,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,8,64,8,128,0.5876905918121338,57.77734375,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.gate_proj,8,128,8,128,0.68519526720047,56.888671875,58720256,4.865220934501604,NA,3.931487663066946e-5
+15,mlp.up_proj,2,32,8,128,30.786827087402344,17.5546875,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,2,64,8,128,35.99784469604492,15.77734375,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,2,128,8,128,40.7634391784668,14.888671875,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,3,32,8,128,13.163134574890137,24.5546875,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,3,64,8,128,15.429580688476562,22.77734375,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,3,128,8,128,17.48986053466797,21.888671875,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,4,32,8,128,6.141441822052002,31.5546875,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,4,64,8,128,7.195915222167969,29.77734375,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,4,128,8,128,8.165613174438477,28.888671875,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,8,32,8,128,0.3731957972049713,59.5546875,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,8,64,8,128,0.4366535246372223,57.77734375,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.up_proj,8,128,8,128,0.4933501482009887,56.888671875,58720256,3.2533507748499355,NA,3.931487663066946e-5
+15,mlp.down_proj,2,32,8,128,30.46190452575684,17.5546875,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,2,64,8,128,35.60262680053711,15.77734375,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,2,128,8,128,40.28842544555664,14.888671875,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,3,32,8,128,13.020027160644531,24.5546875,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,3,64,8,128,15.25452709197998,22.77734375,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,3,128,8,128,17.293224334716797,21.888671875,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,4,32,8,128,6.07711935043335,31.5546875,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,4,64,8,128,7.117532253265381,29.77734375,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,4,128,8,128,8.078948974609375,28.888671875,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,8,32,8,128,0.3692785799503326,59.5546875,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,8,64,8,128,0.4319080412387848,57.77734375,58720256,3.9654243276798735,NA,3.272089088568464e-5
+15,mlp.down_proj,8,128,8,128,0.4883196055889129,56.888671875,58720256,3.9654243276798735,NA,3.272089088568464e-5
+16,self_attn.q_proj,2,32,8,128,27.64369010925293,5.015625,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,2,64,8,128,33.14297103881836,4.5078125,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,2,128,8,128,38.94926834106445,4.25390625,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,3,32,8,128,11.870041847229004,7.015625,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,3,64,8,128,14.29415798187256,6.5078125,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,3,128,8,128,17.090879440307617,6.25390625,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,4,32,8,128,5.538924217224121,9.015625,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,4,64,8,128,6.677006721496582,8.5078125,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,4,128,8,128,7.995426177978516,8.25390625,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,8,32,8,128,0.3354693651199341,17.015625,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,8,64,8,128,0.4038237333297729,16.5078125,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.q_proj,8,128,8,128,0.4819633960723877,16.25390625,16777216,5.060737921458923,NA,2.7760812372434884e-5
+16,self_attn.v_proj,2,32,8,128,6.684208869934082,1.25390625,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,2,64,8,128,7.885555267333984,1.126953125,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,2,128,8,128,8.979830741882324,1.0634765625,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,3,32,8,128,2.866643190383911,1.75390625,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,3,64,8,128,3.392282724380493,1.626953125,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,3,128,8,128,3.8811380863189697,1.5634765625,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,4,32,8,128,1.3368695974349976,2.25390625,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,4,64,8,128,1.5830122232437134,2.126953125,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,4,128,8,128,1.8121806383132928,2.0634765625,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,8,32,8,128,0.080979585647583,4.25390625,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,8,64,8,128,0.0957918316125869,4.126953125,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.v_proj,8,128,8,128,0.1093247085809707,4.0634765625,4194304,3.809677120066478,NA,2.7760812372434884e-5
+16,self_attn.k_proj,2,32,8,128,21.50868034362793,1.25390625,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,2,64,8,128,25.74004554748535,1.126953125,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,2,128,8,128,29.626441955566406,1.0634765625,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,3,32,8,128,9.239008903503418,1.75390625,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,3,64,8,128,11.116647720336914,1.626953125,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,3,128,8,128,12.926946640014648,1.5634765625,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,4,32,8,128,4.311690807342529,2.25390625,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,4,64,8,128,5.187531471252441,2.126953125,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,4,128,8,128,6.0370965003967285,2.0634765625,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,8,32,8,128,0.261018842458725,4.25390625,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,8,64,8,128,0.3137146830558777,4.126953125,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,self_attn.k_proj,8,128,8,128,0.3635620474815368,4.0634765625,4194304,4.557167140836658,NA,2.7760812372434884e-5
+16,mlp.gate_proj,2,32,8,128,41.20309066772461,17.5546875,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,2,64,8,128,49.11593246459961,15.77734375,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,2,128,8,128,56.7407341003418,14.888671875,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,3,32,8,128,17.65849494934082,24.5546875,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,3,64,8,128,21.15079498291016,22.77734375,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,3,128,8,128,24.60658264160156,21.888671875,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,4,32,8,128,8.242731094360352,31.5546875,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,4,64,8,128,9.868480682373049,29.77734375,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,4,128,8,128,11.499907493591309,28.888671875,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,8,32,8,128,0.499927282333374,59.5546875,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,8,64,8,128,0.5988888740539551,57.77734375,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.gate_proj,8,128,8,128,0.6959277391433716,56.888671875,58720256,4.2383498603177,NA,4.0284980059368536e-5
+16,mlp.up_proj,2,32,8,128,30.287851333618164,17.5546875,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,2,64,8,128,35.38748550415039,15.77734375,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,2,128,8,128,40.03034591674805,14.888671875,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,3,32,8,128,12.94804573059082,24.5546875,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,3,64,8,128,15.161388397216797,22.77734375,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,3,128,8,128,17.161760330200195,21.888671875,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,4,32,8,128,6.040839195251465,31.5546875,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,4,64,8,128,7.070522785186768,29.77734375,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,4,128,8,128,8.018121719360352,28.888671875,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,8,32,8,128,0.3671192526817322,59.5546875,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,8,64,8,128,0.4291189908981323,57.77734375,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.up_proj,8,128,8,128,0.4842276573181152,56.888671875,58720256,3.2004469731018137,NA,4.0284980059368536e-5
+16,mlp.down_proj,2,32,8,128,29.9908447265625,17.5546875,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,2,64,8,128,35.022098541259766,15.77734375,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,2,128,8,128,39.60578155517578,14.888671875,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,3,32,8,128,12.824692726135254,24.5546875,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,3,64,8,128,15.006463050842283,22.77734375,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,3,128,8,128,16.995845794677734,21.888671875,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,4,32,8,128,5.985208034515381,31.5546875,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,4,64,8,128,7.0030622482299805,29.77734375,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,4,128,8,128,7.940154075622559,28.888671875,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,8,32,8,128,0.3637315034866333,59.5546875,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,8,64,8,128,0.4250793755054474,57.77734375,58720256,4.25649855417861,NA,3.478123471722938e-5
+16,mlp.down_proj,8,128,8,128,0.4799773395061493,56.888671875,58720256,4.25649855417861,NA,3.478123471722938e-5
+17,self_attn.q_proj,2,32,8,128,28.243900299072266,5.015625,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,2,64,8,128,33.9013671875,4.5078125,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,2,128,8,128,39.67461013793945,4.25390625,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,3,32,8,128,12.116202354431152,7.015625,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,3,64,8,128,14.606330871582031,6.5078125,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,3,128,8,128,17.453651428222656,6.25390625,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,4,32,8,128,5.654839515686035,9.015625,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,4,64,8,128,6.818025588989258,8.5078125,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,4,128,8,128,8.15267562866211,8.25390625,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,8,32,8,128,0.3423314392566681,17.015625,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,8,64,8,128,0.4120905995368957,16.5078125,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.q_proj,8,128,8,128,0.491163969039917,16.25390625,16777216,4.516747289014367,NA,2.093178591167089e-5
+17,self_attn.v_proj,2,32,8,128,7.274535655975342,1.25390625,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,2,64,8,128,8.557072639465332,1.126953125,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,2,128,8,128,9.730944633483888,1.0634765625,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,3,32,8,128,3.112748146057129,1.75390625,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,3,64,8,128,3.664875745773315,1.626953125,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,3,128,8,128,4.174811363220215,1.5634765625,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,4,32,8,128,1.4522428512573242,2.25390625,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,4,64,8,128,1.7108559608459473,2.126953125,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,4,128,8,128,1.9488672018051147,2.0634765625,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,8,32,8,128,0.0879412665963172,4.25390625,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,8,64,8,128,0.1034611463546752,4.126953125,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.v_proj,8,128,8,128,0.1174446046352386,4.0634765625,4194304,3.41969594431638,NA,2.093178591167089e-5
+17,self_attn.k_proj,2,32,8,128,21.450056076049805,1.25390625,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,2,64,8,128,25.589683532714844,1.126953125,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,2,128,8,128,29.36025619506836,1.0634765625,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,3,32,8,128,9.210640907287598,1.75390625,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,3,64,8,128,11.032175064086914,1.626953125,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,3,128,8,128,12.762304306030272,1.5634765625,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,4,32,8,128,4.297592639923096,2.25390625,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,4,64,8,128,5.15066385269165,2.126953125,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,4,128,8,128,5.959950923919678,2.0634765625,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,8,32,8,128,0.2601512372493744,4.25390625,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,8,64,8,128,0.3113722205162048,4.126953125,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,self_attn.k_proj,8,128,8,128,0.3592627048492431,4.0634765625,4194304,4.530582257065935,NA,2.093178591167089e-5
+17,mlp.gate_proj,2,32,8,128,41.69158172607422,17.5546875,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,2,64,8,128,49.69167709350586,15.77734375,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,2,128,8,128,57.42879104614258,14.888671875,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,3,32,8,128,17.86820411682129,24.5546875,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,3,64,8,128,21.40752601623535,22.77734375,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,3,128,8,128,24.90820503234864,21.888671875,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,4,32,8,128,8.338918685913086,31.5546875,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,4,64,8,128,9.987314224243164,29.77734375,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,4,128,8,128,11.63386344909668,28.888671875,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,8,32,8,128,0.5058061480522156,59.5546875,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,8,64,8,128,0.6063711047172546,57.77734375,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.gate_proj,8,128,8,128,0.7045861482620239,56.888671875,58720256,4.273503957144466,NA,4.6294422645587474e-5
+17,mlp.up_proj,2,32,8,128,30.13922119140625,17.5546875,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,2,64,8,128,35.19236755371094,15.77734375,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,2,128,8,128,39.78667068481445,14.888671875,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,3,32,8,128,12.884716033935549,24.5546875,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,3,64,8,128,15.078737258911133,22.77734375,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,3,128,8,128,17.062938690185547,21.888671875,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,4,32,8,128,6.011682033538818,31.5546875,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,4,64,8,128,7.033123970031738,29.77734375,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,4,128,8,128,7.97081708908081,28.888671875,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,8,32,8,128,0.3654176890850067,59.5546875,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,8,64,8,128,0.4268718361854553,57.77734375,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.up_proj,8,128,8,128,0.4815108478069305,56.888671875,58720256,3.252461937929888,NA,4.6294422645587474e-5
+17,mlp.down_proj,2,32,8,128,29.92352867126465,17.5546875,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,2,64,8,128,34.939910888671875,15.77734375,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,2,128,8,128,39.52711868286133,14.888671875,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,3,32,8,128,12.802757263183594,24.5546875,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,3,64,8,128,14.989115715026855,22.77734375,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,3,128,8,128,16.984521865844727,21.888671875,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,4,32,8,128,5.975630760192871,31.5546875,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,4,64,8,128,6.997411727905273,29.77734375,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,4,128,8,128,7.946293830871582,28.888671875,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,8,32,8,128,0.3631942868232727,59.5546875,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,8,64,8,128,0.4247721433639526,57.77734375,58720256,4.2237623557773,NA,4.277548941900022e-5
+17,mlp.down_proj,8,128,8,128,0.4803014993667602,56.888671875,58720256,4.2237623557773,NA,4.277548941900022e-5
+18,self_attn.q_proj,2,32,8,128,27.30513381958008,5.015625,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,2,64,8,128,32.50996017456055,4.5078125,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,2,128,8,128,38.30387878417969,4.25390625,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,3,32,8,128,11.690193176269531,7.015625,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,3,64,8,128,13.941518783569336,6.5078125,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,3,128,8,128,16.589494705200195,6.25390625,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,4,32,8,128,5.45534086227417,9.015625,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,4,64,8,128,6.506054401397705,8.5078125,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,4,128,8,128,7.744174957275391,8.25390625,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,8,32,8,128,0.3303576409816742,17.015625,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,8,64,8,128,0.3934611678123474,16.5078125,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.q_proj,8,128,8,128,0.4666202068328857,16.25390625,16777216,4.067242092398344,NA,1.4197165000950918e-5
+18,self_attn.v_proj,2,32,8,128,6.611334323883057,1.25390625,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,2,64,8,128,7.799045085906982,1.126953125,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,2,128,8,128,8.893885612487793,1.0634765625,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,3,32,8,128,2.832592725753784,1.75390625,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,3,64,8,128,3.348790168762207,1.626953125,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,3,128,8,128,3.829371929168701,1.5634765625,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,4,32,8,128,1.3221431970596311,2.25390625,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,4,64,8,128,1.5629265308380127,2.126953125,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,4,128,8,128,1.788077473640442,2.0634765625,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,8,32,8,128,0.0800507664680481,4.25390625,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,8,64,8,128,0.0945601016283035,4.126953125,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.v_proj,8,128,8,128,0.107764333486557,4.0634765625,4194304,3.7779785892985496,NA,1.4197165000950918e-5
+18,self_attn.k_proj,2,32,8,128,21.83268165588379,1.25390625,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,2,64,8,128,25.942413330078125,1.126953125,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,2,128,8,128,29.68288993835449,1.0634765625,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,3,32,8,128,9.358898162841797,1.75390625,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,3,64,8,128,11.15013599395752,1.626953125,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,3,128,8,128,12.824159622192385,1.5634765625,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,4,32,8,128,4.364651203155518,2.25390625,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,4,64,8,128,5.203103542327881,2.126953125,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,4,128,8,128,5.986807823181152,2.0634765625,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,8,32,8,128,0.264183759689331,4.25390625,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,8,64,8,128,0.3146951198577881,4.126953125,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,self_attn.k_proj,8,128,8,128,0.3608006238937378,4.0634765625,4194304,4.21079745969506,NA,1.4197165000950918e-5
+18,mlp.gate_proj,2,32,8,128,41.45539855957031,17.5546875,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,2,64,8,128,49.28993988037109,15.77734375,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,2,128,8,128,56.8601188659668,14.888671875,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,3,32,8,128,17.751710891723633,24.5546875,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,3,64,8,128,21.1910343170166,22.77734375,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,3,128,8,128,24.55527877807617,21.888671875,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,4,32,8,128,8.285884857177734,31.5546875,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,4,64,8,128,9.88524055480957,29.77734375,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,4,128,8,128,11.470847129821776,28.888671875,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,8,32,8,128,0.5024477243423462,59.5546875,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,8,64,8,128,0.5998983979225159,57.77734375,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.gate_proj,8,128,8,128,0.6943526268005371,56.888671875,58720256,4.032466993585882,NA,4.586353679769673e-5
+18,mlp.up_proj,2,32,8,128,29.890323638916016,17.5546875,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,2,64,8,128,34.86293029785156,15.77734375,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,2,128,8,128,39.36655426025391,14.888671875,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,3,32,8,128,12.7736177444458,24.5546875,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,3,64,8,128,14.925217628479004,22.77734375,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,3,128,8,128,16.863479614257812,21.888671875,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,4,32,8,128,5.9599432945251465,31.5546875,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,4,64,8,128,6.961752891540527,29.77734375,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,4,128,8,128,7.874423027038574,28.888671875,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,8,32,8,128,0.3622696101665497,59.5546875,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,8,64,8,128,0.4225781559944153,57.77734375,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.up_proj,8,128,8,128,0.4757608473300934,56.888671875,58720256,3.1302849701083124,NA,4.586353679769673e-5
+18,mlp.down_proj,2,32,8,128,29.774751663208008,17.5546875,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,2,64,8,128,34.73874282836914,15.77734375,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,2,128,8,128,39.24348831176758,14.888671875,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,3,32,8,128,12.72853183746338,24.5546875,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,3,64,8,128,14.876253128051758,22.77734375,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,3,128,8,128,16.82144546508789,21.888671875,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,4,32,8,128,5.939826488494873,31.5546875,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,4,64,8,128,6.940883636474609,29.77734375,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,4,128,8,128,7.857932090759277,28.888671875,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,8,32,8,128,0.3610455095767975,59.5546875,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,8,64,8,128,0.4213594198226928,57.77734375,58720256,3.9032884957564695,NA,4.10507018386852e-5
+18,mlp.down_proj,8,128,8,128,0.4748960435390472,56.888671875,58720256,3.9032884957564695,NA,4.10507018386852e-5
+19,self_attn.q_proj,2,32,8,128,27.51787567138672,5.015625,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,2,64,8,128,32.85622024536133,4.5078125,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,2,128,8,128,38.62761688232422,4.25390625,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,3,32,8,128,11.786128997802734,7.015625,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,3,64,8,128,14.105334281921388,6.5078125,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,3,128,8,128,16.762516021728516,6.25390625,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,4,32,8,128,5.501605987548828,9.015625,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,4,64,8,128,6.584290027618408,8.5078125,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,4,128,8,128,7.82921838760376,8.25390625,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,8,32,8,128,0.333046406507492,17.015625,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,8,64,8,128,0.3981630206108093,16.5078125,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.q_proj,8,128,8,128,0.4716943204402923,16.25390625,16777216,4.185006916688276,NA,1.2588428035087418e-5
+19,self_attn.v_proj,2,32,8,128,7.030470371246338,1.25390625,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,2,64,8,128,8.304581642150879,1.126953125,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,2,128,8,128,9.479228973388672,1.0634765625,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,3,32,8,128,3.008863925933838,1.75390625,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,3,64,8,128,3.5585362911224365,1.626953125,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,3,128,8,128,4.066886425018311,1.5634765625,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,4,32,8,128,1.404317021369934,2.25390625,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,4,64,8,128,1.6601585149765017,2.126953125,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,4,128,8,128,1.8971848487854004,2.0634765625,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,8,32,8,128,0.0850284621119499,4.25390625,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,8,64,8,128,0.1004456281661987,4.126953125,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.v_proj,8,128,8,128,0.1143800020217895,4.0634765625,4194304,3.63569114970576,NA,1.2588428035087418e-5
+19,self_attn.k_proj,2,32,8,128,21.01363182067871,1.25390625,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,2,64,8,128,24.98497200012207,1.126953125,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,2,128,8,128,28.669771194458008,1.0634765625,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,3,32,8,128,8.993736267089844,1.75390625,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,3,64,8,128,10.7388277053833,1.626953125,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,3,128,8,128,12.39926528930664,1.5634765625,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,4,32,8,128,4.197534561157227,2.25390625,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,4,64,8,128,5.010700225830078,2.126953125,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,4,128,8,128,5.782396793365479,2.0634765625,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,8,32,8,128,0.2539751529693603,4.25390625,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,8,64,8,128,0.3030913770198822,4.126953125,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,self_attn.k_proj,8,128,8,128,0.3485586941242218,4.0634765625,4194304,4.165694323069783,NA,1.2588428035087418e-5
+19,mlp.gate_proj,2,32,8,128,41.56752014160156,17.5546875,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,2,64,8,128,49.3490104675293,15.77734375,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,2,128,8,128,56.8859748840332,14.888671875,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,3,32,8,128,17.795673370361328,24.5546875,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,3,64,8,128,21.20685195922852,22.77734375,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,3,128,8,128,24.54552459716797,21.888671875,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,4,32,8,128,8.306742668151855,31.5546875,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,4,64,8,128,9.892951965332031,29.77734375,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,4,128,8,128,11.465137481689451,28.888671875,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,8,32,8,128,0.5038571357727051,59.5546875,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,8,64,8,128,0.6006820201873779,57.77734375,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.gate_proj,8,128,8,128,0.6943200826644897,56.888671875,58720256,3.989818499236077,NA,4.83258263557218e-5
+19,mlp.up_proj,2,32,8,128,29.724327087402344,17.5546875,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,2,64,8,128,34.64796829223633,15.77734375,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,2,128,8,128,39.09792709350586,14.888671875,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,3,32,8,128,12.704089164733888,24.5546875,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,3,64,8,128,14.834226608276367,22.77734375,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,3,128,8,128,16.749486923217773,21.888671875,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,4,32,8,128,5.928028583526611,31.5546875,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,4,64,8,128,6.918476581573486,29.77734375,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,4,128,8,128,7.819451332092285,28.888671875,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,8,32,8,128,0.3603351414203644,59.5546875,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,8,64,8,128,0.4199293255805969,57.77734375,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.up_proj,8,128,8,128,0.4725494384765625,56.888671875,58720256,3.118681134978947,NA,4.83258263557218e-5
+19,mlp.down_proj,2,32,8,128,29.674684524536133,17.5546875,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,2,64,8,128,34.607879638671875,15.77734375,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,2,128,8,128,39.08812713623047,14.888671875,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,3,32,8,128,12.686997413635254,24.5546875,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,3,64,8,128,14.819061279296877,22.77734375,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,3,128,8,128,16.75285530090332,21.888671875,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,4,32,8,128,5.920626640319824,31.5546875,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,4,64,8,128,6.914460182189941,29.77734375,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,4,128,8,128,7.822765827178955,28.888671875,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,8,32,8,128,0.3599112629890442,59.5546875,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,8,64,8,128,0.4198633134365082,57.77734375,58720256,4.5472740868368415,NA,4.149312735535205e-5
+19,mlp.down_proj,8,128,8,128,0.4729660749435425,56.888671875,58720256,4.5472740868368415,NA,4.149312735535205e-5
+20,self_attn.q_proj,2,32,8,128,27.31611633300781,5.015625,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,2,64,8,128,32.593658447265625,4.5078125,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,2,128,8,128,38.2834587097168,4.25390625,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,3,32,8,128,11.700910568237305,7.015625,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,3,64,8,128,13.989947319030762,6.5078125,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,3,128,8,128,16.647871017456055,6.25390625,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,4,32,8,128,5.460706233978272,9.015625,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,4,64,8,128,6.527448654174805,8.5078125,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,4,128,8,128,7.773484230041504,8.25390625,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,8,32,8,128,0.3306195735931396,17.015625,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,8,64,8,128,0.3947963714599609,16.5078125,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.q_proj,8,128,8,128,0.4683714509010315,16.25390625,16777216,4.228404247253315,NA,1.5595796867273748e-5
+20,self_attn.v_proj,2,32,8,128,7.3568525314331055,1.25390625,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,2,64,8,128,8.683716773986816,1.126953125,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,2,128,8,128,9.915620803833008,1.0634765625,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,3,32,8,128,3.148019313812256,1.75390625,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,3,64,8,128,3.718897819519043,1.626953125,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,3,128,8,128,4.255496978759766,1.5634765625,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,4,32,8,128,1.4689884185791016,2.25390625,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,4,64,8,128,1.7367335557937622,2.126953125,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,4,128,8,128,1.987268686294556,2.0634765625,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,8,32,8,128,0.0889342427253723,4.25390625,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,8,64,8,128,0.1050533503293991,4.126953125,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.v_proj,8,128,8,128,0.1197745874524116,4.0634765625,4194304,3.548836089411498,NA,1.5595796867273748e-5
+20,self_attn.k_proj,2,32,8,128,20.810379028320312,1.25390625,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,2,64,8,128,24.79256057739257,1.126953125,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,2,128,8,128,28.46826171875,1.0634765625,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,3,32,8,128,8.924111366271973,1.75390625,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,3,64,8,128,10.67470359802246,1.626953125,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,3,128,8,128,12.336373329162598,1.5634765625,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,4,32,8,128,4.164855480194092,2.25390625,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,4,64,8,128,4.982593536376953,2.126953125,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,4,128,8,128,5.7602858543396,2.0634765625,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,8,32,8,128,0.2521738708019256,4.25390625,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,8,64,8,128,0.301372081041336,4.126953125,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,self_attn.k_proj,8,128,8,128,0.3471800684928894,4.0634765625,4194304,4.455881985347597,NA,1.5595796867273748e-5
+20,mlp.gate_proj,2,32,8,128,41.40095901489258,17.5546875,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,2,64,8,128,49.0767936706543,15.77734375,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,2,128,8,128,56.47530364990234,14.888671875,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,3,32,8,128,17.720191955566406,24.5546875,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,3,64,8,128,21.07521247863769,22.77734375,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,3,128,8,128,24.336673736572266,21.888671875,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,4,32,8,128,8.272833824157715,31.5546875,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,4,64,8,128,9.831510543823242,29.77734375,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,4,128,8,128,11.366402626037598,28.888671875,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,8,32,8,128,0.5016068816184998,59.5546875,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,8,64,8,128,0.5967675447463989,57.77734375,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.gate_proj,8,128,8,128,0.6880964636802673,56.888671875,58720256,3.837938664425802,NA,5.25682553416118e-5
+20,mlp.up_proj,2,32,8,128,29.80850410461425,17.5546875,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,2,64,8,128,34.74061584472656,15.77734375,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,2,128,8,128,39.19353103637695,14.888671875,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,3,32,8,128,12.740331649780272,24.5546875,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,3,64,8,128,14.869338989257812,22.77734375,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,3,128,8,128,16.785167694091797,21.888671875,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,4,32,8,128,5.9447431564331055,31.5546875,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,4,64,8,128,6.93602991104126,29.77734375,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,4,128,8,128,7.835127353668213,28.888671875,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,8,32,8,128,0.3612865805625915,59.5546875,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,8,64,8,128,0.4209855198860168,57.77734375,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.up_proj,8,128,8,128,0.4734995067119598,56.888671875,58720256,3.113201394317839,NA,5.25682553416118e-5
+20,mlp.down_proj,2,32,8,128,29.76346969604492,17.5546875,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,2,64,8,128,34.7084846496582,15.77734375,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,2,128,8,128,39.18354797363281,14.888671875,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,3,32,8,128,12.723167419433594,24.5546875,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,3,64,8,128,14.86019515991211,22.77734375,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,3,128,8,128,16.789318084716797,21.888671875,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,4,32,8,128,5.938924789428711,31.5546875,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,4,64,8,128,6.934266090393066,29.77734375,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,4,128,8,128,7.8432393074035645,28.888671875,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,8,32,8,128,0.3608524799346924,59.5546875,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,8,64,8,128,0.4208720326423645,57.77734375,58720256,3.448078004405742,NA,4.5334993046708405e-5
+20,mlp.down_proj,8,128,8,128,0.473922461271286,56.888671875,58720256,3.448078004405742,NA,4.5334993046708405e-5
+21,self_attn.q_proj,2,32,8,128,26.613637924194336,5.015625,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,2,64,8,128,31.72728157043457,4.5078125,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,2,128,8,128,37.28963851928711,4.25390625,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,3,32,8,128,11.398756980895996,7.015625,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,3,64,8,128,13.612385749816896,6.5078125,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,3,128,8,128,16.119693756103516,6.25390625,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,4,32,8,128,5.320000171661377,9.015625,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,4,64,8,128,6.354912757873535,8.5078125,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,4,128,8,128,7.528256893157959,8.25390625,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,8,32,8,128,0.3220490217208862,17.015625,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,8,64,8,128,0.3843535780906677,16.5078125,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.q_proj,8,128,8,128,0.4536494314670563,16.25390625,16777216,4.090330489017306,NA,2.3958395104273222e-5
+21,self_attn.v_proj,2,32,8,128,7.535129547119141,1.25390625,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,2,64,8,128,8.869972229003906,1.126953125,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,2,128,8,128,10.069236755371094,1.0634765625,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,3,32,8,128,3.236624956130981,1.75390625,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,3,64,8,128,3.824875831604004,1.626953125,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,3,128,8,128,4.362272262573242,1.5634765625,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,4,32,8,128,1.511125922203064,2.25390625,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,4,64,8,128,1.7847682237625122,2.126953125,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,4,128,8,128,2.038510322570801,2.0634765625,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,8,32,8,128,0.091513954102993,4.25390625,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,8,64,8,128,0.1080395504832267,4.126953125,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.v_proj,8,128,8,128,0.122831143438816,4.0634765625,4194304,3.662409406970797,NA,2.3958395104273222e-5
+21,self_attn.k_proj,2,32,8,128,20.66949462890625,1.25390625,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,2,64,8,128,24.51624298095703,1.126953125,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,2,128,8,128,28.033615112304688,1.0634765625,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,3,32,8,128,8.851082801818848,1.75390625,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,3,64,8,128,10.525378227233888,1.626953125,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,3,128,8,128,12.100557327270508,1.5634765625,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,4,32,8,128,4.1329874992370605,2.25390625,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,4,64,8,128,4.914510726928711,2.126953125,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,4,128,8,128,5.648586273193359,2.0634765625,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,8,32,8,128,0.2501528859138489,4.25390625,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,8,64,8,128,0.2973358035087585,4.126953125,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,self_attn.k_proj,8,128,8,128,0.3402147889137268,4.0634765625,4194304,4.12421091177845,NA,2.3958395104273222e-5
+21,mlp.gate_proj,2,32,8,128,41.70660781860352,17.5546875,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,2,64,8,128,49.44601440429688,15.77734375,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,2,128,8,128,56.88743591308594,14.888671875,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,3,32,8,128,17.864152908325195,24.5546875,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,3,64,8,128,21.24855232238769,22.77734375,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,3,128,8,128,24.55762481689453,21.888671875,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,4,32,8,128,8.337279319763184,31.5546875,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,4,64,8,128,9.915610313415527,29.77734375,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,4,128,8,128,11.472938537597656,28.888671875,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,8,32,8,128,0.5056385397911072,59.5546875,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,8,64,8,128,0.6018403172492981,57.77734375,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.gate_proj,8,128,8,128,0.6946837902069092,56.888671875,58720256,4.068175323005299,NA,6.226491677807644e-5
+21,mlp.up_proj,2,32,8,128,29.90048789978028,17.5546875,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,2,64,8,128,34.84867858886719,15.77734375,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,2,128,8,128,39.3271598815918,14.888671875,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,3,32,8,128,12.779494285583496,24.5546875,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,3,64,8,128,14.922487258911133,22.77734375,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,3,128,8,128,16.849123001098633,21.888671875,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,4,32,8,128,5.964380264282227,31.5546875,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,4,64,8,128,6.9611639976501465,29.77734375,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,4,128,8,128,7.867311954498291,28.888671875,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,8,32,8,128,0.3624645173549652,59.5546875,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,8,64,8,128,0.4223872125148773,57.77734375,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.up_proj,8,128,8,128,0.4753336608409881,56.888671875,58720256,3.209442464767817,NA,6.226491677807644e-5
+21,mlp.down_proj,2,32,8,128,29.85926628112793,17.5546875,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,2,64,8,128,34.81978225708008,15.77734375,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,2,128,8,128,39.30513381958008,14.888671875,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,3,32,8,128,12.764520645141602,24.5546875,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,3,64,8,128,14.916038513183594,22.77734375,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,3,128,8,128,16.857704162597656,21.888671875,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,4,32,8,128,5.956398010253906,31.5546875,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,4,64,8,128,6.958872318267822,29.77734375,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,4,128,8,128,7.873658180236816,28.888671875,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,8,32,8,128,0.3620364665985107,59.5546875,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,8,64,8,128,0.4224471151828766,57.77734375,58720256,3.86023879886732,NA,5.485265864990651e-5
+21,mlp.down_proj,8,128,8,128,0.4759640395641327,56.888671875,58720256,3.86023879886732,NA,5.485265864990651e-5
+22,self_attn.q_proj,2,32,8,128,25.80609130859375,5.015625,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,2,64,8,128,30.682270050048828,4.5078125,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,2,128,8,128,35.724220275878906,4.25390625,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,3,32,8,128,11.056154251098633,7.015625,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,3,64,8,128,13.17056941986084,6.5078125,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,3,128,8,128,15.422249794006348,6.25390625,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,4,32,8,128,5.15937089920044,9.015625,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,4,64,8,128,6.1462082862854,8.5078125,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,4,128,8,128,7.201276302337647,8.25390625,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,8,32,8,128,0.3124086260795593,17.015625,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,8,64,8,128,0.3716919422149658,16.5078125,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.q_proj,8,128,8,128,0.4336889386177063,16.25390625,16777216,3.870923415807217,NA,1.8626806195243265e-5
+22,self_attn.v_proj,2,32,8,128,7.859790325164795,1.25390625,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,2,64,8,128,9.203939437866213,1.126953125,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,2,128,8,128,10.420348167419434,1.0634765625,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,3,32,8,128,3.3680005073547363,1.75390625,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,3,64,8,128,3.952582597732544,1.626953125,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,3,128,8,128,4.486354351043701,1.5634765625,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,4,32,8,128,1.5703846216201782,2.25390625,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,4,64,8,128,1.8439210653305047,2.126953125,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,4,128,8,128,2.0944957733154297,2.0634765625,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,8,32,8,128,0.0951443091034889,4.25390625,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,8,64,8,128,0.1115146800875663,4.126953125,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.v_proj,8,128,8,128,0.1261604875326156,4.0634765625,4194304,3.464781714938944,NA,1.8626806195243265e-5
+22,self_attn.k_proj,2,32,8,128,20.348981857299805,1.25390625,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,2,64,8,128,24.19514274597168,1.126953125,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,2,128,8,128,27.700634002685547,1.0634765625,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,3,32,8,128,8.726444244384766,1.75390625,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,3,64,8,128,10.408102989196776,1.626953125,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,3,128,8,128,11.971535682678224,1.5634765625,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,4,32,8,128,4.073057651519775,2.25390625,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,4,64,8,128,4.855081081390381,2.126953125,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,4,128,8,128,5.589670181274414,2.0634765625,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,8,32,8,128,0.2463237047195434,4.25390625,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,8,64,8,128,0.2936039268970489,4.126953125,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,self_attn.k_proj,8,128,8,128,0.3368735909461975,4.0634765625,4194304,4.30360387770753,NA,1.8626806195243265e-5
+22,mlp.gate_proj,2,32,8,128,41.4613151550293,17.5546875,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,2,64,8,128,49.01024627685547,15.77734375,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,2,128,8,128,56.24254989624024,14.888671875,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,3,32,8,128,17.741302490234375,24.5546875,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,3,64,8,128,21.031667709350582,22.77734375,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,3,128,8,128,24.197364807128903,21.888671875,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,4,32,8,128,8.280515670776367,31.5546875,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,4,64,8,128,9.813997268676758,29.77734375,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,4,128,8,128,11.302498817443848,28.888671875,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,8,32,8,128,0.502230167388916,59.5546875,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,8,64,8,128,0.595484733581543,57.77734375,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.gate_proj,8,128,8,128,0.6840186715126038,56.888671875,58720256,3.699515267927764,NA,6.48412897135131e-5
+22,mlp.up_proj,2,32,8,128,29.95798110961914,17.5546875,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,2,64,8,128,34.88895797729492,15.77734375,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,2,128,8,128,39.34118270874024,14.888671875,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,3,32,8,128,12.801081657409668,24.5546875,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,3,64,8,128,14.933656692504885,22.77734375,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,3,128,8,128,16.854429244995117,21.888671875,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,4,32,8,128,5.973466396331787,31.5546875,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,4,64,8,128,6.9661478996276855,29.77734375,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,4,128,8,128,7.867184638977051,28.888671875,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,8,32,8,128,0.3629900217056274,59.5546875,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,8,64,8,128,0.4228070378303528,57.77734375,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.up_proj,8,128,8,128,0.4752513766288757,56.888671875,58720256,3.0786883542170176,NA,6.48412897135131e-5
+22,mlp.down_proj,2,32,8,128,29.94915580749512,17.5546875,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,2,64,8,128,34.90878677368164,15.77734375,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,2,128,8,128,39.39927291870117,14.888671875,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,3,32,8,128,12.799161911010742,24.5546875,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,3,64,8,128,14.940771102905272,22.77734375,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,3,128,8,128,16.866147994995117,21.888671875,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,4,32,8,128,5.972724437713623,31.5546875,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,4,64,8,128,6.97109317779541,29.77734375,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,4,128,8,128,7.876532077789307,28.888671875,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,8,32,8,128,0.3630459606647491,59.5546875,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,8,64,8,128,0.4231007993221283,57.77734375,58720256,3.567797308839977,NA,5.678017987520434e-5
+22,mlp.down_proj,8,128,8,128,0.4760325253009796,56.888671875,58720256,3.567797308839977,NA,5.678017987520434e-5
+23,self_attn.q_proj,2,32,8,128,26.0787353515625,5.015625,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,2,64,8,128,30.93896484375,4.5078125,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,2,128,8,128,35.81895446777344,4.25390625,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,3,32,8,128,11.175576210021973,7.015625,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,3,64,8,128,13.291476249694824,6.5078125,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,3,128,8,128,15.451822280883787,6.25390625,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,4,32,8,128,5.21599006652832,9.015625,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,4,64,8,128,6.204405307769775,8.5078125,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,4,128,8,128,7.213257789611816,8.25390625,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,8,32,8,128,0.3157745599746704,17.015625,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,8,64,8,128,0.3752713799476623,16.5078125,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.q_proj,8,128,8,128,0.43470299243927,16.25390625,16777216,3.932731260523578,NA,1.9537032130756415e-5
+23,self_attn.v_proj,2,32,8,128,8.22785472869873,1.25390625,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,2,64,8,128,9.627814292907717,1.126953125,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,2,128,8,128,10.899560928344728,1.0634765625,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,3,32,8,128,3.5217771530151367,1.75390625,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,3,64,8,128,4.127471446990967,1.626953125,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,3,128,8,128,4.680270195007324,1.5634765625,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,4,32,8,128,1.6431550979614258,2.25390625,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,4,64,8,128,1.925819635391236,2.126953125,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,4,128,8,128,2.184685230255127,2.0634765625,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,8,32,8,128,0.0995269864797592,4.25390625,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,8,64,8,128,0.1165014430880546,4.126953125,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.v_proj,8,128,8,128,0.131690114736557,4.0634765625,4194304,3.360367333184239,NA,1.9537032130756415e-5
+23,self_attn.k_proj,2,32,8,128,20.1629695892334,1.25390625,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,2,64,8,128,23.915668487548828,1.126953125,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,2,128,8,128,27.350526809692383,1.0634765625,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,3,32,8,128,8.644986152648926,1.75390625,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,3,64,8,128,10.28492259979248,1.626953125,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,3,128,8,128,11.814826011657717,1.5634765625,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,4,32,8,128,4.0297722816467285,2.25390625,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,4,64,8,128,4.7978715896606445,2.126953125,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,4,128,8,128,5.514894485473633,2.0634765625,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,8,32,8,128,0.2441458851099014,4.25390625,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,8,64,8,128,0.2903361618518829,4.126953125,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,self_attn.k_proj,8,128,8,128,0.3323166370391845,4.0634765625,4194304,4.275003533884144,NA,1.9537032130756415e-5
+23,mlp.gate_proj,2,32,8,128,41.48234558105469,17.5546875,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,2,64,8,128,48.99182891845703,15.77734375,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,2,128,8,128,56.17173767089844,14.888671875,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,3,32,8,128,17.746910095214844,24.5546875,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,3,64,8,128,21.01276206970215,22.77734375,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,3,128,8,128,24.165817260742188,21.888671875,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,4,32,8,128,8.283446311950684,31.5546875,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,4,64,8,128,9.805842399597168,29.77734375,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,4,128,8,128,11.291529655456545,28.888671875,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,8,32,8,128,0.5024151802062988,59.5546875,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,8,64,8,128,0.5952130556106567,57.77734375,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.gate_proj,8,128,8,128,0.6831754446029663,56.888671875,58720256,3.6353830883474023,NA,7.07345170667395e-5
+23,mlp.up_proj,2,32,8,128,30.061702728271484,17.5546875,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,2,64,8,128,34.997066497802734,15.77734375,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,2,128,8,128,39.485137939453125,14.888671875,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,3,32,8,128,12.845873832702637,24.5546875,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,3,64,8,128,14.98180866241455,22.77734375,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,3,128,8,128,16.903629302978516,21.888671875,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,4,32,8,128,5.993279457092285,31.5546875,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,4,64,8,128,6.989481925964356,29.77734375,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,4,128,8,128,7.890745162963867,28.888671875,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,8,32,8,128,0.3642669320106506,59.5546875,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,8,64,8,128,0.4241582751274109,57.77734375,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.up_proj,8,128,8,128,0.476793110370636,56.888671875,58720256,3.0855681203918777,NA,7.07345170667395e-5
+23,mlp.down_proj,2,32,8,128,30.07884788513184,17.5546875,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,2,64,8,128,35.05538558959961,15.77734375,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,2,128,8,128,39.556678771972656,14.888671875,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,3,32,8,128,12.851511001586914,24.5546875,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,3,64,8,128,15.004534721374512,22.77734375,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,3,128,8,128,16.92758560180664,21.888671875,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,4,32,8,128,5.996975898742676,31.5546875,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,4,64,8,128,6.999776363372803,29.77734375,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,4,128,8,128,7.905321598052978,28.888671875,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,8,32,8,128,0.3644680082798004,59.5546875,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,8,64,8,128,0.4247871041297912,57.77734375,58720256,3.516118656088573,NA,5.868830339750275e-5
+23,mlp.down_proj,8,128,8,128,0.4777131676673889,56.888671875,58720256,3.516118656088573,NA,5.868830339750275e-5
+24,self_attn.q_proj,2,32,8,128,25.547338485717773,5.015625,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,2,64,8,128,30.37944602966309,4.5078125,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,2,128,8,128,35.39955139160156,4.25390625,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,3,32,8,128,10.960057258605955,7.015625,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,3,64,8,128,13.067266464233398,6.5078125,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,3,128,8,128,15.332708358764648,6.25390625,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,4,32,8,128,5.114231109619141,9.015625,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,4,64,8,128,6.100057125091553,8.5078125,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,4,128,8,128,7.159446716308594,8.25390625,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,8,32,8,128,0.309750497341156,17.015625,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,8,64,8,128,0.3689931631088257,16.5078125,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.q_proj,8,128,8,128,0.4316083788871765,16.25390625,16777216,4.08087985164355,NA,2.3221307856147178e-5
+24,self_attn.v_proj,2,32,8,128,9.21454620361328,1.25390625,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,2,64,8,128,10.856494903564451,1.126953125,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,2,128,8,128,12.344929695129396,1.0634765625,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,3,32,8,128,3.948546886444092,1.75390625,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,3,64,8,128,4.659796237945557,1.626953125,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,3,128,8,128,5.323941707611084,1.5634765625,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,4,32,8,128,1.8415682315826416,2.25390625,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,4,64,8,128,2.1758296489715576,2.126953125,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,4,128,8,128,2.484900951385498,2.0634765625,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,8,32,8,128,0.1115508377552032,4.25390625,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,8,64,8,128,0.1316062808036804,4.126953125,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.v_proj,8,128,8,128,0.1497008502483368,4.0634765625,4194304,3.656821001946709,NA,2.3221307856147178e-5
+24,self_attn.k_proj,2,32,8,128,19.65592384338379,1.25390625,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,2,64,8,128,23.45724868774414,1.126953125,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,2,128,8,128,26.901391983032227,1.0634765625,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,3,32,8,128,8.4442777633667,1.75390625,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,3,64,8,128,10.13780689239502,1.626953125,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,3,128,8,128,11.737333297729492,1.5634765625,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,4,32,8,128,3.938342332839966,2.25390625,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,4,64,8,128,4.7286696434021,2.126953125,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,4,128,8,128,5.481230735778809,2.0634765625,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,8,32,8,128,0.2383520901203155,4.25390625,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,8,64,8,128,0.2860680222511291,4.126953125,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,self_attn.k_proj,8,128,8,128,0.3303963541984558,4.0634765625,4194304,4.951444601324738,NA,2.3221307856147178e-5
+24,mlp.gate_proj,2,32,8,128,41.63678359985352,17.5546875,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,2,64,8,128,49.15891647338867,15.77734375,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,2,128,8,128,56.340293884277344,14.888671875,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,3,32,8,128,17.812793731689453,24.5546875,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,3,64,8,128,21.092559814453125,22.77734375,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,3,128,8,128,24.236173629760746,21.888671875,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,4,32,8,128,8.314607620239258,31.5546875,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,4,64,8,128,9.837584495544434,29.77734375,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,4,128,8,128,11.327216148376465,28.888671875,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,8,32,8,128,0.5042733550071716,59.5546875,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,8,64,8,128,0.5973262190818787,57.77734375,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.gate_proj,8,128,8,128,0.6853898763656616,56.888671875,58720256,3.6455670560148055,NA,7.768856448819861e-5
+24,mlp.up_proj,2,32,8,128,30.197975158691406,17.5546875,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,2,64,8,128,35.16866683959961,15.77734375,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,2,128,8,128,39.64540481567383,14.888671875,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,3,32,8,128,12.90445613861084,24.5546875,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,3,64,8,128,15.05442237854004,22.77734375,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,3,128,8,128,16.983179092407227,21.888671875,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,4,32,8,128,6.020968437194824,31.5546875,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,4,64,8,128,7.023828983306885,29.77734375,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,4,128,8,128,7.930030822753906,28.888671875,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,8,32,8,128,0.3659334182739258,59.5546875,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,8,64,8,128,0.4261747896671295,57.77734375,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.up_proj,8,128,8,128,0.4791154861450195,56.888671875,58720256,3.115363691160067,NA,7.768856448819861e-5
+24,mlp.down_proj,2,32,8,128,30.21818351745605,17.5546875,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,2,64,8,128,35.216644287109375,15.77734375,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,2,128,8,128,39.71371078491211,14.888671875,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,3,32,8,128,12.912989616394045,24.5546875,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,3,64,8,128,15.066468238830566,22.77734375,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,3,128,8,128,16.998876571655273,21.888671875,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,4,32,8,128,6.024109840393066,31.5546875,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,4,64,8,128,7.027546405792236,29.77734375,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,4,128,8,128,7.935858726501465,28.888671875,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,8,32,8,128,0.3661464154720306,59.5546875,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,8,64,8,128,0.4265152513980865,57.77734375,58720256,3.3736948940606157,NA,6.239259528229013e-5
+24,mlp.down_proj,8,128,8,128,0.4795728325843811,56.888671875,58720256,3.3736948940606157,NA,6.239259528229013e-5
+25,self_attn.q_proj,2,32,8,128,24.947967529296875,5.015625,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,2,64,8,128,29.67112159729004,4.5078125,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,2,128,8,128,34.52857208251953,4.25390625,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,3,32,8,128,10.708093643188477,7.015625,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,3,64,8,128,12.765064239501951,6.5078125,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,3,128,8,128,14.922027587890623,6.25390625,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,4,32,8,128,4.997904300689697,9.015625,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,4,64,8,128,5.958414077758789,8.5078125,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,4,128,8,128,6.966994285583496,8.25390625,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,8,32,8,128,0.302553117275238,17.015625,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,8,64,8,128,0.3603058755397796,16.5078125,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.q_proj,8,128,8,128,0.4198490381240845,16.25390625,16777216,4.108803654292911,NA,3.281717363279313e-5
+25,self_attn.v_proj,2,32,8,128,9.151209831237791,1.25390625,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,2,64,8,128,10.719162940979004,1.126953125,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,2,128,8,128,12.100696563720703,1.0634765625,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,3,32,8,128,3.927787780761719,1.75390625,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,3,64,8,128,4.620433330535889,1.626953125,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,3,128,8,128,5.249710559844971,1.5634765625,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,4,32,8,128,1.8322116136550903,2.25390625,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,4,64,8,128,2.155165195465088,2.126953125,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,4,128,8,128,2.4497387409210205,2.0634765625,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,8,32,8,128,0.1109356805682182,4.25390625,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,8,64,8,128,0.1303529143333435,4.126953125,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.v_proj,8,128,8,128,0.1476238965988159,4.0634765625,4194304,3.679973980730717,NA,3.281717363279313e-5
+25,self_attn.k_proj,2,32,8,128,19.44907188415528,1.25390625,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,2,64,8,128,23.125280380249023,1.126953125,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,2,128,8,128,26.34911727905273,1.0634765625,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,3,32,8,128,8.36483097076416,1.75390625,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,3,64,8,128,10.003403663635254,1.626953125,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,3,128,8,128,11.534846305847168,1.5634765625,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,4,32,8,128,3.904937505722046,2.25390625,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,4,64,8,128,4.670424938201904,2.126953125,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,4,128,8,128,5.3935546875,2.0634765625,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,8,32,8,128,0.2363821864128112,4.25390625,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,8,64,8,128,0.2823867499828338,4.126953125,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,self_attn.k_proj,8,128,8,128,0.3251522481441498,4.0634765625,4194304,4.834827145491792,NA,3.281717363279313e-5
+25,mlp.gate_proj,2,32,8,128,42.0462760925293,17.5546875,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,2,64,8,128,49.69237518310547,15.77734375,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,2,128,8,128,57.04249572753906,14.888671875,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,3,32,8,128,17.991498947143555,24.5546875,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,3,64,8,128,21.332902908325195,22.77734375,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,3,128,8,128,24.555967330932617,21.888671875,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,4,32,8,128,8.395861625671387,31.5546875,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,4,64,8,128,9.950685501098633,29.77734375,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,4,128,8,128,11.46721076965332,28.888671875,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,8,32,8,128,0.5092549324035645,59.5546875,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,8,64,8,128,0.6039854884147644,57.77734375,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.gate_proj,8,128,8,128,0.6943060755729675,56.888671875,58720256,3.6742327030855497,NA,8.866462303558365e-5
+25,mlp.up_proj,2,32,8,128,30.46546936035156,17.5546875,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,2,64,8,128,35.48342514038086,15.77734375,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,2,128,8,128,40.01642608642578,14.888671875,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,3,32,8,128,13.018470764160156,24.5546875,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,3,64,8,128,15.19490909576416,22.77734375,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,3,128,8,128,17.145286560058594,21.888671875,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,4,32,8,128,6.074680805206299,31.5546875,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,4,64,8,128,7.086403846740723,29.77734375,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,4,128,8,128,8.004955291748047,28.888671875,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,8,32,8,128,0.3691857159137726,59.5546875,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,8,64,8,128,0.4300290644168854,57.77734375,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.up_proj,8,128,8,128,0.4836242198944092,56.888671875,58720256,3.097579016625762,NA,8.866462303558365e-5
+25,mlp.down_proj,2,32,8,128,30.48154258728028,17.5546875,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,2,64,8,128,35.51686096191406,15.77734375,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,2,128,8,128,40.075382232666016,14.888671875,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,3,32,8,128,13.021214485168455,24.5546875,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,3,64,8,128,15.201196670532228,22.77734375,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,3,128,8,128,17.143184661865234,21.888671875,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,4,32,8,128,6.07676887512207,31.5546875,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,4,64,8,128,7.08850622177124,29.77734375,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,4,128,8,128,8.005151748657227,28.888671875,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,8,32,8,128,0.3692609965801239,59.5546875,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,8,64,8,128,0.4302140176296234,57.77734375,58720256,3.3385034624872225,NA,6.630196003243327e-5
+25,mlp.down_proj,8,128,8,128,0.4836136102676391,56.888671875,58720256,3.3385034624872225,NA,6.630196003243327e-5
+26,self_attn.q_proj,2,32,8,128,25.341272354125977,5.015625,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,2,64,8,128,30.107913970947266,4.5078125,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,2,128,8,128,35.163570404052734,4.25390625,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,3,32,8,128,10.86099338531494,7.015625,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,3,64,8,128,12.921805381774902,6.5078125,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,3,128,8,128,15.189332008361816,6.25390625,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,4,32,8,128,5.066875457763672,9.015625,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,4,64,8,128,6.032660484313965,8.5078125,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,4,128,8,128,7.090411186218262,8.25390625,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,8,32,8,128,0.3067868947982788,17.015625,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,8,64,8,128,0.3648423254489898,16.5078125,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.q_proj,8,128,8,128,0.4275660812854767,16.25390625,16777216,4.097010854867958,NA,3.806484892265871e-5
+26,self_attn.v_proj,2,32,8,128,9.58985996246338,1.25390625,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,2,64,8,128,11.23852825164795,1.126953125,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,2,128,8,128,12.71847438812256,1.0634765625,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,3,32,8,128,4.117676734924316,1.75390625,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,3,64,8,128,4.841552257537842,1.626953125,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,3,128,8,128,5.503371238708496,1.5634765625,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,4,32,8,128,1.9206961393356323,2.25390625,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,4,64,8,128,2.258723497390747,2.126953125,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,4,128,8,128,2.570795774459839,2.0634765625,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,8,32,8,128,0.116376981139183,4.25390625,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,8,64,8,128,0.1366570144891739,4.126953125,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.v_proj,8,128,8,128,0.1549541652202606,4.0634765625,4194304,3.4497557015797127,NA,3.806484892265871e-5
+26,self_attn.k_proj,2,32,8,128,20.219913482666016,1.25390625,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,2,64,8,128,24.16564559936523,1.126953125,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,2,128,8,128,27.7011833190918,1.0634765625,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,3,32,8,128,8.676671981811523,1.75390625,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,3,64,8,128,10.416951179504396,1.626953125,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,3,128,8,128,12.064611434936523,1.5634765625,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,4,32,8,128,4.051470756530762,2.25390625,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,4,64,8,128,4.862581253051758,2.126953125,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,4,128,8,128,5.62973690032959,2.0634765625,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,8,32,8,128,0.2450635582208633,4.25390625,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,8,64,8,128,0.2940589189529419,4.126953125,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,self_attn.k_proj,8,128,8,128,0.3393743634223938,4.0634765625,4194304,4.9323094970352495,NA,3.806484892265871e-5
+26,mlp.gate_proj,2,32,8,128,42.71823120117188,17.5546875,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,2,64,8,128,50.7114143371582,15.77734375,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,2,128,8,128,58.47052001953125,14.888671875,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,3,32,8,128,18.284013748168945,24.5546875,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,3,64,8,128,21.78992080688477,22.77734375,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,3,128,8,128,25.228498458862305,21.888671875,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,4,32,8,128,8.531761169433594,31.5546875,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,4,64,8,128,10.16638469696045,29.77734375,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,4,128,8,128,11.780739784240724,28.888671875,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,8,32,8,128,0.5176286697387695,59.5546875,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,8,64,8,128,0.6171294450759888,57.77734375,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.gate_proj,8,128,8,128,0.71336430311203,56.888671875,58720256,3.90828748233402,NA,1.0366818605689e-4
+26,mlp.up_proj,2,32,8,128,30.723989486694336,17.5546875,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,2,64,8,128,35.78413772583008,15.77734375,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,2,128,8,128,40.35905456542969,14.888671875,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,3,32,8,128,13.128897666931152,24.5546875,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,3,64,8,128,15.32298755645752,22.77734375,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,3,128,8,128,17.285682678222656,21.888671875,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,4,32,8,128,6.12621021270752,31.5546875,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,4,64,8,128,7.1481451988220215,29.77734375,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,4,128,8,128,8.072976112365723,28.888671875,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,8,32,8,128,0.3722635507583618,59.5546875,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,8,64,8,128,0.4336202144622803,57.77734375,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.up_proj,8,128,8,128,0.4876210391521454,56.888671875,58720256,3.098346061432496,NA,1.0366818605689e-4
+26,mlp.down_proj,2,32,8,128,30.7542724609375,17.5546875,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,2,64,8,128,35.83990478515625,15.77734375,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,2,128,8,128,40.42461395263672,14.888671875,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,3,32,8,128,13.135388374328612,24.5546875,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,3,64,8,128,15.3322172164917,22.77734375,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,3,128,8,128,17.29229736328125,21.888671875,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,4,32,8,128,6.129529476165772,31.5546875,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,4,64,8,128,7.152489185333252,29.77734375,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,4,128,8,128,8.074078559875488,28.888671875,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,8,32,8,128,0.3724960386753082,59.5546875,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,8,64,8,128,0.4339540898799896,57.77734375,58720256,3.439492477428255,NA,7.719753193669021e-5
+26,mlp.down_proj,8,128,8,128,0.4879628419876098,56.888671875,58720256,3.439492477428255,NA,7.719753193669021e-5
+27,self_attn.q_proj,2,32,8,128,24.8642635345459,5.015625,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,2,64,8,128,29.59623336791992,4.5078125,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,2,128,8,128,34.41663360595703,4.25390625,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,3,32,8,128,10.677159309387209,7.015625,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,3,64,8,128,12.733939170837402,6.5078125,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,3,128,8,128,14.900238037109377,6.25390625,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,4,32,8,128,4.981285572052002,9.015625,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,4,64,8,128,5.944271564483643,8.5078125,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,4,128,8,128,6.957520961761475,8.25390625,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,8,32,8,128,0.3015709519386291,17.015625,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,8,64,8,128,0.3596012890338897,16.5078125,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.q_proj,8,128,8,128,0.4192833602428436,16.25390625,16777216,4.158059891517987,NA,5.681679249391891e-5
+27,self_attn.v_proj,2,32,8,128,10.465078353881836,1.25390625,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,2,64,8,128,12.289191246032717,1.126953125,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,2,128,8,128,13.937743186950684,1.0634765625,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,3,32,8,128,4.516843318939209,1.75390625,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,3,64,8,128,5.34511137008667,1.626953125,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,3,128,8,128,6.108231067657471,1.5634765625,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,4,32,8,128,2.109103918075561,2.25390625,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,4,64,8,128,2.497126817703247,2.126953125,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,4,128,8,128,2.859825134277344,2.0634765625,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,8,32,8,128,0.1276155263185501,4.25390625,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,8,64,8,128,0.1510124653577804,4.126953125,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.v_proj,8,128,8,128,0.1722709089517593,4.0634765625,4194304,4.277643019761399,NA,5.681679249391891e-5
+27,self_attn.k_proj,2,32,8,128,20.050647735595703,1.25390625,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,2,64,8,128,23.887086868286133,1.126953125,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,2,128,8,128,27.287803649902344,1.0634765625,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,3,32,8,128,8.62087345123291,1.75390625,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,3,64,8,128,10.342302322387695,1.626953125,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,3,128,8,128,11.975882530212402,1.5634765625,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,4,32,8,128,4.024779319763184,2.25390625,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,4,64,8,128,4.828305244445801,2.126953125,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,4,128,8,128,5.590905666351318,2.0634765625,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,8,32,8,128,0.2436296641826629,4.25390625,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,8,64,8,128,0.2918936610221863,4.126953125,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,self_attn.k_proj,8,128,8,128,0.3370570540428161,4.0634765625,4194304,5.417876234887497,NA,5.681679249391891e-5
+27,mlp.gate_proj,2,32,8,128,43.32671737670898,17.5546875,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,2,64,8,128,51.58456802368164,15.77734375,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,2,128,8,128,59.66764450073242,14.888671875,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,3,32,8,128,18.5451774597168,24.5546875,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,3,64,8,128,22.17163276672364,22.77734375,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,3,128,8,128,25.74264335632324,21.888671875,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,4,32,8,128,8.65266227722168,31.5546875,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,4,64,8,128,10.34215259552002,29.77734375,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,4,128,8,128,12.019047737121582,28.888671875,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,8,32,8,128,0.5250110030174255,59.5546875,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,8,64,8,128,0.6281165480613708,57.77734375,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.gate_proj,8,128,8,128,0.727878987789154,56.888671875,58720256,3.949630922280118,NA,1.2904960021842e-4
+27,mlp.up_proj,2,32,8,128,31.14664077758789,17.5546875,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,2,64,8,128,36.28898239135742,15.77734375,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,2,128,8,128,40.94233322143555,14.888671875,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,3,32,8,128,13.308853149414062,24.5546875,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,3,64,8,128,15.538352966308594,22.77734375,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,3,128,8,128,17.550668716430664,21.888671875,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,4,32,8,128,6.208175659179688,31.5546875,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,4,64,8,128,7.247622489929199,29.77734375,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,4,128,8,128,8.18896770477295,28.888671875,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,8,32,8,128,0.3773232996463775,59.5546875,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,8,64,8,128,0.439748078584671,57.77734375,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.up_proj,8,128,8,128,0.4947958886623382,56.888671875,58720256,3.101340650733902,NA,1.2904960021842e-4
+27,mlp.down_proj,2,32,8,128,31.081233978271484,17.5546875,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,2,64,8,128,36.21077728271485,15.77734375,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,2,128,8,128,40.85276412963867,14.888671875,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,3,32,8,128,13.277050018310549,24.5546875,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,3,64,8,128,15.495223999023438,22.77734375,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,3,128,8,128,17.479244232177734,21.888671875,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,4,32,8,128,6.194422245025635,31.5546875,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,4,64,8,128,7.227962017059326,29.77734375,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,4,128,8,128,8.158595085144043,28.888671875,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,8,32,8,128,0.3764027953147888,59.5546875,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,8,64,8,128,0.4385375380516052,57.77734375,58720256,3.274384355389698,NA,1.0146880958927e-4
+27,mlp.down_proj,8,128,8,128,0.4930305778980255,56.888671875,58720256,3.274384355389698,NA,1.0146880958927e-4
+28,self_attn.q_proj,2,32,8,128,25.128877639770508,5.015625,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,2,64,8,128,29.8079776763916,4.5078125,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,2,128,8,128,34.4616813659668,4.25390625,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,3,32,8,128,10.761868476867676,7.015625,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,3,64,8,128,12.79036808013916,6.5078125,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,3,128,8,128,14.881439208984377,6.25390625,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,4,32,8,128,5.023092746734619,9.015625,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,4,64,8,128,5.970032691955566,8.5078125,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,4,128,8,128,6.946596622467041,8.25390625,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,8,32,8,128,0.3040515482425689,17.015625,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,8,64,8,128,0.3610921204090118,16.5078125,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.q_proj,8,128,8,128,0.4184679388999939,16.25390625,16777216,4.025990376370339,NA,6.343707354972139e-5
+28,self_attn.v_proj,2,32,8,128,10.774648666381836,1.25390625,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,2,64,8,128,12.643866539001465,1.126953125,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,2,128,8,128,14.361855506896973,1.0634765625,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,3,32,8,128,4.6230010986328125,1.75390625,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,3,64,8,128,5.453924655914307,1.626953125,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,3,128,8,128,6.219938278198242,1.5634765625,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,4,32,8,128,2.158798694610596,2.25390625,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,4,64,8,128,2.5453004837036133,2.126953125,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,4,128,8,128,2.905359029769897,2.0634765625,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,8,32,8,128,0.1306397467851638,4.25390625,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,8,64,8,128,0.153981015086174,4.126953125,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.v_proj,8,128,8,128,0.1752317398786544,4.0634765625,4194304,3.530892919231584,NA,6.343707354972139e-5
+28,self_attn.k_proj,2,32,8,128,18.45705986022949,1.25390625,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,2,64,8,128,21.84394264221192,1.126953125,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,2,128,8,128,24.92411422729492,1.0634765625,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,3,32,8,128,7.911220073699951,1.75390625,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,3,64,8,128,9.379950523376465,1.626953125,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,3,128,8,128,10.762051582336426,1.5634765625,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,4,32,8,128,3.691862344741821,2.25390625,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,4,64,8,128,4.379453659057617,2.126953125,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,4,128,8,128,5.02256441116333,2.0634765625,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,8,32,8,128,0.2233436703681945,4.25390625,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,8,64,8,128,0.2647968828678131,4.126953125,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,self_attn.k_proj,8,128,8,128,0.3025440573692322,4.0634765625,4194304,4.394677532187053,NA,6.343707354972139e-5
+28,mlp.gate_proj,2,32,8,128,43.369659423828125,17.5546875,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,2,64,8,128,51.801700592041016,15.77734375,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,2,128,8,128,60.04250717163086,14.888671875,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,3,32,8,128,18.57156181335449,24.5546875,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,3,64,8,128,22.285978317260746,22.77734375,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,3,128,8,128,25.955814361572266,21.888671875,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,4,32,8,128,8.665281295776367,31.5546875,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,4,64,8,128,10.39654541015625,29.77734375,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,4,128,8,128,12.125450134277344,28.888671875,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,8,32,8,128,0.5258702635765076,59.5546875,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,8,64,8,128,0.6312415599822998,57.77734375,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.gate_proj,8,128,8,128,0.7340298295021057,56.888671875,58720256,4.146796028538303,NA,1.7864370602183e-4
+28,mlp.up_proj,2,32,8,128,31.849252700805664,17.5546875,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,2,64,8,128,37.12252426147461,15.77734375,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,2,128,8,128,41.938255310058594,14.888671875,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,3,32,8,128,13.61239528656006,24.5546875,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,3,64,8,128,15.902270317077637,22.77734375,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,3,128,8,128,17.96623420715332,21.888671875,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,4,32,8,128,6.348927974700928,31.5546875,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,4,64,8,128,7.418264865875244,29.77734375,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,4,128,8,128,8.385152816772461,28.888671875,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,8,32,8,128,0.385782241821289,59.5546875,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,8,64,8,128,0.4498964548110962,57.77734375,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.up_proj,8,128,8,128,0.5066384673118591,56.888671875,58720256,3.1735248235517166,NA,1.7864370602183e-4
+28,mlp.down_proj,2,32,8,128,31.62061309814453,17.5546875,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,2,64,8,128,36.8379135131836,15.77734375,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,2,128,8,128,41.56753158569336,14.888671875,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,3,32,8,128,13.507688522338867,24.5546875,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,3,64,8,128,15.76890468597412,22.77734375,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,3,128,8,128,17.79062843322754,21.888671875,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,4,32,8,128,6.302467346191406,31.5546875,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,4,64,8,128,7.356359004974365,29.77734375,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,4,128,8,128,8.305274963378906,28.888671875,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,8,32,8,128,0.3828789293766022,59.5546875,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,8,64,8,128,0.4461589753627777,57.77734375,58720256,3.499416742110099,NA,1.3823156768921e-4
+28,mlp.down_proj,8,128,8,128,0.5018621683120728,56.888671875,58720256,3.499416742110099,NA,1.3823156768921e-4
+29,self_attn.q_proj,2,32,8,128,24.61381721496582,5.015625,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,2,64,8,128,29.170211791992188,4.5078125,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,2,128,8,128,33.989593505859375,4.25390625,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,3,32,8,128,10.543471336364746,7.015625,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,3,64,8,128,12.524491310119627,6.5078125,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,3,128,8,128,14.67572784423828,6.25390625,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,4,32,8,128,4.920046806335449,9.015625,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,4,64,8,128,5.845945358276367,8.5078125,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,4,128,8,128,6.854568004608154,8.25390625,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,8,32,8,128,0.2980112135410309,17.015625,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,8,64,8,128,0.3536786735057831,16.5078125,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.q_proj,8,128,8,128,0.4130273759365082,16.25390625,16777216,4.196303764501026,NA,9.894864342641084e-5
+29,self_attn.v_proj,2,32,8,128,11.208635330200195,1.25390625,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,2,64,8,128,13.142824172973633,1.126953125,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,2,128,8,128,14.882481575012209,1.0634765625,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,3,32,8,128,4.8240885734558105,1.75390625,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,3,64,8,128,5.69713830947876,1.626953125,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,3,128,8,128,6.506002426147461,1.5634765625,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,4,32,8,128,2.253066301345825,2.25390625,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,4,64,8,128,2.6614325046539307,2.126953125,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,4,128,8,128,3.0452027320861816,2.0634765625,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,8,32,8,128,0.136377677321434,4.25390625,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,8,64,8,128,0.1609858870506286,4.126953125,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.v_proj,8,128,8,128,0.1835995316505432,4.0634765625,4194304,3.864623471658886,NA,9.894864342641084e-5
+29,self_attn.k_proj,2,32,8,128,19.53223991394043,1.25390625,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,2,64,8,128,23.18816947937012,1.126953125,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,2,128,8,128,26.52897071838379,1.0634765625,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,3,32,8,128,8.37331771850586,1.75390625,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,3,64,8,128,9.976702690124512,1.626953125,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,3,128,8,128,11.493720054626465,1.5634765625,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,4,32,8,128,3.9071877002716064,2.25390625,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,4,64,8,128,4.657023906707764,2.126953125,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,4,128,8,128,5.365540981292725,2.0634765625,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,8,32,8,128,0.2366570085287094,4.25390625,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,8,64,8,128,0.2816894948482513,4.126953125,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,self_attn.k_proj,8,128,8,128,0.323504775762558,4.0634765625,4194304,4.4899813154534325,NA,9.894864342641084e-5
+29,mlp.gate_proj,2,32,8,128,43.68278884887695,17.5546875,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,2,64,8,128,52.37386703491211,15.77734375,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,2,128,8,128,60.91967010498047,14.888671875,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,3,32,8,128,18.71257972717285,24.5546875,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,3,64,8,128,22.55941390991211,22.77734375,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,3,128,8,128,26.394655227661133,21.888671875,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,4,32,8,128,8.729564666748047,31.5546875,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,4,64,8,128,10.525504112243652,29.77734375,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,4,128,8,128,12.324248313903809,28.888671875,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,8,32,8,128,0.5298333168029785,59.5546875,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,8,64,8,128,0.6389312744140625,57.77734375,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.gate_proj,8,128,8,128,0.7463177442550659,56.888671875,58720256,4.378694827879926,NA,2.5600841036066e-4
+29,mlp.up_proj,2,32,8,128,32.798038482666016,17.5546875,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,2,64,8,128,38.2857666015625,15.77734375,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,2,128,8,128,43.2891960144043,14.888671875,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,3,32,8,128,14.019954681396484,24.5546875,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,3,64,8,128,16.389312744140625,22.77734375,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,3,128,8,128,18.53764533996582,21.888671875,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,4,32,8,128,6.540040493011475,31.5546875,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,4,64,8,128,7.649489879608154,29.77734375,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,4,128,8,128,8.652259826660156,28.888671875,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,8,32,8,128,0.3972397446632385,59.5546875,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,8,64,8,128,0.4637281000614166,57.77734375,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.up_proj,8,128,8,128,0.5229818820953369,56.888671875,58720256,3.2025294741228207,NA,2.5600841036066e-4
+29,mlp.down_proj,2,32,8,128,32.18458938598633,17.5546875,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,2,64,8,128,37.50214767456055,15.77734375,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,2,128,8,128,42.336612701416016,14.888671875,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,3,32,8,128,13.748483657836914,24.5546875,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,3,64,8,128,16.058168411254883,22.77734375,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,3,128,8,128,18.12397766113281,21.888671875,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,4,32,8,128,6.413782119750977,31.5546875,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,4,64,8,128,7.490348815917969,29.77734375,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,4,128,8,128,8.46006965637207,28.888671875,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,8,32,8,128,0.3897242546081543,59.5546875,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,8,64,8,128,0.4543188214302063,57.77734375,58720256,3.719138186035215,NA,2.1439717966131e-4
+29,mlp.down_proj,8,128,8,128,0.5115240216255188,56.888671875,58720256,3.719138186035215,NA,2.1439717966131e-4
+30,self_attn.q_proj,2,32,8,128,23.932138442993164,5.015625,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,2,64,8,128,28.549251556396484,4.5078125,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,2,128,8,128,33.23263168334961,4.25390625,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,3,32,8,128,10.276080131530762,7.015625,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,3,64,8,128,12.305699348449709,6.5078125,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,3,128,8,128,14.515734672546388,6.25390625,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,4,32,8,128,4.796483993530273,9.015625,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,4,64,8,128,5.746490955352783,8.5078125,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,4,128,8,128,6.780810832977295,8.25390625,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,8,32,8,128,0.2903332710266113,17.015625,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,8,64,8,128,0.3475486040115356,16.5078125,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.q_proj,8,128,8,128,0.4086917042732239,16.25390625,16777216,4.736890766635124,NA,2.0991287601646e-4
+30,self_attn.v_proj,2,32,8,128,13.593299865722656,1.25390625,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,2,64,8,128,15.973933219909668,1.126953125,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,2,128,8,128,18.097976684570312,1.0634765625,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,3,32,8,128,5.866329669952393,1.75390625,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,3,64,8,128,6.936065196990967,1.626953125,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,3,128,8,128,7.914339065551758,1.5634765625,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,4,32,8,128,2.7383298873901367,2.25390625,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,4,64,8,128,3.242396831512451,2.126953125,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,4,128,8,128,3.705921411514282,2.0634765625,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,8,32,8,128,0.1659533679485321,4.25390625,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,8,64,8,128,0.1961722820997238,4.126953125,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.v_proj,8,128,8,128,0.2233032137155532,4.0634765625,4194304,4.031878953855771,NA,2.0991287601646e-4
+30,self_attn.k_proj,2,32,8,128,17.307981491088867,1.25390625,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,2,64,8,128,20.61306953430176,1.126953125,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,2,128,8,128,23.60615921020508,1.0634765625,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,3,32,8,128,7.435863971710205,1.75390625,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,3,64,8,128,8.913440704345703,1.626953125,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,3,128,8,128,10.305980682373049,1.5634765625,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,4,32,8,128,3.468794345855713,2.25390625,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,4,64,8,128,4.161543369293213,2.126953125,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,4,128,8,128,4.810798645019531,2.0634765625,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,8,32,8,128,0.2100029587745666,4.25390625,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,8,64,8,128,0.2517099976539612,4.126953125,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,self_attn.k_proj,8,128,8,128,0.2900921702384949,4.0634765625,4194304,6.817786100262908,NA,2.0991287601646e-4
+30,mlp.gate_proj,2,32,8,128,45.67716979980469,17.5546875,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,2,64,8,128,54.90836715698242,15.77734375,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,2,128,8,128,63.91456985473633,14.888671875,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,3,32,8,128,19.56645774841309,24.5546875,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,3,64,8,128,23.636524200439453,22.77734375,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,3,128,8,128,27.691688537597656,21.888671875,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,4,32,8,128,9.128087043762209,31.5546875,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,4,64,8,128,11.03372859954834,29.77734375,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,4,128,8,128,12.926897048950195,28.888671875,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,8,32,8,128,0.5540670156478882,59.5546875,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,8,64,8,128,0.670048713684082,57.77734375,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.gate_proj,8,128,8,128,0.7827392220497131,56.888671875,58720256,4.490591132988391,NA,4.7499820357188e-4
+30,mlp.up_proj,2,32,8,128,33.815696716308594,17.5546875,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,2,64,8,128,39.57140350341797,15.77734375,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,2,128,8,128,44.87066650390625,14.888671875,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,3,32,8,128,14.462481498718262,24.5546875,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,3,64,8,128,16.943021774291992,22.77734375,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,3,128,8,128,19.22920227050781,21.888671875,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,4,32,8,128,6.745366096496582,31.5546875,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,4,64,8,128,7.909011363983154,29.77734375,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,4,128,8,128,8.969450950622559,28.888671875,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,8,32,8,128,0.4095952808856964,59.5546875,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,8,64,8,128,0.4794001579284668,57.77734375,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.up_proj,8,128,8,128,0.5422195196151733,56.888671875,58720256,3.294243091521411,NA,4.7499820357188e-4
+30,mlp.down_proj,2,32,8,128,32.36918640136719,17.5546875,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,2,64,8,128,37.74430847167969,15.77734375,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,2,128,8,128,42.64178085327149,14.888671875,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,3,32,8,128,13.840448379516602,24.5546875,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,3,64,8,128,16.18268394470215,22.77734375,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,3,128,8,128,18.312593460083008,21.888671875,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,4,32,8,128,6.456140041351318,31.5546875,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,4,64,8,128,7.550130367279053,29.77734375,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,4,128,8,128,8.548364639282227,28.888671875,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,8,32,8,128,0.3921487033367157,59.5546875,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,8,64,8,128,0.457781434059143,57.77734375,58720256,3.475028233237564,NA,4.9341743579134e-4
+30,mlp.down_proj,8,128,8,128,0.5167698860168457,56.888671875,58720256,3.475028233237564,NA,4.9341743579134e-4
+31,self_attn.q_proj,2,32,8,128,26.472570419311523,5.015625,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,2,64,8,128,31.61062431335449,4.5078125,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,2,128,8,128,36.75197982788086,4.25390625,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,3,32,8,128,11.368563652038574,7.015625,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,3,64,8,128,13.6343412399292,6.5078125,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,3,128,8,128,16.04696273803711,6.25390625,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,4,32,8,128,5.305564403533936,9.015625,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,4,64,8,128,6.367289066314697,8.5078125,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,4,128,8,128,7.496603488922119,8.25390625,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,8,32,8,128,0.3211942613124847,17.015625,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,8,64,8,128,0.3851563930511474,16.5078125,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.q_proj,8,128,8,128,0.4517876803874969,16.25390625,16777216,5.325704608919926,NA,2.6171901845373e-4
+31,self_attn.v_proj,2,32,8,128,12.27296543121338,1.25390625,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,2,64,8,128,14.527003288269045,1.126953125,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,2,128,8,128,16.583271026611328,1.0634765625,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,3,32,8,128,5.2820329666137695,1.75390625,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,3,64,8,128,6.289672374725342,1.626953125,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,3,128,8,128,7.240473747253418,1.5634765625,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,4,32,8,128,2.46514892578125,2.25390625,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,4,64,8,128,2.937155246734619,2.126953125,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,4,128,8,128,3.386962413787842,2.0634765625,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,8,32,8,128,0.1492158472537994,4.25390625,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,8,64,8,128,0.1775472611188888,4.126953125,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.v_proj,8,128,8,128,0.2040145844221115,4.0634765625,4194304,4.308375838130611,NA,2.6171901845373e-4
+31,self_attn.k_proj,2,32,8,128,18.52001571655273,1.25390625,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,2,64,8,128,21.92378807067871,1.126953125,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,2,128,8,128,25.032684326171875,1.0634765625,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,3,32,8,128,7.956022262573242,1.75390625,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,3,64,8,128,9.463165283203123,1.626953125,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,3,128,8,128,10.881024360656738,1.5634765625,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,4,32,8,128,3.711784601211548,2.25390625,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,4,64,8,128,4.416527271270752,2.126953125,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,4,128,8,128,5.085290431976318,2.0634765625,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,8,32,8,128,0.2247177511453628,4.25390625,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,8,64,8,128,0.2671961486339569,4.126953125,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,self_attn.k_proj,8,128,8,128,0.3062677085399627,4.0634765625,4194304,5.780387350146194,NA,2.6171901845373e-4
+31,mlp.gate_proj,2,32,8,128,51.96072769165039,17.5546875,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,2,64,8,128,62.50411605834961,15.77734375,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,2,128,8,128,72.632171630859375,14.888671875,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,3,32,8,128,22.3032455444336,24.5546875,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,3,64,8,128,26.91626167297364,22.77734375,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,3,128,8,128,31.600067138671875,21.888671875,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,4,32,8,128,10.4017972946167,31.5546875,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,4,64,8,128,12.561721801757812,29.77734375,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,4,128,8,128,14.746207237243652,28.888671875,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,8,32,8,128,0.6317282915115356,59.5546875,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,8,64,8,128,0.7629415392875671,57.77734375,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.gate_proj,8,128,8,128,0.8918254375457764,56.888671875,58720256,4.928933565096192,NA,0.00210070470348
+31,mlp.up_proj,2,32,8,128,39.715599060058594,17.5546875,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,2,64,8,128,46.80196762084961,15.77734375,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,2,128,8,128,53.45743942260742,14.888671875,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,3,32,8,128,17.01172637939453,24.5546875,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,3,64,8,128,20.10135078430176,22.77734375,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,3,128,8,128,23.08347511291504,21.888671875,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,4,32,8,128,7.942127704620361,31.5546875,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,4,64,8,128,9.382147789001465,29.77734375,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,4,128,8,128,10.781952857971191,28.888671875,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,8,32,8,128,0.4815772175788879,59.5546875,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,8,64,8,128,0.5693203806877136,57.77734375,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.up_proj,8,128,8,128,0.6523986458778381,56.888671875,58720256,4.221088632679399,NA,0.00210070470348
+31,mlp.down_proj,2,32,8,128,33.46739196777344,17.5546875,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,2,64,8,128,39.11680221557617,15.77734375,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,2,128,8,128,44.33309173583984,14.888671875,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,3,32,8,128,14.33449649810791,24.5546875,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,3,64,8,128,16.806303024291992,22.77734375,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,3,128,8,128,19.115291595458984,21.888671875,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,4,32,8,128,6.690278053283691,31.5546875,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,4,64,8,128,7.854846954345703,29.77734375,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,4,128,8,128,8.939277648925781,28.888671875,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,8,32,8,128,0.4062495231628418,59.5546875,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,8,64,8,128,0.4761776328086853,57.77734375,58720256,4.448183420812677,NA,0.10236936062574388
+31,mlp.down_proj,8,128,8,128,0.5405846834182739,56.888671875,58720256,4.448183420812677,NA,0.10236936062574388
+0,self_attn.o_proj,2,32,8,128,11.62565803527832,5.015625,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,2,64,8,128,13.760478973388672,4.5078125,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,2,128,8,128,15.775431632995604,4.25390625,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,3,32,8,128,5.015730857849121,7.015625,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,3,64,8,128,5.997352600097656,6.5078125,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,3,128,8,128,6.988894939422607,6.25390625,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,4,32,8,128,2.347808599472046,9.015625,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,4,64,8,128,2.814517021179199,8.5078125,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,4,128,8,128,3.2919631004333496,8.25390625,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,8,32,8,128,0.1421578079462051,17.015625,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,8,64,8,128,0.1704837083816528,16.5078125,16777216,12.816708625097188,NA,2.350435579501209e-7
+0,self_attn.o_proj,8,128,8,128,0.1990951299667358,16.25390625,16777216,12.816708625097188,NA,2.350435579501209e-7
+1,self_attn.o_proj,2,32,8,128,13.320256233215332,5.015625,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,2,64,8,128,15.85250186920166,4.5078125,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,2,128,8,128,18.25271224975586,4.25390625,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,3,32,8,128,5.814366817474365,7.015625,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,3,64,8,128,7.047060966491699,6.5078125,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,3,128,8,128,8.325507164001465,6.25390625,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,4,32,8,128,2.7238576412200928,9.015625,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,4,64,8,128,3.32004714012146,8.5078125,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,4,128,8,128,3.962393045425415,8.25390625,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,8,32,8,128,0.1648536026477813,17.015625,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,8,64,8,128,0.2008322775363922,16.5078125,16777216,10.695647626043575,NA,3.939759949389554e-7
+1,self_attn.o_proj,8,128,8,128,0.2392399460077285,16.25390625,16777216,10.695647626043575,NA,3.939759949389554e-7
+2,self_attn.o_proj,2,32,8,128,12.128545761108398,5.015625,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,2,64,8,128,14.357359886169434,4.5078125,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,2,128,8,128,16.48350715637207,4.25390625,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,3,32,8,128,5.209362030029297,7.015625,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,3,64,8,128,6.204757690429688,6.5078125,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,3,128,8,128,7.207065582275391,6.25390625,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,4,32,8,128,2.432673931121826,9.015625,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,4,64,8,128,2.898388624191284,8.5078125,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,4,128,8,128,3.370327472686768,8.25390625,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,8,32,8,128,0.1472485661506652,17.015625,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,8,64,8,128,0.1753260344266891,16.5078125,16777216,6.804941241649276,NA,9.326020631306164e-7
+2,self_attn.o_proj,8,128,8,128,0.2031807005405426,16.25390625,16777216,6.804941241649276,NA,9.326020631306164e-7
+3,self_attn.o_proj,2,32,8,128,13.677197456359863,5.015625,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,2,64,8,128,16.046985626220703,4.5078125,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,2,128,8,128,18.24164581298828,4.25390625,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,3,32,8,128,5.862359046936035,7.015625,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,3,64,8,128,6.901886463165283,6.5078125,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,3,128,8,128,7.893731594085693,6.25390625,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,4,32,8,128,2.7355918884277344,9.015625,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,4,64,8,128,3.2214348316192627,8.5078125,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,4,128,8,128,3.687077522277832,8.25390625,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,8,32,8,128,0.1656187325716018,17.015625,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,8,64,8,128,0.1948099285364151,16.5078125,16777216,4.685622851443846,NA,1.5665997352698469e-6
+3,self_attn.o_proj,8,128,8,128,0.2220923602581024,16.25390625,16777216,4.685622851443846,NA,1.5665997352698469e-6
+4,self_attn.o_proj,2,32,8,128,14.107036590576172,5.015625,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,2,64,8,128,16.614194869995117,4.5078125,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,2,128,8,128,18.963024139404297,4.25390625,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,3,32,8,128,6.076632499694824,7.015625,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,3,64,8,128,7.223157405853271,6.5078125,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,3,128,8,128,8.366185188293457,6.25390625,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,4,32,8,128,2.836582660675049,9.015625,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,4,64,8,128,3.375967264175415,8.5078125,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,4,128,8,128,3.922086238861084,8.25390625,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,8,32,8,128,0.1717239171266555,17.015625,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,8,64,8,128,0.2041394561529159,16.5078125,16777216,5.640226420128138,NA,3.732759296326549e-6
+4,self_attn.o_proj,8,128,8,128,0.236321359872818,16.25390625,16777216,5.640226420128138,NA,3.732759296326549e-6
+5,self_attn.o_proj,2,32,8,128,12.935108184814451,5.015625,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,2,64,8,128,15.15129852294922,4.5078125,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,2,128,8,128,17.19681167602539,4.25390625,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,3,32,8,128,5.535101413726807,7.015625,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,3,64,8,128,6.496115684509277,6.5078125,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,3,128,8,128,7.393651008605957,6.25390625,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,4,32,8,128,2.583302974700928,9.015625,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,4,64,8,128,3.0311734676361084,8.5078125,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,4,128,8,128,3.4520888328552246,8.25390625,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,8,32,8,128,0.1563338637351989,17.015625,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,8,64,8,128,0.1833816468715667,16.5078125,16777216,4.270565604929899,NA,5.492578566190787e-6
+5,self_attn.o_proj,8,128,8,128,0.2080183327198028,16.25390625,16777216,4.270565604929899,NA,5.492578566190787e-6
+6,self_attn.o_proj,2,32,8,128,13.487937927246094,5.015625,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,2,64,8,128,15.787430763244627,4.5078125,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,2,128,8,128,17.891263961791992,4.25390625,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,3,32,8,128,5.77061653137207,7.015625,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,3,64,8,128,6.7621049880981445,6.5078125,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,3,128,8,128,7.691638946533203,6.25390625,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,4,32,8,128,2.694453716278076,9.015625,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,4,64,8,128,3.1594767570495605,8.5078125,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,4,128,8,128,3.593095302581787,8.25390625,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,8,32,8,128,0.1630697697401046,17.015625,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,8,64,8,128,0.1910372674465179,16.5078125,16777216,6.574584949698578,NA,6.826936896686675e-6
+6,self_attn.o_proj,8,128,8,128,0.2165930122137069,16.25390625,16777216,6.574584949698578,NA,6.826936896686675e-6
+7,self_attn.o_proj,2,32,8,128,13.807162284851074,5.015625,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,2,64,8,128,16.211566925048828,4.5078125,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,2,128,8,128,18.44451904296875,4.25390625,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,3,32,8,128,5.908563613891602,7.015625,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,3,64,8,128,6.952080726623535,6.5078125,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,3,128,8,128,7.942596912384033,6.25390625,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,4,32,8,128,2.758080244064331,9.015625,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,4,64,8,128,3.245979070663452,8.5078125,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,4,128,8,128,3.710340023040772,8.25390625,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,8,32,8,128,0.1669350117444992,17.015625,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,8,64,8,128,0.1963274776935577,16.5078125,16777216,4.22290253365536,NA,8.94385993888136e-6
+7,self_attn.o_proj,8,128,8,128,0.2235273271799087,16.25390625,16777216,4.22290253365536,NA,8.94385993888136e-6
+8,self_attn.o_proj,2,32,8,128,14.093366622924805,5.015625,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,2,64,8,128,16.5416202545166,4.5078125,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,2,128,8,128,18.809246063232425,4.25390625,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,3,32,8,128,6.036656856536865,7.015625,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,3,64,8,128,7.104803085327148,6.5078125,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,3,128,8,128,8.118633270263672,6.25390625,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,4,32,8,128,2.819232702255249,9.015625,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,4,64,8,128,3.32045841217041,8.5078125,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,4,128,8,128,3.797840833663941,8.25390625,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,8,32,8,128,0.1706764101982116,17.015625,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,8,64,8,128,0.2007231414318084,16.5078125,16777216,6.142698948322791,NA,1.259053260582732e-5
+8,self_attn.o_proj,8,128,8,128,0.2288123369216919,16.25390625,16777216,6.142698948322791,NA,1.259053260582732e-5
+9,self_attn.o_proj,2,32,8,128,15.094277381896973,5.015625,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,2,64,8,128,17.70538902282715,4.5078125,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,2,128,8,128,20.105255126953125,4.25390625,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,3,32,8,128,6.4606523513793945,7.015625,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,3,64,8,128,7.59069013595581,6.5078125,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,3,128,8,128,8.645902633666992,6.25390625,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,4,32,8,128,3.014559030532837,9.015625,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,4,64,8,128,3.5421299934387207,8.5078125,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,4,128,8,128,4.038563251495361,8.25390625,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,8,32,8,128,0.1824861317873001,17.015625,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,8,64,8,128,0.2141808569431305,16.5078125,16777216,4.588205024655082,NA,1.545005761727225e-5
+9,self_attn.o_proj,8,128,8,128,0.2432269304990768,16.25390625,16777216,4.588205024655082,NA,1.545005761727225e-5
+10,self_attn.o_proj,2,32,8,128,13.89360523223877,5.015625,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,2,64,8,128,16.27947998046875,4.5078125,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,2,128,8,128,18.45605659484864,4.25390625,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,3,32,8,128,5.949365139007568,7.015625,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,3,64,8,128,6.983646392822266,6.5078125,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,3,128,8,128,7.942622184753418,6.25390625,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,4,32,8,128,2.7777304649353027,9.015625,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,4,64,8,128,3.262352705001831,8.5078125,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,4,128,8,128,3.7130494117736816,8.25390625,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,8,32,8,128,0.1681594401597976,17.015625,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,8,64,8,128,0.1972410529851913,16.5078125,16777216,5.74682408435817,NA,1.5661613360862248e-5
+10,self_attn.o_proj,8,128,8,128,0.2237548381090164,16.25390625,16777216,5.74682408435817,NA,1.5661613360862248e-5
+11,self_attn.o_proj,2,32,8,128,14.244765281677246,5.015625,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,2,64,8,128,16.716487884521484,4.5078125,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,2,128,8,128,18.98614883422852,4.25390625,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,3,32,8,128,6.097167015075684,7.015625,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,3,64,8,128,7.168283939361572,6.5078125,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,3,128,8,128,8.164941787719727,6.25390625,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,4,32,8,128,2.84542179107666,9.015625,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,4,64,8,128,3.345279693603516,8.5078125,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,4,128,8,128,3.812110424041748,8.25390625,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,8,32,8,128,0.1722400337457656,17.015625,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,8,64,8,128,0.2022191286087036,16.5078125,16777216,4.2034837806221095,NA,1.598008384462446e-5
+11,self_attn.o_proj,8,128,8,128,0.2296272665262222,16.25390625,16777216,4.2034837806221095,NA,1.598008384462446e-5
+12,self_attn.o_proj,2,32,8,128,15.051139831542969,5.015625,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,2,64,8,128,17.665584564208984,4.5078125,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,2,128,8,128,20.06760025024414,4.25390625,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,3,32,8,128,6.443262100219727,7.015625,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,3,64,8,128,7.575900077819824,6.5078125,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,3,128,8,128,8.629935264587402,6.25390625,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,4,32,8,128,3.008702754974365,9.015625,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,4,64,8,128,3.538581371307373,8.5078125,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,4,128,8,128,4.036471843719482,8.25390625,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,8,32,8,128,0.1820550709962844,17.015625,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,8,64,8,128,0.2139701545238495,16.5078125,16777216,5.081191454227569,NA,1.6118803614517674e-5
+12,self_attn.o_proj,8,128,8,128,0.243266299366951,16.25390625,16777216,5.081191454227569,NA,1.6118803614517674e-5
+13,self_attn.o_proj,2,32,8,128,14.635335922241213,5.015625,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,2,64,8,128,17.123037338256836,4.5078125,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,2,128,8,128,19.390033721923828,4.25390625,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,3,32,8,128,6.264823913574219,7.015625,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,3,64,8,128,7.340836524963379,6.5078125,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,3,128,8,128,8.326494216918945,6.25390625,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,4,32,8,128,2.9257869720458984,9.015625,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,4,64,8,128,3.4308698177337646,8.5078125,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,4,128,8,128,3.8957786560058594,8.25390625,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,8,32,8,128,0.1771737784147262,17.015625,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,8,64,8,128,0.2075408399105072,16.5078125,16777216,8.476714934470609,NA,2.202295399911236e-5
+13,self_attn.o_proj,8,128,8,128,0.2349102646112442,16.25390625,16777216,8.476714934470609,NA,2.202295399911236e-5
+14,self_attn.o_proj,2,32,8,128,14.63621425628662,5.015625,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,2,64,8,128,17.19491958618164,4.5078125,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,2,128,8,128,19.55340003967285,4.25390625,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,3,32,8,128,6.268218994140625,7.015625,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,3,64,8,128,7.380595684051514,6.5078125,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,3,128,8,128,8.436356544494629,6.25390625,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,4,32,8,128,2.9249253273010254,9.015625,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,4,64,8,128,3.444727659225464,8.5078125,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,4,128,8,128,3.938780069351196,8.25390625,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,8,32,8,128,0.1770697385072708,17.015625,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,8,64,8,128,0.208248421549797,16.5078125,16777216,4.002373111636598,NA,2.678215787454974e-5
+14,self_attn.o_proj,8,128,8,128,0.2372278124094009,16.25390625,16777216,4.002373111636598,NA,2.678215787454974e-5
+15,self_attn.o_proj,2,32,8,128,15.051722526550291,5.015625,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,2,64,8,128,17.624523162841797,4.5078125,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,2,128,8,128,20.003849029541016,4.25390625,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,3,32,8,128,6.447397708892822,7.015625,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,3,64,8,128,7.569221019744873,6.5078125,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,3,128,8,128,8.61890983581543,6.25390625,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,4,32,8,128,3.008628368377685,9.015625,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,4,64,8,128,3.5341615676879883,8.5078125,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,4,128,8,128,4.029312610626221,8.25390625,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,8,32,8,128,0.1822375059127807,17.015625,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,8,64,8,128,0.2137708812952041,16.5078125,16777216,6.085332035790016,NA,2.328735718037933e-5
+15,self_attn.o_proj,8,128,8,128,0.2428483068943023,16.25390625,16777216,6.085332035790016,NA,2.328735718037933e-5
+16,self_attn.o_proj,2,32,8,128,14.671588897705078,5.015625,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,2,64,8,128,17.20387840270996,4.5078125,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,2,128,8,128,19.533960342407227,4.25390625,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,3,32,8,128,6.29518985748291,7.015625,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,3,64,8,128,7.406421661376953,6.5078125,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,3,128,8,128,8.455559730529785,6.25390625,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,4,32,8,128,2.938764333724976,9.015625,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,4,64,8,128,3.4616949558258057,8.5078125,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,4,128,8,128,3.9625065326690674,8.25390625,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,8,32,8,128,0.1780003607273101,17.015625,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,8,64,8,128,0.2093121707439422,16.5078125,16777216,6.877278363318378,NA,2.7760812372434884e-5
+16,self_attn.o_proj,8,128,8,128,0.2387027740478515,16.25390625,16777216,6.877278363318378,NA,2.7760812372434884e-5
+17,self_attn.o_proj,2,32,8,128,15.159659385681152,5.015625,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,2,64,8,128,17.726320266723633,4.5078125,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,2,128,8,128,20.09735679626465,4.25390625,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,3,32,8,128,6.492598056793213,7.015625,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,3,64,8,128,7.610992431640625,6.5078125,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,3,128,8,128,8.64831256866455,6.25390625,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,4,32,8,128,3.031862258911133,9.015625,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,4,64,8,128,3.556722640991211,8.5078125,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,4,128,8,128,4.0480780601501465,8.25390625,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,8,32,8,128,0.1835595369338989,17.015625,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,8,64,8,128,0.2150303274393081,16.5078125,16777216,6.177168616937917,NA,2.093178591167089e-5
+17,self_attn.o_proj,8,128,8,128,0.2438358813524246,16.25390625,16777216,6.177168616937917,NA,2.093178591167089e-5
+18,self_attn.o_proj,2,32,8,128,14.78054141998291,5.015625,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,2,64,8,128,17.31252098083496,4.5078125,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,2,128,8,128,19.65506362915039,4.25390625,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,3,32,8,128,6.328427791595459,7.015625,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,3,64,8,128,7.425669193267822,6.5078125,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,3,128,8,128,8.456360816955566,6.25390625,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,4,32,8,128,2.95471715927124,9.015625,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,4,64,8,128,3.467207908630371,8.5078125,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,4,128,8,128,3.9520959854125977,8.25390625,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,8,32,8,128,0.1788134425878524,17.015625,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,8,64,8,128,0.2096264213323593,16.5078125,16777216,7.978689698077938,NA,1.4197165000950918e-5
+18,self_attn.o_proj,8,128,8,128,0.2381150126457214,16.25390625,16777216,7.978689698077938,NA,1.4197165000950918e-5
+19,self_attn.o_proj,2,32,8,128,14.957073211669922,5.015625,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,2,64,8,128,17.497554779052734,4.5078125,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,2,128,8,128,19.854581832885746,4.25390625,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,3,32,8,128,6.4000678062438965,7.015625,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,3,64,8,128,7.502575874328613,6.5078125,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,3,128,8,128,8.5342864990234375,6.25390625,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,4,32,8,128,2.9878108501434326,9.015625,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,4,64,8,128,3.504570245742798,8.5078125,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,4,128,8,128,3.99132251739502,8.25390625,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,8,32,8,128,0.1809410005807876,17.015625,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,8,64,8,128,0.2119685560464859,16.5078125,16777216,5.787306369736095,NA,1.2588428035087418e-5
+19,self_attn.o_proj,8,128,8,128,0.2405126243829727,16.25390625,16777216,5.787306369736095,NA,1.2588428035087418e-5
+20,self_attn.o_proj,2,32,8,128,14.878053665161133,5.015625,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,2,64,8,128,17.47359275817871,4.5078125,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,2,128,8,128,19.893165588378903,4.25390625,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,3,32,8,128,6.376826763153076,7.015625,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,3,64,8,128,7.518867492675781,6.5078125,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,3,128,8,128,8.61892318725586,6.25390625,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,4,32,8,128,2.976961612701416,9.015625,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,4,64,8,128,3.511537551879883,8.5078125,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,4,128,8,128,4.029510498046875,8.25390625,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,8,32,8,128,0.1802045851945877,17.015625,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,8,64,8,128,0.2123738974332809,16.5078125,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+20,self_attn.o_proj,8,128,8,128,0.2428355813026428,16.25390625,16777216,6.2495268494118434,NA,1.5595796867273748e-5
+21,self_attn.o_proj,2,32,8,128,15.174367904663086,5.015625,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,2,64,8,128,17.784578323364258,4.5078125,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,2,128,8,128,20.190414428710938,4.25390625,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,3,32,8,128,6.5373382568359375,7.015625,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,3,64,8,128,7.720632553100586,6.5078125,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,3,128,8,128,8.863565444946289,6.25390625,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,4,32,8,128,3.054070234298706,9.015625,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,4,64,8,128,3.612856864929199,8.5078125,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,4,128,8,128,4.1695098876953125,8.25390625,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,8,32,8,128,0.1848398596048355,17.015625,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,8,64,8,128,0.2183932512998581,16.5078125,16777216,5.118433747281833,NA,2.3958395104273222e-5
+21,self_attn.o_proj,8,128,8,128,0.2512237429618835,16.25390625,16777216,5.118433747281833,NA,2.3958395104273222e-5
+22,self_attn.o_proj,2,32,8,128,15.403858184814451,5.015625,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,2,64,8,128,17.991758346557617,4.5078125,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,2,128,8,128,20.357152938842773,4.25390625,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,3,32,8,128,6.59965705871582,7.015625,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,3,64,8,128,7.729466915130615,6.5078125,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,3,128,8,128,8.768677711486816,6.25390625,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,4,32,8,128,3.078189849853516,9.015625,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,4,64,8,128,3.607653617858887,8.5078125,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,4,128,8,128,4.097971439361572,8.25390625,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,8,32,8,128,0.1864184588193893,17.015625,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,8,64,8,128,0.2181044220924377,16.5078125,16777216,4.379309348149095,NA,1.8626806195243265e-5
+22,self_attn.o_proj,8,128,8,128,0.2469657659530639,16.25390625,16777216,4.379309348149095,NA,1.8626806195243265e-5
+23,self_attn.o_proj,2,32,8,128,15.661473274230955,5.015625,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,2,64,8,128,18.266996383666992,4.5078125,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,2,128,8,128,20.64474296569824,4.25390625,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,3,32,8,128,6.695975303649902,7.015625,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,3,64,8,128,7.823517322540283,6.5078125,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,3,128,8,128,8.849101066589355,6.25390625,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,4,32,8,128,3.1255362033843994,9.015625,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,4,64,8,128,3.651583194732666,8.5078125,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,4,128,8,128,4.133560180664063,8.25390625,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,8,32,8,128,0.1891371309757232,17.015625,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,8,64,8,128,0.2207475155591964,16.5078125,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+23,self_attn.o_proj,8,128,8,128,0.2490492910146713,16.25390625,16777216,3.6975582201599577,NA,1.9537032130756415e-5
+24,self_attn.o_proj,2,32,8,128,16.394994735717773,5.015625,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,2,64,8,128,19.14152336120605,4.5078125,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,2,128,8,128,21.64337539672852,4.25390625,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,3,32,8,128,7.011787414550781,7.015625,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,3,64,8,128,8.197203636169434,6.5078125,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,3,128,8,128,9.272076606750488,6.25390625,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,4,32,8,128,3.272226572036743,9.015625,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,4,64,8,128,3.825253486633301,8.5078125,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,4,128,8,128,4.328807830810547,8.25390625,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,8,32,8,128,0.1980541795492172,17.015625,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,8,64,8,128,0.2311242371797561,16.5078125,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+24,self_attn.o_proj,8,128,8,128,0.2609146535396576,16.25390625,16777216,3.3829084966717327,NA,2.3221307856147178e-5
+25,self_attn.o_proj,2,32,8,128,16.53202247619629,5.015625,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,2,64,8,128,19.339879989624023,4.5078125,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,2,128,8,128,21.89888572692871,4.25390625,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,3,32,8,128,7.078935623168945,7.015625,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,3,64,8,128,8.309752464294434,6.5078125,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,3,128,8,128,9.464712142944336,6.25390625,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,4,32,8,128,3.3040571212768555,9.015625,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,4,64,8,128,3.879425287246704,8.5078125,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,4,128,8,128,4.418932914733887,8.25390625,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,8,32,8,128,0.1999684274196624,17.015625,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,8,64,8,128,0.2344671934843063,16.5078125,16777216,3.8142517078071942,NA,3.281717363279313e-5
+25,self_attn.o_proj,8,128,8,128,0.2664281129837036,16.25390625,16777216,3.8142517078071942,NA,3.281717363279313e-5
+26,self_attn.o_proj,2,32,8,128,16.810836791992188,5.015625,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,2,64,8,128,19.624425888061523,4.5078125,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,2,128,8,128,22.199460983276367,4.25390625,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,3,32,8,128,7.196712493896484,7.015625,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,3,64,8,128,8.41883659362793,6.5078125,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,3,128,8,128,9.55011749267578,6.25390625,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,4,32,8,128,3.3594369888305664,9.015625,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,4,64,8,128,3.9312922954559326,8.5078125,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,4,128,8,128,4.46165943145752,8.25390625,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,8,32,8,128,0.2032840251922607,17.015625,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,8,64,8,128,0.237574964761734,16.5078125,16777216,3.9272371580725527,NA,3.806484892265871e-5
+26,self_attn.o_proj,8,128,8,128,0.2688663601875305,16.25390625,16777216,3.9272371580725527,NA,3.806484892265871e-5
+27,self_attn.o_proj,2,32,8,128,17.266559600830078,5.015625,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,2,64,8,128,20.1784782409668,4.5078125,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,2,128,8,128,22.853853225708008,4.25390625,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,3,32,8,128,7.411194324493408,7.015625,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,3,64,8,128,8.691489219665527,6.5078125,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,3,128,8,128,9.894498825073242,6.25390625,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,4,32,8,128,3.457504749298096,9.015625,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,4,64,8,128,4.058493137359619,8.5078125,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,4,128,8,128,4.6237874031066895,8.25390625,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,8,32,8,128,0.2092214375734329,17.015625,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,8,64,8,128,0.2453083097934723,16.5078125,16777216,5.168380628218723,NA,5.681679249391891e-5
+27,self_attn.o_proj,8,128,8,128,0.2786425650119781,16.25390625,16777216,5.168380628218723,NA,5.681679249391891e-5
+28,self_attn.o_proj,2,32,8,128,17.700225830078125,5.015625,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,2,64,8,128,20.673864364624023,4.5078125,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,2,128,8,128,23.37679672241211,4.25390625,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,3,32,8,128,7.58986759185791,7.015625,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,3,64,8,128,8.88255786895752,6.5078125,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,3,128,8,128,10.08473014831543,6.25390625,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,4,32,8,128,3.5435383319854736,9.015625,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,4,64,8,128,4.151835441589356,8.5078125,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,4,128,8,128,4.720759391784668,8.25390625,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,8,32,8,128,0.2144432812929153,17.015625,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,8,64,8,128,0.250990092754364,16.5078125,16777216,6.208068661807829,NA,6.343707354972139e-5
+28,self_attn.o_proj,8,128,8,128,0.2844915986061096,16.25390625,16777216,6.208068661807829,NA,6.343707354972139e-5
+29,self_attn.o_proj,2,32,8,128,18.33413887023925,5.015625,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,2,64,8,128,21.427490234375,4.5078125,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,2,128,8,128,24.24564552307129,4.25390625,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,3,32,8,128,7.850170612335205,7.015625,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,3,64,8,128,9.1857271194458,6.5078125,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,3,128,8,128,10.42802619934082,6.25390625,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,4,32,8,128,3.662515878677368,9.015625,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,4,64,8,128,4.287515640258789,8.5078125,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,4,128,8,128,4.868528366088867,8.25390625,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,8,32,8,128,0.2216504961252212,17.015625,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,8,64,8,128,0.2591716349124908,16.5078125,16777216,3.8188944230133473,NA,9.894864342641084e-5
+29,self_attn.o_proj,8,128,8,128,0.2935357689857483,16.25390625,16777216,3.8188944230133473,NA,9.894864342641084e-5
+30,self_attn.o_proj,2,32,8,128,19.64671897888184,5.015625,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,2,64,8,128,22.98756217956543,4.5078125,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,2,128,8,128,26.05941200256348,4.25390625,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,3,32,8,128,8.41417121887207,7.015625,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,3,64,8,128,9.869516372680664,6.5078125,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,3,128,8,128,11.22982120513916,6.25390625,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,4,32,8,128,3.9267284870147705,9.015625,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,4,64,8,128,4.603414535522461,8.5078125,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,4,128,8,128,5.240401744842529,8.25390625,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,8,32,8,128,0.2375096082687378,17.015625,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,8,64,8,128,0.2784024477005005,16.5078125,16777216,3.916996248805428,NA,2.0991287601646e-4
+30,self_attn.o_proj,8,128,8,128,0.3158771097660064,16.25390625,16777216,3.916996248805428,NA,2.0991287601646e-4
+31,self_attn.o_proj,2,32,8,128,19.6539363861084,5.015625,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,2,64,8,128,23.17051887512207,4.5078125,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,2,128,8,128,26.426538467407227,4.25390625,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,3,32,8,128,8.46374797821045,7.015625,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,3,64,8,128,10.066619873046877,6.5078125,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,3,128,8,128,11.667180061340332,6.25390625,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,4,32,8,128,3.952760696411133,9.015625,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,4,64,8,128,4.707706928253174,8.5078125,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,4,128,8,128,5.475996017456055,8.25390625,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,8,32,8,128,0.2390454709529876,17.015625,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,8,64,8,128,0.2846476435661316,16.5078125,16777216,5.591712502474565,NA,2.6171901845373e-4
+31,self_attn.o_proj,8,128,8,128,0.3298292756080627,16.25390625,16777216,5.591712502474565,NA,2.6171901845373e-4
diff --git a/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-L-14-laion2B-s32B-b82K.csv b/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-L-14-laion2B-s32B-b82K.csv
new file mode 100644
index 0000000000000000000000000000000000000000..0c6f7f4fb28f74c490ed94826434e6150eeaab8a
--- /dev/null
+++ b/lm-quant-toolkit/src/data/kurtosis-CLIP-ViT-L-14-laion2B-s32B-b82K.csv
@@ -0,0 +1,73 @@
+module,layer,kurtosis
+vision.mlp.c_fc,0,82.60432030937712
+vision.mlp.c_fc,1,12.176342485174775
+vision.mlp.c_fc,2,9.386186464371631
+vision.mlp.c_fc,3,6.0384986521099115
+vision.mlp.c_fc,4,5.121874436272312
+vision.mlp.c_fc,5,4.184969377709542
+vision.mlp.c_fc,6,4.256117717855523
+vision.mlp.c_fc,7,3.953278547908379
+vision.mlp.c_fc,8,3.923302040046366
+vision.mlp.c_fc,9,3.640527020435186
+vision.mlp.c_fc,10,3.8766116638521972
+vision.mlp.c_fc,11,3.364467358103123
+vision.mlp.c_fc,12,3.2501048983174172
+vision.mlp.c_fc,13,3.144904609090658
+vision.mlp.c_fc,14,3.1176022019209833
+vision.mlp.c_fc,15,3.170790033142059
+vision.mlp.c_fc,16,3.1901952176386157
+vision.mlp.c_fc,17,3.1728736833856566
+vision.mlp.c_fc,18,3.1092420753715064
+vision.mlp.c_fc,19,3.1289105323616995
+vision.mlp.c_fc,20,3.1633778054509616
+vision.mlp.c_fc,21,3.1948778928908625
+vision.mlp.c_fc,22,3.153364851937131
+vision.mlp.c_fc,23,3.1481862277653847
+vision.mlp.c_proj,0,48.91602226406029
+vision.mlp.c_proj,1,7.994586159177927
+vision.mlp.c_proj,2,9.259903305960076
+vision.mlp.c_proj,3,12.161504492865403
+vision.mlp.c_proj,4,11.137208824577082
+vision.mlp.c_proj,5,4.50118105140538
+vision.mlp.c_proj,6,3.9747365034192206
+vision.mlp.c_proj,7,4.846873977315018
+vision.mlp.c_proj,8,4.914742513787588
+vision.mlp.c_proj,9,5.0506453475209065
+vision.mlp.c_proj,10,13.838509154283804
+vision.mlp.c_proj,11,4.258906842824289
+vision.mlp.c_proj,12,3.4482411935554316
+vision.mlp.c_proj,13,3.4563538182296156
+vision.mlp.c_proj,14,3.1506796697623156
+vision.mlp.c_proj,15,3.140821870180245
+vision.mlp.c_proj,16,3.1870137578290736
+vision.mlp.c_proj,17,3.1622263243928037
+vision.mlp.c_proj,18,3.1727003318990294
+vision.mlp.c_proj,19,3.126420350320576
+vision.mlp.c_proj,20,3.214422564736524
+vision.mlp.c_proj,21,3.5410765909033515
+vision.mlp.c_proj,22,3.885907825953595
+vision.mlp.c_proj,23,3.241221353196137
+text.mlp.c_fc,0,3.111321629333239
+text.mlp.c_fc,1,3.1040094243226832
+text.mlp.c_fc,2,3.0476798189406455
+text.mlp.c_fc,3,3.0567396405641647
+text.mlp.c_fc,4,3.072398311749821
+text.mlp.c_fc,5,3.0581450022889753
+text.mlp.c_fc,6,3.053470998692992
+text.mlp.c_fc,7,3.0531601497538126
+text.mlp.c_fc,8,3.0696775698776992
+text.mlp.c_fc,9,3.065453985089178
+text.mlp.c_fc,10,3.067657384247557
+text.mlp.c_fc,11,3.086129858871849
+text.mlp.c_proj,0,16.411416535361464
+text.mlp.c_proj,1,10.636934397271798
+text.mlp.c_proj,2,5.467879831695406
+text.mlp.c_proj,3,3.117101945617834
+text.mlp.c_proj,4,3.162125318135728
+text.mlp.c_proj,5,3.114353715482094
+text.mlp.c_proj,6,3.1560610360682673
+text.mlp.c_proj,7,3.230989551532942
+text.mlp.c_proj,8,3.273957420333387
+text.mlp.c_proj,9,3.322224898837691
+text.mlp.c_proj,10,3.252371370612472
+text.mlp.c_proj,11,3.2207147468979427
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit.egg-info/SOURCES.txt b/lm-quant-toolkit/src/lm_quant_toolkit.egg-info/SOURCES.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c82b074ffb4889c82b50d3738a1d11bf0f6f3178
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit.egg-info/SOURCES.txt
@@ -0,0 +1,46 @@
+LICENSE
+MANIFEST.in
+README.md
+pyproject.toml
+docs/Makefile
+docs/source/CHANGELOG.rst
+docs/source/conf.py
+docs/source/index.rst
+docs/source/modules/index.rst
+src/cli.py
+src/dump.py
+src/lm_quant_toolkit/__init__.py
+src/lm_quant_toolkit.egg-info/PKG-INFO
+src/lm_quant_toolkit.egg-info/SOURCES.txt
+src/lm_quant_toolkit.egg-info/dependency_links.txt
+src/lm_quant_toolkit.egg-info/requires.txt
+src/lm_quant_toolkit.egg-info/top_level.txt
+src/lm_quant_toolkit/adapter/__init__.py
+src/lm_quant_toolkit/adapter/autoawq.py
+src/lm_quant_toolkit/adapter/autogptq.py
+src/lm_quant_toolkit/adapter/awq.py
+src/lm_quant_toolkit/adapter/bnb.py
+src/lm_quant_toolkit/adapter/common.py
+src/lm_quant_toolkit/adapter/fp16.py
+src/lm_quant_toolkit/adapter/hqq.py
+src/lm_quant_toolkit/adapter/mxq.py
+src/lm_quant_toolkit/adapter/vit/hqq.py
+src/lm_quant_toolkit/eval/__init__.py
+src/lm_quant_toolkit/eval/bench.py
+src/lm_quant_toolkit/eval/bench_vit.py
+src/lm_quant_toolkit/eval/clipbenchmark.py
+src/lm_quant_toolkit/eval/common.py
+src/lm_quant_toolkit/eval/leaderboard.py
+src/lm_quant_toolkit/eval/perplexity.py
+src/lm_quant_toolkit/misc/__init__.py
+src/lm_quant_toolkit/misc/lp_norm.py
+src/lm_quant_toolkit/misc/quant_sim.py
+src/lm_quant_toolkit/misc/qweight.py
+src/lm_quant_toolkit/prep/__init__.py
+src/lm_quant_toolkit/prep/fnorm.py
+src/lm_quant_toolkit/prep/sensitivity.py
+src/lm_quant_toolkit/prep/wdist.py
+src/lm_quant_toolkit/utils/__init__.py
+src/lm_quant_toolkit/utils/hub.py
+src/lm_quant_toolkit/utils/pickle.py
+src/lm_quant_toolkit/utils/safetensors.py
\ No newline at end of file
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit.egg-info/dependency_links.txt b/lm-quant-toolkit/src/lm_quant_toolkit.egg-info/dependency_links.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit.egg-info/requires.txt b/lm-quant-toolkit/src/lm_quant_toolkit.egg-info/requires.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8ad4e4c540bbe19c713691d686b5f8ec2b6c2446
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit.egg-info/requires.txt
@@ -0,0 +1,28 @@
+accelerate>=0.30.1
+bitsandbytes>0.37.0
+antlr4-python3-runtime==4.11.0
+datasets==2.20.0
+Jinja2==3.1.4
+langdetect==1.0.9
+nltk==3.9.1
+numpy==1.26.4
+optimum>=1.21.4
+pandas==2.2.2
+safetensors==0.4.3
+scikit-learn==1.4.2
+scipy==1.13.0
+sentencepiece==0.2.0
+tokenizers>=0.19.1
+torch>=2.1.0
+tqdm==4.66.4
+transformers>=4.41.2
+
+[dev]
+bumpver
+pip-tools
+
+[doc]
+sphinx
+
+[test]
+tox
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/__init__.py b/lm-quant-toolkit/src/lm_quant_toolkit/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4693c4432186db740b4ce2ca95ae018f3994fdb0
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/__init__.py
@@ -0,0 +1,8 @@
+"""Packages for LLM Quantization Evaluation."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+ __version__ = version("llm_quant_eval")
+except PackageNotFoundError:
+ __version__ = "unknown version"
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/__pycache__/__init__.cpython-310.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f1a4adb6aaccfa6ad9072193dd15aea5ce2c62f5
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/__pycache__/__init__.cpython-310.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/__pycache__/__init__.cpython-311.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c35de46ed5459dba7ac07e6f839cd492dfb27c9e
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/__pycache__/__init__.cpython-311.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autoawq.py b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autoawq.py
new file mode 100644
index 0000000000000000000000000000000000000000..601128e6acedc080764ca766c4d8b10f0872ce2c
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autoawq.py
@@ -0,0 +1,66 @@
+import os
+import time
+
+import torch
+import transformers
+from awq import AutoAWQForCausalLM
+from transformers import AutoConfig
+
+from lm_quant_toolkit.adapter.common import get_model_storage_size
+
+
+def create_autoawq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+ model_file_size = 0
+ quantized = False
+ quant_path = f"{save_dir}/{model_id}-{config_id}-awq"
+
+ config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
+ # To avoid OOM after huggingface transformers 4.36.2
+ config.use_cache = False
+ if load_quantized and os.path.exists(quant_path):
+ model = AutoAWQForCausalLM.from_quantized(
+ quant_path,
+ device_map="auto",
+ offload_state_dict=False,
+ config=config,
+ )
+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+ quantized = True
+ model_file_size = get_model_storage_size(quant_path)
+ model = model.cuda()
+ else:
+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+ # max_memory={0: "18GiB", "cpu": "60GiB"},
+ # )
+ model = AutoAWQForCausalLM.from_pretrained(
+ model_id,
+ device_map="auto",
+ offload_state_dict=False,
+ torch_dtype=torch.float16,
+ max_memory={0: "18GiB", "cpu": "60GiB"},
+ config=config,
+ )
+ return model, tokenizer, quantized, model_file_size
+
+
+def quantize_autoawq_model(
+ model, tokenizer, quant_config, model_id, config_id, save_dir
+):
+ t1 = time.time()
+ model.quantize(tokenizer, quant_config=quant_config)
+ t2 = time.time()
+ print("Took " + str(t2 - t1) + " seconds to quantize the model with AutoAWQ")
+ quant_path = f"{save_dir}/{model_id}-{config_id}-awq"
+ model.save_quantized(quant_path)
+ tokenizer.save_pretrained(quant_path)
+ # persistent the quantized model
+ os.sync()
+ return model, t2 - t1, _get_model_file_size(quant_path)
+
+
+def _get_model_file_size(quant_path):
+ quant_fp_pt = os.path.join(quant_path, "qmodel.pth")
+ if os.path.exists(quant_fp_pt):
+ return os.path.getsize(quant_fp_pt)
+ else:
+ return get_model_storage_size(quant_path)
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autogptq.py b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autogptq.py
new file mode 100644
index 0000000000000000000000000000000000000000..6a2822005935cc164ad3e02693f0e12d99b5a2bb
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/autogptq.py
@@ -0,0 +1,85 @@
+import gc
+import os
+import random
+import time
+
+import torch
+import transformers
+from auto_gptq import AutoGPTQForCausalLM
+from datasets import load_dataset
+from tqdm import tqdm
+
+
+# Adapted from: https://towardsdatascience.com/4-bit-quantization-with-gptq-36b0f4f02c34
+def prepare_model(model, tokenizer, n_samples=1024, max_tokens=512, use_triton=False):
+ # Load data and tokenize examples
+ data = load_dataset(
+ "allenai/c4",
+ data_files="en/c4-train.00001-of-01024.json.gz",
+ split=f"train[:{n_samples}]",
+ )
+ # ~536K tokens
+ tokenized_data = torch.cat(
+ [
+ tokenizer(data[i]["text"], return_tensors="pt").input_ids
+ for i in tqdm(range(len(data)))
+ ],
+ axis=-1,
+ )
+
+ # Format tokenized examples
+ random.seed(1)
+ examples_ids = []
+ for _ in range(n_samples):
+ i = random.randint(0, tokenized_data.shape[1] - max_tokens - 1)
+ j = i + max_tokens
+ input_ids = tokenized_data[:, i:j]
+ attention_mask = torch.ones_like(input_ids)
+ examples_ids.append({"input_ids": input_ids, "attention_mask": attention_mask})
+
+ print("Using " + str(len(examples_ids)) + " samples for calibration.")
+ model.quantize(examples_ids, batch_size=1, use_triton=use_triton)
+ # model = model.cuda()
+ # with torch.no_grad():
+ # x = model(input_ids.to('cuda'))
+ # del examples_ids, x
+ del examples_ids
+ torch.cuda.empty_cache()
+ gc.collect()
+ return model
+
+
+def create_autogptq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+ model_file_size = 0
+ quantized = False
+ quant_path = f"{save_dir}/{model_id}-{config_id}-gptq"
+ if load_quantized and os.path.exists(quant_path):
+ model = AutoGPTQForCausalLM.from_quantized(quant_path, device="cuda:0")
+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+ quantized = True
+ model_file_size = _get_model_file_size(quant_path, quant_config)
+ else:
+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+ model = AutoGPTQForCausalLM.from_pretrained(model_id, quant_config)
+ return model, tokenizer, quantized, model_file_size
+
+
+def quantize_autogptq_model(
+ model, tokenizer, quant_config, model_id, config_id, save_dir
+):
+ t1 = time.time()
+ model = prepare_model(model, tokenizer)
+ t2 = time.time()
+ print("Took " + str(t2 - t1) + " seconds to quantize the model with AutoGPTQ")
+ quant_path = f"{save_dir}/{model_id}-{config_id}-gptq"
+ model.save_quantized(quant_path, use_safetensors=True)
+ # persistent the quantized model
+ os.sync()
+ return model, t2 - t1, _get_model_file_size(quant_path, quant_config)
+
+
+def _get_model_file_size(quant_path, quant_config):
+ b = quant_config.bits
+ g = quant_config.group_size
+ fp = os.path.join(quant_path, f"gptq_model-{b}bit-{g}g.safetensors")
+ return os.path.getsize(fp)
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/adapter/awq.py b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/awq.py
new file mode 100644
index 0000000000000000000000000000000000000000..d40e8a962118a79dbeac89860282482590eb9642
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/awq.py
@@ -0,0 +1,91 @@
+import os
+import time
+
+import torch
+from accelerate import (
+ infer_auto_device_map,
+ init_empty_weights,
+ load_checkpoint_in_model,
+)
+from awq.quantize.pre_quant import apply_awq, run_awq
+from awq.quantize.quantizer import real_quantize_model_weight
+from awq.utils.utils import simple_dispatch_model
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+
+
+def create_awq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+ quantized = False
+ quant_path = f"{save_dir}/{model_id}-{config_id}-awq"
+
+ tokenizer = AutoTokenizer.from_pretrained(
+ model_id, use_fast=False, trust_remote_code=True
+ )
+ config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
+ # Note (Haotian): To avoid OOM after huggingface transformers 4.36.2
+ config.use_cache = False
+ if load_quantized and os.path.exists(f"{quant_path}/qmodel.pth"):
+ with init_empty_weights():
+ model = AutoModelForCausalLM.from_config(
+ config=config, torch_dtype=torch.float16, trust_remote_code=True
+ )
+ max_memory = {0: "20GiB", "cpu": "60GiB"}
+ # Infer device map
+ kwargs = {"max_memory": max_memory} if len(max_memory) else {}
+ device_map = infer_auto_device_map(
+ model,
+ no_split_module_classes=[
+ "OPTDecoderLayer",
+ "LlamaDecoderLayer",
+ "BloomBlock",
+ "MPTBlock",
+ "DecoderLayer",
+ ],
+ **kwargs,
+ )
+ # Load checkpoint in the model
+ load_checkpoint_in_model(
+ model,
+ checkpoint=quant_path,
+ device_map=device_map,
+ offload_state_dict=False,
+ )
+ # Dispatch model
+ model = simple_dispatch_model(model, device_map=device_map)
+ quantized = True
+ model.eval()
+ else:
+ kwargs = {"torch_dtype": torch.float16, "low_cpu_mem_usage": True}
+ model = AutoModelForCausalLM.from_pretrained(
+ model_id, config=config, trust_remote_code=True, **kwargs
+ )
+ return model, tokenizer, quantized, 0
+
+
+def quantize_awq_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
+ t1 = time.time()
+ nbits = quant_config.pop("w_bit")
+ awq_results = run_awq(
+ model,
+ tokenizer,
+ w_bit=nbits,
+ q_config=quant_config,
+ n_samples=128,
+ seqlen=512,
+ )
+ intermediate_fp = f"{save_dir}/{model_id}-{config_id}-awq/intermediate.pth"
+ dirpath = os.path.dirname(intermediate_fp)
+ os.makedirs(dirpath, exist_ok=True)
+ torch.save(awq_results, intermediate_fp)
+ awq_results = torch.load(intermediate_fp, map_location="cpu")
+ apply_awq(model, awq_results)
+ real_quantize_model_weight(model, w_bit=nbits, q_config=quant_config)
+
+ t2 = time.time()
+ print("Took " + str(t2 - t1) + " seconds to quantize the model with AWQ")
+ quant_path = f"{save_dir}/{model_id}-{config_id}-awq"
+ quant_fp = os.path.join(quant_path, "qmodel.pth")
+ torch.save(model.cpu().state_dict(), quant_fp)
+ tokenizer.save_pretrained(quant_path)
+
+ model_file_size = os.path.getsize(quant_fp)
+ return model, t2 - t1, model_file_size
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/adapter/bnb.py b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/bnb.py
new file mode 100644
index 0000000000000000000000000000000000000000..71a6234bd9fc1834630b015212eea46e7fcf6038
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/bnb.py
@@ -0,0 +1,37 @@
+import os
+import time
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from lm_quant_toolkit.adapter.common import get_model_storage_size
+
+
+def create_bnb_model(model_id, quant_config, config_id, load_quantized, save_dir):
+ quantized = False
+ model_file_size = 0
+ quant_path = f"{save_dir}/{model_id}-{config_id}-bnb"
+ if load_quantized and os.path.exists(quant_path):
+ model = AutoModelForCausalLM.from_pretrained(quant_path)
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
+ quantized = True
+ model_file_size = get_model_storage_size(quant_path)
+ else:
+ model = None
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
+ return model, tokenizer, quantized, model_file_size
+
+
+def quantize_bnb_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
+ model_file_size = 0
+ t1 = time.time()
+ model = AutoModelForCausalLM.from_pretrained(
+ model_id, quantization_config=quant_config
+ )
+ t2 = time.time()
+ print("Took " + str(t2 - t1) + " seconds to quantize the model with BnB")
+ quant_path = f"{save_dir}/{model_id}-{config_id}-bnb"
+ model.save_pretrained(quant_path)
+ # persistent the quantized model
+ os.sync()
+ model_file_size = get_model_storage_size(quant_path)
+ return model, t2 - t1, model_file_size
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/adapter/common.py b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..9a323216de087a42049a2c903f89eaca5ab81cd9
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/common.py
@@ -0,0 +1,20 @@
+import json
+import os
+
+
+def get_model_storage_size(
+ base_dir,
+ index_file="model.safetensors.index.json",
+ model_file="model.safetensors",
+):
+ size = 0
+ index_file = os.path.join(base_dir, index_file)
+ if os.path.exists(index_file):
+ # model is split into shards
+ with open(index_file, "r") as f:
+ index = json.load(f)
+ for shard in set(index["weight_map"].values()):
+ size += os.path.getsize(os.path.join(base_dir, shard))
+ else:
+ size = os.path.getsize(os.path.join(base_dir, model_file))
+ return size
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/adapter/fp16.py b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/fp16.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ba54fff22b16a191deda2520ce680eb0b3eb8bb
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/fp16.py
@@ -0,0 +1,17 @@
+import torch
+import transformers
+from transformers import AutoModelForCausalLM
+
+from lm_quant_toolkit.adapter.common import get_model_storage_size
+from lm_quant_toolkit.utils.hub import get_hf_model_storge_base_dir
+
+
+def create_fp16_model(model_id, quant_config, config_id, load_quantized, save_dir):
+ model_file_size = 0
+ model = AutoModelForCausalLM.from_pretrained(
+ model_id, device_map="auto", torch_dtype=torch.float16
+ )
+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+ base_dir = get_hf_model_storge_base_dir(model_id)
+ model_file_size = get_model_storage_size(base_dir)
+ return model, tokenizer, False, model_file_size
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/adapter/hqq.py b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/hqq.py
new file mode 100644
index 0000000000000000000000000000000000000000..5c8a09e699f843376071c8430aa3e86419b2aab4
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/hqq.py
@@ -0,0 +1,34 @@
+import os
+import time
+
+from hqq.engine.hf import AutoTokenizer as hggAutoTokenizer
+from hqq.engine.hf import HQQModelForCausalLM
+
+
+def create_hqq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+ quantized = False
+ model_file_size = 0
+ quant_path = f"{save_dir}/{model_id}-{config_id}-hqq"
+ if load_quantized and os.path.exists(quant_path):
+ model = HQQModelForCausalLM.from_quantized(quant_path)
+ tokenizer = hggAutoTokenizer.from_pretrained(model_id)
+ quantized = True
+ model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt"))
+ else:
+ model = HQQModelForCausalLM.from_pretrained(model_id)
+ tokenizer = hggAutoTokenizer.from_pretrained(model_id)
+ return model, tokenizer, quantized, model_file_size
+
+
+def quantize_hqq_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
+ model_file_size = 0
+ t1 = time.time()
+ model.quantize_model(quant_config=quant_config)
+ t2 = time.time()
+ print("Took " + str(t2 - t1) + " seconds to quantize the model with HQQ")
+ quant_path = f"{save_dir}/{model_id}-{config_id}-hqq"
+ model.save_quantized(quant_path)
+ # persistent the quantized model
+ os.sync()
+ model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt"))
+ return model, t2 - t1, model_file_size
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/adapter/mxq.py b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/mxq.py
new file mode 100644
index 0000000000000000000000000000000000000000..3baebbf79d6078c0ed293d50fbc87d0acc4296ee
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/mxq.py
@@ -0,0 +1,34 @@
+import os
+import time
+
+from hqq.engine.hf import AutoTokenizer as hggAutoTokenizer
+from hqq.engine.hf import HQQModelForCausalLM
+
+
+def create_mxq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+ quantized = False
+ model_file_size = 0
+ quant_path = f"{save_dir}/{model_id}-{config_id}-mxq"
+ if load_quantized and os.path.exists(quant_path):
+ model = HQQModelForCausalLM.from_quantized(quant_path)
+ tokenizer = hggAutoTokenizer.from_pretrained(model_id)
+ quantized = True
+ model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt"))
+ else:
+ model = HQQModelForCausalLM.from_pretrained(model_id)
+ tokenizer = hggAutoTokenizer.from_pretrained(model_id)
+ return model, tokenizer, quantized, model_file_size
+
+
+def quantize_mxq_model(model, tokenizer, quant_config, model_id, config_id, save_dir):
+ model_file_size = 0
+ t1 = time.time()
+ model.quantize_model(quant_config=quant_config)
+ t2 = time.time()
+ print("Took " + str(t2 - t1) + " seconds to quantize the model with MXQ")
+ quant_path = f"{save_dir}/{model_id}-{config_id}-mxq"
+ model.save_quantized(quant_path)
+ # persistent the quantized model
+ os.sync()
+ model_file_size = os.path.getsize(os.path.join(quant_path, "qmodel.pt"))
+ return model, t2 - t1, model_file_size
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/adapter/vit/hqq.py b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/vit/hqq.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e3d0887d92ea80c12df01bcdc61ca2d60f9686c
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/adapter/vit/hqq.py
@@ -0,0 +1,27 @@
+import os
+import time
+from pathlib import Path
+
+from hqq.engine.open_clip import HQQOpenCLIP
+
+
+def create_hqq_model(model_id, quant_config, config_id, load_quantized, save_dir):
+ quantized = False
+ quant_path = f"{save_dir}/{model_id}-{config_id}-hqq"
+ if load_quantized and os.path.exists(quant_path):
+ model = HQQOpenCLIP.from_quantized(quant_path)
+ quantized = True
+ else:
+ model = HQQOpenCLIP.create_model(model_id, device="cpu")
+ return model, quantized
+
+
+def quantize_hqq_model(model, quant_config, model_id, config_id, save_dir):
+ t1 = time.time()
+ model.quantize_model(quant_config=quant_config)
+ t2 = time.time()
+ print("Took " + str(t2 - t1) + " seconds to quantize the model with HQQ")
+ quant_path = f"{save_dir}/{model_id}-{config_id}-hqq"
+ Path(quant_path).mkdir(parents=True, exist_ok=True)
+ model.save_quantized(save_dir=quant_path)
+ return model, t2 - t1
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/eval/__init__.py b/lm-quant-toolkit/src/lm_quant_toolkit/eval/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench.py b/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a10102ef0afe1a16dbfb1fbad1240d1b71faeeb
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench.py
@@ -0,0 +1,759 @@
+import copy
+import logging
+import os
+
+# from adapter.awq import create_awq_model
+# from adapter.awq import quantize_awq_model
+from datetime import datetime
+from pathlib import Path
+
+import pandas as pd
+import torch
+from auto_gptq import BaseQuantizeConfig as GPTQQuantConfig
+from hqq.core.quantize import BaseQuantizeConfig as HQQQuantConfig
+from transformers import BitsAndBytesConfig
+
+from lm_quant_toolkit.adapter.autoawq import (
+ create_autoawq_model,
+ quantize_autoawq_model,
+)
+from lm_quant_toolkit.adapter.autogptq import (
+ create_autogptq_model,
+ quantize_autogptq_model,
+)
+from lm_quant_toolkit.adapter.bnb import create_bnb_model, quantize_bnb_model
+from lm_quant_toolkit.adapter.fp16 import create_fp16_model
+from lm_quant_toolkit.adapter.hqq import create_hqq_model, quantize_hqq_model
+from lm_quant_toolkit.adapter.mxq import create_mxq_model, quantize_mxq_model
+from lm_quant_toolkit.eval.common import (
+ HQQ_CONFIGS,
+ _dump_cuda_mem_snapshot,
+ _reset_peak_memory_stats,
+ cleanup,
+ combine_metrics,
+ get_memory_metrics,
+ get_mxq_quant_meta_data_file,
+ persist_progress,
+ save_partial_metric,
+)
+from lm_quant_toolkit.eval.leaderboard import eval_llm_leaderboard
+from lm_quant_toolkit.eval.perplexity import eval_ppls
+
+ALL_MODELS = [
+ "meta-llama/Llama-2-7b-hf",
+ "meta-llama/Llama-2-13b-hf",
+ "meta-llama/Meta-Llama-3-8B",
+ "meta-llama/Llama-3.1-8B",
+]
+
+MXQ_CONFIGS = [
+ (
+ f"{bits:.2f}".replace(".", "_"),
+ HQQQuantConfig(mixed=True, budget=bits, quant_scale=True),
+ )
+ for bits in [5.00, 4.75, 4.50, 4.25, 4.01, 3.76, 3.50, 3.00, 2.75, 2.48]
+]
+
+BNB_CONFIGS = [
+ (
+ "b4g64",
+ BitsAndBytesConfig(
+ load_in_4bit=True,
+ bnb_4bit_quant_type="nf4",
+ bnb_4bit_compute_dtype="float16",
+ bnb_4bit_use_double_quant=True,
+ ),
+ ),
+ (
+ "b8g128",
+ BitsAndBytesConfig(
+ load_in_8bit=True,
+ bnb_4bit_compute_dtype="float16",
+ ),
+ ),
+]
+
+
+AUTOAWQ_CONFIGS = [
+ ("b4g32", {"w_bit": 4, "q_group_size": 32, "zero_point": True, "version": "GEMM"}),
+ ("b4g64", {"w_bit": 4, "q_group_size": 64, "zero_point": True, "version": "GEMM"}),
+ (
+ "b4g128",
+ {"w_bit": 4, "q_group_size": 128, "zero_point": True, "version": "GEMM"},
+ ),
+ # 3-bit not supported by AutoAWQ right now
+ # ("b3g64", {"w_bit": 3, "q_group_size": 64, "zero_point": True, 'version':'gemv_fast'}),
+ # ("b3g128", {"w_bit": 3, "q_group_size": 128, "zero_point": True, 'version':'gemv_fast'}),
+]
+
+GPTQ_CONFIGS = [
+ (
+ "b8g32",
+ GPTQQuantConfig(bits=8, group_size=32, damp_percent=0.01, desc_act=False),
+ ),
+ (
+ "b8g64",
+ GPTQQuantConfig(bits=8, group_size=64, damp_percent=0.01, desc_act=False),
+ ),
+ (
+ "b8g128",
+ GPTQQuantConfig(bits=8, group_size=128, damp_percent=0.01, desc_act=False),
+ ),
+ (
+ "b4g32",
+ GPTQQuantConfig(bits=4, group_size=32, damp_percent=0.01, desc_act=False),
+ ),
+ (
+ "b4g64",
+ GPTQQuantConfig(bits=4, group_size=64, damp_percent=0.01, desc_act=False),
+ ),
+ (
+ "b4g128",
+ GPTQQuantConfig(bits=4, group_size=128, damp_percent=0.01, desc_act=False),
+ ),
+ (
+ "b3g32",
+ GPTQQuantConfig(bits=3, group_size=32, damp_percent=0.01, desc_act=False),
+ ),
+ (
+ "b3g64",
+ GPTQQuantConfig(bits=3, group_size=64, damp_percent=0.01, desc_act=False),
+ ),
+ (
+ "b3g128",
+ GPTQQuantConfig(bits=3, group_size=128, damp_percent=0.01, desc_act=False),
+ ),
+]
+
+
+def gen_experiment_items(models, tasks):
+ dikts = []
+ for algo, spec in tasks.items():
+ configs = spec["configs"]
+ for config in configs:
+ for model_id in models:
+ dikts.append(
+ {
+ "model": model_id,
+ "cfg": config[0],
+ "task_type": spec["type"],
+ "algo": algo,
+ }
+ )
+ return pd.DataFrame(dikts)
+
+
+def _setup_fn(algo, spec):
+ match algo:
+ case "fp16":
+ spec["create_fn"] = create_fp16_model
+ spec["quantize_fn"] = None
+ case "awq":
+ spec["create_fn"] = create_autoawq_model
+ spec["quantize_fn"] = quantize_autoawq_model
+ case "gptq":
+ spec["create_fn"] = create_autogptq_model
+ spec["quantize_fn"] = quantize_autogptq_model
+ case "hqq":
+ spec["create_fn"] = create_hqq_model
+ spec["quantize_fn"] = quantize_hqq_model
+ case "bnb":
+ spec["create_fn"] = create_bnb_model
+ spec["quantize_fn"] = quantize_bnb_model
+ case "mxq":
+ spec["create_fn"] = create_mxq_model
+ spec["quantize_fn"] = quantize_mxq_model
+ case _:
+ raise ValueError(f"Invalid algo: {algo}")
+
+
+def do_expermient(
+ experiment_name,
+ models,
+ tasks,
+ quant_dir="snapshots",
+ result_dir="results",
+ log_dir="logs",
+ track_cuda_memory=False,
+ **kwargs,
+):
+ df_all = gen_experiment_items(models, tasks)
+ progress_path = os.path.join(result_dir, experiment_name, "progress.csv")
+ if Path(progress_path).exists():
+ df_saved = pd.read_csv(progress_path)
+ df_all = df_all.merge(
+ df_saved, how="left", on=["model", "cfg", "task_type", "algo"]
+ )
+ # filter already processed repos, equivalent to SQL is null
+ df_todo = df_all.query("status != status or status != 1")
+ else:
+ df_all["status"] = 0
+ df_all["completion_time"] = ""
+ df_todo = df_all
+ print("*" * 72)
+ print("Sub-task list:")
+ print(df_all)
+ cnt_todo, cnt_tot = len(df_todo), len(df_all)
+ print(f"Todo:{cnt_todo}, Done: {cnt_tot - cnt_todo}, Total: {cnt_tot}")
+ if cnt_todo == 0:
+ print("Tasks completed!")
+ print("*" * 72)
+ if cnt_todo == 0:
+ return
+
+ df_todo = df_todo.sort_values(by=["model", "cfg"], ascending=False)
+ for idx, row in df_todo.iterrows():
+ model_id = row["model"]
+ algo = row["algo"]
+ task_type = row["task_type"]
+ cfg = row["cfg"]
+ spec = tasks[algo]
+ _setup_fn(algo, spec)
+ config = [c for c in spec["configs"] if c[0] == cfg][0]
+ quant_fn = spec["quantize_fn"]
+ metric = _init_metrics(model_id, algo, config)
+ print("*" * 72)
+ if task_type == "quant":
+ print(f"Quantizing {algo} on {model_id} w/ config: {cfg}...")
+ elif task_type == "eval_ppl":
+ print(f"Evaluating {algo} PPL on {model_id} w/ config: {cfg}...")
+ elif task_type == "eval_leaderboard":
+ print(
+ f"Evaluating {algo} LLM Leaderboard benchmarks on {model_id} w/ config: {cfg}..."
+ )
+ else:
+ print(
+ f"Evaluating {algo} model storage metrics on {model_id} w/ config: {cfg}..."
+ )
+ print("*" * 72)
+
+ if track_cuda_memory:
+ torch.cuda.memory._record_memory_history()
+ _reset_peak_memory_stats()
+ if task_type != "eval_leaderboard":
+ create_fn = spec["create_fn"]
+ model, tokenizer, quantized, model_file_size = create_fn(
+ model_id, config[1], cfg, quant_fn is not None, quant_dir
+ )
+
+ if not quantized and quant_fn:
+ # avoid interventions between models
+ quant_config = copy.deepcopy(config[1])
+ if algo == "mxq":
+ ok, metric_fp = get_mxq_quant_meta_data_file(model_id)
+ if not ok:
+ print(
+ f"Quantization meta data file: {metric_fp} doesn't exists!"
+ )
+ return
+ quant_config["quant_metrics_file"] = metric_fp
+ quant_config["weight_algo"] = kwargs.get("weight_algo", None)
+ quant_config["boost_layers"] = kwargs.get("boost_layers", None)
+ quant_config["decline_layers"] = kwargs.get("decline_layers", None)
+ quant_config["boost_stop"] = kwargs.get("boost_stop", None)
+ quant_config["decline_stop"] = kwargs.get("decline_stop", None)
+ quant_config["ablation"] = kwargs.get("ablation", None)
+ quant_config["top_m_layer"] = kwargs.get("top_m_layer", None)
+ quant_config["factor"] = kwargs.get("factor", None)
+ model, duration, model_file_size = quant_fn(
+ model,
+ tokenizer,
+ quant_config,
+ model_id,
+ cfg,
+ quant_dir,
+ )
+ metric["quant_duration"] = duration
+ if task_type == "eval_model_storage":
+ allot, reserved = get_memory_metrics()
+ metric["load_mem_allot"] = allot
+ metric["load_mem_reserved"] = reserved
+ metric["model_storage_size"] = model_file_size
+
+ elif task_type == "eval_ppl":
+ # Evaluate the quantized model
+ metric = eval_ppls(model, tokenizer, metric)
+ metric["ppl_mem_allot"], metric["ppl_mem_reserved"] = (
+ get_memory_metrics()
+ )
+ if track_cuda_memory:
+ _dump_cuda_mem_snapshot(experiment_name, model_id, algo, result_dir)
+ cleanup(model)
+ else:
+ metric = eval_llm_leaderboard(
+ experiment_name,
+ model_id,
+ algo,
+ cfg,
+ quant_fn is not None,
+ metric,
+ quant_dir,
+ result_dir,
+ )
+ metric["leaderboard_mem_allot"], metric["leaderboard_mem_reserved"] = (
+ get_memory_metrics()
+ )
+ if track_cuda_memory:
+ _dump_cuda_mem_snapshot(experiment_name, model_id, algo, result_dir)
+ save_partial_metric(experiment_name, algo, model_id, cfg, metric, result_dir)
+ df_all.loc[
+ (df_all["model"] == model_id)
+ & (df_all["cfg"] == cfg)
+ & (df_all["algo"] == algo)
+ & (df_all["task_type"] == task_type),
+ ["status", "completion_time"],
+ ] = 1, datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+ persist_progress(df_all, progress_path)
+ # combine metrics
+ combine_metrics(experiment_name, result_dir)
+
+
+def _init_metrics(model_id, algo, config):
+ return {
+ "model": model_id.split("/")[1],
+ "algo": algo,
+ "config": config[0],
+ "config_detail": str(config[1]).replace("\n", ""),
+ "quant_duration": 0,
+ "model_storage_size": 0,
+ "load_mem_allot": 0,
+ "load_mem_reserved": 0,
+ "ppl_mem_allot": 0,
+ "ppl_mem_reserved": 0,
+ "leaderboard_mem_allot": 0,
+ "leaderboard_mem_reserved": 0,
+ "quant_duration": 0,
+ "ppl_wikitext": 0,
+ "ppl_c4": 0,
+ "duration_wikitext": 0,
+ "duration_c4": 0,
+ "duration_leaderboard": 0,
+ "ifeval": 0,
+ "bbh": 0,
+ "mathlevel5": 0,
+ "gpqa": 0,
+ "musr": 0,
+ "mmlupro": 0,
+ }
+
+
+def do_expermient_fdata(
+ experiment_name,
+ models,
+ tasks,
+ track_cuda_memory=False,
+):
+ do_expermient(
+ experiment_name,
+ models,
+ tasks,
+ quant_dir="/fdata/llm/mxq/snapshots",
+ result_dir="/fdata/llm/mxq/results",
+ log_dir="/fdata/llm/mxq/logs",
+ track_cuda_memory=track_cuda_memory,
+ )
+
+
+########################################################################
+# Quantization experiments
+########################################################################
+
+
+def experiment_quant_hqq():
+ models = ALL_MODELS
+ tasks = {
+ "hqq": {
+ "type": "quant",
+ "configs": HQQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(
+ "quant_hqq",
+ models,
+ tasks,
+ )
+
+
+def experiment_quant_mxq():
+ models = ALL_MODELS
+ type = "quant"
+ algo = "mxq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": MXQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}_mxq", models, tasks)
+
+
+def experiment_quant_awq():
+ # models = [ALL_MODELS[0], ALL_MODELS[2]]
+ models = [ALL_MODELS[1]]
+ type = "quant"
+ algo = "awq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": AUTOAWQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+def experiment_quant_gptq():
+ models = ALL_MODELS
+ type = "quant"
+ algo = "gptq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": GPTQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+def experiment_quantize_405B():
+ models = [
+ "meta-llama/Meta-Llama-3.1-405B-Instruct",
+ ]
+
+ tasks = {
+ "hqq": {
+ "type": "quant",
+ "configs": HQQ_CONFIGS[1:2],
+ },
+ }
+ do_expermient(
+ "quant_hqq_405B",
+ models,
+ tasks,
+ quant_dir="/data/gqq-eval/snapshots/",
+ )
+
+
+########################################################################
+# Perplexity evaluation experiments
+########################################################################
+
+
+def experiment_ppl_eval_fp16():
+ models = ALL_MODELS
+ type = "eval_ppl"
+ algo = "fp16"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": [
+ ("base", {}),
+ ],
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+def experiment_ppl_eval_awq():
+ models = [ALL_MODELS[0], ALL_MODELS[2]]
+ type = "eval_ppl"
+ algo = "awq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": AUTOAWQ_CONFIGS[1:2],
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+def experiment_ppl_eval_gptq():
+ models = ALL_MODELS
+ type = "eval_ppl"
+ algo = "gptq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": GPTQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+def experiment_ppl_eval_hqq():
+ models = ALL_MODELS
+ type = "eval_ppl"
+ algo = "hqq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": HQQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+########################################################################
+# Open LLM Leaderboard evaluation experiments
+########################################################################
+
+
+def experiment_llm_leaderboard_fp16():
+ models = [ALL_MODELS[0], ALL_MODELS[2]]
+ type = "eval_leaderboard"
+ algo = "fp16"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": [
+ ("base", {}),
+ ],
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+def experiment_llm_leaderboard_autogptq():
+ models = ALL_MODELS
+ type = "eval_leaderboard"
+ algo = "gptq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": GPTQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+def experiment_llm_leaderboard_hqq():
+ models = ALL_MODELS
+ type = "eval_leaderboard"
+ algo = "hqq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": HQQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+def experiment_llm_leaderboard_mxq():
+ models = ALL_MODELS
+ type = "eval_leaderboard"
+ algo = "mxq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": MXQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+def experiment_llm_leaderboard_autoawq():
+ models = ALL_MODELS[0:1]
+ type = "eval_leaderboard"
+ algo = "awq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": AUTOAWQ_CONFIGS[0:2],
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+########################################################################
+# Mixed Quant Eval experiments
+########################################################################
+
+
+def experiment_quant_ppl_eval_mxq_comprise():
+ models = ALL_MODELS
+ equiv_mxq_configs = []
+ nbits = [4.06, 4.10, 4.15, 4.19, 4.24, 4.28, 4.33]
+ for bits in nbits:
+ cfg_name = f"mxq-{str(bits).replace('.', '_')}"
+ equiv_mxq_configs.append(
+ (cfg_name, HQQQuantConfig(mixed=True, budget=bits, quant_scale=True))
+ )
+ quant_tasks = {
+ "hqq": {
+ "type": "quant",
+ "configs": equiv_mxq_configs,
+ },
+ }
+ ppl_tasks = {
+ "hqq": {
+ "type": "eval_ppl",
+ "configs": equiv_mxq_configs,
+ },
+ }
+ do_expermient_fdata("quant_mxq_compromise", models, quant_tasks)
+ do_expermient_fdata("eval_mxq_compromise", models, ppl_tasks)
+
+
+########################################################################
+# Misc experiments
+########################################################################
+
+
+def experiment_fp16_llama3_8B_OOM():
+ models = ALL_MODELS[-1:]
+ type = "eval_ppl"
+ algo = "fp16"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": [
+ ("base", {}),
+ ],
+ },
+ }
+ do_expermient_fdata(
+ f"{type}_llama3_8B_OOM_{algo}",
+ models,
+ tasks,
+ track_cuda_memory=True,
+ )
+
+
+def experiment_fp16_vs_hqq_eval_gpu_mem():
+ models = ALL_MODELS[-1:]
+ type = "eval_ppl"
+ algo = "fp16"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": [
+ ("base", {}),
+ ],
+ },
+ }
+ do_expermient_fdata("experiment_fp16_vs_hqq_eval_gpu_mem", models, tasks)
+ algo = "hqq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": HQQ_CONFIGS[1:2],
+ },
+ }
+ do_expermient_fdata("experiment_fp16_vs_hqq_eval_gpu_mem", models, tasks)
+
+
+def experiment_eval_model_storage():
+ models = ALL_MODELS
+ type = "eval_model_storage"
+ tasks = {
+ "fp16": {
+ "type": type,
+ "configs": [
+ ("base", {}),
+ ],
+ },
+ "mxq": {
+ "type": type,
+ "configs": MXQ_CONFIGS,
+ },
+ "hqq": {
+ "type": type,
+ "configs": HQQ_CONFIGS,
+ },
+ "awq": {
+ "type": type,
+ "configs": AUTOAWQ_CONFIGS,
+ },
+ "gptq": {"type": type, "configs": GPTQ_CONFIGS},
+ }
+ for i in range(5):
+ do_expermient_fdata(f"eval_model_storge_{i}", models, tasks)
+
+
+def experiment_eval_ppl_all():
+ models = ALL_MODELS
+ type = "eval_ppl"
+ tasks = {
+ "fp16": {
+ "type": type,
+ "configs": [
+ ("base", {}),
+ ],
+ },
+ "mxq": {
+ "type": type,
+ "configs": MXQ_CONFIGS,
+ },
+ "hqq": {
+ "type": type,
+ "configs": HQQ_CONFIGS,
+ },
+ "awq": {
+ "type": type,
+ "configs": AUTOAWQ_CONFIGS,
+ },
+ "gptq": {"type": type, "configs": GPTQ_CONFIGS},
+ }
+ do_expermient_fdata("experiment_eval_ppl_all", models, tasks)
+
+
+def experiment_debug_quant_hqq():
+ models = [ALL_MODELS[1]]
+ type = "eval_model_storage"
+ algo = "hqq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": HQQ_CONFIGS[1:2],
+ },
+ }
+
+ do_expermient(
+ f"debug_{type}_{algo}",
+ models,
+ tasks,
+ quant_dir="/fdata/llm/mxq/snapshots-debug",
+ result_dir="/fdata/llm/mxq/results",
+ )
+
+
+def main():
+ logging.basicConfig(
+ format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+ level=logging.INFO,
+ datefmt="%Y-%m-%d %H:%M:%S",
+ )
+
+ # experiment_llm_leaderboard_autogptq()
+ # experiment_llm_leaderboard_fp16()
+ # experiment_llm_leaderboard_hqq()
+ # experiment_llm_leaderboard_autoawq()
+ # experiment_quant_hqq()
+ # experiment_quant_mxq()
+ # experiment_quant_awq()
+ # experiment_quant_gptq()
+ # experiment_ppl_eval_fp16()
+ # experiment_ppl_eval_hqq()
+ # experiment_ppl_eval_gptq()
+ # experiment_ppl_eval_awq()
+ # experiment_fp16_llama3_8B_OOM()
+ # experiment_fp16_vs_hqq_eval_gpu_mem()
+ # experiment_debug_quant_hqq()
+ # experiment_eval_model_storage()
+ # experiment_eval_ppl_all()
+ experiment_debug_quant_hqq()
+
+
+if __name__ == "__main__":
+ # os.environ['HF_DATASETS_OFFLINE'] = '1'
+
+ max_threads = str(min(8, os.cpu_count()))
+ os.environ["OMP_NUM_THREADS"] = max_threads
+ os.environ["OPENBLAS_NUM_THREADS"] = max_threads
+ os.environ["MKL_NUM_THREADS"] = max_threads
+ os.environ["VECLIB_MAXIMUM_THREADS"] = max_threads
+ os.environ["NUMEXPR_NUM_THREADS"] = max_threads
+ os.environ["NUMEXPR_MAX_THREADS"] = max_threads
+ os.environ["HF_HOME"] = "/data/hugginface/"
+
+ main()
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench_vit.py b/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench_vit.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a30d5efc33f98cb994d46acd57c79028e7b7fa4
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/eval/bench_vit.py
@@ -0,0 +1,451 @@
+import copy
+import logging
+import os
+from datetime import datetime
+from pathlib import Path
+
+import pandas as pd
+import torch
+from hqq.core.quantize import BaseQuantizeConfig as HQQQuantConfig
+
+from lm_quant_toolkit.eval.clipbenchmark import (
+ eval_linear_probe,
+ eval_zeroshot_classification,
+)
+from lm_quant_toolkit.eval.common import (
+ HQQ_CONFIGS,
+ _dump_cuda_mem_snapshot,
+ _reset_peak_memory_stats,
+ calc_bits,
+ combine_metrics,
+ get_memory_metrics,
+ get_mxq_quant_meta_data_file,
+ persist_progress,
+ save_partial_metric,
+)
+
+ALL_MODELS = [
+ "laion/CLIP-ViT-B-32-laion2B-s34B-b79K",
+ "laion/CLIP-ViT-H-14-laion2B-s32B-b79K",
+ "laion/CLIP-ViT-L-14-laion2B-s32B-b82K",
+]
+
+MXQ_CONFIGS = [
+ (
+ f"{bits:.2f}".replace(".", "_"),
+ HQQQuantConfig(mixed=True, budget=bits, quant_scale=True),
+ )
+ for bits in [
+ 7.80,
+ 7.72,
+ 7.64,
+ 7.56,
+ 7.48,
+ 7.40,
+ 7.32,
+ 5.00,
+ 4.95,
+ 4.90,
+ 4.86,
+ 4.82,
+ 4.78,
+ 4.73,
+ 4.00,
+ 3.96,
+ 3.92,
+ 3.00,
+ ]
+]
+
+
+def gen_experiment_items(models, tasks):
+ dikts = []
+ for algo, spec in tasks.items():
+ configs = spec["configs"]
+ for config in configs:
+ for model_id in models:
+ dikts.append(
+ {
+ "model": model_id,
+ "cfg": config[0],
+ "task_type": spec["type"],
+ "algo": algo,
+ }
+ )
+ return pd.DataFrame(dikts)
+
+
+def do_expermient(
+ experiment_name,
+ models,
+ tasks,
+ quant_dir="snapshots",
+ result_dir="results",
+ track_cuda_memory=False,
+ **kwargs,
+):
+ df_all = gen_experiment_items(models, tasks)
+ progress_path = os.path.join(result_dir, experiment_name, "progress.csv")
+ if Path(progress_path).exists():
+ df_saved = pd.read_csv(progress_path)
+ df_all = df_all.merge(
+ df_saved, how="left", on=["model", "cfg", "task_type", "algo"]
+ )
+ # filter already processed repos, equivalent to SQL is null
+ df_todo = df_all.query("status != status or status != 1")
+ else:
+ df_all["status"] = 0
+ df_all["completion_time"] = ""
+ df_todo = df_all
+ print("*" * 72)
+ print("Sub-task list:")
+ print(df_all)
+ cnt_todo, cnt_tot = len(df_todo), len(df_all)
+ print(f"Todo:{cnt_todo}, Done: {cnt_tot - cnt_todo}, Total: {cnt_tot}")
+ if cnt_todo == 0:
+ print("Tasks completed!")
+ print("*" * 72)
+ if cnt_todo == 0:
+ return
+
+ df_todo = df_todo.sort_values(by=["model", "cfg"], ascending=False)
+ for idx, row in df_todo.iterrows():
+ model_id = row["model"]
+ algo = row["algo"]
+ task_type = row["task_type"]
+ cfg = row["cfg"]
+ spec = tasks[algo]
+ config = [c for c in spec["configs"] if c[0] == cfg][0]
+ metric = _init_metrics(model_id, algo, config)
+ print("*" * 72)
+ # if task_type == "quant":
+ # print(f"Quantizing {algo} on {model_id} w/ config: {cfg}...")
+ # elif task_type == "eval_zeroshot_cls":
+ if task_type == "eval_zeroshot_cls":
+ print(
+ f"Evaluating {algo} zero-shot classification on {model_id} w/ config: {cfg}..."
+ )
+ else:
+ print(f"Evaluating {algo} linear-probe on {model_id} w/ config: {cfg}...")
+ print("*" * 72)
+
+ if track_cuda_memory:
+ torch.cuda.memory._record_memory_history()
+ _reset_peak_memory_stats()
+
+ quant_config = None
+ if algo != "fp16":
+ quant_config = copy.deepcopy(config[1])
+ if algo == "mxq":
+ ok, metric_fp = get_mxq_quant_meta_data_file(model_id)
+ if not ok:
+ print(f"Quantization meta data file: {metric_fp} doesn't exists!")
+ return
+ quant_config["quant_metrics_file"] = metric_fp
+ quant_config["weight_algo"] = kwargs.get("weight_algo", None)
+ quant_config["boost_stop"] = kwargs.get("boost_stop", None)
+ quant_config["decline_stop"] = kwargs.get("decline_stop", None)
+ quant_config["top_m_layer"] = kwargs.get("top_m_layer", None)
+ if task_type == "eval_zeroshot_cls":
+ # avoid interventions between models
+ metric = eval_zeroshot_classification(
+ metric,
+ model_id,
+ result_dir,
+ quant_dir,
+ quant_config,
+ )
+ (
+ metric["zeroshot_mem_allot"],
+ metric["zeroshot_mem_reserved"],
+ ) = get_memory_metrics()
+ else:
+ # Make model with different quantization configs don't share the
+ # pre-calated features vectors, thus they are evaluated
+ # separately.
+ feature_root = os.path.join(quant_dir, "features", cfg)
+ Path(feature_root).mkdir(parents=True, exist_ok=True)
+ metric = eval_linear_probe(
+ metric,
+ model_id,
+ result_dir,
+ quant_dir,
+ quant_config,
+ feature_root,
+ )
+ (
+ metric["linear_probe_mem_allot"],
+ metric["linear_probe_mem_reserved"],
+ ) = get_memory_metrics()
+
+ if track_cuda_memory:
+ _dump_cuda_mem_snapshot(experiment_name, model_id, algo, result_dir)
+ save_partial_metric(experiment_name, algo, model_id, cfg, metric, result_dir)
+ df_all.loc[
+ (df_all["model"] == model_id)
+ & (df_all["cfg"] == cfg)
+ & (df_all["algo"] == algo)
+ & (df_all["task_type"] == task_type),
+ ["status", "completion_time"],
+ ] = 1, datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+ persist_progress(df_all, progress_path)
+ # combine metrics
+ combine_metrics(experiment_name, result_dir)
+
+
+def _init_metrics(model_id, algo, config):
+ quant_config = copy.deepcopy(config[1])
+ mixed = quant_config.get("mixed", False)
+ if mixed:
+ quant_config.pop("weight_quant_params", None)
+ quant_config.pop("scale_quant_params", None)
+ quant_config.pop("zero_quant_params", None)
+
+ return {
+ "model": model_id.split("/")[1],
+ "algo": algo,
+ "config": config[0],
+ "config_detail": quant_config,
+ "quant_duration": 0,
+ "load_mem_allot": 0,
+ "load_mem_reserved": 0,
+ "zeroshot_mem_allot": 0,
+ "zeroshot_mem_reserved": 0,
+ "linear_probe_mem_allot": 0,
+ "linear_probe_mem_reserved": 0,
+ "acc1_zeroshot_cls": 0,
+ "acc5_zeroshot_cls": 0,
+ "recall_zeroshot_cls": 0,
+ "duration_zeroshot_cls": 0,
+ "acc1_linear_probe": 0,
+ "acc5_linear_probe": 0,
+ "recall_linear_probe": 0,
+ "duration_linear_probe": 0,
+ }
+
+
+def get_mxq_bits(reduction_pcts=[3, 5, 8]):
+ nbits = []
+ for cfg in HQQ_CONFIGS:
+ bpp = calc_bits(
+ cfg[1]["weight_quant_params"]["nbits"],
+ cfg[1]["weight_quant_params"]["group_size"],
+ 8,
+ 128,
+ )
+ nbits.extend([round(bpp * (1 - pct / 100), 2) for pct in reduction_pcts])
+ return sorted(list(set(nbits)), reverse=True)
+
+
+def do_expermient_fdata(
+ experiment_name,
+ models,
+ tasks,
+ track_cuda_memory=False,
+):
+ do_expermient(
+ experiment_name,
+ models,
+ tasks,
+ quant_dir="/fdata/llm/mxq/snapshots",
+ result_dir="/fdata/llm/mxq/results",
+ track_cuda_memory=track_cuda_memory,
+ )
+
+
+########################################################################
+# Quantization experiments
+########################################################################
+
+
+def experiment_quant_hqq():
+ models = ALL_MODELS
+ tasks = {
+ "hqq": {
+ "type": "quant",
+ "configs": HQQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(
+ "quant_hqq",
+ models,
+ tasks,
+ )
+
+
+def experiment_quant_mxq():
+ models = ALL_MODELS
+ type = "quant"
+ algo = "hqq"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": HQQ_CONFIGS[9:],
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}_vit", models, tasks)
+
+
+########################################################################
+# zero-shot image classification evaluation experiments
+########################################################################
+
+
+def experiment_zeroshot_eval_fp16():
+ models = ALL_MODELS
+ type = "eval_zeroshot_cls"
+ algo = "fp16"
+ tasks = {
+ algo: {
+ "type": type,
+ "configs": [
+ ("base", {}),
+ ],
+ },
+ }
+ do_expermient_fdata(f"{type}_{algo}", models, tasks)
+
+
+def experiment_eval_fp16_combined():
+ models = ALL_MODELS
+ algo = "fp16"
+ tasks = {
+ algo: {
+ "type": "eval_zeroshot_cls",
+ "configs": [
+ ("base", {}),
+ ],
+ },
+ }
+ linear_probe_tasks = {
+ algo: {
+ "type": "eval_linear_probe",
+ "configs": [
+ ("base", {}),
+ ],
+ },
+ }
+ do_expermient_fdata(f"eval_linear_probe_{algo}2", models, linear_probe_tasks)
+ do_expermient_fdata(f"eval_zeroshot_cls_{algo}2", models, tasks)
+
+
+def experiment_eval_hqq_comprehensive():
+ models = ALL_MODELS
+ tasks = {
+ "hqq": {
+ "type": "eval_zeroshot_cls",
+ "configs": HQQ_CONFIGS,
+ }
+ }
+ linear_probe_tasks = {
+ "hqq": {
+ "type": "eval_linear_probe",
+ "configs": HQQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata("eval_lp_hqq_comprehensive4", models, linear_probe_tasks)
+ do_expermient_fdata("eval_zs_hqq_comprehensive5", models, tasks)
+
+
+def experiment_eval_mxq_358_memory_saving():
+ nbits = get_mxq_bits()
+ # budget 2.07 is infeasible
+ nbits = [bit for bit in nbits if bit > 2.07]
+ mxq_configs = [
+ (
+ f"{bits:.2f}".replace(".", "_"),
+ HQQQuantConfig(mixed=True, budget=bits, quant_scale=True),
+ )
+ for bits in nbits
+ ]
+ models = ALL_MODELS
+ tasks = {
+ "mxq": {
+ "type": "eval_zeroshot_cls",
+ "configs": mxq_configs,
+ }
+ }
+ linear_probe_tasks = {
+ "mxq": {
+ "type": "eval_linear_probe",
+ "configs": mxq_configs,
+ },
+ }
+ do_expermient_fdata("eval_lp_mxq_358_memory_saving4", models, linear_probe_tasks)
+ do_expermient_fdata("eval_zs_mxq_358_memory_saving5", models, tasks)
+
+
+def experiment_eval_mxq_comprehensive():
+ models = ALL_MODELS
+ zeroshot_tasks = {
+ "mxq": {
+ "type": "eval_zeroshot_cls",
+ "configs": MXQ_CONFIGS,
+ },
+ }
+ linear_probe_tasks = {
+ "mxq": {
+ "type": "eval_linear_probe",
+ "configs": MXQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata("eval_zs_mxq_comprehensive2", models, zeroshot_tasks)
+ do_expermient_fdata("eval_lp_mxq_comprehensive2", models, linear_probe_tasks)
+
+
+def experiment_zeroshot_eval_mxq():
+ models = ALL_MODELS
+ type = "eval_zeroshot_cls"
+ tasks = {
+ "mxq": {
+ "type": type,
+ "configs": MXQ_CONFIGS,
+ },
+ }
+ do_expermient_fdata(f"{type}_mxq", models, tasks)
+
+
+def experiment_eval_mxq_combined():
+ models = ALL_MODELS
+ equiv_mxq_configs = [
+ (
+ f"{bits:.2f}".replace(".", "_"),
+ HQQQuantConfig(mixed=True, budget=bits, quant_scale=True),
+ )
+ for bits in [4.06, 4.10, 4.15, 4.19, 4.24, 4.28, 4.33]
+ ]
+ zeroshot_tasks = {
+ "mxq": {
+ "type": "eval_zeroshot_cls",
+ "configs": equiv_mxq_configs,
+ },
+ }
+ linear_probe_tasks = {
+ "mxq": {
+ "type": "eval_linear_probe",
+ "configs": equiv_mxq_configs,
+ },
+ }
+ do_expermient_fdata("eval_linear_probe_mxq2", models, linear_probe_tasks)
+ do_expermient_fdata("eval_zeroshot_cls_mxq2", models, zeroshot_tasks)
+
+
+def main():
+ logging.basicConfig(
+ format="%(asctime)s %(levelname)s [%(name)s] %(message)s",
+ level=logging.INFO,
+ datefmt="%Y-%m-%d %H:%M:%S",
+ )
+
+ # experiment_quant_hqq()
+ # experiment_quant_mxq()
+ # experiment_zeroshot_eval_fp16()
+ # experiment_zeroshot_eval_mxq()
+ # experiment_eval_mxq_combined()
+ # experiment_eval_fp16_combined()
+ experiment_eval_hqq_comprehensive()
+ experiment_eval_mxq_358_memory_saving()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/eval/clipbenchmark.py b/lm-quant-toolkit/src/lm_quant_toolkit/eval/clipbenchmark.py
new file mode 100644
index 0000000000000000000000000000000000000000..a54f52c164ae0a9d1396c8d971037cd90dea5dfe
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/eval/clipbenchmark.py
@@ -0,0 +1,109 @@
+import os
+import sys
+import time
+from pathlib import Path
+
+from clip_benchmark.cli import get_parser_args, run
+
+
+def eval_clip_benchmark(
+ task,
+ model_id,
+ result_dir,
+ quant_dir,
+ quant_config,
+ additional_args=[],
+):
+ # build args for clip_benchmark
+ model_type = "open_clip_hqq" if quant_config else "open_clip"
+ comps = model_id.split("/")
+ elems = comps[1].split("-")
+ model_name = "-".join(elems[1:4])
+ pretrained = "-".join(elems[4:])
+ extra_args = ""
+ if quant_config:
+ mixed = quant_config.get("mixed", None)
+ if mixed:
+ budget = quant_config["budget"]
+ qmf = quant_config["quant_metrics_file"]
+ weight_algo = quant_config["weight_algo"]
+ boost_stop = quant_config["boost_stop"]
+ top_m_layer = quant_config["top_m_layer"]
+ extra_args = f"budget={budget},quant_metrics_file={qmf},weight_algo={weight_algo},boost_stop={boost_stop},top_m_layer={top_m_layer}"
+ else:
+ b = quant_config["weight_quant_params"]["nbits"]
+ g = quant_config["weight_quant_params"]["group_size"]
+ extra_args = f"nbits={b},group_size={g}"
+ wds_cache_dir = os.path.join(quant_dir, "wds-dataset")
+ Path(wds_cache_dir).mkdir(parents=True, exist_ok=True)
+ out_file = f"benchmark_{{dataset}}_{{pretrained}}_{{model}}_{{language}}_{{task}}-{model_type}.json"
+ out_fp = os.path.join(result_dir, out_file)
+ args_str = [
+ "eval",
+ f"--task={task}",
+ f"--model_type={model_type}",
+ "--dataset=wds/imagenet1k",
+ "--dataset_root=https://huggingface.co/datasets/clip-benchmark/wds_{dataset_cleaned}/tree/main",
+ f"--output={out_fp}",
+ f"--wds_cache_dir={wds_cache_dir}",
+ f"--extra_args={extra_args}",
+ ]
+ if len(additional_args) > 0:
+ args_str.extend(additional_args)
+ t1 = time.time()
+
+ # clear the command line args for main python program
+ # so that they are not passed to clip_benchmark CLI
+ sys.argv = [sys.argv[0]]
+ parser, _ = get_parser_args()
+ cli_args = parser.parse_args(args_str)
+ cli_args.dataset = cli_args.dataset[0]
+ if any(["--train_split" in arg for arg in additional_args]):
+ cli_args.train_split = cli_args.train_split[0]
+ cli_args.model = model_name
+ cli_args.pretrained = pretrained
+ result_dict = run(cli_args)
+ t2 = time.time()
+ result_dict["__lm_quant_tk_duration"] = t2 - t1
+ return result_dict
+
+
+def eval_zeroshot_classification(metric, model_id, result_dir, quant_dir, quant_config):
+ result_dict = eval_clip_benchmark(
+ "zeroshot_classification",
+ model_id,
+ result_dir,
+ quant_dir,
+ quant_config,
+ )
+ metric["acc1_zeroshot_cls"] = result_dict["metrics"]["acc1"]
+ metric["acc5_zeroshot_cls"] = result_dict["metrics"]["acc5"]
+ metric["recall_zeroshot_cls"] = result_dict["metrics"]["mean_per_class_recall"]
+ metric["duration_zeroshot_cls"] = result_dict["__lm_quant_tk_duration"]
+ return metric
+
+
+def eval_linear_probe(
+ metric, model_id, result_dir, quant_dir, quant_config, feature_root
+):
+ additional_args = [
+ "--batch_size=512",
+ "--fewshot_lr=0.1",
+ "--fewshot_epochs=20",
+ "--train_split=train",
+ "--test_split=test",
+ f"--feature_root={feature_root}",
+ ]
+ result_dict = eval_clip_benchmark(
+ "linear_probe",
+ model_id,
+ result_dir,
+ quant_dir,
+ quant_config,
+ additional_args,
+ )
+ metric["acc1_linear_probe"] = result_dict["metrics"]["lp_acc1"]
+ metric["acc5_linear_probe"] = result_dict["metrics"]["lp_acc5"]
+ metric["recall_linear_probe"] = result_dict["metrics"]["lp_mean_per_class_recall"]
+ metric["duration_linear_probe"] = result_dict["__lm_quant_tk_duration"]
+ return metric
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/eval/common.py b/lm-quant-toolkit/src/lm_quant_toolkit/eval/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..315e2f4030be4c08a887b5fdb7e3f05da05ab23a
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/eval/common.py
@@ -0,0 +1,239 @@
+import gc
+import glob
+import os
+from datetime import datetime
+from pathlib import Path
+
+import pandas as pd
+import torch
+from hqq.core.quantize import BaseQuantizeConfig as HQQQuantConfig
+from hqq.utils.optimizer import find_optimal_configs
+
+from lm_quant_toolkit.utils.hub import LLAMA_MODELS, VIT_OPENCLIP_MODELS
+
+HQQ_CONFIGS = [
+ ("b8g32", HQQQuantConfig(nbits=8, group_size=32, quant_scale=True)),
+ ("b8g64", HQQQuantConfig(nbits=8, group_size=64, quant_scale=True)),
+ ("b8g128", HQQQuantConfig(nbits=8, group_size=128, quant_scale=True)),
+ ("b4g32", HQQQuantConfig(nbits=4, group_size=32, quant_scale=True)),
+ ("b4g64", HQQQuantConfig(nbits=4, group_size=64, quant_scale=True)),
+ ("b4g128", HQQQuantConfig(nbits=4, group_size=128, quant_scale=True)),
+ ("b3g32", HQQQuantConfig(nbits=3, group_size=32, quant_scale=True)),
+ ("b3g64", HQQQuantConfig(nbits=3, group_size=64, quant_scale=True)),
+ ("b3g128", HQQQuantConfig(nbits=3, group_size=128, quant_scale=True)),
+ ("b2g16", HQQQuantConfig(nbits=2, group_size=16, quant_scale=True)),
+ ("b2g32", HQQQuantConfig(nbits=2, group_size=32, quant_scale=True)),
+ ("b2g64", HQQQuantConfig(nbits=2, group_size=64, quant_scale=True)),
+]
+
+
+def get_mxq_quant_meta_data_file(model_id):
+ short_id = model_id.split("/")[1]
+ data_dir = os.path.join(os.path.dirname(__file__), "..", "..", "data")
+ fp = os.path.join(data_dir, f"fnorm-{short_id}.csv")
+ return os.path.exists(fp), os.path.abspath(fp)
+
+
+def persist_progress(df, progress_path):
+ """Save the progress of experiment for resumption."""
+ Path(progress_path).parent.mkdir(parents=True, exist_ok=True)
+ df.to_csv(progress_path, index=False)
+
+
+def save_partial_metric(experiment_name, algo, model_id, config, metric, result_dir):
+ metrics = [metric]
+ df = pd.DataFrame(metrics)
+ result_dir = os.path.join(result_dir, experiment_name)
+ os.makedirs(result_dir, exist_ok=True)
+ model_short_id = model_id.split("/")[1]
+ file_name = f"{result_dir}/partial-{algo}-{model_short_id}-{config}.csv"
+ df.to_csv(file_name, index=False)
+
+
+def _dump_cuda_mem_snapshot(experiment_name, model_id, algo, result_dir):
+ mem_fp = f"{result_dir}/{experiment_name}/mem-snapshot-{algo}-{model_id.split('/')[1]}.pickle"
+ torch.cuda.memory._dump_snapshot(mem_fp)
+
+
+def combine_metrics(experiment_name, result_dir):
+ dfs = []
+ iters = glob.iglob(f"{result_dir}/{experiment_name}/partial-*.csv")
+ for it in iters:
+ df = pd.read_csv(it)
+ dfs.append(df)
+ combined = pd.concat(dfs)
+ ts_str = datetime.now().strftime("%Y%m%d%H%M%S")
+ file_name = f"{result_dir}/{experiment_name}/result-{experiment_name}-{ts_str}.csv"
+ Path(file_name).parent.mkdir(parents=True, exist_ok=True)
+ combined.to_csv(file_name, index=False)
+
+
+def cleanup(model):
+ del model
+ torch.cuda.empty_cache()
+ gc.collect()
+
+
+def calc_bits(b1, g1, b2=8, g2=128):
+ return b1 + 2 * b2 / g1 + 32 / g1 / g2
+
+
+def get_mxq_bits(reduction_pcts=[3, 5, 8]):
+ nbits = []
+ for cfg in HQQ_CONFIGS:
+ bpp = calc_bits(
+ cfg[1]["weight_quant_params"]["nbits"],
+ cfg[1]["weight_quant_params"]["group_size"],
+ 8,
+ 128,
+ )
+ nbits.extend([round(bpp * (1 - pct / 100), 2) for pct in reduction_pcts])
+ return sorted(list(set(nbits)), reverse=True)
+
+
+def _reset_peak_memory_stats():
+ return torch.cuda.reset_peak_memory_stats()
+
+
+def get_memory_metrics():
+ return torch.cuda.max_memory_allocated(), torch.cuda.max_memory_reserved()
+
+
+def plan_eval_bit_budgets(
+ model_arch="ViT",
+ points=5,
+ step=1,
+ bases=[4.51],
+ include_base=False,
+):
+ for base in bases:
+ ideals, solvables = get_eval_plan(model_arch, base, points, step, include_base)
+ print("*" * 72)
+ print(f"base: {base}")
+ for t in zip(ideals, solvables):
+ print(f"ideal: {t[0]:.2f}, solvable: {t[1]:.2f}")
+ print("*" * 72)
+
+
+def get_eval_plan(model_arch, base, points, step, include_base):
+ ideals = []
+ solvables = []
+ start = 0 if include_base else 1
+ for point in range(start, points + 1):
+ tentative = round(base + point * step, 2)
+ ideals.append(tentative)
+ ret = try_solvable(model_arch, tentative, step)
+ if ret is not None:
+ solvables.append(ret)
+ else:
+ solvables.append(0.0)
+ return ideals, solvables
+
+
+def try_solvable(model_arch, bit_budget, step):
+ if model_arch == "ViT":
+ model_ids = VIT_OPENCLIP_MODELS.keys()
+ else:
+ model_ids = [
+ key for key in LLAMA_MODELS if LLAMA_MODELS[key].get("experiment", False)
+ ]
+
+ dikt = {}
+ feasible_budget = round(bit_budget, 2)
+ for model_id in model_ids:
+ attempts = 1
+ _, fp = get_mxq_quant_meta_data_file(model_id)
+ while True:
+ try:
+ # find_optimal_configs(fp, feasible_budget, time_limit=200)
+ find_optimal_configs(
+ fp,
+ feasible_budget,
+ time_limit=200,
+ weight_algo="sensi-milp",
+ )
+ dikt[model_id] = feasible_budget
+ break
+ except ValueError:
+ print(f"Warning: {feasible_budget:.2f} unsolvable for model {model_id}")
+ if attempts > 3:
+ return None
+ feasible_budget += -0.01 if step < 0 else 0.01
+ attempts += 1
+ if len(set(dikt.values())) > 1:
+ for model_id, budget in dikt.items():
+ if abs(budget - feasible_budget) > 0.01:
+ try:
+ fp = get_mxq_quant_meta_data_file(model_id)
+ find_optimal_configs(fp, feasible_budget, time_limit=200)
+ except ValueError:
+ print(
+ f"Warning: {feasible_budget:.2f} unsolvable for model {model_id}"
+ )
+ return None
+ return feasible_budget
+
+
+def debug_milp_solvable(model_id, bit_budget):
+ _, fp = get_mxq_quant_meta_data_file(model_id)
+ configs = find_optimal_configs(fp, bit_budget, time_limit=200)
+ print(configs)
+
+
+def plan_432_bits():
+ bases = [4.51, 4.25, 4.13]
+ plan_eval_bit_budgets(
+ model_arch="llm", points=5, step=0.02, bases=bases, include_base=True
+ )
+ plan_eval_bit_budgets(
+ model_arch="llm", points=5, step=-0.02, bases=bases, include_base=True
+ )
+ bases = [3.51, 3.25, 3.13, 2.51, 2.25, 2.13]
+ plan_eval_bit_budgets(
+ model_arch="llm", points=3, step=0.02, bases=bases, include_base=True
+ )
+ plan_eval_bit_budgets(
+ model_arch="llm", points=3, step=-0.02, bases=bases, include_base=True
+ )
+
+
+def plan_567_bits():
+ best_bit_budget = calc_bits(8, 32, 8, 128)
+ save_objs = [10, 20, 30, 40]
+ bases = [best_bit_budget * (100 - obj) / 100 for obj in save_objs]
+ plan_eval_bit_budgets(
+ model_arch="llm", points=5, step=0.02, bases=bases, include_base=True
+ )
+ plan_eval_bit_budgets(
+ model_arch="llm", points=5, step=-0.02, bases=bases, include_base=True
+ )
+
+
+def fill_budget_gap(start, stop, step=0.02):
+ result = []
+ d = start + step
+ while d < stop:
+ result.append(round(d, 2))
+ d += step
+ return result
+
+
+def fill_gaps():
+ gaps = []
+ # gaps.extend(fill_budget_gap(3.57, 4.03))
+ # gaps.extend(fill_budget_gap(3.29, 3.45))
+ # gaps.extend(fill_budget_gap(6.92, 7.56))
+ # gaps.extend(fill_budget_gap(4.61, 5.00))
+ # gaps.extend(fill_budget_gap(5.20, 5.86))
+ gaps.extend(fill_budget_gap(6.01, 6.70))
+ # gaps.extend([4.39, 4.37])
+ print(gaps)
+ plan_eval_bit_budgets(
+ model_arch="llm", points=0, step=0.02, bases=gaps, include_base=True
+ )
+
+
+if __name__ == "__main__":
+ # fill_gaps()
+ plan_432_bits()
+ plan_567_bits()
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/eval/leaderboard.py b/lm-quant-toolkit/src/lm_quant_toolkit/eval/leaderboard.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ce24adcbeaae001784397c0335eabd63d30314b
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/eval/leaderboard.py
@@ -0,0 +1,304 @@
+import json
+import os
+import shutil
+import time
+from pathlib import Path
+
+import numpy as np
+from lm_eval import evaluator
+from lm_eval.tasks import TaskManager
+
+from lm_quant_toolkit.utils.hub import get_hf_model_storge_base_dir
+
+HIGHER_IS_BETTER_SYMBOLS = {
+ True: "↑",
+ False: "↓",
+}
+
+
+def make_table(result_dict, column: str = "results", sort_results: bool = False):
+ """Generate table of results."""
+ from pytablewriter import LatexTableWriter, MarkdownTableWriter
+
+ if column == "results":
+ column_name = "Tasks"
+ elif column == "groups":
+ column_name = "Groups"
+
+ all_headers = [
+ column_name,
+ "Version",
+ "Filter",
+ "n-shot",
+ "Metric",
+ "",
+ "Value",
+ "",
+ "Stderr",
+ ]
+
+ md_writer = MarkdownTableWriter()
+ latex_writer = LatexTableWriter()
+ md_writer.headers = all_headers
+ latex_writer.headers = all_headers
+
+ values = []
+
+ keys = result_dict[column].keys()
+ if sort_results:
+ # sort entries alphabetically by task or group name.
+ # NOTE: we default here to false, because order matters
+ # for multi-level table printing a la mmlu.
+ # sorting here would mess that up
+ keys = sorted(keys)
+ for k in keys:
+ dic = result_dict[column][k]
+ version = result_dict["versions"].get(k, " N/A")
+ n = str(result_dict.get("n-shot", " ").get(k, " "))
+ higher_is_better = result_dict.get("higher_is_better", {}).get(k, {})
+
+ if "alias" in dic:
+ k = dic.pop("alias")
+
+ metric_items = dic.items()
+ metric_items = sorted(metric_items)
+
+ for (mf), v in metric_items:
+ m, _, f = mf.partition(",")
+ if m.endswith("_stderr"):
+ continue
+
+ hib = HIGHER_IS_BETTER_SYMBOLS.get(higher_is_better.get(m), "")
+
+ v = "%.4f" % v if isinstance(v, float) else v
+
+ if m + "_stderr" + "," + f in dic:
+ se = dic[m + "_stderr" + "," + f]
+ se = " N/A" if se == "N/A" else "%.4f" % se
+ values.append([k, version, f, n, m, hib, v, "±", se])
+ else:
+ values.append([k, version, f, n, m, hib, v, "", ""])
+ k = ""
+ version = ""
+ md_writer.value_matrix = values
+ latex_writer.value_matrix = values
+
+ # todo: make latex table look good
+ # print(latex_writer.dumps())
+
+ return md_writer.dumps()
+
+
+def handle_non_serializable(o):
+ if isinstance(o, np.int64) or isinstance(o, np.int32):
+ return int(o)
+ elif isinstance(o, set):
+ return list(o)
+ else:
+ return str(o)
+
+
+def eval_llm_leaderboard(
+ experiment_name,
+ model_id,
+ quant_method,
+ confg_name,
+ quantized,
+ metric,
+ quant_base_dir,
+ result_dir,
+ verbosity="INFO",
+):
+ if quantized:
+ quant_dir = os.path.join(
+ quant_base_dir, f"{model_id}-{confg_name}-{quant_method.lower()}"
+ )
+ # prepare the quantized model by copying tokenizer files
+ _prepare_tokenizer_files(model_id, quant_dir)
+ model_args = f"pretrained={quant_dir},quant_method={quant_method}"
+ else:
+ model_args = f"pretrained={model_id}"
+
+ t1 = time.time()
+ task_manager = TaskManager(verbosity)
+ results = evaluator.simple_evaluate(
+ model="hf",
+ model_args=model_args,
+ tasks="leaderboard",
+ # num_fewshot=args.num_fewshot,
+ batch_size="auto:16",
+ max_batch_size=16,
+ device="cuda:0",
+ # use_cache=True,
+ # check_integrity=True,
+ write_out=False,
+ log_samples=True,
+ system_instruction=None,
+ # apply_chat_template=args.apply_chat_template,
+ fewshot_as_multiturn=False,
+ # gen_kwargs=args.gen_kwargs,
+ task_manager=task_manager,
+ verbosity=verbosity,
+ predict_only=False,
+ random_seed=0,
+ numpy_random_seed=1234,
+ torch_random_seed=1234,
+ fewshot_random_seed=1234,
+ )
+ t2 = time.time()
+
+ if results is not None:
+ lm_eval_result_fp = os.path.join(
+ result_dir,
+ experiment_name,
+ f"{model_id.split('/')[1]}-{confg_name}-{quant_method.lower()}-results.json",
+ )
+ Path(lm_eval_result_fp).parent.mkdir(parents=True, exist_ok=True)
+
+ results.pop("samples", None)
+ with open(lm_eval_result_fp, "w") as fh:
+ json.dump(
+ results,
+ fh,
+ indent=2,
+ default=handle_non_serializable,
+ ensure_ascii=False,
+ )
+ print(make_table(results))
+
+ ifeval, bbh, mathlevel5, gpqa, musr, mmlupro = _cal_leaderboard_score(results)
+ metric["ifeval"] = ifeval
+ metric["bbh"] = bbh
+ metric["mathlevel5"] = mathlevel5
+ metric["gpqa"] = gpqa
+ metric["musr"] = musr
+ metric["mmlupro"] = mmlupro
+ metric["duration_leaderboard"] = t2 - t1
+ return metric
+
+
+def _cal_leaderboard_score(results):
+ ifeval = _cal_leaderboard_ifeval_score(results)
+ bbh = _cal_leaderboard_bbh(results)
+ mathlevel5 = _cal_leaderboard_mathlevel5(results)
+ gpqa = _cal_leaderboard_gpqa(results)
+ musr = _cal_leaderboard_musr(results)
+ mmlupro = _cal_leaderboard_mmlu_pro(results)
+ return ifeval, bbh, mathlevel5, gpqa, musr, mmlupro
+
+
+def _cal_leaderboard_gpqa(results):
+ res_gpqa = results["results"]["leaderboard_gpqa"]
+ value = res_gpqa.get("acc_norm,none", None)
+ # aggregate data from sub tasks
+ if value is None:
+ subtask_names = results["group_subtasks"]["leaderboard_gpqa"]
+ metrics = {}
+ for name in subtask_names:
+ metrics[name] = len(results["configs"][name]["doc_to_choice"])
+ scores = []
+ for metric, choices in metrics.items():
+ value = results["results"][metric]["acc_norm,none"]
+ scores.append(_cal_normalized_score(value, 1 / choices, 1.0))
+ return sum(scores) / len(scores)
+
+ else:
+ return _cal_normalized_score(value, 1 / 4, 1.0)
+
+
+def _cal_leaderboard_mathlevel5(results):
+ res_math_hard = results["results"]["leaderboard_math_hard"]
+ value = res_math_hard.get("exact_match,none", None)
+ # aggregate data from sub tasks
+ if value is None:
+ subtask_names = results["group_subtasks"]["leaderboard_math_hard"]
+ scores = []
+ for metric in subtask_names:
+ value = results["results"][metric]["exact_match,none"]
+ scores.append(_cal_normalized_score(value, 0, 1.0))
+ return sum(scores) / len(scores)
+ else:
+ return _cal_normalized_score(value, 0, 1.0)
+
+
+def _cal_leaderboard_bbh(results):
+ subtask_names = results["group_subtasks"]["leaderboard_bbh"]
+ metrics = {}
+ for name in subtask_names:
+ metrics[name] = len(results["configs"][name]["doc_to_choice"])
+ scores = []
+ for metric, choices in metrics.items():
+ value = results["results"][metric]["acc_norm,none"]
+ scores.append(_cal_normalized_score(value, 1 / choices, 1.0))
+ return sum(scores) / len(scores)
+
+
+# noqa refer to: https://huggingface.co/docs/leaderboards/open_llm_leaderboard/normalization#example-normalizing-musr-scores
+def _cal_leaderboard_musr(results):
+ metrics = {
+ "leaderboard_musr_murder_mysteries": 2,
+ "leaderboard_musr_object_placements": 5,
+ "leaderboard_musr_team_allocation": 3,
+ }
+ scores = []
+ for metric, choices in metrics.items():
+ value = results["results"][metric]["acc_norm,none"]
+ scores.append(_cal_normalized_score(value, 1 / choices, 1.0))
+ return sum(scores) / len(scores)
+
+
+def _cal_leaderboard_mmlu_pro(results):
+ value = results["results"]["leaderboard_mmlu_pro"]["acc,none"]
+ return _cal_normalized_score(value, 0.1, 1.0)
+
+
+def _cal_leaderboard_ifeval_score(results):
+ scores = []
+ value1 = results["results"]["leaderboard_ifeval"]["inst_level_strict_acc,none"]
+ value2 = results["results"]["leaderboard_ifeval"]["prompt_level_strict_acc,none"]
+ scores.append(_cal_normalized_score(value1, 0, 1.0))
+ scores.append(_cal_normalized_score(value2, 0, 1.0))
+ return sum(scores) / len(scores)
+
+
+def _cal_normalized_score(value, lower_bound, higher_bound=1.0):
+ if value < lower_bound:
+ return 0
+ return 100 * (value - lower_bound) / (higher_bound - lower_bound)
+
+
+def _prepare_tokenizer_files(model_id, quant_dir):
+ files = [
+ "config.json",
+ "tokenizer.model",
+ "tokenizer.json",
+ "special_tokens_map.json",
+ "tokenizer_config.json",
+ ]
+
+ base_dir = get_hf_model_storge_base_dir(model_id)
+ for f in files:
+ src_fp = os.path.join(base_dir, f)
+ if os.path.exists(src_fp):
+ dst_fp = os.path.join(quant_dir, f)
+ shutil.copyfile(src_fp, dst_fp, follow_symlinks=True)
+
+
+def test_run(fp):
+ with open(fp) as fh:
+ results = json.load(fh)
+ ifeval, bbh, mathlevel5, gpqa, musr, mmlupro = _cal_leaderboard_score(results)
+ avg = (ifeval + bbh + mathlevel5 + gpqa + musr + mmlupro) / 6
+ print(f"avg={avg:.2f}")
+ print(f"ifeval={ifeval:.2f}")
+ print(f"bbh={bbh:.2f}")
+ print(f"mathlevel5={mathlevel5:.2f}")
+ print(f"gpqa={gpqa:.2f}")
+ print(f"musr={musr:.2f}")
+ print(f"mmlupro={mmlupro:.2f}")
+
+
+if __name__ == "__main__":
+ # noqa result3.json is a copy of https://huggingface.co/datasets/open-llm-leaderboard/meta-llama__Llama-2-7b-hf-details/raw/main/meta-llama__Llama-2-7b-hf/results_2024-06-16T18-52-55.970021.json
+ test_run("results/fp16_leaderboard/Llama-2-7b-hf-base-fp16-results.json")
+ test_run("logs/result3.json")
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/eval/perplexity.py b/lm-quant-toolkit/src/lm_quant_toolkit/eval/perplexity.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a4b9b955d7d76e1753fcef1176ba67d45ef1b99
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/eval/perplexity.py
@@ -0,0 +1,125 @@
+import gc
+import time
+
+import numpy as np
+import torch
+from datasets import load_dataset
+from tqdm import tqdm
+
+
+def cleanup():
+ torch.cuda.empty_cache()
+ gc.collect()
+
+
+def eval_ptb(model, tokenizer, max_length=1024, stride=512, verbose=True):
+ dataset = load_dataset("ptb_text_only", "penn_treebank", split="test")
+ return eval_ppl(
+ "ptb",
+ model,
+ tokenizer,
+ dataset,
+ text_column="sentence",
+ max_length=max_length,
+ stride=stride,
+ verbose=verbose,
+ )
+
+
+def eval_c4(model, tokenizer, max_length=1024, stride=512, verbose=True):
+ dataset = load_dataset(
+ "allenai/c4",
+ data_files={"validation": "en/c4-validation.00000-of-00008.json.gz"},
+ split="validation",
+ download_mode="reuse_dataset_if_exists",
+ )
+ # pick first 1100
+ dataset = dataset[:1100]
+ return eval_ppl(
+ "C4",
+ model,
+ tokenizer,
+ dataset,
+ text_column="text",
+ max_length=max_length,
+ stride=stride,
+ verbose=verbose,
+ )
+
+
+def eval_wikitext2(model, tokenizer, max_length=1024, stride=512, verbose=True):
+ dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")
+ return eval_ppl(
+ "wikitext",
+ model,
+ tokenizer,
+ dataset,
+ text_column="text",
+ max_length=max_length,
+ stride=stride,
+ verbose=verbose,
+ )
+
+
+# Adapted from https://huggingface.co/transformers/v4.2.2/perplexity.html
+def eval_ppl(
+ ds_type,
+ model,
+ tokenizer,
+ dataset,
+ text_column="text",
+ max_length=1024,
+ stride=512,
+ verbose=True,
+):
+ model.eval()
+ tokenizer.pad_token = tokenizer.eos_token
+ tokenizer.padding_side = "right"
+ tokenizer.add_eos_token = False
+
+ encodings = tokenizer("\n\n".join(dataset[text_column]), return_tensors="pt")
+
+ encodings["input_ids"] = encodings["input_ids"].to("cuda")
+
+ lls, t = [], []
+ for i in tqdm(
+ range(0, encodings["input_ids"].size(1), stride),
+ desc=ds_type,
+ disable=not verbose,
+ ):
+ begin_loc = max(i + stride - max_length, 0)
+ end_loc = min(i + stride, encodings["input_ids"].size(1))
+ trg_len = end_loc - i
+ input_ids = encodings["input_ids"][:, begin_loc:end_loc]
+ target_ids = input_ids.clone()
+ target_ids[:, :-trg_len] = -100 # ignore context
+
+ t1 = time.time()
+ with torch.no_grad():
+ log_likelihood = model(input_ids, labels=target_ids).loss * trg_len
+ torch.cuda.synchronize()
+ t2 = time.time()
+ t.append((t2 - t1))
+ lls.append(log_likelihood)
+
+ del input_ids, target_ids
+
+ ppl = np.round(float(torch.exp(torch.stack(lls).sum() / end_loc)), 4)
+ pred_time = np.round(np.mean(t), 3)
+ if verbose:
+ print(f"{ds_type} perplexity: {ppl}, time: {pred_time} sec")
+
+ del encodings
+ cleanup()
+
+ return ppl, pred_time
+
+
+def eval_ppls(model, tokenizer, metric):
+ ppl_wikitext, duration_wikitext = eval_wikitext2(model, tokenizer, verbose=True)
+ ppl_c4, duration_c4 = eval_c4(model, tokenizer, verbose=True)
+ metric["ppl_wikitext"] = ppl_wikitext
+ metric["ppl_c4"] = ppl_c4
+ metric["duration_wikitext"] = duration_wikitext
+ metric["duration_c4"] = duration_c4
+ return metric
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/misc/__init__.py b/lm-quant-toolkit/src/lm_quant_toolkit/misc/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/misc/lp_norm.py b/lm-quant-toolkit/src/lm_quant_toolkit/misc/lp_norm.py
new file mode 100644
index 0000000000000000000000000000000000000000..abc89cdf9fd83e44ac6bfeb44eda7acb4b0ba75d
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/misc/lp_norm.py
@@ -0,0 +1,51 @@
+import numpy as np
+import pandas as pd
+import torch
+from hqq.core.quantize import Quantizer as hQuant
+
+from lm_quant_toolkit.utils.hub import get_hf_model_storge_base_dir
+from lm_quant_toolkit.utils.safetensors import get_tensor
+
+
+def quant_hqq(tensor, nbits, group_size=64, lp_norm=0.7, optimize=True):
+ opt_params = {
+ "lp_norm": lp_norm,
+ "iters": 100,
+ "early_stop": False,
+ }
+
+ wq, meta = hQuant.quantize(
+ tensor, nbits=nbits, group_size=group_size, optimize=optimize, **opt_params
+ )
+ return hQuant.dequantize(wq, meta)
+
+
+if __name__ == "__main__":
+ model_id = "meta-llama/Llama-2-7b-hf"
+ base_dir = get_hf_model_storge_base_dir(model_id)
+
+ layer = 31
+ module = "self_attn.o_proj"
+ matrix_name = f"model.layers.{layer}.{module}.weight"
+ w = get_tensor(matrix_name, base_dir)
+
+ dick = []
+ for lp_norm in np.linspace(0.1, 0.9, 9):
+ for b in [3, 4, 8]:
+ for g in [32, 64, 128]:
+ wq_hqq = quant_hqq(w, nbits=b, lp_norm=lp_norm, group_size=g)
+ norm_hqq = torch.norm(w - wq_hqq)
+ dick.append(
+ {
+ "model": model_id.split("/")[1],
+ "layer": layer,
+ "module": module,
+ "lp_norm": lp_norm,
+ "b": b,
+ "g": g,
+ "fnorm": norm_hqq.item(),
+ }
+ )
+ print(f"FNorm HQQ(lp_norm={lp_norm:.2f}): {norm_hqq}")
+ df = pd.DataFrame(dick)
+ df.to_csv("lp_norm_tuning.csv", index=False)
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/misc/quant_sim.py b/lm-quant-toolkit/src/lm_quant_toolkit/misc/quant_sim.py
new file mode 100644
index 0000000000000000000000000000000000000000..f13699aace4854ed8d014d7899f2e050a007e572
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/misc/quant_sim.py
@@ -0,0 +1,64 @@
+import pandas as pd
+from hqq.utils.optimizer import find_optimal_configs
+
+from lm_quant_toolkit.eval.common import get_mxq_quant_meta_data_file
+
+
+def dump_mxq_objectives(model_ids, bit_budgets, csv_fp="mxq-objectives.csv"):
+ dikt = []
+ for model_id in model_ids:
+ short_id = model_id.split("/")[1]
+ _, fp = get_mxq_quant_meta_data_file(model_id)
+ for bit_budget in bit_budgets:
+ _, objective = find_optimal_configs(fp, bit_budget, time_limit=200)
+ dikt.append(
+ {
+ "model": short_id,
+ "bpp": bit_budget,
+ "fnorm": objective,
+ }
+ )
+
+ df = pd.DataFrame(dikt)
+ df.to_csv(csv_fp, index=False)
+
+
+def dump_mxq_configs(model_ids, bit_budgets, csv_fp, weight_algo, factor):
+ dikt = []
+ for model_id in model_ids:
+ short_id = model_id.split("/")[1]
+ for bit_budget in bit_budgets:
+ try:
+ _, fp = get_mxq_quant_meta_data_file(model_id)
+ kwargs = {"weight_algo": weight_algo, "factor": factor}
+ configs, objective = find_optimal_configs(
+ fp,
+ bit_budget,
+ time_limit=200,
+ **kwargs,
+ )
+ for k, v in configs.items():
+ comps = k.split(".", 1)
+ layer, module = comps[0], comps[1]
+ dikt.append(
+ {
+ "model": short_id,
+ "module": module,
+ "layer": layer,
+ "memmb": 0, # to work with the plot program
+ "bit_budget": bit_budget,
+ "b1": v[0],
+ "g1": v[1],
+ "b2": v[2],
+ "g2": v[3],
+ }
+ )
+ except ValueError:
+ print(f"Warning: {bit_budget:.2f} unsolvable for model {model_id}")
+ df = pd.DataFrame(dikt)
+ df.to_csv(csv_fp, index=False)
+
+
+if __name__ == "__main__":
+ bit_budgets = [4.51, 4.25, 4.13]
+ dump_mxq_configs(["meta-llama/Llama-2-7b-hf"], bit_budgets)
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/misc/qweight.py b/lm-quant-toolkit/src/lm_quant_toolkit/misc/qweight.py
new file mode 100644
index 0000000000000000000000000000000000000000..87cbfc89495b18c18e7ad4a4289acf6b23eaf2c8
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/misc/qweight.py
@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+
+
+import re
+
+import pandas as pd
+import torch
+from hqq.core.quantize import Quantizer as hQuant
+from safetensors.torch import save_file as safe_save
+from scipy.stats import kurtosis
+from torch import uint8
+
+from lm_quant_toolkit.eval.common import calc_bits
+from lm_quant_toolkit.utils.safetensors import get_tensor, get_tensor_dual
+
+
+def load_weight(matrix_name, base_dir):
+ m = f"{matrix_name}.weight"
+ return get_tensor(m, base_dir)
+
+
+def load_weight_dual(prefix, base_dir, st_file):
+ o = f"{prefix}.weight"
+ q = f"{prefix}.qweight"
+ return get_tensor_dual(o, q, base_dir, st_file)
+
+
+def dequantize(wq, meta):
+ # Zero/Scale packed together
+ if "zero_scale" in meta:
+ zero_scale = meta["zero_scale"]
+
+ if zero_scale.dtype == uint8:
+ meta["zero_q"], meta["scale_q"] = zero_scale[0], zero_scale[1]
+ else:
+ meta["zero"], meta["scale"] = zero_scale[0], zero_scale[1]
+
+ if meta["quant_zero"]:
+ meta["zero"] = hQuant.dequantize(meta["zero_q"], meta["meta_zero"])
+
+ if meta["quant_scale"]:
+ meta["scale"] = hQuant.dequantize(meta["scale_q"], meta["meta_scale"])
+ return hQuant.dequantize(wq, meta)
+
+
+def restore_weight(matrix, state_dict):
+ key = matrix
+ if key in state_dict:
+ m_dikt = state_dict[key]
+ if "meta" in m_dikt:
+ meta_dict = m_dikt["meta"]
+ meta_scale_dict = meta_dict.get("meta_scale", None)
+ b1 = meta_dict["nbits"]
+ g1 = meta_dict["group_size"]
+ b2 = meta_scale_dict["nbits"] if meta_scale_dict else 8
+ g2 = meta_scale_dict["group_size"] if meta_scale_dict else 128
+ quant_config = {
+ "b1": b1,
+ "g1": g1,
+ "b2": b2,
+ "g2": g2,
+ }
+ wq = dequantize(m_dikt["W_q"], meta_dict)
+ return wq, quant_config
+ else:
+ return None, None
+ else:
+ return None, None
+
+
+def save_compare_pair(
+ base_dir, quant_base_dir, quant_cfg, model_id, layers, output_dir
+):
+ file_path = f"{quant_base_dir}/{model_id}-{quant_cfg}-hqq/qmodel.pt"
+ state_dict = torch.load(file_path, map_location="cpu")
+
+ tensors = {}
+ metadata = {}
+ # walk thru the linear layers
+ # for each layer
+ # for each linear module
+ # load the original weight
+ # load the quantized weight and dequantized
+ # save the two matrix into a combined safetensors
+ for layer in range(layers):
+ matricies = [
+ f"model.layers.{layer}.mlp.down_proj",
+ f"model.layers.{layer}.mlp.gate_proj",
+ f"model.layers.{layer}.mlp.up_proj",
+ f"model.layers.{layer}.self_attn.k_proj",
+ f"model.layers.{layer}.self_attn.o_proj",
+ f"model.layers.{layer}.self_attn.q_proj",
+ f"model.layers.{layer}.self_attn.v_proj",
+ ]
+ for matrix in matricies:
+ wq, quant_cfg = restore_weight(matrix, state_dict)
+ if wq is None:
+ # skip unquantized matrix
+ continue
+ wo = load_weight(matrix, base_dir)
+ tensors[f"{matrix}.weight"] = wo
+ tensors[f"{matrix}.qweight"] = wq
+ metadata[f"{matrix}.quant_cfg.b1"] = str(quant_cfg["b1"])
+ metadata[f"{matrix}.quant_cfg.b2"] = str(quant_cfg["b2"])
+ metadata[f"{matrix}.quant_cfg.g1"] = str(quant_cfg["g1"])
+ metadata[f"{matrix}.quant_cfg.g2"] = str(quant_cfg["g2"])
+
+ output_fp = f"{output_dir}/{model_id}-cmp.safetensors"
+ safe_save(tensors, output_fp, metadata=metadata)
+
+
+def compare_pair(model_id, layers, output_dir):
+ st_file = f"{output_dir}/{model_id}-cmp.safetensors"
+ for layer in range(layers):
+ matricies = [
+ f"model.layers.{layer}.mlp.down_proj",
+ f"model.layers.{layer}.mlp.gate_proj",
+ f"model.layers.{layer}.mlp.up_proj",
+ f"model.layers.{layer}.self_attn.k_proj",
+ f"model.layers.{layer}.self_attn.o_proj",
+ f"model.layers.{layer}.self_attn.q_proj",
+ f"model.layers.{layer}.self_attn.v_proj",
+ ]
+ for matrix in matricies:
+ wo, wq = load_weight_dual(matrix, output_dir, st_file)
+ diff = torch.norm(wo - wq).item()
+ kurt_peason = kurtosis(
+ wo.numpy(), axis=None, fisher=False, bias=True, nan_policy="omit"
+ )
+ kurt_fisher = kurtosis(
+ wo.numpy(), axis=None, fisher=True, bias=True, nan_policy="omit"
+ )
+ # print(f"{matrix} FNorm Diff: {diff:.5f} Kurtosis: {kurt:.2f}")
+ print(f"{matrix},{diff:.5f},{kurt_fisher:.3f},{kurt_peason:.3f}")
+
+
+def is_linear_module(key):
+ self_attns = ["q_proj", "v_proj", "k_proj", "o_proj"]
+ mlps = ["gate_proj", "up_proj", "down_proj"]
+ modules = self_attns + mlps
+ for module in modules:
+ if module in key:
+ return True
+ return False
+
+
+def extract_quant_config(base_dir, model_id, config, algo="hqq"):
+ file_path = f"{base_dir}/{model_id}-{config}-{algo}/qmodel.pt"
+ dikt = torch.load(file_path, map_location="cpu")
+ quant_configs = {}
+ mem_fp16_all_total = 0
+ mem_all_total = 0
+ mem_quant_total = 0
+ param_quant_total = 0
+ param_all_total = 0
+ # search quantized linear module with meta
+ for key in dikt.keys():
+ m_dikt = dikt[key]
+ if is_linear_module(key):
+ if "meta" in m_dikt:
+ meta_dict = m_dikt["meta"]
+ meta_scale_dict = meta_dict.get("meta_scale", None)
+ shape = meta_dict["shape"]
+ b1 = meta_dict["nbits"]
+ g1 = meta_dict["group_size"]
+ b2 = meta_scale_dict["nbits"] if meta_scale_dict else 8
+ g2 = meta_scale_dict["group_size"] if meta_scale_dict else 128
+ param_count = shape[0] * shape[1]
+ param_quant_total += param_count
+ param_all_total += param_count
+ memmb = (
+ (b1 + 2 * b2 / g1 + 32 / (g1 * g2)) * param_count / 8 / 1024 / 1024
+ )
+ mem_fp16_all_total += param_count * 2 / 1024 / 1024
+ mem_quant_total += memmb
+ mem_all_total += memmb
+ quant_configs[key] = {
+ "b1": b1,
+ "g1": g1,
+ "b2": b2,
+ "g2": g2,
+ "memmb": memmb,
+ "param_cnt": param_count,
+ }
+ else:
+ w = m_dikt["weight"]
+ mem_all_total += w.numel() * 2 / 1024 / 1024
+ mem_fp16_all_total += w.numel() * 2 / 1024 / 1024
+ param_all_total += w.numel()
+ return (
+ quant_configs,
+ mem_quant_total,
+ mem_all_total,
+ mem_fp16_all_total,
+ param_quant_total,
+ param_all_total,
+ )
+
+
+def get_mem_usage_df(model_ids, confs, base_dir):
+ dikts = []
+ for model_id in model_ids:
+ for conf in confs:
+ configs, mem_quant_total, mem_all_total, mem_fp16_all_total, _, _ = (
+ extract_quant_config(base_dir, model_id, conf)
+ )
+ dikt = {
+ "model": model_id.split("/")[1],
+ "config": conf,
+ "mem_quant_total": mem_quant_total,
+ "mem_all_total": mem_all_total,
+ "mem_fp16_all_total": mem_fp16_all_total,
+ }
+ dikts.append(dikt)
+ df = pd.DataFrame(dikts)
+ return df
+
+
+def dump_quant_allocation(
+ quant_dir, model_ids, confs, csv_fp="mxq-allot.csv", attempts=None, algo="mxq"
+):
+ dikt = []
+ pat = re.compile(r"model\.layers\.(\d+)\.(.+)")
+ pat2 = re.compile(r"b(\d+)g(\d+)")
+ loops = ["mxq1"] if attempts is None else attempts
+ for attempt in loops:
+ snapshot_dir = f"{quant_dir}/{attempt}"
+ for model_id in model_ids:
+ for conf in confs:
+ m2 = re.match(pat2, conf)
+ if m2:
+ b1 = int(m2.group(1))
+ g1 = int(m2.group(2))
+ bit_budget = round(calc_bits(b1, g1), 2)
+ else:
+ bit_budget = conf.replace("_", ".")
+
+ (
+ configs,
+ mem_quant_total,
+ mem_all_total,
+ mem_fp16_all_total,
+ param_quant_total,
+ param_all_total,
+ ) = extract_quant_config(snapshot_dir, model_id, conf, algo=algo)
+ for key, val in configs.items():
+ matcher = re.match(pat, key)
+ if matcher:
+ layer = matcher.group(1)
+ module = matcher.group(2)
+ val["model"] = model_id.split("/")[1]
+ val["layer"] = layer
+ val["module"] = module
+ val["bit_budget"] = bit_budget
+ val["params_quant_tot"] = param_quant_total
+ val["params_all_tot"] = param_all_total
+ dikt.append(val)
+ columns = [
+ "model",
+ "module",
+ "layer",
+ "memmb",
+ "param_cnt",
+ "bit_budget",
+ "b1",
+ "g1",
+ "b2",
+ "g2",
+ "params_quant_tot",
+ "params_all_tot",
+ ]
+ df = pd.DataFrame(dikt)
+ df.to_csv(csv_fp, index=False, columns=columns)
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/prep/__init__.py b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/__init__.cpython-310.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b7de98eb89df052ae2e74fedd15a71202546b0a1
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/__init__.cpython-310.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/__init__.cpython-311.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8a27c14dd7e61d12e5601a4a512970796a8f3fb1
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/__init__.cpython-311.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/fnorm.cpython-310.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/fnorm.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6cbd995889f3b26dce48d57fa3fb0d94535bdd8e
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/fnorm.cpython-310.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/fnorm.cpython-311.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/fnorm.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0f73a33712fd861e1209267c78f87e5b3fbea9c4
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/fnorm.cpython-311.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/sensitivity.cpython-311.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/sensitivity.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9dc54d57a77c639b2bcd92482632fda1e1916e00
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/sensitivity.cpython-311.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/wdist.cpython-311.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/wdist.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2c095618e785a3d0be4c7c94eb26d7262741decf
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/prep/__pycache__/wdist.cpython-311.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/prep/fnorm.py b/lm-quant-toolkit/src/lm_quant_toolkit/prep/fnorm.py
new file mode 100644
index 0000000000000000000000000000000000000000..75d7abd215d4339a0ca5ac9444cf30027417b406
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/prep/fnorm.py
@@ -0,0 +1,298 @@
+import os
+from timeit import default_timer as timer
+
+import pandas as pd
+import torch
+from hqq.core.quantize import Quantizer as hQuant
+
+from lm_quant_toolkit.utils.hub import (
+ LLAMA_MODELS,
+ VIT_OPENCLIP_MODELS,
+ get_hf_model_storge_base_dir,
+)
+from lm_quant_toolkit.utils.safetensors import get_tensor
+
+
+def quant_hqq(tensor, nbits, group_size=64, optimize=True):
+ wq, meta = hQuant.quantize(
+ tensor, nbits=nbits, group_size=group_size, optimize=optimize
+ )
+ return hQuant.dequantize(wq, meta)
+
+
+def calc_fnorm_vit(
+ state_dict,
+ model_type,
+ prefix,
+ layer,
+ module,
+ suffix,
+ nbits1,
+ gsizes1,
+ nbits2,
+ gsizes2,
+):
+ dikts = []
+ matrix_name = f"{prefix}.{layer}.{module}.{suffix}"
+ w = state_dict[matrix_name]
+ params = w.numel()
+ for nbit1 in nbits1:
+ for gsize1 in gsizes1:
+ for nbit2 in nbits2:
+ for gsize2 in gsizes2:
+ wq_hqq = quant_hqq(w, nbits=nbit1, group_size=gsize1, optimize=True)
+ norm_hqq = torch.norm(w - wq_hqq).item()
+ bpp = nbit1 + 2 * nbit2 / gsize1 + 32 / (gsize1 * gsize2)
+ memmb = bpp * params / 8 / (1024**2)
+ dikt = {
+ "layer": layer,
+ "module": f"{model_type}.{module}",
+ "nbit1": nbit1,
+ "gsize1": gsize1,
+ "nbit2": nbit2,
+ "gsize2": gsize2,
+ "fnorm": norm_hqq,
+ "memmb": memmb,
+ "params": params,
+ }
+ dikts.append(dikt)
+ return dikts
+
+
+def calc_fnorm(
+ base_dir, prefix, layer, module, suffix, nbits1, gsizes1, nbits2, gsizes2
+):
+ dikts = []
+ matrix_name = f"{prefix}.{layer}.{module}.{suffix}"
+ w = get_tensor(matrix_name, base_dir)
+ params = w.numel()
+ for nbit1 in nbits1:
+ for gsize1 in gsizes1:
+ for nbit2 in nbits2:
+ for gsize2 in gsizes2:
+ wq_hqq = quant_hqq(w, nbits=nbit1, group_size=gsize1, optimize=True)
+ norm_hqq = torch.norm(w - wq_hqq).item()
+ bpp = nbit1 + 2 * nbit2 / gsize1 + 32 / (gsize1 * gsize2)
+ memmb = bpp * params / 8 / (1024**2)
+ dikt = {
+ "layer": layer,
+ "module": module,
+ "nbit1": nbit1,
+ "gsize1": gsize1,
+ "nbit2": nbit2,
+ "gsize2": gsize2,
+ "fnorm": norm_hqq,
+ "memmb": memmb,
+ "params": params,
+ }
+ dikts.append(dikt)
+ return dikts
+
+
+def calc_fnorm_for_vit_model(model_id, base_dir, layer_cfg):
+ # self_attns = ["out_proj"]
+ self_attns = []
+ mlps = ["c_fc", "c_proj"]
+ nbits1 = [2, 3, 4, 8]
+ gsizes1 = [32, 64, 128]
+ nbits2 = [8]
+ gsizes2 = [128]
+ prefixes = [
+ "visual.transformer.resblocks",
+ "transformer.resblocks",
+ ]
+ suffix = "weight"
+ dikts = []
+ state_dict = torch.load(
+ os.path.join(base_dir, "open_clip_pytorch_model.bin"),
+ weights_only=True,
+ )
+ for prefix in prefixes:
+ if "visual" in prefix:
+ model_type = "vision"
+ layers = layer_cfg["vlayers"]
+ else:
+ model_type = "text"
+ layers = layer_cfg["tlayers"]
+ for layer in range(layers):
+ for attn in self_attns:
+ ds = calc_fnorm_vit(
+ state_dict,
+ model_type,
+ prefix,
+ layer,
+ f"attn.{attn}",
+ suffix,
+ nbits1,
+ gsizes1,
+ nbits2,
+ gsizes2,
+ )
+ dikts.extend(ds)
+ for mlp in mlps:
+ ds = calc_fnorm_vit(
+ state_dict,
+ model_type,
+ prefix,
+ layer,
+ f"mlp.{mlp}",
+ suffix,
+ nbits1,
+ gsizes1,
+ nbits2,
+ gsizes2,
+ )
+ dikts.extend(ds)
+ df = pd.DataFrame(dikts)
+ file_name = f"data/fnorm-{model_id.split('/')[1]}.csv"
+ df.to_csv(
+ file_name,
+ columns=[
+ "layer",
+ "module",
+ "nbit1",
+ "gsize1",
+ "nbit2",
+ "gsize2",
+ "fnorm",
+ "memmb",
+ "params",
+ ],
+ index=False,
+ )
+
+
+def calc_fnorm_for_model(model_id, base_dir, layers, output_dir="data"):
+ self_attns = ["q_proj", "v_proj", "k_proj", "o_proj"]
+ mlps = ["gate_proj", "up_proj", "down_proj"]
+ nbits1 = [2, 3, 4, 8]
+ gsizes1 = [32, 64, 128]
+ nbits2 = [8]
+ gsizes2 = [128]
+ prefix = "model.layers"
+ suffix = "weight"
+ dikts = []
+ for layer in range(layers):
+ for attn in self_attns:
+ ds = calc_fnorm(
+ base_dir,
+ prefix,
+ layer,
+ f"self_attn.{attn}",
+ suffix,
+ nbits1,
+ gsizes1,
+ nbits2,
+ gsizes2,
+ )
+ dikts.extend(ds)
+ for mlp in mlps:
+ ds = calc_fnorm(
+ base_dir,
+ prefix,
+ layer,
+ f"mlp.{mlp}",
+ suffix,
+ nbits1,
+ gsizes1,
+ nbits2,
+ gsizes2,
+ )
+ dikts.extend(ds)
+
+ df = pd.DataFrame(dikts)
+ file_name = f"{output_dir}/fnorm-{model_id.split('/')[1]}.csv"
+ df.to_csv(
+ file_name,
+ columns=[
+ "layer",
+ "module",
+ "nbit1",
+ "gsize1",
+ "nbit2",
+ "gsize2",
+ "fnorm",
+ "memmb",
+ "params",
+ ],
+ index=False,
+ )
+
+
+def main_vit():
+ for model_id, layer_cfg in VIT_OPENCLIP_MODELS.items():
+ t1 = timer()
+ base_dir = layer_cfg.get("base_dir", None)
+ model_base_dir = get_hf_model_storge_base_dir(model_id, base_dir)
+ calc_fnorm_for_vit_model(model_id, model_base_dir, layer_cfg)
+ t2 = timer()
+ print(f"Finished {model_id} metrics calc in {t2 - t1} seconds")
+
+
+def main():
+ for model_id, model in LLAMA_MODELS.items():
+ if model_id != "meta-llama/Llama-3.1-8B":
+ continue
+ t1 = timer()
+ base_dir = model.get("base_dir", None)
+ model_base_dir = get_hf_model_storge_base_dir(model_id, base_dir)
+ calc_fnorm_for_model(model_id, model_base_dir, model["layers"])
+ t2 = timer()
+ print(f"Finished {model_id} metrics calc in {t2 - t1} seconds")
+
+
+def join_kurtosis():
+ for model_id, model in LLAMA_MODELS.items():
+ exp = model.get("experiment", False)
+ if not exp or model_id == "meta-llama/Llama-3.1-8B":
+ continue
+ t1 = timer()
+ model_short_id = model_id.split("/")[1]
+ df_fnorm = pd.read_csv(f"data/fnorm-{model_short_id}.csv")
+ df_wdist = pd.read_csv(f"data/wdist-{model_short_id}.csv")
+ # calculate scaled kurtosis
+ df_kurt_agg = df_wdist.groupby("module").agg(
+ kurt_max=pd.NamedAgg(column="kurtosis", aggfunc="max"),
+ kurt_min=pd.NamedAgg(column="kurtosis", aggfunc="min"),
+ )
+ df_wdist = df_wdist.merge(df_kurt_agg, how="left", on="module")
+ df_wdist["kurtosis_scaled"] = (df_wdist["kurtosis"] - df_wdist["kurt_min"]) / (
+ df_wdist["kurt_max"] - df_wdist["kurt_min"]
+ )
+ df_fnorm = df_fnorm[
+ [
+ "layer",
+ "module",
+ "nbit1",
+ "gsize1",
+ "nbit2",
+ "gsize2",
+ "fnorm",
+ "memmb",
+ "params",
+ ]
+ ]
+ df_fnorm = pd.merge(df_fnorm, df_wdist, how="inner", on=["module", "layer"])
+ df_fnorm = df_fnorm[
+ [
+ "layer",
+ "module",
+ "nbit1",
+ "gsize1",
+ "nbit2",
+ "gsize2",
+ "fnorm",
+ "memmb",
+ "params",
+ "kurtosis",
+ "kurtosis_scaled",
+ ]
+ ]
+
+ df_fnorm.to_csv(f"fnorm-{model_short_id}.csv", index=False)
+ t2 = timer()
+ print(f"Finished {model_id} Kurtosis data join in {t2 - t1} seconds")
+
+
+if __name__ == "__main__":
+ join_kurtosis()
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/prep/sensitivity.py b/lm-quant-toolkit/src/lm_quant_toolkit/prep/sensitivity.py
new file mode 100644
index 0000000000000000000000000000000000000000..63e886df292c1b9bdc4441447a327dae211a16fe
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/prep/sensitivity.py
@@ -0,0 +1,453 @@
+import functools
+import gc
+import inspect
+import os
+import re
+import time
+from collections import defaultdict
+from typing import List
+
+import pandas as pd
+import torch
+import torch.nn as nn
+from bitsandbytes.functional import dequantize_nf4
+from bitsandbytes.nn import Params4bit
+from datasets import load_dataset
+from hqq.core.quantize import Quantizer as hQuant
+from tqdm import tqdm
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers.models.llama.modeling_llama import LlamaDecoderLayer
+
+
+# nbits is defined as placeholder to be consistent with other quant methods
+def quant_nf4(tensor, nbits=4, group_size=64):
+ qw = Params4bit(tensor, blocksize=group_size, quant_type="nf4").to(tensor.device)
+ return dequantize_nf4(qw.data, qw.quant_state)
+
+
+def quant_hqq(tensor, nbits, group_size=64, optimize=True):
+ wq, meta = hQuant.quantize(
+ tensor, nbits=nbits, group_size=group_size, optimize=optimize
+ )
+ return hQuant.dequantize(wq, meta)
+
+
+def get_named_linears(module):
+ return {name: m for name, m in module.named_modules() if isinstance(m, nn.Linear)}
+
+
+def get_layers_for_scaling(module: LlamaDecoderLayer, input_feat, module_kwargs):
+ layers = []
+
+ # attention input
+ layers.append(
+ dict(
+ part="attn_in",
+ prev_op=module.input_layernorm,
+ layers=[
+ module.self_attn.q_proj,
+ module.self_attn.k_proj,
+ module.self_attn.v_proj,
+ ],
+ inp=input_feat["self_attn.q_proj"],
+ module2inspect=module.self_attn,
+ kwargs=module_kwargs,
+ )
+ )
+
+ # attention out
+ # Please refer to https://github.com/mit-han-lab/llm-awq/pull/67#issue-1850622696
+ if module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape:
+ layers.append(
+ dict(
+ part="attn_out",
+ prev_op=module.self_attn.v_proj,
+ layers=[module.self_attn.o_proj],
+ inp=input_feat["self_attn.o_proj"],
+ )
+ )
+
+ # linear 1
+ layers.append(
+ dict(
+ part="mlp_gate",
+ prev_op=module.post_attention_layernorm,
+ layers=[module.mlp.gate_proj, module.mlp.up_proj],
+ inp=input_feat["mlp.gate_proj"],
+ module2inspect=module.mlp,
+ )
+ )
+
+ # linear 2
+ layers.append(
+ dict(
+ part="mlp_down",
+ prev_op=module.mlp.up_proj,
+ layers=[module.mlp.down_proj],
+ inp=input_feat["mlp.down_proj"],
+ )
+ )
+
+ return layers
+
+
+def clear_memory(weight=None):
+ if weight is not None:
+ del weight
+ gc.collect()
+ torch.cuda.empty_cache()
+
+
+def get_best_device(idx=None):
+ if os.environ.get("USE_CPU_FOR_SENSITIVITY", None) == "1":
+ return "cpu"
+ if torch.cuda.is_available():
+ if idx is None:
+ return "cuda:0"
+ else:
+ return "cuda:" + str(idx % torch.cuda.device_count())
+ elif torch.backends.mps.is_available():
+ return "mps"
+ else:
+ return "cpu"
+
+
+def get_calib_dataset(
+ data: str = "pileval",
+ tokenizer=None,
+ n_samples=512,
+ block_size=512,
+ split="train",
+ text_column="text",
+):
+ if data == "pileval":
+ dataset = load_dataset("mit-han-lab/pile-val-backup", split="validation")
+ elif data == "wikitext":
+ dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="validation")
+ elif data == "bos":
+ dataset = load_dataset("schnell18/branch-of-science", split="train")
+ elif data == "c4":
+ dataset = load_dataset(
+ "allenai/c4",
+ data_files={"validation": "en/c4-validation.00000-of-00008.json.gz"},
+ split="validation",
+ download_mode="reuse_dataset_if_exists",
+ )
+ else:
+ dataset = load_dataset(data, split=split)
+
+ dataset = dataset.shuffle(seed=42)
+
+ samples = []
+ n_run = 0
+ for data in dataset:
+ if isinstance(data, list):
+ line_encoded = data
+ else:
+ line = data[text_column]
+ line = line.strip()
+ line_encoded = tokenizer.encode(line)
+ if len(line_encoded) > 512:
+ continue
+ sample = torch.tensor([line_encoded])
+ if sample.numel() == 0:
+ continue
+ samples.append(sample)
+ n_run += 1
+ if n_run == n_samples:
+ break
+ # now concatenate all samples and split according to block size
+ cat_samples = torch.cat(samples, dim=1)
+ n_split = cat_samples.shape[1] // block_size
+ return [
+ cat_samples[:, i * block_size : (i + 1) * block_size] for i in range(n_split)
+ ]
+
+
+class SensitiveLayerFinder:
+ def __init__(
+ self,
+ model,
+ model_name,
+ w_bit,
+ group_size,
+ tokenizer,
+ calib_data="pileval",
+ split="train",
+ text_column="text",
+ quant_method="hqq",
+ ) -> None:
+ self.model = model
+ self.model_name = model_name
+ self.tokenizer = tokenizer
+ self.w_bit = w_bit
+ self.group_size = group_size
+ self.calib_data = calib_data
+ self.split = split
+ self.text_column = text_column
+ self.quant_method = quant_method
+ self.layers, self.module_kwargs, self.inps = self.init_quant()
+ if self.quant_method == "rtn":
+ self.quant_func = functools.partial(quant_hqq, optimize=False)
+ elif self.quant_method == "hqq":
+ self.quant_func = functools.partial(quant_hqq, optimize=True)
+ elif self.quant_method == "bnb":
+ self.quant_func = quant_nf4
+
+ @torch.no_grad()
+ def measure(self, csv_fp):
+ dikts = []
+ cfg = f"b{self.w_bit}g{self.group_size}"
+ for i in tqdm(
+ range(len(self.layers)), desc=f"{self.model_name}-{cfg}-{self.calib_data}"
+ ):
+ # Move module and inputs to correct device
+ common_device = next(self.layers[i].parameters()).device
+ if common_device is None or str(common_device) == "cpu":
+ best_device = get_best_device(i)
+ self.layers[i] = self.layers[i].to(best_device)
+ common_device = next(self.layers[i].parameters()).device
+
+ if self.module_kwargs.get("position_ids") is not None:
+ self.module_kwargs["position_ids"] = self.module_kwargs[
+ "position_ids"
+ ].to(common_device)
+
+ if self.module_kwargs.get("attention_mask") is not None:
+ self.module_kwargs["attention_mask"] = self.module_kwargs[
+ "attention_mask"
+ ].to(common_device)
+
+ self.inps = self.inps.to(common_device)
+ named_linears = get_named_linears(self.layers[i])
+ input_feat = self._get_input_feat(self.layers[i], named_linears)
+ clear_memory()
+
+ module_config = get_layers_for_scaling(
+ self.layers[i], input_feat, self.module_kwargs
+ )
+
+ for layer in module_config:
+ part = layer.pop("part", "Unknown")
+ mse = self._measure_layer_sensitivity(self.layers[i], **layer)
+ dikts.append(
+ {
+ "dataset": self.calib_data,
+ "part": part,
+ "model": self.model_name,
+ "nbits": self.w_bit,
+ "group_size": self.group_size,
+ "layer": i,
+ "sensitivity": mse,
+ }
+ )
+
+ del module_config
+ del input_feat
+ clear_memory()
+ return dikts
+
+ def _measure_layer_sensitivity(
+ self,
+ module,
+ prev_op,
+ layers: List[nn.Linear],
+ inp: torch.Tensor,
+ module2inspect=None,
+ kwargs={},
+ ):
+ if module2inspect is None:
+ assert len(layers) == 1
+ module2inspect = layers[0]
+
+ # Put x on the right device
+ inp = inp.to(next(module2inspect.parameters()).device)
+
+ module_kwargs = self._sanitize_kwargs(kwargs, module2inspect)
+
+ fp16_output = module2inspect(inp, **module_kwargs)
+ if isinstance(fp16_output, tuple):
+ fp16_output = fp16_output[0]
+
+ # Quantize the weights
+ for fc in layers:
+ # call quantization function
+ fc.weight.data = self.quant_func(
+ fc.weight.data, self.w_bit, self.group_size
+ )
+
+ # W * X
+ int_w_output = module2inspect(inp, **module_kwargs)
+ if isinstance(int_w_output, tuple):
+ int_w_output = int_w_output[0]
+
+ # compute mean squared error (L2 norm)
+ mse = (fp16_output - int_w_output).float().pow(2).mean().item()
+ del fp16_output
+ del int_w_output
+ clear_memory()
+ return mse
+
+ def init_quant(self, n_samples=128, seqlen=512):
+ modules = self.model.model.layers
+ samples = get_calib_dataset(
+ data=self.calib_data,
+ tokenizer=self.tokenizer,
+ n_samples=n_samples,
+ block_size=seqlen,
+ split=self.split,
+ text_column=self.text_column,
+ )
+ samples = torch.cat(samples, dim=0)
+
+ inps = []
+ layer_kwargs = {}
+
+ best_device = get_best_device()
+ modules[0] = modules[0].to(best_device)
+ self.model.model.embed_tokens = self.model.model.embed_tokens.to(best_device)
+
+ # get input and kwargs to layer 0
+ # with_kwargs is only supported in PyTorch 2.0
+ # use this Catcher hack for now
+ class Catcher(nn.Module):
+ def __init__(self, module):
+ super().__init__()
+ self.module = module
+
+ def forward(self, *args, **kwargs):
+ # assume first input to forward is hidden states
+ if len(args) > 0:
+ hidden_states = args[0]
+ del args
+ else:
+ first_key = list(kwargs.keys())[0]
+ hidden_states = kwargs.pop(first_key)
+
+ inps.append(hidden_states)
+ layer_kwargs.update(kwargs)
+ raise ValueError # early exit to break later inference
+
+ # patch layer 0 to catch input and kwargs
+ modules[0] = Catcher(modules[0])
+ try:
+ self.model(samples.to(next(self.model.parameters()).device))
+ except ValueError: # work with early exit
+ pass
+ modules[0] = modules[0].module # restore
+
+ # Update the layer kwargs with `prepare_inputs_for_generation` method
+ # that takes care of everything to avoid unexpected errors.
+ layer_kwargs = self.model.prepare_inputs_for_generation(samples, **layer_kwargs)
+ # Pop the input_ids as they are not needed at all.
+ layer_kwargs.pop("input_ids")
+
+ del samples
+ inps = inps[0]
+
+ modules[0] = modules[0].cpu()
+ self.model.model.embed_tokens = self.model.model.embed_tokens.to("cpu")
+
+ clear_memory()
+
+ if layer_kwargs.get("attention_mask") is not None:
+ layer_kwargs["attention_mask"] = layer_kwargs["attention_mask"].to(
+ best_device
+ )
+
+ return modules, layer_kwargs, inps
+
+ def _get_input_feat(self, layer, named_linears):
+ # firstly, get input features of all linear layers
+ def cache_input_hook(m, x, y, name, feat_dict):
+ x = x[0]
+ x = x.detach().cpu()
+ feat_dict[name].append(x)
+
+ input_feat = defaultdict(list)
+ handles = []
+
+ for name in named_linears:
+ handles.append(
+ named_linears[name].register_forward_hook(
+ functools.partial(cache_input_hook, name=name, feat_dict=input_feat)
+ )
+ )
+ self.inps = self.inps.to(next(layer.parameters()).device) # in case multi-gpu
+ # get output as next layer's input
+
+ # Sanitize the kwargs in case we use transformers version that contains
+ # kwargs that are not handled by the module.
+ # Useful for trust_remote_code models.
+ module_kwargs = self._sanitize_kwargs(self.module_kwargs, layer)
+
+ self.inps = layer(self.inps, **module_kwargs)[0]
+ for h in handles:
+ h.remove()
+ # now solve for scaling and clipping
+ input_feat = {k: torch.cat(v, dim=0) for k, v in input_feat.items()}
+
+ return input_feat
+
+ def _sanitize_kwargs(self, inputs_kwargs, module):
+ """
+ Remove the arguments that are not supported in the module's
+ forward pass to avoid breaking behaviour between different versions
+ of transformers.
+
+ Args:
+ inputs_kwargs (`dict`):
+ The input dictionary to pass to the model layer
+ module (`torch.nn.Module`):
+ Target module to quantize.
+ """
+ module_signature = inspect.signature(module.forward).parameters
+ sanitized_kwargs = {}
+ for k, v in inputs_kwargs.items():
+ if k in module_signature:
+ sanitized_kwargs[k] = v
+ return sanitized_kwargs
+
+
+def measure_sensitivity(models, quant_method, cfgs, calib_datasets, csv_fp):
+ pat = re.compile(r"b(\d)g(\d+)")
+ bgs = []
+ for cfg in cfgs:
+ m = re.match(pat, cfg)
+ if m:
+ bgs.append((int(m.group(1)), int(m.group(2))))
+ dikts = []
+ for ds in calib_datasets:
+ for bg in bgs:
+ for model_path in models:
+ short_name = model_path.split("/")[-1]
+ model = AutoModelForCausalLM.from_pretrained(
+ model_path,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ offload_state_dict=False,
+ max_memory={0: "18GiB", "cpu": "60GiB"},
+ )
+ tokenizer = AutoTokenizer.from_pretrained(model_path, legacy=False)
+ finder = SensitiveLayerFinder(
+ model,
+ short_name,
+ bg[0],
+ bg[1],
+ tokenizer,
+ ds,
+ quant_method=quant_method,
+ )
+ dikts.extend(finder.measure(csv_fp))
+ clear_memory()
+ time.sleep(2)
+
+ df = pd.DataFrame(dikts)
+ df.to_csv(csv_fp, index=False)
+
+
+# with profile(
+# activities=[ProfilerActivity.CPU], profile_memory=True, record_shapes=True
+# ) as prof:
+# finder.identify(csv_fp)
+# print(prof.key_averages().table(sort_by="self_cpu_memory_usage", row_limit=10))
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/prep/wdist.py b/lm-quant-toolkit/src/lm_quant_toolkit/prep/wdist.py
new file mode 100644
index 0000000000000000000000000000000000000000..08e2e34362958a335001bed8640de75a6333b5e9
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/prep/wdist.py
@@ -0,0 +1,292 @@
+import re
+
+import numpy as np
+import pandas as pd
+from scipy.stats import kurtosis
+
+from lm_quant_toolkit.utils.hub import (
+ LLAMA_MODELS,
+ VIT_OPENCLIP_MODELS,
+ get_hf_model_storge_base_dir,
+)
+from lm_quant_toolkit.utils.pickle import load_state_dict
+from lm_quant_toolkit.utils.safetensors import get_tensor
+
+
+def calculate_kurtosis_llm(model_id, base_dir, layers, output_dir):
+ modules = [
+ "self_attn.q_proj",
+ "self_attn.k_proj",
+ "self_attn.v_proj",
+ "self_attn.o_proj",
+ "mlp.gate_proj",
+ "mlp.down_proj",
+ "mlp.up_proj",
+ ]
+ dikts = []
+ for layer in range(layers):
+ for module in modules:
+ full_name = f"model.layers.{layer}.{module}.weight"
+ w = get_tensor(full_name, base_dir)
+ print(full_name, w.shape)
+ param_count = w.numel()
+ w = w.flatten().float().numpy()
+ kurt_pearson = kurtosis(
+ w, axis=None, fisher=False, bias=True, nan_policy="omit"
+ )
+ dikt = {
+ "module": module,
+ "layer": layer,
+ "param_count": param_count,
+ "kurtosis": kurt_pearson,
+ }
+ dikts.append(dikt)
+ df = pd.DataFrame(dikts)
+ short_id = model_id.split("/")[1]
+ csv_fp = f"{output_dir}/kurtosis-{short_id}.csv"
+ df.to_csv(csv_fp, index=False)
+
+
+def calculate_kurtosis_vit(model_id, model_cfg, output_dir):
+ modules = [
+ "mlp.c_fc",
+ "mlp.c_proj",
+ ]
+ dikts = []
+ state_dict = load_state_dict(model_id)
+ for layer_type, layers in model_cfg.items():
+ if layer_type == "vlayers":
+ model_type = "vision"
+ prefix = "visual.transformer"
+ else:
+ model_type = "text"
+ prefix = "transformer"
+ for i, module in enumerate(modules):
+ for layer in range(layers):
+ full_name = f"{prefix}.resblocks.{layer}.{module}.weight"
+ w = state_dict[full_name]
+ w = w.flatten().float().numpy()
+ kurt_pearson = kurtosis(
+ w, axis=None, fisher=False, bias=True, nan_policy="omit"
+ )
+ dikt = {
+ "module": f"{model_type}.{module}",
+ "layer": layer,
+ "kurtosis": kurt_pearson,
+ }
+ dikts.append(dikt)
+ df = pd.DataFrame(dikts)
+ short_id = model_id.split("/")[1]
+ csv_fp = f"{output_dir}/kurtosis-{short_id}.csv"
+ df.to_csv(csv_fp, index=False)
+
+
+def summarize_vit_percentiles(
+ model_id,
+ model_cfg,
+ csv_fp,
+ wt_file="open_clip_pytorch_model.bin",
+):
+ modules = [
+ "attn.in_proj_bias",
+ "attn.in_proj_weight",
+ "attn.out_proj.bias",
+ "attn.out_proj.weight",
+ "ln_1.bias",
+ "ln_1.weight",
+ "ln_2.bias",
+ "ln_2.weight",
+ "mlp.c_fc.bias",
+ "mlp.c_fc.weight",
+ "mlp.c_proj.bias",
+ "mlp.c_proj.weight",
+ ]
+ dikts = []
+ state_dict = load_state_dict(model_id)
+ for layer_type, layers in model_cfg.items():
+ if layer_type == "vlayers":
+ model_type = "vision"
+ prefix = "visual.transformer"
+ else:
+ model_type = "text"
+ prefix = "transformer"
+ for i, module in enumerate(modules):
+ for layer in range(layers):
+ full_name = f"{prefix}.resblocks.{layer}.{module}"
+ w = state_dict[full_name]
+ param_count = w.numel()
+ w = w.flatten().float().numpy()
+ percentiles = np.percentile(np.abs(w), [0, 99, 99.9, 99.99, 100])
+ kurt_pearson = kurtosis(
+ w, axis=None, fisher=False, bias=True, nan_policy="omit"
+ )
+ dikt = {
+ "type": model_type,
+ "module": module,
+ "param_count": param_count,
+ "layer": layer,
+ "percentile_0": percentiles[0],
+ "percentile_99": percentiles[1],
+ "percentile_999": percentiles[2],
+ "percentile_9999": percentiles[3],
+ "percentile_100": percentiles[4],
+ "kurtosis": kurt_pearson,
+ }
+ dikts.append(dikt)
+ df = pd.DataFrame(dikts)
+ df.to_csv(csv_fp, index=False)
+
+
+def summarize_llama_quantable_params(base_dir, layers, fp, llama2=True):
+ modules = {
+ "norm": {"layerwise": False, "quant": False},
+ "lm_head": {"layerwise": False, "quant": False, "prefix": ""},
+ "embed_tokens": {"layerwise": False, "quant": False},
+ "input_layernorm": {"layerwise": True, "quant": False},
+ "post_attention_layernorm": {"layerwise": True, "quant": False},
+ "mlp.down_proj": {"layerwise": True, "quant": True},
+ "mlp.gate_proj": {"layerwise": True, "quant": True},
+ "mlp.up_proj": {"layerwise": True, "quant": True},
+ "self_attn.k_proj": {"layerwise": True, "quant": True},
+ "self_attn.o_proj": {"layerwise": True, "quant": True},
+ "self_attn.q_proj": {"layerwise": True, "quant": True},
+ "self_attn.v_proj": {"layerwise": True, "quant": True},
+ }
+ if llama2:
+ modules["self_attn.rotary_emb"] = {
+ "layerwise": True,
+ "quant": False,
+ "suffix": "inv_freq",
+ }
+ dikts = []
+ for module, data in modules.items():
+ suffix = data.get("suffix", "weight")
+ if data["layerwise"]:
+ for layer in range(layers):
+ full_name = f"model.layers.{layer}.{module}.{suffix}"
+ w = get_tensor(full_name, base_dir)
+ param_count = w.numel()
+ dikt = {
+ "module": module,
+ "layer": layer,
+ "param_count": param_count,
+ "quant_count": param_count if data["quant"] else 0,
+ }
+ dikts.append(dikt)
+ else:
+ prefix = data.get("prefix", "model")
+ if prefix == "":
+ full_name = f"{module}.{suffix}"
+ else:
+ full_name = f"{prefix}.{module}.{suffix}"
+ w = get_tensor(full_name, base_dir)
+ param_count = w.numel()
+ dikt = {
+ "module": module,
+ "layer": 0,
+ "param_count": param_count,
+ "quant_count": param_count if data["quant"] else 0,
+ }
+ dikts.append(dikt)
+ df = pd.DataFrame(dikts)
+ df.to_csv(fp, index=False)
+
+
+def summarize_vit_quantable_params(
+ model_id,
+ csv_fp,
+ wt_file="open_clip_pytorch_model.bin",
+):
+ quantables = [
+ "mlp.c_fc.weight",
+ "mlp.c_proj.weight",
+ ]
+ dikts = []
+ state_dict = load_state_dict(model_id)
+ pat = r"(.*)\.resblocks\.(\d+)\.(.*)"
+ for key in state_dict:
+ m = re.match(pat, key)
+ if m is not None:
+ layer = int(m.group(2))
+ if "visual" in m.group(1):
+ type = "vision"
+ elif "transformer" in m.group(1):
+ type = "text"
+ else:
+ type = ""
+ module = m.group(3)
+ else:
+ layer = 0
+ module = key
+ if "visual" in key:
+ type = "vision"
+ elif "transformer" in key:
+ type = "text"
+ else:
+ type = ""
+ quantable = any([qnt in key for qnt in quantables])
+ w = state_dict[key]
+ param_count = w.numel()
+ dikt = {
+ "type": type,
+ "module": module,
+ "layer": layer,
+ "param_count": param_count,
+ "quant_count": param_count if quantable else 0,
+ }
+ dikts.append(dikt)
+ df = pd.DataFrame(dikts)
+ df.to_csv(csv_fp, index=False)
+
+
+def aggregate_quantable_parameters(models, base_dir="data", unit=1_000_000_000):
+ ret = {}
+ for model in models:
+ csv_file = f"{base_dir}/quantable-{model}.csv"
+ df = pd.read_csv(csv_file)
+ total = df["param_count"].sum()
+ quant = df["quant_count"].sum()
+ pct = quant / total
+ ret[model] = (total / unit, quant / unit, pct * 100)
+ return ret
+
+
+def summarize_known_llama_quantable_params(base_dir="data", skip_models=None):
+ for model_id, cfg in LLAMA_MODELS.items():
+ if skip_models is not None and len(skip_models) > 0:
+ if any([skip_model in model_id for skip_model in skip_models]):
+ continue
+ model_base_dir = get_hf_model_storge_base_dir(
+ model_id, cfg.get("base_dir", None)
+ )
+ layers = cfg["layers"]
+ llama2 = cfg.get("llama2", True)
+ csv_file = f"{base_dir}/quantable-{model_id.split('/')[1]}.csv"
+ summarize_llama_quantable_params(
+ model_base_dir, layers, csv_file, llama2=llama2
+ )
+
+
+def summarize_known_vit_quantable_params(base_dir="data", skip_models=None):
+ for model_id, cfg in VIT_OPENCLIP_MODELS.items():
+ if skip_models is not None and len(skip_models) > 0:
+ if any([skip_model in model_id for skip_model in skip_models]):
+ continue
+ csv_file = f"{base_dir}/quantable-{model_id.split('/')[1]}.csv"
+ summarize_vit_quantable_params(model_id, csv_file)
+
+
+def summarize_vit_wdist(base_dir="data"):
+ for model_id, cfg in VIT_OPENCLIP_MODELS.items():
+ csv_file = f"data/wdist-{model_id.split('/')[1]}.csv"
+ summarize_vit_percentiles(model_id, cfg, csv_file)
+
+
+def calculate_vit_kurtosis():
+ for model_id, cfg in VIT_OPENCLIP_MODELS.items():
+ calculate_kurtosis_vit(model_id, cfg, "data")
+
+
+if __name__ == "__main__":
+ calculate_vit_kurtosis()
+ # summarize_known_vit_quantable_params()
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/utils/__init__.py b/lm-quant-toolkit/src/lm_quant_toolkit/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/utils/__pycache__/__init__.cpython-311.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/utils/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4719b5af25b25ba91af707a2b3b53eeab288eac1
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/utils/__pycache__/__init__.cpython-311.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/utils/__pycache__/pickle.cpython-311.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/utils/__pycache__/pickle.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3801b2d732576f44d7e79369e342ac10eeebbb33
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/utils/__pycache__/pickle.cpython-311.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/utils/__pycache__/safetensors.cpython-311.pyc b/lm-quant-toolkit/src/lm_quant_toolkit/utils/__pycache__/safetensors.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2e72334dbb40cd95a5a935040b32db090e4fbe43
Binary files /dev/null and b/lm-quant-toolkit/src/lm_quant_toolkit/utils/__pycache__/safetensors.cpython-311.pyc differ
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/utils/hub.py b/lm-quant-toolkit/src/lm_quant_toolkit/utils/hub.py
new file mode 100644
index 0000000000000000000000000000000000000000..6cc7d8e095981662980911be81b89eb09345440d
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/utils/hub.py
@@ -0,0 +1,100 @@
+import os
+
+from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
+
+LLAMA_MODELS = {
+ "meta-llama/Llama-2-7b-hf": {
+ "layers": 32,
+ "experiment": True,
+ },
+ "meta-llama/Llama-2-13b-hf": {
+ "layers": 40,
+ "experiment": True,
+ },
+ "meta-llama/Meta-Llama-3-8B": {
+ "layers": 32,
+ "llama2": False,
+ "experiment": True,
+ },
+ "meta-llama/Llama-3.1-8B": {
+ "layers": 32,
+ "llama2": False,
+ "experiment": False,
+ },
+ "meta-llama/Meta-Llama-3-70B": {
+ "layers": 80,
+ "llama2": False,
+ "base_dir": "/data/hugginface/hub",
+ },
+ "meta-llama/Llama-2-70b-hf": {
+ "layers": 80,
+ "base_dir": "/data/hugginface/hub",
+ },
+ "meta-llama/Meta-Llama-3-70B-Instruct": {
+ "layers": 80,
+ "base_dir": "/data/hugginface/hub",
+ "llama2": False,
+ },
+ "meta-llama/Meta-Llama-3.1-405B-Instruct": {
+ "layers": 126,
+ "base_dir": "/data/hugginface/hub",
+ "llama2": False,
+ },
+ "../models/Llama-2-13b-hf": {
+ "layers": 40,
+ "base_dir": "../models/Llama-2-13b-hf",
+ "llama2": False,
+ },
+ "../models/Qwen3-8B": {
+ "layers": 36,
+ "base_dir": "../models/Qwen3-8B",
+ "llama2": False,
+ },
+ "../models/Qwen3-4B": {
+ "layers": 36,
+ "base_dir": "../models/Qwen3-4B",
+ "llama2": False,
+ },
+ "../models/Llama-3.2-3B-Instruct": {
+ "layers": 28,
+ "base_dir": "../models/Llama-3.2-3B-Instruct",
+ "llama2": False,
+ },
+ "../models/Mistral-7B-Instruct-v0.3": {
+ "layers": 32,
+ "base_dir": "../models/Mistral-7B-Instruct-v0.3",
+ "llama2": False,
+ },
+ "../models/Llama-3.1-70B-Instruct": {
+ "layers": 80,
+ "base_dir": "../models/Llama-3.1-70B-Instruct",
+ "llama2": False,
+ },
+
+}
+
+VIT_OPENCLIP_MODELS = {
+ "laion/CLIP-ViT-B-32-laion2B-s34B-b79K": {"vlayers": 12, "tlayers": 12},
+ "laion/CLIP-ViT-H-14-laion2B-s32B-b79K": {"vlayers": 32, "tlayers": 24},
+ "laion/CLIP-ViT-L-14-laion2B-s32B-b82K": {"vlayers": 24, "tlayers": 12},
+}
+
+SENSITIVITY_MODELS = [
+ "meta-llama/Llama-2-7b-hf",
+ "meta-llama/Llama-2-7b-chat-hf",
+ "epfl-llm/meditron-7b",
+ "meta-llama/Llama-3.1-8B",
+ "meta-llama/Llama-3.1-8B-Instruct",
+]
+
+
+def get_hf_model_storge_base_dir(model_id, hf_hub_dir=None):
+ # model_id_x = model_id.replace("/", "--")
+ # cache_dir = hf_hub_dir if hf_hub_dir else HUGGINGFACE_HUB_CACHE
+ # hf_model_dir = os.path.join(cache_dir, f"models--{model_id_x}")
+ # ref_main_fp = os.path.join(hf_model_dir, "refs", "main")
+ # with open(ref_main_fp) as fh:
+ # commit_sha = fh.read().strip()
+ # return os.path.join(hf_model_dir, "snapshots", commit_sha)
+ return os.path.join('../models', model_id.split("/")[-1])
+
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/utils/pickle.py b/lm-quant-toolkit/src/lm_quant_toolkit/utils/pickle.py
new file mode 100644
index 0000000000000000000000000000000000000000..67408bb639b04573c1d76af0156bae1d571b7140
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/utils/pickle.py
@@ -0,0 +1,18 @@
+import os
+
+import torch
+
+from .hub import get_hf_model_storge_base_dir
+
+
+def dump_vit_weight_names(model_id, wt_file="open_clip_pytorch_model.bin"):
+ base_dir = get_hf_model_storge_base_dir(model_id)
+ state_dict = torch.load(os.path.join(base_dir, wt_file), weights_only=True)
+ for name in state_dict:
+ print(name)
+
+
+def load_state_dict(model_id, wt_file="open_clip_pytorch_model.bin"):
+ base_dir = get_hf_model_storge_base_dir(model_id)
+ wt_file_fp = os.path.join(base_dir, wt_file)
+ return torch.load(wt_file_fp, weights_only=True)
diff --git a/lm-quant-toolkit/src/lm_quant_toolkit/utils/safetensors.py b/lm-quant-toolkit/src/lm_quant_toolkit/utils/safetensors.py
new file mode 100644
index 0000000000000000000000000000000000000000..118fded2ca2b6af3128aa16c669d906a66994705
--- /dev/null
+++ b/lm-quant-toolkit/src/lm_quant_toolkit/utils/safetensors.py
@@ -0,0 +1,44 @@
+import json
+import os
+
+from safetensors import safe_open
+
+
+def get_tensor(matrix_name, base_dir, index_json="model.safetensors.index.json"):
+ fp = os.path.join(base_dir, index_json)
+ with open(fp, "r") as fh:
+ index = json.load(fh)
+ try:
+ st_file = index["weight_map"][matrix_name]
+ mp = os.path.join(base_dir, st_file)
+ with safe_open(mp, framework="pt", device="cpu") as f:
+ print(f.keys())
+ return f.get_tensor(matrix_name)
+ except Exception:
+ raise ValueError(f"Invalid key {matrix_name}")
+
+
+def get_tensor2(matrix_name, st_file_fp):
+ try:
+ with safe_open(st_file_fp, framework="pt", device="cpu") as f:
+ return f.get_tensor(matrix_name)
+ except Exception:
+ raise ValueError("Fail to retrieve tensor keys")
+
+
+def get_tensor_keys(base_dir, st_file):
+ try:
+ mp = os.path.join(base_dir, st_file)
+ with safe_open(mp, framework="pt", device="cpu") as f:
+ return f.keys()
+ except Exception:
+ raise ValueError("Fail to retrieve tensor keys")
+
+
+def get_tensor_dual(matrix1, matrix2, base_dir, st_file):
+ try:
+ mp = os.path.join(base_dir, st_file)
+ with safe_open(mp, framework="pt", device="cpu") as f:
+ return f.get_tensor(matrix1), f.get_tensor(matrix2)
+ except Exception:
+ raise ValueError(f"Invalid key {matrix1}")
diff --git a/lm-quant-toolkit/tmp/kurtosis-dump/Llama-2-13b-hf/kurtosis-models.csv b/lm-quant-toolkit/tmp/kurtosis-dump/Llama-2-13b-hf/kurtosis-models.csv
new file mode 100644
index 0000000000000000000000000000000000000000..9a472a8af3c26849647308e117a086d101e83647
--- /dev/null
+++ b/lm-quant-toolkit/tmp/kurtosis-dump/Llama-2-13b-hf/kurtosis-models.csv
@@ -0,0 +1,281 @@
+module,layer,param_count,kurtosis
+self_attn.q_proj,0,26214400,130.83713963078688
+self_attn.k_proj,0,26214400,64.52939795597361
+self_attn.v_proj,0,26214400,6.886821964622624
+self_attn.o_proj,0,26214400,27.589738715545355
+mlp.gate_proj,0,70778880,6.313392220689957
+mlp.down_proj,0,70778880,14.052663070569466
+mlp.up_proj,0,70778880,3.249017058311768
+self_attn.q_proj,1,26214400,13.08326186837005
+self_attn.k_proj,1,26214400,11.090631672369472
+self_attn.v_proj,1,26214400,4.5346238983368945
+self_attn.o_proj,1,26214400,12.902235283117223
+mlp.gate_proj,1,70778880,3.722426098049931
+mlp.down_proj,1,70778880,6.31968660475201
+mlp.up_proj,1,70778880,3.1117683059926806
+self_attn.q_proj,2,26214400,7.525079013280852
+self_attn.k_proj,2,26214400,5.773346598858214
+self_attn.v_proj,2,26214400,3.2075231160174624
+self_attn.o_proj,2,26214400,5.9170687265908235
+mlp.gate_proj,2,70778880,3.2308532400372494
+mlp.down_proj,2,70778880,3.275261780947932
+mlp.up_proj,2,70778880,3.080849335928324
+self_attn.q_proj,3,26214400,8.390562250303343
+self_attn.k_proj,3,26214400,5.7891896210301095
+self_attn.v_proj,3,26214400,3.588921793712654
+self_attn.o_proj,3,26214400,47.64313079806701
+mlp.gate_proj,3,70778880,3.4501241375690457
+mlp.down_proj,3,70778880,9.500845348313662
+mlp.up_proj,3,70778880,3.2669610023884306
+self_attn.q_proj,4,26214400,4.4551653725123055
+self_attn.k_proj,4,26214400,4.573635068411043
+self_attn.v_proj,4,26214400,3.234219982779673
+self_attn.o_proj,4,26214400,4.84968615849468
+mlp.gate_proj,4,70778880,3.1955631490833403
+mlp.down_proj,4,70778880,3.1665068249588635
+mlp.up_proj,4,70778880,3.095126478043274
+self_attn.q_proj,5,26214400,4.949500834374297
+self_attn.k_proj,5,26214400,4.8544486363668495
+self_attn.v_proj,5,26214400,3.181611634064576
+self_attn.o_proj,5,26214400,4.859526520116928
+mlp.gate_proj,5,70778880,3.2403906037879353
+mlp.down_proj,5,70778880,3.2328986745567874
+mlp.up_proj,5,70778880,3.1187659555331035
+self_attn.q_proj,6,26214400,4.49736449180947
+self_attn.k_proj,6,26214400,4.544626995182409
+self_attn.v_proj,6,26214400,3.1369344191217348
+self_attn.o_proj,6,26214400,4.214262948729349
+mlp.gate_proj,6,70778880,3.2443207674755223
+mlp.down_proj,6,70778880,3.2297373677424512
+mlp.up_proj,6,70778880,3.108046386120236
+self_attn.q_proj,7,26214400,4.21756324679348
+self_attn.k_proj,7,26214400,4.362283595211108
+self_attn.v_proj,7,26214400,3.133032125857189
+self_attn.o_proj,7,26214400,4.71536300013941
+mlp.gate_proj,7,70778880,3.3936300161835935
+mlp.down_proj,7,70778880,4.681598217347249
+mlp.up_proj,7,70778880,3.1168342824347484
+self_attn.q_proj,8,26214400,4.2598895714391
+self_attn.k_proj,8,26214400,4.405596656467856
+self_attn.v_proj,8,26214400,3.1597576159760687
+self_attn.o_proj,8,26214400,5.974815297237448
+mlp.gate_proj,8,70778880,3.349456358964778
+mlp.down_proj,8,70778880,3.625380400230104
+mlp.up_proj,8,70778880,3.169466057423486
+self_attn.q_proj,9,26214400,4.848214892015664
+self_attn.k_proj,9,26214400,4.253390650850058
+self_attn.v_proj,9,26214400,3.204364379777786
+self_attn.o_proj,9,26214400,4.723760427466953
+mlp.gate_proj,9,70778880,3.3583231350160534
+mlp.down_proj,9,70778880,3.644571413205425
+mlp.up_proj,9,70778880,3.1575336738495885
+self_attn.q_proj,10,26214400,4.303988247000512
+self_attn.k_proj,10,26214400,4.424770246783603
+self_attn.v_proj,10,26214400,3.2424239952907317
+self_attn.o_proj,10,26214400,4.358131861560106
+mlp.gate_proj,10,70778880,3.3639723293555432
+mlp.down_proj,10,70778880,3.7832580141204364
+mlp.up_proj,10,70778880,3.152513533783804
+self_attn.q_proj,11,26214400,4.002172186447971
+self_attn.k_proj,11,26214400,4.13048267998045
+self_attn.v_proj,11,26214400,3.1947096505061916
+self_attn.o_proj,11,26214400,3.9787550573897312
+mlp.gate_proj,11,70778880,3.4940067849359866
+mlp.down_proj,11,70778880,3.5196599441868432
+mlp.up_proj,11,70778880,3.178196654496327
+self_attn.q_proj,12,26214400,4.006244550550894
+self_attn.k_proj,12,26214400,4.189627180144341
+self_attn.v_proj,12,26214400,3.196419036679571
+self_attn.o_proj,12,26214400,4.060156170436918
+mlp.gate_proj,12,70778880,3.521030510832614
+mlp.down_proj,12,70778880,3.6228070645744403
+mlp.up_proj,12,70778880,3.141626426727454
+self_attn.q_proj,13,26214400,4.4299117684795135
+self_attn.k_proj,13,26214400,4.447280781844841
+self_attn.v_proj,13,26214400,3.255516211444412
+self_attn.o_proj,13,26214400,4.708008374501372
+mlp.gate_proj,13,70778880,3.4971410209159255
+mlp.down_proj,13,70778880,3.3872878087348695
+mlp.up_proj,13,70778880,3.1080582609263234
+self_attn.q_proj,14,26214400,3.9606859057598074
+self_attn.k_proj,14,26214400,4.124556259461563
+self_attn.v_proj,14,26214400,3.17076299282502
+self_attn.o_proj,14,26214400,4.786041526677701
+mlp.gate_proj,14,70778880,3.397050603770907
+mlp.down_proj,14,70778880,3.3989561080039383
+mlp.up_proj,14,70778880,3.1233856872096504
+self_attn.q_proj,15,26214400,3.900748740594978
+self_attn.k_proj,15,26214400,4.1431620457492055
+self_attn.v_proj,15,26214400,3.210370125926212
+self_attn.o_proj,15,26214400,3.6813562849865957
+mlp.gate_proj,15,70778880,3.522446408932607
+mlp.down_proj,15,70778880,3.367632157102179
+mlp.up_proj,15,70778880,3.1566055735186187
+self_attn.q_proj,16,26214400,4.015306292045239
+self_attn.k_proj,16,26214400,4.115485197768421
+self_attn.v_proj,16,26214400,3.208079856248501
+self_attn.o_proj,16,26214400,6.091573402332714
+mlp.gate_proj,16,70778880,3.4666247117190028
+mlp.down_proj,16,70778880,3.2793004568888255
+mlp.up_proj,16,70778880,3.093354071104304
+self_attn.q_proj,17,26214400,3.8968048183620883
+self_attn.k_proj,17,26214400,4.119076165517264
+self_attn.v_proj,17,26214400,3.1923282596666613
+self_attn.o_proj,17,26214400,4.002000642357157
+mlp.gate_proj,17,70778880,3.4159640675062617
+mlp.down_proj,17,70778880,3.9286620564322408
+mlp.up_proj,17,70778880,3.133833524652865
+self_attn.q_proj,18,26214400,3.7631441598780113
+self_attn.k_proj,18,26214400,3.909797088561795
+self_attn.v_proj,18,26214400,3.1055448499499003
+self_attn.o_proj,18,26214400,3.8858433468481586
+mlp.gate_proj,18,70778880,3.353831686213448
+mlp.down_proj,18,70778880,3.206519627778468
+mlp.up_proj,18,70778880,3.1039488066796554
+self_attn.q_proj,19,26214400,3.9938806997987055
+self_attn.k_proj,19,26214400,4.052765581853566
+self_attn.v_proj,19,26214400,3.1198698064368555
+self_attn.o_proj,19,26214400,3.9114055177110076
+mlp.gate_proj,19,70778880,3.3847912025490117
+mlp.down_proj,19,70778880,3.0971674608421758
+mlp.up_proj,19,70778880,3.089232912233438
+self_attn.q_proj,20,26214400,3.7925515574012776
+self_attn.k_proj,20,26214400,3.947109397028816
+self_attn.v_proj,20,26214400,3.116102509566827
+self_attn.o_proj,20,26214400,4.149188151567589
+mlp.gate_proj,20,70778880,3.376316700726662
+mlp.down_proj,20,70778880,3.2922558293785413
+mlp.up_proj,20,70778880,3.1405187056523607
+self_attn.q_proj,21,26214400,3.984176450445595
+self_attn.k_proj,21,26214400,4.1638067788358715
+self_attn.v_proj,21,26214400,3.090195036594181
+self_attn.o_proj,21,26214400,4.137223077108645
+mlp.gate_proj,21,70778880,3.3018414645837937
+mlp.down_proj,21,70778880,3.223733999187667
+mlp.up_proj,21,70778880,3.048963194698438
+self_attn.q_proj,22,26214400,3.605910793407125
+self_attn.k_proj,22,26214400,3.7222669512437996
+self_attn.v_proj,22,26214400,3.0891454541465455
+self_attn.o_proj,22,26214400,3.7049986475082526
+mlp.gate_proj,22,70778880,3.26893401295517
+mlp.down_proj,22,70778880,3.2725552041768626
+mlp.up_proj,22,70778880,3.064189693523624
+self_attn.q_proj,23,26214400,3.7204906793103816
+self_attn.k_proj,23,26214400,3.910896881454506
+self_attn.v_proj,23,26214400,3.117658875099658
+self_attn.o_proj,23,26214400,3.7576727840829602
+mlp.gate_proj,23,70778880,3.238691402749831
+mlp.down_proj,23,70778880,3.3791804748048095
+mlp.up_proj,23,70778880,3.0380139216222206
+self_attn.q_proj,24,26214400,3.9068847347530693
+self_attn.k_proj,24,26214400,3.788283257206423
+self_attn.v_proj,24,26214400,3.0860331622448647
+self_attn.o_proj,24,26214400,8.464742855236834
+mlp.gate_proj,24,70778880,3.181030610905902
+mlp.down_proj,24,70778880,3.179330304089504
+mlp.up_proj,24,70778880,3.051218091213576
+self_attn.q_proj,25,26214400,3.7105399549947515
+self_attn.k_proj,25,26214400,3.8724433992574077
+self_attn.v_proj,25,26214400,3.081358429754507
+self_attn.o_proj,25,26214400,3.3975478304093554
+mlp.gate_proj,25,70778880,3.1751834555934093
+mlp.down_proj,25,70778880,3.0716347899566943
+mlp.up_proj,25,70778880,3.048168807880293
+self_attn.q_proj,26,26214400,3.710783805147631
+self_attn.k_proj,26,26214400,4.002086270347984
+self_attn.v_proj,26,26214400,3.1605120215671305
+self_attn.o_proj,26,26214400,4.555290016948518
+mlp.gate_proj,26,70778880,3.1475936238919386
+mlp.down_proj,26,70778880,3.2339799975026913
+mlp.up_proj,26,70778880,3.0361184315449607
+self_attn.q_proj,27,26214400,3.6128496367100977
+self_attn.k_proj,27,26214400,3.7650393075529607
+self_attn.v_proj,27,26214400,3.0835869745815465
+self_attn.o_proj,27,26214400,3.1842078993627765
+mlp.gate_proj,27,70778880,3.157584369714205
+mlp.down_proj,27,70778880,3.0598856754149213
+mlp.up_proj,27,70778880,3.0251719970979862
+self_attn.q_proj,28,26214400,3.7117594247355523
+self_attn.k_proj,28,26214400,3.814974524914145
+self_attn.v_proj,28,26214400,3.0727206019189923
+self_attn.o_proj,28,26214400,13.607847890990024
+mlp.gate_proj,28,70778880,3.1952669014106267
+mlp.down_proj,28,70778880,3.1706288688125244
+mlp.up_proj,28,70778880,3.0860375775511315
+self_attn.q_proj,29,26214400,3.5461431186652543
+self_attn.k_proj,29,26214400,3.6549022643653717
+self_attn.v_proj,29,26214400,3.0545580329471944
+self_attn.o_proj,29,26214400,3.3875154692277696
+mlp.gate_proj,29,70778880,3.1642828381065327
+mlp.down_proj,29,70778880,3.2209807909367822
+mlp.up_proj,29,70778880,3.030916472520534
+self_attn.q_proj,30,26214400,3.578559751573899
+self_attn.k_proj,30,26214400,3.663534054993735
+self_attn.v_proj,30,26214400,3.0946710011424896
+self_attn.o_proj,30,26214400,3.422926202998674
+mlp.gate_proj,30,70778880,3.1901158459098817
+mlp.down_proj,30,70778880,3.146555838960171
+mlp.up_proj,30,70778880,3.0423417803431696
+self_attn.q_proj,31,26214400,3.5746784321998972
+self_attn.k_proj,31,26214400,3.7005669290359076
+self_attn.v_proj,31,26214400,3.082027691980877
+self_attn.o_proj,31,26214400,3.1539103185663726
+mlp.gate_proj,31,70778880,3.231870053797732
+mlp.down_proj,31,70778880,3.1999752212382955
+mlp.up_proj,31,70778880,3.026023815726937
+self_attn.q_proj,32,26214400,3.648626260402023
+self_attn.k_proj,32,26214400,3.8127163060765277
+self_attn.v_proj,32,26214400,3.0790647819921895
+self_attn.o_proj,32,26214400,3.1787468600069664
+mlp.gate_proj,32,70778880,3.2412875372910315
+mlp.down_proj,32,70778880,3.218702971060462
+mlp.up_proj,32,70778880,3.037189293635506
+self_attn.q_proj,33,26214400,3.606294432238728
+self_attn.k_proj,33,26214400,3.7002149416408994
+self_attn.v_proj,33,26214400,3.064640666796211
+self_attn.o_proj,33,26214400,3.782551729012757
+mlp.gate_proj,33,70778880,3.2633987551023353
+mlp.down_proj,33,70778880,3.368253222560591
+mlp.up_proj,33,70778880,3.0444602722176484
+self_attn.q_proj,34,26214400,3.7008888716634907
+self_attn.k_proj,34,26214400,3.9016793791258264
+self_attn.v_proj,34,26214400,3.0699658643467256
+self_attn.o_proj,34,26214400,3.089490962048976
+mlp.gate_proj,34,70778880,3.276419555666429
+mlp.down_proj,34,70778880,3.0853296262737886
+mlp.up_proj,34,70778880,3.052189834731244
+self_attn.q_proj,35,26214400,3.695354537166732
+self_attn.k_proj,35,26214400,3.877189056760186
+self_attn.v_proj,35,26214400,3.08790764923557
+self_attn.o_proj,35,26214400,3.151090877754406
+mlp.gate_proj,35,70778880,3.2620195372275735
+mlp.down_proj,35,70778880,3.0694624239201405
+mlp.up_proj,35,70778880,3.0455443318297197
+self_attn.q_proj,36,26214400,3.642855649545628
+self_attn.k_proj,36,26214400,3.833528981728686
+self_attn.v_proj,36,26214400,3.1403323886474728
+self_attn.o_proj,36,26214400,3.2896862307779187
+mlp.gate_proj,36,70778880,3.330972969715487
+mlp.down_proj,36,70778880,3.175252880494978
+mlp.up_proj,36,70778880,3.056325074580746
+self_attn.q_proj,37,26214400,3.957392821774528
+self_attn.k_proj,37,26214400,4.254668164665942
+self_attn.v_proj,37,26214400,3.1504427946267484
+self_attn.o_proj,37,26214400,5.5332901959324925
+mlp.gate_proj,37,70778880,3.6163712417859415
+mlp.down_proj,37,70778880,3.3536157623483787
+mlp.up_proj,37,70778880,3.093066664166201
+self_attn.q_proj,38,26214400,4.022298271040198
+self_attn.k_proj,38,26214400,4.238062894882317
+self_attn.v_proj,38,26214400,3.168706955380703
+self_attn.o_proj,38,26214400,4.620147934426817
+mlp.gate_proj,38,70778880,4.20085133702986
+mlp.down_proj,38,70778880,4.5711958175866725
+mlp.up_proj,38,70778880,3.990713763191592
+self_attn.q_proj,39,26214400,4.085587322066365
+self_attn.k_proj,39,26214400,4.530434964667962
+self_attn.v_proj,39,26214400,5.760929836951725
+self_attn.o_proj,39,26214400,62.18514136779736
+mlp.gate_proj,39,70778880,3.535894804400329
+mlp.down_proj,39,70778880,9.784047895918558
+mlp.up_proj,39,70778880,4.174917959911426
diff --git a/lm-quant-toolkit/tmp/kurtosis-dump/Llama-3.2-3B-Instruct/kurtosis-models.csv b/lm-quant-toolkit/tmp/kurtosis-dump/Llama-3.2-3B-Instruct/kurtosis-models.csv
new file mode 100644
index 0000000000000000000000000000000000000000..4a31b74531b56141c86a7d3635c8e48e6e06e54c
--- /dev/null
+++ b/lm-quant-toolkit/tmp/kurtosis-dump/Llama-3.2-3B-Instruct/kurtosis-models.csv
@@ -0,0 +1,197 @@
+module,layer,param_count,kurtosis
+self_attn.q_proj,0,9437184,11.4032211843563
+self_attn.k_proj,0,3145728,9.217216064850914
+self_attn.v_proj,0,3145728,4.604718236650218
+self_attn.o_proj,0,9437184,7.428493409904912
+mlp.gate_proj,0,25165824,3.6447236427271266
+mlp.down_proj,0,25165824,5.195673629524258
+mlp.up_proj,0,25165824,3.168665653876268
+self_attn.q_proj,1,9437184,5.336984544376062
+self_attn.k_proj,1,3145728,5.339282288365327
+self_attn.v_proj,1,3145728,3.449248833133815
+self_attn.o_proj,1,9437184,8.447688192842369
+mlp.gate_proj,1,25165824,3.729562089597947
+mlp.down_proj,1,25165824,3.879427336599383
+mlp.up_proj,1,25165824,4.796355138141482
+self_attn.q_proj,2,9437184,4.726275595963493
+self_attn.k_proj,2,3145728,5.145624941450599
+self_attn.v_proj,2,3145728,3.567766889871732
+self_attn.o_proj,2,9437184,6.051975745724625
+mlp.gate_proj,2,25165824,3.2955092298418887
+mlp.down_proj,2,25165824,3.5025943217859434
+mlp.up_proj,2,25165824,3.2351502886702233
+self_attn.q_proj,3,9437184,3.993118150375115
+self_attn.k_proj,3,3145728,4.215884819215548
+self_attn.v_proj,3,3145728,3.373603778282327
+self_attn.o_proj,3,9437184,3.8148777949613097
+mlp.gate_proj,3,25165824,3.4553490308716364
+mlp.down_proj,3,25165824,3.414461769201579
+mlp.up_proj,3,25165824,3.180254174978485
+self_attn.q_proj,4,9437184,4.797976217859671
+self_attn.k_proj,4,3145728,4.362223404314293
+self_attn.v_proj,4,3145728,3.514366319804521
+self_attn.o_proj,4,9437184,4.9843821269550315
+mlp.gate_proj,4,25165824,3.410751839695408
+mlp.down_proj,4,25165824,3.5918015152473406
+mlp.up_proj,4,25165824,3.3529052861946558
+self_attn.q_proj,5,9437184,4.339168210869567
+self_attn.k_proj,5,3145728,5.438107981986459
+self_attn.v_proj,5,3145728,3.4258499734306973
+self_attn.o_proj,5,9437184,3.7633597064690534
+mlp.gate_proj,5,25165824,3.3675098503179886
+mlp.down_proj,5,25165824,4.021735193594413
+mlp.up_proj,5,25165824,3.354675336596755
+self_attn.q_proj,6,9437184,4.304909559019118
+self_attn.k_proj,6,3145728,3.9087506524535036
+self_attn.v_proj,6,3145728,3.3224016981195676
+self_attn.o_proj,6,9437184,4.813807567855305
+mlp.gate_proj,6,25165824,3.5562124097739107
+mlp.down_proj,6,25165824,3.6839778066410807
+mlp.up_proj,6,25165824,3.2063531610040625
+self_attn.q_proj,7,9437184,4.14425321547239
+self_attn.k_proj,7,3145728,3.83665458898216
+self_attn.v_proj,7,3145728,3.4097019150751646
+self_attn.o_proj,7,9437184,4.071268973837191
+mlp.gate_proj,7,25165824,3.5619345762898806
+mlp.down_proj,7,25165824,4.128608048496753
+mlp.up_proj,7,25165824,3.3197500242676075
+self_attn.q_proj,8,9437184,5.083228592536028
+self_attn.k_proj,8,3145728,4.933864215659103
+self_attn.v_proj,8,3145728,3.459105005603344
+self_attn.o_proj,8,9437184,4.828394335682302
+mlp.gate_proj,8,25165824,3.635435117109145
+mlp.down_proj,8,25165824,4.230877631022458
+mlp.up_proj,8,25165824,3.3031920857660277
+self_attn.q_proj,9,9437184,4.5056030674949215
+self_attn.k_proj,9,3145728,4.388605882481324
+self_attn.v_proj,9,3145728,4.08458374830089
+self_attn.o_proj,9,9437184,5.335885282015746
+mlp.gate_proj,9,25165824,4.011775204572054
+mlp.down_proj,9,25165824,4.422385465392906
+mlp.up_proj,9,25165824,3.261307151555541
+self_attn.q_proj,10,9437184,4.190483363257681
+self_attn.k_proj,10,3145728,4.471920030782345
+self_attn.v_proj,10,3145728,3.3628256210092813
+self_attn.o_proj,10,9437184,4.858982345051063
+mlp.gate_proj,10,25165824,4.164930822539018
+mlp.down_proj,10,25165824,3.994863567807714
+mlp.up_proj,10,25165824,3.331300026222553
+self_attn.q_proj,11,9437184,3.6605895049660413
+self_attn.k_proj,11,3145728,3.781987600908594
+self_attn.v_proj,11,3145728,3.367136852897026
+self_attn.o_proj,11,9437184,4.732796054584504
+mlp.gate_proj,11,25165824,4.4816087273591485
+mlp.down_proj,11,25165824,4.579134940117831
+mlp.up_proj,11,25165824,3.4530226917969373
+self_attn.q_proj,12,9437184,4.280057698006496
+self_attn.k_proj,12,3145728,4.260322470713546
+self_attn.v_proj,12,3145728,3.3579212687862294
+self_attn.o_proj,12,9437184,5.615457798561787
+mlp.gate_proj,12,25165824,5.21698959165818
+mlp.down_proj,12,25165824,3.9803392578392494
+mlp.up_proj,12,25165824,3.4240661871737776
+self_attn.q_proj,13,9437184,4.805448877679935
+self_attn.k_proj,13,3145728,5.008741304556607
+self_attn.v_proj,13,3145728,3.605621521083981
+self_attn.o_proj,13,9437184,3.8377425308410866
+mlp.gate_proj,13,25165824,5.669409923820501
+mlp.down_proj,13,25165824,3.707847987843797
+mlp.up_proj,13,25165824,3.3740919725070517
+self_attn.q_proj,14,9437184,5.397043406179097
+self_attn.k_proj,14,3145728,3.9848544637994494
+self_attn.v_proj,14,3145728,3.743967311249583
+self_attn.o_proj,14,9437184,4.767701736577938
+mlp.gate_proj,14,25165824,5.436430464617039
+mlp.down_proj,14,25165824,3.814413261649823
+mlp.up_proj,14,25165824,3.303605454593828
+self_attn.q_proj,15,9437184,4.343450146189656
+self_attn.k_proj,15,3145728,4.247057657270448
+self_attn.v_proj,15,3145728,3.3421669840866213
+self_attn.o_proj,15,9437184,4.471233770374361
+mlp.gate_proj,15,25165824,4.668346492114722
+mlp.down_proj,15,25165824,3.759846315048496
+mlp.up_proj,15,25165824,3.2303436121538205
+self_attn.q_proj,16,9437184,4.092051448835192
+self_attn.k_proj,16,3145728,3.939123893324522
+self_attn.v_proj,16,3145728,3.7094585678481136
+self_attn.o_proj,16,9437184,3.847456643547808
+mlp.gate_proj,16,25165824,4.305557947658462
+mlp.down_proj,16,25165824,3.3472790936606245
+mlp.up_proj,16,25165824,3.1454203456990655
+self_attn.q_proj,17,9437184,3.9403248593554827
+self_attn.k_proj,17,3145728,3.9961727553105577
+self_attn.v_proj,17,3145728,3.57452603224689
+self_attn.o_proj,17,9437184,3.673095352438195
+mlp.gate_proj,17,25165824,4.256418582501942
+mlp.down_proj,17,25165824,3.5008090128907003
+mlp.up_proj,17,25165824,3.122437316169248
+self_attn.q_proj,18,9437184,4.13760287931356
+self_attn.k_proj,18,3145728,4.117287003194707
+self_attn.v_proj,18,3145728,3.640980196818294
+self_attn.o_proj,18,9437184,4.1199245891609655
+mlp.gate_proj,18,25165824,4.111593695564576
+mlp.down_proj,18,25165824,3.3429763082000705
+mlp.up_proj,18,25165824,3.1350019911216296
+self_attn.q_proj,19,9437184,3.9681580243051333
+self_attn.k_proj,19,3145728,3.997291293601032
+self_attn.v_proj,19,3145728,3.6345569361798717
+self_attn.o_proj,19,9437184,4.69568353837042
+mlp.gate_proj,19,25165824,4.437889470708911
+mlp.down_proj,19,25165824,3.3541633659673322
+mlp.up_proj,19,25165824,3.1664970542228215
+self_attn.q_proj,20,9437184,3.6756522367795283
+self_attn.k_proj,20,3145728,4.177356067023282
+self_attn.v_proj,20,3145728,3.3579493445239987
+self_attn.o_proj,20,9437184,3.511770789734999
+mlp.gate_proj,20,25165824,3.8802067231606756
+mlp.down_proj,20,25165824,3.3007848688559345
+mlp.up_proj,20,25165824,3.0892652007971364
+self_attn.q_proj,21,9437184,3.9055666156672095
+self_attn.k_proj,21,3145728,4.663671888308388
+self_attn.v_proj,21,3145728,3.6269540386765518
+self_attn.o_proj,21,9437184,3.302676320854945
+mlp.gate_proj,21,25165824,3.982608115286393
+mlp.down_proj,21,25165824,3.2153136506660056
+mlp.up_proj,21,25165824,3.1039616693207326
+self_attn.q_proj,22,9437184,3.8724598254264104
+self_attn.k_proj,22,3145728,4.848868949184333
+self_attn.v_proj,22,3145728,3.5763252277625086
+self_attn.o_proj,22,9437184,3.6434205824217774
+mlp.gate_proj,22,25165824,3.9866902376930704
+mlp.down_proj,22,25165824,3.2381283067493736
+mlp.up_proj,22,25165824,3.0964274714411517
+self_attn.q_proj,23,9437184,3.7866807346221556
+self_attn.k_proj,23,3145728,4.657839106717557
+self_attn.v_proj,23,3145728,3.3948936786560213
+self_attn.o_proj,23,9437184,3.53536334220555
+mlp.gate_proj,23,25165824,4.4101644829295665
+mlp.down_proj,23,25165824,3.277864016193387
+mlp.up_proj,23,25165824,3.0891617346230746
+self_attn.q_proj,24,9437184,3.821498221369848
+self_attn.k_proj,24,3145728,5.055560228160958
+self_attn.v_proj,24,3145728,3.938261907857065
+self_attn.o_proj,24,9437184,3.8114061937731276
+mlp.gate_proj,24,25165824,4.431189069071876
+mlp.down_proj,24,25165824,3.3296718549579887
+mlp.up_proj,24,25165824,3.125399920822586
+self_attn.q_proj,25,9437184,3.857153584854403
+self_attn.k_proj,25,3145728,4.191612359270162
+self_attn.v_proj,25,3145728,3.563188037769977
+self_attn.o_proj,25,9437184,3.859718890470012
+mlp.gate_proj,25,25165824,4.909699692731088
+mlp.down_proj,25,25165824,3.47029265274409
+mlp.up_proj,25,25165824,3.324190164533464
+self_attn.q_proj,26,9437184,4.738310324868295
+self_attn.k_proj,26,3145728,7.148714223921746
+self_attn.v_proj,26,3145728,4.042092376483989
+self_attn.o_proj,26,9437184,3.8786019603066673
+mlp.gate_proj,26,25165824,5.097509598802841
+mlp.down_proj,26,25165824,3.482956792876306
+mlp.up_proj,26,25165824,3.334885545608837
+self_attn.q_proj,27,9437184,4.46184149642568
+self_attn.k_proj,27,3145728,6.246512356716073
+self_attn.v_proj,27,3145728,4.385324021982905
+self_attn.o_proj,27,9437184,10.343792571161947
+mlp.gate_proj,27,25165824,7.146536693645115
+mlp.down_proj,27,25165824,4.31996183970937
+mlp.up_proj,27,25165824,8.474028631272057
diff --git a/lm-quant-toolkit/tmp/kurtosis-dump/Qwen3-4B/kurtosis-models.csv b/lm-quant-toolkit/tmp/kurtosis-dump/Qwen3-4B/kurtosis-models.csv
new file mode 100644
index 0000000000000000000000000000000000000000..2395943201f4664d9a2aecc056aedcf8b60190dc
--- /dev/null
+++ b/lm-quant-toolkit/tmp/kurtosis-dump/Qwen3-4B/kurtosis-models.csv
@@ -0,0 +1,253 @@
+module,layer,param_count,kurtosis
+self_attn.q_proj,0,10485760,4.253318805190092
+self_attn.k_proj,0,2621440,3.7371099967768204
+self_attn.v_proj,0,2621440,3.2497736282064644
+self_attn.o_proj,0,10485760,5.790140317872634
+mlp.gate_proj,0,24903680,3.329197014750964
+mlp.down_proj,0,24903680,4.0266069238996876
+mlp.up_proj,0,24903680,3.2023532325987625
+self_attn.q_proj,1,10485760,3.942629984731793
+self_attn.k_proj,1,2621440,3.6934736819533662
+self_attn.v_proj,1,2621440,3.3844862716334356
+self_attn.o_proj,1,10485760,4.179547296478103
+mlp.gate_proj,1,24903680,5.586620360872403
+mlp.down_proj,1,24903680,8.229085632921839
+mlp.up_proj,1,24903680,6.531721577664805
+self_attn.q_proj,2,10485760,3.8477474438324237
+self_attn.k_proj,2,2621440,4.837453806648587
+self_attn.v_proj,2,2621440,3.266851845861394
+self_attn.o_proj,2,10485760,3.5707977078519932
+mlp.gate_proj,2,24903680,6.541330336381953
+mlp.down_proj,2,24903680,9.076448257748993
+mlp.up_proj,2,24903680,7.3873809123276715
+self_attn.q_proj,3,10485760,4.034066461648582
+self_attn.k_proj,3,2621440,4.373236014254
+self_attn.v_proj,3,2621440,3.4195500265551213
+self_attn.o_proj,3,10485760,4.104747081041226
+mlp.gate_proj,3,24903680,4.414650644593131
+mlp.down_proj,3,24903680,5.596984747479932
+mlp.up_proj,3,24903680,4.988010863238269
+self_attn.q_proj,4,10485760,3.9839481305394617
+self_attn.k_proj,4,2621440,5.93174281605637
+self_attn.v_proj,4,2621440,3.2583588658220624
+self_attn.o_proj,4,10485760,3.804437427409464
+mlp.gate_proj,4,24903680,3.4044513453376832
+mlp.down_proj,4,24903680,4.970867305396444
+mlp.up_proj,4,24903680,3.566975321400291
+self_attn.q_proj,5,10485760,4.24665578233011
+self_attn.k_proj,5,2621440,4.467094170549161
+self_attn.v_proj,5,2621440,3.3754332335521346
+self_attn.o_proj,5,10485760,4.264408945563392
+mlp.gate_proj,5,24903680,4.169316352335702
+mlp.down_proj,5,24903680,4.490748530077453
+mlp.up_proj,5,24903680,3.3983461139482207
+self_attn.q_proj,6,10485760,4.78159224013428
+self_attn.k_proj,6,2621440,3.9222697917200775
+self_attn.v_proj,6,2621440,3.9689292399896097
+self_attn.o_proj,6,10485760,7.616362648534986
+mlp.gate_proj,6,24903680,3.5972980032689454
+mlp.down_proj,6,24903680,4.675597348123977
+mlp.up_proj,6,24903680,3.3313798413563074
+self_attn.q_proj,7,10485760,4.243212815144079
+self_attn.k_proj,7,2621440,5.103183171507923
+self_attn.v_proj,7,2621440,3.416362924014573
+self_attn.o_proj,7,10485760,4.081488377017616
+mlp.gate_proj,7,24903680,3.54359890210121
+mlp.down_proj,7,24903680,3.5973531932627076
+mlp.up_proj,7,24903680,3.320837974969858
+self_attn.q_proj,8,10485760,4.014793538243717
+self_attn.k_proj,8,2621440,4.015997455633962
+self_attn.v_proj,8,2621440,3.280119535296401
+self_attn.o_proj,8,10485760,3.445478826724856
+mlp.gate_proj,8,24903680,3.357522659128114
+mlp.down_proj,8,24903680,3.71810535694023
+mlp.up_proj,8,24903680,3.310220050292468
+self_attn.q_proj,9,10485760,4.629543178424452
+self_attn.k_proj,9,2621440,8.965207445713958
+self_attn.v_proj,9,2621440,3.6839053999905995
+self_attn.o_proj,9,10485760,3.8164099674064618
+mlp.gate_proj,9,24903680,3.61624275395672
+mlp.down_proj,9,24903680,3.58709824156339
+mlp.up_proj,9,24903680,3.3481070511248343
+self_attn.q_proj,10,10485760,4.054727966254171
+self_attn.k_proj,10,2621440,4.134356050553003
+self_attn.v_proj,10,2621440,3.6043397323828397
+self_attn.o_proj,10,10485760,4.294208658053437
+mlp.gate_proj,10,24903680,3.6116888754558714
+mlp.down_proj,10,24903680,3.7485942827259757
+mlp.up_proj,10,24903680,3.435034460221979
+self_attn.q_proj,11,10485760,4.41484425679344
+self_attn.k_proj,11,2621440,6.083418245745321
+self_attn.v_proj,11,2621440,3.5624439133681007
+self_attn.o_proj,11,10485760,3.7315716395629126
+mlp.gate_proj,11,24903680,3.570623248031814
+mlp.down_proj,11,24903680,3.6825707299419514
+mlp.up_proj,11,24903680,3.3858486914346173
+self_attn.q_proj,12,10485760,4.427213829781026
+self_attn.k_proj,12,2621440,7.296089233803497
+self_attn.v_proj,12,2621440,3.7063235775051075
+self_attn.o_proj,12,10485760,5.012833730390846
+mlp.gate_proj,12,24903680,3.757955111229807
+mlp.down_proj,12,24903680,3.7382495596725835
+mlp.up_proj,12,24903680,3.537895450693348
+self_attn.q_proj,13,10485760,4.8806529865632555
+self_attn.k_proj,13,2621440,6.451500768119467
+self_attn.v_proj,13,2621440,3.5069281136305546
+self_attn.o_proj,13,10485760,3.6343029197668013
+mlp.gate_proj,13,24903680,3.7548432972482586
+mlp.down_proj,13,24903680,4.135041849098481
+mlp.up_proj,13,24903680,3.685431973947491
+self_attn.q_proj,14,10485760,4.351478606099228
+self_attn.k_proj,14,2621440,4.469622069015039
+self_attn.v_proj,14,2621440,3.4747846605518307
+self_attn.o_proj,14,10485760,4.526016888838636
+mlp.gate_proj,14,24903680,3.855260589561603
+mlp.down_proj,14,24903680,3.9954921445816183
+mlp.up_proj,14,24903680,3.5459054423578538
+self_attn.q_proj,15,10485760,5.627859979724338
+self_attn.k_proj,15,2621440,5.805651718774689
+self_attn.v_proj,15,2621440,3.394097529023669
+self_attn.o_proj,15,10485760,3.765554931953106
+mlp.gate_proj,15,24903680,4.0279168795439615
+mlp.down_proj,15,24903680,3.9766343695390787
+mlp.up_proj,15,24903680,3.5198072799360385
+self_attn.q_proj,16,10485760,5.302763201855311
+self_attn.k_proj,16,2621440,4.696036919700942
+self_attn.v_proj,16,2621440,3.614740771424773
+self_attn.o_proj,16,10485760,4.382096656824375
+mlp.gate_proj,16,24903680,4.374222756717472
+mlp.down_proj,16,24903680,6.292924036145322
+mlp.up_proj,16,24903680,5.803099081183349
+self_attn.q_proj,17,10485760,5.942481236572444
+self_attn.k_proj,17,2621440,5.071122315045897
+self_attn.v_proj,17,2621440,3.4557598166852856
+self_attn.o_proj,17,10485760,4.323181727578074
+mlp.gate_proj,17,24903680,4.19045568959615
+mlp.down_proj,17,24903680,4.124297523417064
+mlp.up_proj,17,24903680,3.4984194517750513
+self_attn.q_proj,18,10485760,5.143495214136145
+self_attn.k_proj,18,2621440,4.630349722317864
+self_attn.v_proj,18,2621440,3.4565480345594852
+self_attn.o_proj,18,10485760,4.228194103482593
+mlp.gate_proj,18,24903680,4.452818287196711
+mlp.down_proj,18,24903680,4.137954364863396
+mlp.up_proj,18,24903680,3.6653413801096675
+self_attn.q_proj,19,10485760,6.156970481363907
+self_attn.k_proj,19,2621440,9.514631158311941
+self_attn.v_proj,19,2621440,3.5988781514916783
+self_attn.o_proj,19,10485760,5.042880026498917
+mlp.gate_proj,19,24903680,4.730264747276834
+mlp.down_proj,19,24903680,4.614209494227822
+mlp.up_proj,19,24903680,3.9555483604462243
+self_attn.q_proj,20,10485760,6.59965259282837
+self_attn.k_proj,20,2621440,5.483598728177798
+self_attn.v_proj,20,2621440,3.540609599035421
+self_attn.o_proj,20,10485760,4.020478558908942
+mlp.gate_proj,20,24903680,4.520518312141517
+mlp.down_proj,20,24903680,3.7423437748184076
+mlp.up_proj,20,24903680,3.595160140097595
+self_attn.q_proj,21,10485760,6.187137720629267
+self_attn.k_proj,21,2621440,4.504119299231147
+self_attn.v_proj,21,2621440,4.201733302381091
+self_attn.o_proj,21,10485760,3.896160900875543
+mlp.gate_proj,21,24903680,4.838096921067465
+mlp.down_proj,21,24903680,3.888235896152907
+mlp.up_proj,21,24903680,3.5228712349838562
+self_attn.q_proj,22,10485760,6.3355227881392775
+self_attn.k_proj,22,2621440,12.613814972556066
+self_attn.v_proj,22,2621440,3.9252707508114186
+self_attn.o_proj,22,10485760,4.384712562556812
+mlp.gate_proj,22,24903680,5.118203605095857
+mlp.down_proj,22,24903680,4.313057636717954
+mlp.up_proj,22,24903680,3.7222658558112247
+self_attn.q_proj,23,10485760,6.52927060854941
+self_attn.k_proj,23,2621440,4.485732045868077
+self_attn.v_proj,23,2621440,4.444195641325555
+self_attn.o_proj,23,10485760,4.417506729656154
+mlp.gate_proj,23,24903680,4.560108842536819
+mlp.down_proj,23,24903680,3.8324032214373376
+mlp.up_proj,23,24903680,3.4538487288781394
+self_attn.q_proj,24,10485760,5.974853878953764
+self_attn.k_proj,24,2621440,4.62668324370435
+self_attn.v_proj,24,2621440,5.126785856091666
+self_attn.o_proj,24,10485760,5.709521123102804
+mlp.gate_proj,24,24903680,3.994252119079401
+mlp.down_proj,24,24903680,3.654858687857337
+mlp.up_proj,24,24903680,3.3325618038239067
+self_attn.q_proj,25,10485760,4.831917126821248
+self_attn.k_proj,25,2621440,3.8713755841214708
+self_attn.v_proj,25,2621440,3.6512967981925644
+self_attn.o_proj,25,10485760,3.3817981090847415
+mlp.gate_proj,25,24903680,3.9102743512880505
+mlp.down_proj,25,24903680,3.576846619783876
+mlp.up_proj,25,24903680,3.3311342676021254
+self_attn.q_proj,26,10485760,5.215155365400787
+self_attn.k_proj,26,2621440,4.011276779838502
+self_attn.v_proj,26,2621440,3.6586861947919216
+self_attn.o_proj,26,10485760,3.4929708163415008
+mlp.gate_proj,26,24903680,3.7119532755340074
+mlp.down_proj,26,24903680,3.4152116880555203
+mlp.up_proj,26,24903680,3.3121801001427125
+self_attn.q_proj,27,10485760,4.9087214834395825
+self_attn.k_proj,27,2621440,4.062868162499059
+self_attn.v_proj,27,2621440,3.7261971312783952
+self_attn.o_proj,27,10485760,3.5152002190643685
+mlp.gate_proj,27,24903680,3.8329750462969594
+mlp.down_proj,27,24903680,3.4492738567671597
+mlp.up_proj,27,24903680,3.2706982379691376
+self_attn.q_proj,28,10485760,5.028487416639257
+self_attn.k_proj,28,2621440,4.132598809251543
+self_attn.v_proj,28,2621440,3.4877280761684477
+self_attn.o_proj,28,10485760,4.140234735555428
+mlp.gate_proj,28,24903680,4.031584990478492
+mlp.down_proj,28,24903680,3.571757279182854
+mlp.up_proj,28,24903680,3.3392348434510946
+self_attn.q_proj,29,10485760,5.4952640318982
+self_attn.k_proj,29,2621440,4.871049355168381
+self_attn.v_proj,29,2621440,4.750744895131683
+self_attn.o_proj,29,10485760,3.3339451284660084
+mlp.gate_proj,29,24903680,3.546133325620335
+mlp.down_proj,29,24903680,3.3899004322640396
+mlp.up_proj,29,24903680,3.1871673316863087
+self_attn.q_proj,30,10485760,4.730655064060836
+self_attn.k_proj,30,2621440,4.632320354065156
+self_attn.v_proj,30,2621440,4.394996303744601
+self_attn.o_proj,30,10485760,3.492055475356061
+mlp.gate_proj,30,24903680,3.602159781929627
+mlp.down_proj,30,24903680,3.5621977207994426
+mlp.up_proj,30,24903680,3.1851493464622727
+self_attn.q_proj,31,10485760,4.9058135026015375
+self_attn.k_proj,31,2621440,4.7837391724163325
+self_attn.v_proj,31,2621440,4.112966861606464
+self_attn.o_proj,31,10485760,3.4326840474904055
+mlp.gate_proj,31,24903680,3.6962994155383138
+mlp.down_proj,31,24903680,3.5249842068242243
+mlp.up_proj,31,24903680,3.2153184207783476
+self_attn.q_proj,32,10485760,5.091440713475687
+self_attn.k_proj,32,2621440,4.97617998198066
+self_attn.v_proj,32,2621440,4.374498148427975
+self_attn.o_proj,32,10485760,3.646425850829135
+mlp.gate_proj,32,24903680,3.711555350478091
+mlp.down_proj,32,24903680,3.5280634434025653
+mlp.up_proj,32,24903680,3.2358785926400304
+self_attn.q_proj,33,10485760,5.777486554347043
+self_attn.k_proj,33,2621440,5.405199789685683
+self_attn.v_proj,33,2621440,4.114427297432304
+self_attn.o_proj,33,10485760,3.5286788440751633
+mlp.gate_proj,33,24903680,3.8670866270601767
+mlp.down_proj,33,24903680,3.5283809922628975
+mlp.up_proj,33,24903680,3.298083607556517
+self_attn.q_proj,34,10485760,5.3403229010976965
+self_attn.k_proj,34,2621440,5.07820337966361
+self_attn.v_proj,34,2621440,4.218392452067374
+self_attn.o_proj,34,10485760,4.023512068728858
+mlp.gate_proj,34,24903680,4.188207735041519
+mlp.down_proj,34,24903680,5.3964053047744445
+mlp.up_proj,34,24903680,3.4059445156948485
+self_attn.q_proj,35,10485760,4.242149977485459
+self_attn.k_proj,35,2621440,4.630365394693358
+self_attn.v_proj,35,2621440,3.7290173678813856
+self_attn.o_proj,35,10485760,5.29634254574227
+mlp.gate_proj,35,24903680,5.177992249494421
+mlp.down_proj,35,24903680,4.805567501337388
+mlp.up_proj,35,24903680,3.7029729235110427
diff --git a/lm-quant-toolkit/tmp/kurtosis-dump/Qwen3-8B/kurtosis-models.csv b/lm-quant-toolkit/tmp/kurtosis-dump/Qwen3-8B/kurtosis-models.csv
new file mode 100644
index 0000000000000000000000000000000000000000..5e72d90deaa0fb234882ae1381f5da44925915db
--- /dev/null
+++ b/lm-quant-toolkit/tmp/kurtosis-dump/Qwen3-8B/kurtosis-models.csv
@@ -0,0 +1,253 @@
+module,layer,param_count,kurtosis
+self_attn.q_proj,0,16777216,3.864092441642656
+self_attn.k_proj,0,4194304,3.7046821497232765
+self_attn.v_proj,0,4194304,3.2112015050970926
+self_attn.o_proj,0,16777216,6.02550824325884
+mlp.gate_proj,0,50331648,3.451993675099092
+mlp.down_proj,0,50331648,4.284477258372157
+mlp.up_proj,0,50331648,3.1851647403250443
+self_attn.q_proj,1,16777216,3.9510820911231272
+self_attn.k_proj,1,4194304,3.6922393304518124
+self_attn.v_proj,1,4194304,3.370965129122138
+self_attn.o_proj,1,16777216,4.229415165768315
+mlp.gate_proj,1,50331648,7.558844466927812
+mlp.down_proj,1,50331648,18.76067331126324
+mlp.up_proj,1,50331648,8.104475258985456
+self_attn.q_proj,2,16777216,3.792258387483579
+self_attn.k_proj,2,4194304,4.550589064703088
+self_attn.v_proj,2,4194304,3.222944276465546
+self_attn.o_proj,2,16777216,3.4884198215917497
+mlp.gate_proj,2,50331648,12.566648713678198
+mlp.down_proj,2,50331648,16.303430607179564
+mlp.up_proj,2,50331648,15.339967350428621
+self_attn.q_proj,3,16777216,3.900521003649076
+self_attn.k_proj,3,4194304,4.201288472124406
+self_attn.v_proj,3,4194304,3.40065694488233
+self_attn.o_proj,3,16777216,4.112928211364129
+mlp.gate_proj,3,50331648,5.81741559064988
+mlp.down_proj,3,50331648,7.779045245176896
+mlp.up_proj,3,50331648,6.42665684847997
+self_attn.q_proj,4,16777216,3.845767181647875
+self_attn.k_proj,4,4194304,4.663126335735003
+self_attn.v_proj,4,4194304,3.245405426270338
+self_attn.o_proj,4,16777216,3.88178673017547
+mlp.gate_proj,4,50331648,3.8585211311274463
+mlp.down_proj,4,50331648,5.846305503008854
+mlp.up_proj,4,50331648,3.802223003786185
+self_attn.q_proj,5,16777216,4.04483246498864
+self_attn.k_proj,5,4194304,4.237208793433962
+self_attn.v_proj,5,4194304,3.353828417782037
+self_attn.o_proj,5,16777216,4.493066764251958
+mlp.gate_proj,5,50331648,3.972785760948491
+mlp.down_proj,5,50331648,5.243240152428123
+mlp.up_proj,5,50331648,3.3468623009892142
+self_attn.q_proj,6,16777216,5.350845246887842
+self_attn.k_proj,6,4194304,3.8276161626518665
+self_attn.v_proj,6,4194304,3.8167932297528466
+self_attn.o_proj,6,16777216,7.400905146186701
+mlp.gate_proj,6,50331648,3.4979813846441097
+mlp.down_proj,6,50331648,4.634067294025263
+mlp.up_proj,6,50331648,3.2160654785516463
+self_attn.q_proj,7,16777216,4.001356305219397
+self_attn.k_proj,7,4194304,4.4968353985746585
+self_attn.v_proj,7,4194304,3.379157663453864
+self_attn.o_proj,7,16777216,4.17571815659844
+mlp.gate_proj,7,50331648,3.399427866197641
+mlp.down_proj,7,50331648,3.7749875564720017
+mlp.up_proj,7,50331648,3.221106657746529
+self_attn.q_proj,8,16777216,3.8439585516931345
+self_attn.k_proj,8,4194304,3.8628969764973724
+self_attn.v_proj,8,4194304,3.229045442100881
+self_attn.o_proj,8,16777216,3.5750367537706147
+mlp.gate_proj,8,50331648,3.275527798293939
+mlp.down_proj,8,50331648,3.898880790779743
+mlp.up_proj,8,50331648,3.2403912192924857
+self_attn.q_proj,9,16777216,4.253052488418564
+self_attn.k_proj,9,4194304,6.984014207565668
+self_attn.v_proj,9,4194304,3.570361043992625
+self_attn.o_proj,9,16777216,4.921920154929393
+mlp.gate_proj,9,50331648,3.5751091391749426
+mlp.down_proj,9,50331648,3.539601146325055
+mlp.up_proj,9,50331648,3.3090845704394174
+self_attn.q_proj,10,16777216,3.8791462629715294
+self_attn.k_proj,10,4194304,3.950424005274065
+self_attn.v_proj,10,4194304,3.60113894746074
+self_attn.o_proj,10,16777216,4.694606308382895
+mlp.gate_proj,10,50331648,3.5787595561518257
+mlp.down_proj,10,50331648,3.963220370658456
+mlp.up_proj,10,50331648,3.3566668853646946
+self_attn.q_proj,11,16777216,4.171904103162667
+self_attn.k_proj,11,4194304,5.477098532133951
+self_attn.v_proj,11,4194304,3.460314959936971
+self_attn.o_proj,11,16777216,4.014968557915078
+mlp.gate_proj,11,50331648,3.513567430029673
+mlp.down_proj,11,50331648,3.6699457962470516
+mlp.up_proj,11,50331648,3.310197921587732
+self_attn.q_proj,12,16777216,4.170024604197322
+self_attn.k_proj,12,4194304,6.292485690498306
+self_attn.v_proj,12,4194304,3.569516304652959
+self_attn.o_proj,12,16777216,5.443451952413357
+mlp.gate_proj,12,50331648,3.634063393975273
+mlp.down_proj,12,50331648,3.8744168119735614
+mlp.up_proj,12,50331648,3.471399540298646
+self_attn.q_proj,13,16777216,4.515867970285675
+self_attn.k_proj,13,4194304,6.121190886633632
+self_attn.v_proj,13,4194304,3.4512616918857684
+self_attn.o_proj,13,16777216,3.6229957907669896
+mlp.gate_proj,13,50331648,3.6896096066771666
+mlp.down_proj,13,50331648,4.088560542019208
+mlp.up_proj,13,50331648,3.4907927800002545
+self_attn.q_proj,14,16777216,4.228741782016346
+self_attn.k_proj,14,4194304,4.241718210866317
+self_attn.v_proj,14,4194304,3.4085052837160448
+self_attn.o_proj,14,16777216,4.778758308061709
+mlp.gate_proj,14,50331648,3.6803733046000815
+mlp.down_proj,14,50331648,3.795151182835799
+mlp.up_proj,14,50331648,3.4604004365790315
+self_attn.q_proj,15,16777216,4.993077372618623
+self_attn.k_proj,15,4194304,5.37010841336525
+self_attn.v_proj,15,4194304,3.339555544152251
+self_attn.o_proj,15,16777216,3.7038482807547015
+mlp.gate_proj,15,50331648,3.8055435446227164
+mlp.down_proj,15,50331648,3.9767018841915047
+mlp.up_proj,15,50331648,3.4954030506759253
+self_attn.q_proj,16,16777216,4.766900604821532
+self_attn.k_proj,16,4194304,4.447584658567737
+self_attn.v_proj,16,4194304,3.567621309536996
+self_attn.o_proj,16,16777216,4.631544998338787
+mlp.gate_proj,16,50331648,4.052078972953341
+mlp.down_proj,16,50331648,6.65422052074798
+mlp.up_proj,16,50331648,3.80745646002702
+self_attn.q_proj,17,16777216,5.12356027665885
+self_attn.k_proj,17,4194304,4.8179813728227705
+self_attn.v_proj,17,4194304,3.3701153108243833
+self_attn.o_proj,17,16777216,4.571589099452379
+mlp.gate_proj,17,50331648,3.906622497385505
+mlp.down_proj,17,50331648,4.085787488778317
+mlp.up_proj,17,50331648,3.4277146096010616
+self_attn.q_proj,18,16777216,4.665349375537282
+self_attn.k_proj,18,4194304,4.4824421762637865
+self_attn.v_proj,18,4194304,3.4143379238489984
+self_attn.o_proj,18,16777216,4.306203688701428
+mlp.gate_proj,18,50331648,4.1348462657591405
+mlp.down_proj,18,50331648,4.186923646591098
+mlp.up_proj,18,50331648,3.543188240673786
+self_attn.q_proj,19,16777216,5.48824974250715
+self_attn.k_proj,19,4194304,8.833747900332906
+self_attn.v_proj,19,4194304,3.622880827123495
+self_attn.o_proj,19,16777216,5.651608392805515
+mlp.gate_proj,19,50331648,4.31161530455121
+mlp.down_proj,19,50331648,4.6553119530523634
+mlp.up_proj,19,50331648,3.728648726502985
+self_attn.q_proj,20,16777216,5.501777436448975
+self_attn.k_proj,20,4194304,5.1242559619562895
+self_attn.v_proj,20,4194304,3.5151964552625405
+self_attn.o_proj,20,16777216,3.991504438233053
+mlp.gate_proj,20,50331648,4.111297786337425
+mlp.down_proj,20,50331648,3.672418903504559
+mlp.up_proj,20,50331648,3.425080519569344
+self_attn.q_proj,21,16777216,5.44368131838779
+self_attn.k_proj,21,4194304,4.308970341186873
+self_attn.v_proj,21,4194304,4.275413597960965
+self_attn.o_proj,21,16777216,3.8560239268660252
+mlp.gate_proj,21,50331648,4.4178484272689555
+mlp.down_proj,21,50331648,3.864780148761345
+mlp.up_proj,21,50331648,3.405279664459031
+self_attn.q_proj,22,16777216,5.410679361978139
+self_attn.k_proj,22,4194304,9.386629080473162
+self_attn.v_proj,22,4194304,3.7696404174251024
+self_attn.o_proj,22,16777216,4.368047008944573
+mlp.gate_proj,22,50331648,4.515261077821607
+mlp.down_proj,22,50331648,4.2652255273597675
+mlp.up_proj,22,50331648,3.592689794198662
+self_attn.q_proj,23,16777216,5.265475135555726
+self_attn.k_proj,23,4194304,4.217328324467858
+self_attn.v_proj,23,4194304,4.332256874285605
+self_attn.o_proj,23,16777216,4.349143654441271
+mlp.gate_proj,23,50331648,4.073948311380071
+mlp.down_proj,23,50331648,3.807942478949522
+mlp.up_proj,23,50331648,3.3598549404746665
+self_attn.q_proj,24,16777216,4.981582379776875
+self_attn.k_proj,24,4194304,4.3863485207069735
+self_attn.v_proj,24,4194304,4.725765711052312
+self_attn.o_proj,24,16777216,5.098466972534927
+mlp.gate_proj,24,50331648,3.7347976397594636
+mlp.down_proj,24,50331648,3.587169330562164
+mlp.up_proj,24,50331648,3.2697257596461746
+self_attn.q_proj,25,16777216,4.445979585333972
+self_attn.k_proj,25,4194304,3.731637101164556
+self_attn.v_proj,25,4194304,3.606963310283395
+self_attn.o_proj,25,16777216,3.353301568862519
+mlp.gate_proj,25,50331648,3.7010141500045375
+mlp.down_proj,25,50331648,3.5062903410028596
+mlp.up_proj,25,50331648,3.2634803496549942
+self_attn.q_proj,26,16777216,4.525310941496539
+self_attn.k_proj,26,4194304,3.8516712676890608
+self_attn.v_proj,26,4194304,3.5550430098566586
+self_attn.o_proj,26,16777216,3.5029344281455024
+mlp.gate_proj,26,50331648,3.516841003730707
+mlp.down_proj,26,50331648,3.3988291312774117
+mlp.up_proj,26,50331648,3.2873835373735623
+self_attn.q_proj,27,16777216,4.364207553415131
+self_attn.k_proj,27,4194304,3.9453304969666907
+self_attn.v_proj,27,4194304,3.59525526384917
+self_attn.o_proj,27,16777216,3.466064411679335
+mlp.gate_proj,27,50331648,3.6298698945332153
+mlp.down_proj,27,50331648,3.4241784298113966
+mlp.up_proj,27,50331648,3.23329138782689
+self_attn.q_proj,28,16777216,4.342530233862741
+self_attn.k_proj,28,4194304,3.980842105404422
+self_attn.v_proj,28,4194304,3.404486681984997
+self_attn.o_proj,28,16777216,4.171793926922165
+mlp.gate_proj,28,50331648,3.7995056716172737
+mlp.down_proj,28,50331648,3.6278368893081145
+mlp.up_proj,28,50331648,3.2814764188955365
+self_attn.q_proj,29,16777216,4.57412224703693
+self_attn.k_proj,29,4194304,4.467150070024177
+self_attn.v_proj,29,4194304,4.324573662599141
+self_attn.o_proj,29,16777216,3.2883411701062375
+mlp.gate_proj,29,50331648,3.3632119145747152
+mlp.down_proj,29,50331648,3.3714387559734638
+mlp.up_proj,29,50331648,3.1470439628898874
+self_attn.q_proj,30,16777216,4.43120527525527
+self_attn.k_proj,30,4194304,4.211077263482161
+self_attn.v_proj,30,4194304,4.090042170082016
+self_attn.o_proj,30,16777216,3.4963261887536063
+mlp.gate_proj,30,50331648,3.446688163532399
+mlp.down_proj,30,50331648,3.5756297172888334
+mlp.up_proj,30,50331648,3.1448239675706744
+self_attn.q_proj,31,16777216,4.328104446565007
+self_attn.k_proj,31,4194304,4.3340755999187985
+self_attn.v_proj,31,4194304,3.8122841253068893
+self_attn.o_proj,31,16777216,3.346408056546486
+mlp.gate_proj,31,50331648,3.437135881438056
+mlp.down_proj,31,50331648,3.417398979440722
+mlp.up_proj,31,50331648,3.1753507665237564
+self_attn.q_proj,32,16777216,4.554424243476571
+self_attn.k_proj,32,4194304,4.486103847418639
+self_attn.v_proj,32,4194304,4.091581226872238
+self_attn.o_proj,32,16777216,3.6065577573925807
+mlp.gate_proj,32,50331648,3.442294596940012
+mlp.down_proj,32,50331648,3.465572932550378
+mlp.up_proj,32,50331648,3.172980374361728
+self_attn.q_proj,33,16777216,5.060775206222359
+self_attn.k_proj,33,4194304,4.680090592211514
+self_attn.v_proj,33,4194304,3.896278664855261
+self_attn.o_proj,33,16777216,3.3469264028743453
+mlp.gate_proj,33,50331648,3.540755340032798
+mlp.down_proj,33,50331648,3.401107733564681
+mlp.up_proj,33,50331648,3.227168304783581
+self_attn.q_proj,34,16777216,4.754147419649401
+self_attn.k_proj,34,4194304,4.570964463523233
+self_attn.v_proj,34,4194304,3.987365679503545
+self_attn.o_proj,34,16777216,4.135442527602238
+mlp.gate_proj,34,50331648,3.7358651692181097
+mlp.down_proj,34,50331648,6.449407651623265
+mlp.up_proj,34,50331648,3.265422178613862
+self_attn.q_proj,35,16777216,4.045388579347707
+self_attn.k_proj,35,4194304,3.789734028923394
+self_attn.v_proj,35,4194304,3.5914908167972004
+self_attn.o_proj,35,16777216,7.439195686502025
+mlp.gate_proj,35,50331648,3.8274763070771414
+mlp.down_proj,35,50331648,11.664499612717798
+mlp.up_proj,35,50331648,3.5060495766237336
diff --git a/lm-quant-toolkit/tmp/kurtosis-dump/kurtosis-models.csv b/lm-quant-toolkit/tmp/kurtosis-dump/kurtosis-models.csv
new file mode 100644
index 0000000000000000000000000000000000000000..5e72d90deaa0fb234882ae1381f5da44925915db
--- /dev/null
+++ b/lm-quant-toolkit/tmp/kurtosis-dump/kurtosis-models.csv
@@ -0,0 +1,253 @@
+module,layer,param_count,kurtosis
+self_attn.q_proj,0,16777216,3.864092441642656
+self_attn.k_proj,0,4194304,3.7046821497232765
+self_attn.v_proj,0,4194304,3.2112015050970926
+self_attn.o_proj,0,16777216,6.02550824325884
+mlp.gate_proj,0,50331648,3.451993675099092
+mlp.down_proj,0,50331648,4.284477258372157
+mlp.up_proj,0,50331648,3.1851647403250443
+self_attn.q_proj,1,16777216,3.9510820911231272
+self_attn.k_proj,1,4194304,3.6922393304518124
+self_attn.v_proj,1,4194304,3.370965129122138
+self_attn.o_proj,1,16777216,4.229415165768315
+mlp.gate_proj,1,50331648,7.558844466927812
+mlp.down_proj,1,50331648,18.76067331126324
+mlp.up_proj,1,50331648,8.104475258985456
+self_attn.q_proj,2,16777216,3.792258387483579
+self_attn.k_proj,2,4194304,4.550589064703088
+self_attn.v_proj,2,4194304,3.222944276465546
+self_attn.o_proj,2,16777216,3.4884198215917497
+mlp.gate_proj,2,50331648,12.566648713678198
+mlp.down_proj,2,50331648,16.303430607179564
+mlp.up_proj,2,50331648,15.339967350428621
+self_attn.q_proj,3,16777216,3.900521003649076
+self_attn.k_proj,3,4194304,4.201288472124406
+self_attn.v_proj,3,4194304,3.40065694488233
+self_attn.o_proj,3,16777216,4.112928211364129
+mlp.gate_proj,3,50331648,5.81741559064988
+mlp.down_proj,3,50331648,7.779045245176896
+mlp.up_proj,3,50331648,6.42665684847997
+self_attn.q_proj,4,16777216,3.845767181647875
+self_attn.k_proj,4,4194304,4.663126335735003
+self_attn.v_proj,4,4194304,3.245405426270338
+self_attn.o_proj,4,16777216,3.88178673017547
+mlp.gate_proj,4,50331648,3.8585211311274463
+mlp.down_proj,4,50331648,5.846305503008854
+mlp.up_proj,4,50331648,3.802223003786185
+self_attn.q_proj,5,16777216,4.04483246498864
+self_attn.k_proj,5,4194304,4.237208793433962
+self_attn.v_proj,5,4194304,3.353828417782037
+self_attn.o_proj,5,16777216,4.493066764251958
+mlp.gate_proj,5,50331648,3.972785760948491
+mlp.down_proj,5,50331648,5.243240152428123
+mlp.up_proj,5,50331648,3.3468623009892142
+self_attn.q_proj,6,16777216,5.350845246887842
+self_attn.k_proj,6,4194304,3.8276161626518665
+self_attn.v_proj,6,4194304,3.8167932297528466
+self_attn.o_proj,6,16777216,7.400905146186701
+mlp.gate_proj,6,50331648,3.4979813846441097
+mlp.down_proj,6,50331648,4.634067294025263
+mlp.up_proj,6,50331648,3.2160654785516463
+self_attn.q_proj,7,16777216,4.001356305219397
+self_attn.k_proj,7,4194304,4.4968353985746585
+self_attn.v_proj,7,4194304,3.379157663453864
+self_attn.o_proj,7,16777216,4.17571815659844
+mlp.gate_proj,7,50331648,3.399427866197641
+mlp.down_proj,7,50331648,3.7749875564720017
+mlp.up_proj,7,50331648,3.221106657746529
+self_attn.q_proj,8,16777216,3.8439585516931345
+self_attn.k_proj,8,4194304,3.8628969764973724
+self_attn.v_proj,8,4194304,3.229045442100881
+self_attn.o_proj,8,16777216,3.5750367537706147
+mlp.gate_proj,8,50331648,3.275527798293939
+mlp.down_proj,8,50331648,3.898880790779743
+mlp.up_proj,8,50331648,3.2403912192924857
+self_attn.q_proj,9,16777216,4.253052488418564
+self_attn.k_proj,9,4194304,6.984014207565668
+self_attn.v_proj,9,4194304,3.570361043992625
+self_attn.o_proj,9,16777216,4.921920154929393
+mlp.gate_proj,9,50331648,3.5751091391749426
+mlp.down_proj,9,50331648,3.539601146325055
+mlp.up_proj,9,50331648,3.3090845704394174
+self_attn.q_proj,10,16777216,3.8791462629715294
+self_attn.k_proj,10,4194304,3.950424005274065
+self_attn.v_proj,10,4194304,3.60113894746074
+self_attn.o_proj,10,16777216,4.694606308382895
+mlp.gate_proj,10,50331648,3.5787595561518257
+mlp.down_proj,10,50331648,3.963220370658456
+mlp.up_proj,10,50331648,3.3566668853646946
+self_attn.q_proj,11,16777216,4.171904103162667
+self_attn.k_proj,11,4194304,5.477098532133951
+self_attn.v_proj,11,4194304,3.460314959936971
+self_attn.o_proj,11,16777216,4.014968557915078
+mlp.gate_proj,11,50331648,3.513567430029673
+mlp.down_proj,11,50331648,3.6699457962470516
+mlp.up_proj,11,50331648,3.310197921587732
+self_attn.q_proj,12,16777216,4.170024604197322
+self_attn.k_proj,12,4194304,6.292485690498306
+self_attn.v_proj,12,4194304,3.569516304652959
+self_attn.o_proj,12,16777216,5.443451952413357
+mlp.gate_proj,12,50331648,3.634063393975273
+mlp.down_proj,12,50331648,3.8744168119735614
+mlp.up_proj,12,50331648,3.471399540298646
+self_attn.q_proj,13,16777216,4.515867970285675
+self_attn.k_proj,13,4194304,6.121190886633632
+self_attn.v_proj,13,4194304,3.4512616918857684
+self_attn.o_proj,13,16777216,3.6229957907669896
+mlp.gate_proj,13,50331648,3.6896096066771666
+mlp.down_proj,13,50331648,4.088560542019208
+mlp.up_proj,13,50331648,3.4907927800002545
+self_attn.q_proj,14,16777216,4.228741782016346
+self_attn.k_proj,14,4194304,4.241718210866317
+self_attn.v_proj,14,4194304,3.4085052837160448
+self_attn.o_proj,14,16777216,4.778758308061709
+mlp.gate_proj,14,50331648,3.6803733046000815
+mlp.down_proj,14,50331648,3.795151182835799
+mlp.up_proj,14,50331648,3.4604004365790315
+self_attn.q_proj,15,16777216,4.993077372618623
+self_attn.k_proj,15,4194304,5.37010841336525
+self_attn.v_proj,15,4194304,3.339555544152251
+self_attn.o_proj,15,16777216,3.7038482807547015
+mlp.gate_proj,15,50331648,3.8055435446227164
+mlp.down_proj,15,50331648,3.9767018841915047
+mlp.up_proj,15,50331648,3.4954030506759253
+self_attn.q_proj,16,16777216,4.766900604821532
+self_attn.k_proj,16,4194304,4.447584658567737
+self_attn.v_proj,16,4194304,3.567621309536996
+self_attn.o_proj,16,16777216,4.631544998338787
+mlp.gate_proj,16,50331648,4.052078972953341
+mlp.down_proj,16,50331648,6.65422052074798
+mlp.up_proj,16,50331648,3.80745646002702
+self_attn.q_proj,17,16777216,5.12356027665885
+self_attn.k_proj,17,4194304,4.8179813728227705
+self_attn.v_proj,17,4194304,3.3701153108243833
+self_attn.o_proj,17,16777216,4.571589099452379
+mlp.gate_proj,17,50331648,3.906622497385505
+mlp.down_proj,17,50331648,4.085787488778317
+mlp.up_proj,17,50331648,3.4277146096010616
+self_attn.q_proj,18,16777216,4.665349375537282
+self_attn.k_proj,18,4194304,4.4824421762637865
+self_attn.v_proj,18,4194304,3.4143379238489984
+self_attn.o_proj,18,16777216,4.306203688701428
+mlp.gate_proj,18,50331648,4.1348462657591405
+mlp.down_proj,18,50331648,4.186923646591098
+mlp.up_proj,18,50331648,3.543188240673786
+self_attn.q_proj,19,16777216,5.48824974250715
+self_attn.k_proj,19,4194304,8.833747900332906
+self_attn.v_proj,19,4194304,3.622880827123495
+self_attn.o_proj,19,16777216,5.651608392805515
+mlp.gate_proj,19,50331648,4.31161530455121
+mlp.down_proj,19,50331648,4.6553119530523634
+mlp.up_proj,19,50331648,3.728648726502985
+self_attn.q_proj,20,16777216,5.501777436448975
+self_attn.k_proj,20,4194304,5.1242559619562895
+self_attn.v_proj,20,4194304,3.5151964552625405
+self_attn.o_proj,20,16777216,3.991504438233053
+mlp.gate_proj,20,50331648,4.111297786337425
+mlp.down_proj,20,50331648,3.672418903504559
+mlp.up_proj,20,50331648,3.425080519569344
+self_attn.q_proj,21,16777216,5.44368131838779
+self_attn.k_proj,21,4194304,4.308970341186873
+self_attn.v_proj,21,4194304,4.275413597960965
+self_attn.o_proj,21,16777216,3.8560239268660252
+mlp.gate_proj,21,50331648,4.4178484272689555
+mlp.down_proj,21,50331648,3.864780148761345
+mlp.up_proj,21,50331648,3.405279664459031
+self_attn.q_proj,22,16777216,5.410679361978139
+self_attn.k_proj,22,4194304,9.386629080473162
+self_attn.v_proj,22,4194304,3.7696404174251024
+self_attn.o_proj,22,16777216,4.368047008944573
+mlp.gate_proj,22,50331648,4.515261077821607
+mlp.down_proj,22,50331648,4.2652255273597675
+mlp.up_proj,22,50331648,3.592689794198662
+self_attn.q_proj,23,16777216,5.265475135555726
+self_attn.k_proj,23,4194304,4.217328324467858
+self_attn.v_proj,23,4194304,4.332256874285605
+self_attn.o_proj,23,16777216,4.349143654441271
+mlp.gate_proj,23,50331648,4.073948311380071
+mlp.down_proj,23,50331648,3.807942478949522
+mlp.up_proj,23,50331648,3.3598549404746665
+self_attn.q_proj,24,16777216,4.981582379776875
+self_attn.k_proj,24,4194304,4.3863485207069735
+self_attn.v_proj,24,4194304,4.725765711052312
+self_attn.o_proj,24,16777216,5.098466972534927
+mlp.gate_proj,24,50331648,3.7347976397594636
+mlp.down_proj,24,50331648,3.587169330562164
+mlp.up_proj,24,50331648,3.2697257596461746
+self_attn.q_proj,25,16777216,4.445979585333972
+self_attn.k_proj,25,4194304,3.731637101164556
+self_attn.v_proj,25,4194304,3.606963310283395
+self_attn.o_proj,25,16777216,3.353301568862519
+mlp.gate_proj,25,50331648,3.7010141500045375
+mlp.down_proj,25,50331648,3.5062903410028596
+mlp.up_proj,25,50331648,3.2634803496549942
+self_attn.q_proj,26,16777216,4.525310941496539
+self_attn.k_proj,26,4194304,3.8516712676890608
+self_attn.v_proj,26,4194304,3.5550430098566586
+self_attn.o_proj,26,16777216,3.5029344281455024
+mlp.gate_proj,26,50331648,3.516841003730707
+mlp.down_proj,26,50331648,3.3988291312774117
+mlp.up_proj,26,50331648,3.2873835373735623
+self_attn.q_proj,27,16777216,4.364207553415131
+self_attn.k_proj,27,4194304,3.9453304969666907
+self_attn.v_proj,27,4194304,3.59525526384917
+self_attn.o_proj,27,16777216,3.466064411679335
+mlp.gate_proj,27,50331648,3.6298698945332153
+mlp.down_proj,27,50331648,3.4241784298113966
+mlp.up_proj,27,50331648,3.23329138782689
+self_attn.q_proj,28,16777216,4.342530233862741
+self_attn.k_proj,28,4194304,3.980842105404422
+self_attn.v_proj,28,4194304,3.404486681984997
+self_attn.o_proj,28,16777216,4.171793926922165
+mlp.gate_proj,28,50331648,3.7995056716172737
+mlp.down_proj,28,50331648,3.6278368893081145
+mlp.up_proj,28,50331648,3.2814764188955365
+self_attn.q_proj,29,16777216,4.57412224703693
+self_attn.k_proj,29,4194304,4.467150070024177
+self_attn.v_proj,29,4194304,4.324573662599141
+self_attn.o_proj,29,16777216,3.2883411701062375
+mlp.gate_proj,29,50331648,3.3632119145747152
+mlp.down_proj,29,50331648,3.3714387559734638
+mlp.up_proj,29,50331648,3.1470439628898874
+self_attn.q_proj,30,16777216,4.43120527525527
+self_attn.k_proj,30,4194304,4.211077263482161
+self_attn.v_proj,30,4194304,4.090042170082016
+self_attn.o_proj,30,16777216,3.4963261887536063
+mlp.gate_proj,30,50331648,3.446688163532399
+mlp.down_proj,30,50331648,3.5756297172888334
+mlp.up_proj,30,50331648,3.1448239675706744
+self_attn.q_proj,31,16777216,4.328104446565007
+self_attn.k_proj,31,4194304,4.3340755999187985
+self_attn.v_proj,31,4194304,3.8122841253068893
+self_attn.o_proj,31,16777216,3.346408056546486
+mlp.gate_proj,31,50331648,3.437135881438056
+mlp.down_proj,31,50331648,3.417398979440722
+mlp.up_proj,31,50331648,3.1753507665237564
+self_attn.q_proj,32,16777216,4.554424243476571
+self_attn.k_proj,32,4194304,4.486103847418639
+self_attn.v_proj,32,4194304,4.091581226872238
+self_attn.o_proj,32,16777216,3.6065577573925807
+mlp.gate_proj,32,50331648,3.442294596940012
+mlp.down_proj,32,50331648,3.465572932550378
+mlp.up_proj,32,50331648,3.172980374361728
+self_attn.q_proj,33,16777216,5.060775206222359
+self_attn.k_proj,33,4194304,4.680090592211514
+self_attn.v_proj,33,4194304,3.896278664855261
+self_attn.o_proj,33,16777216,3.3469264028743453
+mlp.gate_proj,33,50331648,3.540755340032798
+mlp.down_proj,33,50331648,3.401107733564681
+mlp.up_proj,33,50331648,3.227168304783581
+self_attn.q_proj,34,16777216,4.754147419649401
+self_attn.k_proj,34,4194304,4.570964463523233
+self_attn.v_proj,34,4194304,3.987365679503545
+self_attn.o_proj,34,16777216,4.135442527602238
+mlp.gate_proj,34,50331648,3.7358651692181097
+mlp.down_proj,34,50331648,6.449407651623265
+mlp.up_proj,34,50331648,3.265422178613862
+self_attn.q_proj,35,16777216,4.045388579347707
+self_attn.k_proj,35,4194304,3.789734028923394
+self_attn.v_proj,35,4194304,3.5914908167972004
+self_attn.o_proj,35,16777216,7.439195686502025
+mlp.gate_proj,35,50331648,3.8274763070771414
+mlp.down_proj,35,50331648,11.664499612717798
+mlp.up_proj,35,50331648,3.5060495766237336
diff --git a/lm-quant-toolkit/utils/combine-fnorm-llama.R b/lm-quant-toolkit/utils/combine-fnorm-llama.R
new file mode 100644
index 0000000000000000000000000000000000000000..ffeca2579759586fb6c8e59040b381382d547d42
--- /dev/null
+++ b/lm-quant-toolkit/utils/combine-fnorm-llama.R
@@ -0,0 +1,41 @@
+library(dplyr)
+library(readr)
+
+strip_name <- function(name) {
+ start <- nchar("fnorm-") + 1
+ stop <- nchar(name) - 4
+ return(substr(name, start, stop))
+}
+
+fnorm_data_dir <- "../src/data/tmp"
+fnorm_dir <- normalizePath(fnorm_data_dir)
+fnorm_fps <- dir(
+ path = fnorm_dir,
+ pattern = paste0("fnorm-.*", "\\.csv$"),
+ full.names = TRUE
+)
+names(fnorm_fps) <- sapply((basename(fnorm_fps)), strip_name)
+df_fnorm <- plyr::ldply(
+ fnorm_fps,
+ read.csv,
+ stringsAsFactors = FALSE,
+ .id = "model"
+)
+
+k_cols <- c(
+ "model",
+ "module",
+ "layer",
+ "cfg",
+ "memmb",
+ "params",
+ "fnorm",
+ "sensitivity",
+ "kurtosis"
+)
+df_fnorm <- df_fnorm |>
+ dplyr::mutate(
+ cfg = paste0("b", nbit1, "g", gsize1)
+ ) |>
+ select(all_of(k_cols))
+write_csv(df_fnorm, "llama-fnorm.csv")
diff --git a/lm-quant-toolkit/utils/dense.txt b/lm-quant-toolkit/utils/dense.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ec991a0c6025dbce52d7883cf2b086a3276c944e
--- /dev/null
+++ b/lm-quant-toolkit/utils/dense.txt
@@ -0,0 +1,259 @@
+8.51
+8.25
+8.13
+7.76
+7.74
+7.72
+7.7
+7.68
+7.66
+7.63
+7.62
+7.6
+7.57
+7.56
+7.54
+7.53
+7.5
+7.48
+7.47
+7.44
+7.42
+7.41
+7.38
+7.36
+7.34
+7.32
+7.3
+7.29
+7.26
+7.24
+7.22
+7.2
+7.19
+7.16
+7.14
+7.13
+7.1
+7.08
+7.06
+7.04
+7.02
+7.01
+6.98
+6.96
+6.95
+6.92
+6.89
+6.87
+6.86
+6.83
+6.81
+6.78
+6.77
+6.75
+6.72
+6.71
+6.69
+6.68
+6.65
+6.63
+6.61
+6.59
+6.57
+6.55
+6.53
+6.51
+6.49
+6.47
+6.45
+6.43
+6.41
+6.39
+6.37
+6.35
+6.33
+6.31
+6.29
+6.27
+6.25
+6.23
+6.21
+6.19
+6.17
+6.15
+6.13
+6.11
+6.09
+6.07
+6.06
+6.05
+6.04
+6.03
+6.02
+6
+5.98
+5.96
+5.94
+5.92
+5.9
+5.88
+5.86
+5.84
+5.82
+5.8
+5.78
+5.76
+5.74
+5.72
+5.7
+5.68
+5.66
+5.64
+5.62
+5.6
+5.58
+5.56
+5.54
+5.52
+5.5
+5.48
+5.46
+5.44
+5.42
+5.4
+5.38
+5.36
+5.34
+5.32
+5.3
+5.28
+5.26
+5.24
+5.22
+5.2
+5.18
+5.16
+5.14
+5.12
+5.1
+5.08
+5.06
+5.04
+5.02
+5
+4.99
+4.97
+4.95
+4.93
+4.91
+4.89
+4.87
+4.85
+4.83
+4.81
+4.79
+4.77
+4.75
+4.73
+4.71
+4.69
+4.67
+4.65
+4.63
+4.61
+4.59
+4.57
+4.55
+4.53
+4.51
+4.49
+4.47
+4.45
+4.43
+4.41
+4.39
+4.37
+4.35
+4.33
+4.31
+4.29
+4.27
+4.25
+4.23
+4.21
+4.19
+4.17
+4.15
+4.13
+4.11
+4.09
+4.07
+4.05
+4.03
+4.01
+3.99
+3.97
+3.95
+3.93
+3.91
+3.89
+3.87
+3.85
+3.83
+3.81
+3.79
+3.77
+3.75
+3.73
+3.71
+3.69
+3.67
+3.65
+3.63
+3.61
+3.59
+3.57
+3.55
+3.53
+3.51
+3.49
+3.47
+3.45
+3.43
+3.42
+3.41
+3.39
+3.37
+3.35
+3.33
+3.31
+3.29
+3.27
+3.25
+3.23
+3.21
+3.19
+3.17
+3.15
+3.13
+3.11
+3.09
+3.07
+2.57
+2.55
+2.53
+2.51
+2.49
+2.47
+2.45
+2.31
+2.29
+2.27
+2.25
+2.23
+2.21
+2.19
+2.17
+2.15
+2.13
diff --git a/lm-quant-toolkit/utils/gen-ds-bos/debug-llama2-7b.sh b/lm-quant-toolkit/utils/gen-ds-bos/debug-llama2-7b.sh
new file mode 100644
index 0000000000000000000000000000000000000000..db309055a0a9ef8d503b10a9968c4c726aebcc35
--- /dev/null
+++ b/lm-quant-toolkit/utils/gen-ds-bos/debug-llama2-7b.sh
@@ -0,0 +1,5 @@
+python -m pdb generate.py \
+ --model="meta-llama/Llama-2-7b-chat-hf" \
+ --variant="vLLM" \
+ --topic-file=topics-l1.txt \
+ --trace
diff --git a/lm-quant-toolkit/utils/gen-ds-bos/generate.py b/lm-quant-toolkit/utils/gen-ds-bos/generate.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4b92a3f7f49627e8ebd53ef010ef10d65c073d5
--- /dev/null
+++ b/lm-quant-toolkit/utils/gen-ds-bos/generate.py
@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+
+import re
+from argparse import ArgumentParser
+from datetime import datetime
+from timeit import default_timer as timer
+
+import pandas as pd
+from langchain.prompts import PromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain_community.llms import VLLMOpenAI
+from langchain_core.output_parsers.list import CommaSeparatedListOutputParser
+
+
+def generate_level1_topic(model, variant, topic, count=10):
+ output_parser = CommaSeparatedListOutputParser()
+ format_instructions = output_parser.get_format_instructions()
+ template = """
+Please divide the {topic} into {count} most important sub-topics.
+Number each sub-topic. Give answers without any extra description
+or explanation. {format_instructions}
+"""
+ prompt = PromptTemplate(
+ template=template,
+ input_variables=["topic", "count"],
+ partial_variables={"format_instructions": format_instructions},
+ )
+
+ llm = _get_llm_model(model, variant)
+
+ chain = prompt | llm | output_parser
+ result = chain.invoke({"count": count, "topic": topic})
+ return result
+
+
+def generate_level2_topic(model, variant, topic):
+ template = """
+Please explain {topic} concisely in around 100 to 120 words.
+Give the answer directly without any extra words such as
+"sure", "certainly", "of course" etc. Don't begin the answer with
+words such as "system", "expert" etc to indicate role.
+"""
+ prompt = PromptTemplate(
+ template=template,
+ input_variables=["topic"],
+ )
+
+ llm = _get_llm_model(model, variant)
+
+ chain = prompt | llm | StrOutputParser()
+ result = chain.invoke({"topic": topic})
+
+ return result
+
+
+def parse_level1_output(topic, output):
+ pat = re.compile(r"^\d+\.\s+")
+ ret = []
+ for topic in output:
+ m = re.match(pat, topic)
+ if m:
+ ret.append(topic[m.end(0) :])
+ return ret
+
+
+def parse_output(model, topic, text):
+ instructions = text.split("\n")
+ dict_list = []
+ for inst in instructions:
+ if len(inst) < 60:
+ continue
+ ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+ new_task = {
+ "topic": topic,
+ "timestamp": ts,
+ "text": inst,
+ }
+ dict_list.append(new_task)
+ return dict_list
+
+
+def _get_llm_model(model, variant):
+ return VLLMOpenAI(
+ model_name=model,
+ openai_api_key="token-abc123",
+ openai_api_base="http://localhost:8000/v1",
+ temperature=0.6,
+ )
+
+
+def save_llm_raw_output(llm_output_dir, model, variant, topic, text):
+ short_id = model.split("/")[1]
+ topic_str = topic.replace(" ", "-").replace("/", "-")
+ file_name = f"{llm_output_dir}/output-{topic_str}-{short_id}-{variant}.log"
+ with open(file_name, "w") as writer:
+ writer.write(text)
+
+
+def generate(plans, llm_output_dir, model="llama", variant="7b", trace=True):
+ t0 = timer()
+ dict_list = []
+ for plan in plans:
+ ti0 = timer()
+ topic = plan["topic"]
+ output = generate_level1_topic(model, variant, topic, count=plan["count"])
+ ti1 = timer()
+ sub_topics = parse_level1_output(topic, output)
+ for sub_topic in sub_topics:
+ output = generate_level2_topic(model, variant, sub_topic)
+ if trace:
+ save_llm_raw_output(
+ llm_output_dir,
+ model,
+ variant,
+ sub_topic,
+ output,
+ )
+ print(f"{sub_topic} generation took {(ti1 - ti0):.2f} seconds!")
+ records = parse_output(model, sub_topic, output)
+ dict_list.extend(records)
+ df = pd.DataFrame(dict_list)
+ short_id = model.split("/")[1]
+ file_name = f"data-{short_id}-{variant}.csv"
+ df.to_csv(
+ file_name, columns=["topic", "text", "timestamp"], index=False, doublequote=True
+ )
+
+ t1 = timer()
+ if trace:
+ print(f"All instructions generation took {t1 - t0} seconds")
+
+
+def _parse_args():
+ # Create the parser
+ parser = ArgumentParser(description="Kubernetes SFT Dataset Generator")
+
+ # Add arguments
+ parser.add_argument(
+ "-m",
+ "--model",
+ required=True,
+ help="LLM model name, eg: claude, gpt, mistral, llama2",
+ )
+ parser.add_argument(
+ "-v", "--variant", required=True, help="model variant such as opus, sonnet"
+ )
+ parser.add_argument(
+ "-d", "--trace", action="store_true", default=False, help="Print trace messages"
+ )
+ parser.add_argument(
+ "--topic-file",
+ default="topics.txt",
+ help="The Kubernetes topics for generated instructions",
+ )
+ parser.add_argument(
+ "--llm-output-dir",
+ default="results",
+ help="The directory to save raw LLM output",
+ )
+
+ # Parse the arguments
+ args = parser.parse_args()
+ return args
+
+
+def _load_topics(file):
+ plans = []
+ with open(file, "r") as f:
+ for line in f.readlines():
+ topic, count = line.split(":")
+ topic = topic.replace('"', "")
+ plans.append(
+ {
+ "topic": topic,
+ "count": int(count.rstrip()),
+ }
+ )
+ return plans
+
+
+if __name__ == "__main__":
+ args = _parse_args()
+ print(args)
+
+ model = args.model
+ variant = args.variant
+ trace = args.trace
+ llm_output_dir = args.llm_output_dir
+ plans = _load_topics(args.topic_file)
+
+ generate(plans, llm_output_dir, model, variant, trace)
diff --git a/lm-quant-toolkit/utils/join.R b/lm-quant-toolkit/utils/join.R
new file mode 100644
index 0000000000000000000000000000000000000000..0344fdfc11e1f00b90df8fa1955a84e6733228d4
--- /dev/null
+++ b/lm-quant-toolkit/utils/join.R
@@ -0,0 +1,19 @@
+#!/usr/bin/env Rscript
+
+library(tidyverse)
+library(dplyr)
+library(readr)
+
+join_kurt <- function(model_id) {
+ fnorm_csv_fp <- paste0("fnorm-", model_id, ".csv")
+ kurt_csv_fp <- paste0("kurtosis-", model_id, ".csv")
+ df1 <- read_csv(fnorm_csv_fp)
+ df2 <- read_csv(kurt_csv_fp)
+ df <- df1 |>
+ left_join(df2, by = c("module", "layer"))
+ write_csv(df, fnorm_csv_fp)
+}
+
+join_kurt("CLIP-ViT-B-32-laion2B-s34B-b79K")
+join_kurt("CLIP-ViT-L-14-laion2B-s32B-b82K")
+join_kurt("CLIP-ViT-H-14-laion2B-s32B-b79K")
diff --git a/lm-quant-toolkit/utils/pre-commit b/lm-quant-toolkit/utils/pre-commit
new file mode 100644
index 0000000000000000000000000000000000000000..38cdc1a3b54613ef0544f8f662292e5febd55cd9
--- /dev/null
+++ b/lm-quant-toolkit/utils/pre-commit
@@ -0,0 +1,11 @@
+#!/bin/sh -e
+
+rootdir=`git rev-parse --show-toplevel`
+(
+ cd $rootdir
+ python -m venv .pre-commit-venv
+ .pre-commit-venv/bin/python -m pip install tox
+ rm -rf dist .cache *.egg-info _build
+ find ./ -name '*.pyc' -delete
+ .pre-commit-venv/bin/tox $@
+)
diff --git a/lm-quant-toolkit/utils/typeset-bit-budget.py b/lm-quant-toolkit/utils/typeset-bit-budget.py
new file mode 100644
index 0000000000000000000000000000000000000000..db3cc4b538812d95b34ccab1bb47e550dccb5e15
--- /dev/null
+++ b/lm-quant-toolkit/utils/typeset-bit-budget.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+
+def to_latex(bits, nbit, cols=10):
+ v = len(bits) // cols
+ rows = v if len(bits) % cols == 0 else v + 1
+ for i, bit in enumerate(bits):
+ if i == 0:
+ print("\\midrule")
+ print(
+ "\\multirowcell{" + str(rows) + "}{\\textbf{" + str(nbit) + "-bit}} ",
+ end="",
+ )
+ print(f"& {bit:.2f} ", end="")
+ if (i + 1) % cols == 0:
+ print(r" \\")
+ elif i == len(bits) - 1:
+ padding = " &" * (cols - (i + 1) % cols)
+ print(f"{padding} \\\\")
+
+
+def main(ncols=10):
+ head = f"""
+\\begin{{table*}}
+\\centering
+\\caption[SensiMiLP/KurtMiLP Experiment Settings]{{
+The SensiMiLP/KurtMiLP dense experiment bit budget settings.
+}}
+\\label{{tab:sensi-kurt-milp-settings}}
+\\begin{{tabular}}{{r{'l' * ncols}}}
+\\toprule
+"""
+
+ heading = r"\textbf{Group} & "
+ fields = [f" \\textbf{{V{i}}}" for i in range(1, ncols + 1)]
+ heading += "&".join(fields)
+ heading += r" \\"
+
+ tail = r"""
+\bottomrule
+\end{tabular}
+\end{table*}
+"""
+
+ with open("dense.txt", "r") as fh:
+ bits = [float(f) for f in fh.readlines()]
+ bits8 = [f for f in bits if f >= 8.0]
+ bits7 = [f for f in bits if f >= 7.0 and f < 8.0]
+ bits6 = [f for f in bits if f >= 6.0 and f < 7.0]
+ bits5 = [f for f in bits if f >= 5.0 and f < 6.0]
+ bits4 = [f for f in bits if f >= 4.0 and f < 5.0]
+ bits3 = [f for f in bits if f >= 3.0 and f < 4.0]
+ bits2 = [f for f in bits if f >= 2.0 and f < 3.0]
+
+ print(head)
+ print(heading)
+ to_latex(bits8, 8, ncols)
+ to_latex(bits7, 7, ncols)
+ to_latex(bits6, 6, ncols)
+ to_latex(bits5, 5, ncols)
+ to_latex(bits4, 4, ncols)
+ to_latex(bits3, 3, ncols)
+ to_latex(bits2, 2, ncols)
+ print(tail)
+
+
+if __name__ == "__main__":
+ main(12)
diff --git a/models/Qwen/Qwen2.5-14B/model-00008-of-00008.safetensors b/models/Qwen/Qwen2.5-14B/model-00008-of-00008.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..dd1309620ebee47f63ac40b9b235b7ae5f9711e6
--- /dev/null
+++ b/models/Qwen/Qwen2.5-14B/model-00008-of-00008.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b693615965d8548eab4d1dc6bb578aa063e8656e2b9d25125ad5b8c7f59565f
+size 1698724408
diff --git a/models/Qwen/Qwen2.5-7B/tokenizer.json b/models/Qwen/Qwen2.5-7B/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..443909a61d429dff23010e5bddd28ff530edda00
--- /dev/null
+++ b/models/Qwen/Qwen2.5-7B/tokenizer.json
@@ -0,0 +1,303282 @@
+{
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [
+ {
+ "id": 151643,
+ "content": "<|endoftext|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 151644,
+ "content": "<|im_start|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 151645,
+ "content": "<|im_end|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 151646,
+ "content": "<|object_ref_start|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151647,
+ "content": "<|object_ref_end|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151648,
+ "content": "<|box_start|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151649,
+ "content": "<|box_end|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151650,
+ "content": "<|quad_start|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151651,
+ "content": "<|quad_end|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151652,
+ "content": "<|vision_start|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151653,
+ "content": "<|vision_end|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151654,
+ "content": "<|vision_pad|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151655,
+ "content": "<|image_pad|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151656,
+ "content": "<|video_pad|>",
+ "normalized": false,
+ "lstrip": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ {
+ "id": 151657,
+ "content": "
&": 5789,
+ "CON": 5790,
+ "Ġrepl": 5791,
+ "Ġregular": 5792,
+ "Storage": 5793,
+ "ramework": 5794,
+ "Ġgoal": 5795,
+ "Ġtouch": 5796,
+ ".widget": 5797,
+ "Ġbuilt": 5798,
+ "des": 5799,
+ "Part": 5800,
+ "(re": 5801,
+ "Ġworth": 5802,
+ "hib": 5803,
+ "game": 5804,
+ "Ġв": 5805,
+ "acion": 5806,
+ "ĠWhite": 5807,
+ "(type": 5808,
+ "(`": 5809,
+ "Ġnatural": 5810,
+ "Ġinj": 5811,
+ "Ġcalcul": 5812,
+ "ĠApril": 5813,
+ ".List": 5814,
+ "Ġassociated": 5815,
+ "ĉSystem": 5816,
+ "~~": 5817,
+ "=[": 5818,
+ "Ġstorage": 5819,
+ "Ġbytes": 5820,
+ "Ġtravel": 5821,
+ "Ġsou": 5822,
+ "Ġpassed": 5823,
+ "!=": 5824,
+ "ascript": 5825,
+ ".open": 5826,
+ "Ġgrid": 5827,
+ "Ġbus": 5828,
+ "Ġrecogn": 5829,
+ "Ab": 5830,
+ "Ġhon": 5831,
+ "ĠCenter": 5832,
+ "Ġprec": 5833,
+ "build": 5834,
+ "HTML": 5835,
+ "ĠSan": 5836,
+ "Ġcountries": 5837,
+ "aled": 5838,
+ "token": 5839,
+ "kt": 5840,
+ "Ġqual": 5841,
+ "Last": 5842,
+ "adow": 5843,
+ "Ġmanufact": 5844,
+ "idad": 5845,
+ "jango": 5846,
+ "Next": 5847,
+ "xf": 5848,
+ ".a": 5849,
+ "Ġporno": 5850,
+ "ĠPM": 5851,
+ "erve": 5852,
+ "iting": 5853,
+ "_th": 5854,
+ "ci": 5855,
+ "=None": 5856,
+ "gs": 5857,
+ "Ġlogin": 5858,
+ "atives": 5859,
+ "']);Ċ": 5860,
+ "Äħ": 5861,
+ "Ġill": 5862,
+ "IA": 5863,
+ "children": 5864,
+ "DO": 5865,
+ "Ġlevels": 5866,
+ "Ġ{{": 5867,
+ "Ġlooks": 5868,
+ "Ġ\"#": 5869,
+ "ToString": 5870,
+ "Ġnecessary": 5871,
+ "ĠĠĠĊ": 5872,
+ "cell": 5873,
+ "Entry": 5874,
+ "Ġ'#": 5875,
+ "Ġextrem": 5876,
+ "Selector": 5877,
+ "Ġplaceholder": 5878,
+ "Load": 5879,
+ "Ġreleased": 5880,
+ "ORE": 5881,
+ "Enumer": 5882,
+ "ĠTV": 5883,
+ "SET": 5884,
+ "inq": 5885,
+ "Press": 5886,
+ "ĠDepartment": 5887,
+ "Ġproperties": 5888,
+ "Ġrespond": 5889,
+ "Search": 5890,
+ "ael": 5891,
+ "Ġrequ": 5892,
+ "ĠBook": 5893,
+ "/Ċ": 5894,
+ "(st": 5895,
+ "Ġfinancial": 5896,
+ "icket": 5897,
+ "_input": 5898,
+ "Ġthreat": 5899,
+ "(in": 5900,
+ "Strip": 5901,
+ "ìĿ": 5902,
+ "ção": 5903,
+ "Ġevidence": 5904,
+ "));": 5905,
+ "ĠBro": 5906,
+ "Ġ[];Ċ": 5907,
+ "Ġou": 5908,
+ "buf": 5909,
+ "Script": 5910,
+ "dat": 5911,
+ "Ġrule": 5912,
+ "#import": 5913,
+ "=\"/": 5914,
+ "Serial": 5915,
+ "Ġstarting": 5916,
+ "[index": 5917,
+ "ae": 5918,
+ "Ġcontrib": 5919,
+ "session": 5920,
+ "_new": 5921,
+ "utable": 5922,
+ "ober": 5923,
+ "Ġ\"./": 5924,
+ "Ġlogger": 5925,
+ "Ġrecently": 5926,
+ "Ġreturned": 5927,
+ "ččĊ": 5928,
+ ")))Ċ": 5929,
+ "itions": 5930,
+ "Ġseek": 5931,
+ "Ġcommunic": 5932,
+ "Ġ\".": 5933,
+ "Ġusername": 5934,
+ "ECT": 5935,
+ "DS": 5936,
+ "Ġotherwise": 5937,
+ "ĠGerman": 5938,
+ ".aw": 5939,
+ "Adapter": 5940,
+ "ixel": 5941,
+ "Ġsystems": 5942,
+ "Ġdrop": 5943,
+ "Ġstructure": 5944,
+ "Ġ$(\"#": 5945,
+ "encies": 5946,
+ "anning": 5947,
+ "ĠLink": 5948,
+ "ĠResponse": 5949,
+ "Ġstri": 5950,
+ "ż": 5951,
+ "ĠDB": 5952,
+ "æĹ": 5953,
+ "android": 5954,
+ "submit": 5955,
+ "otion": 5956,
+ "(@": 5957,
+ ".test": 5958,
+ "ĊĊĊĊĊĊĊĊ": 5959,
+ "];čĊ": 5960,
+ "Ġdirectly": 5961,
+ "Ġ\"%": 5962,
+ "ris": 5963,
+ "elta": 5964,
+ "AIL": 5965,
+ "){čĊ": 5966,
+ "mine": 5967,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 5968,
+ "(k": 5969,
+ "bon": 5970,
+ "asic": 5971,
+ "pite": 5972,
+ "___": 5973,
+ "Max": 5974,
+ "Ġerrors": 5975,
+ "ĠWhile": 5976,
+ "Ġarguments": 5977,
+ "Ġensure": 5978,
+ "Right": 5979,
+ "-based": 5980,
+ "Web": 5981,
+ "Ġ-=": 5982,
+ "Ġintrodu": 5983,
+ "ĠInst": 5984,
+ "ĠWash": 5985,
+ "ordin": 5986,
+ "join": 5987,
+ "Database": 5988,
+ "Ġgrad": 5989,
+ "Ġusually": 5990,
+ "ITE": 5991,
+ "Props": 5992,
+ "?>Ċ": 5993,
+ "ĠGo": 5994,
+ "@Override": 5995,
+ "REF": 5996,
+ "Ġip": 5997,
+ "ĠAustral": 5998,
+ "Ġist": 5999,
+ "ViewById": 6000,
+ "Ġserious": 6001,
+ "Ġcustomer": 6002,
+ ".prototype": 6003,
+ "odo": 6004,
+ "cor": 6005,
+ "Ġdoor": 6006,
+ "ĠWITHOUT": 6007,
+ "Ġplant": 6008,
+ "Ġbegan": 6009,
+ "Ġdistance": 6010,
+ "()).": 6011,
+ "Ġchance": 6012,
+ "Ġord": 6013,
+ "came": 6014,
+ "pragma": 6015,
+ "Ġprotect": 6016,
+ "ragment": 6017,
+ "ĠNode": 6018,
+ "ening": 6019,
+ "Ñĩ": 6020,
+ "Ġroute": 6021,
+ "ĠSchool": 6022,
+ "hi": 6023,
+ "Ġneighb": 6024,
+ "After": 6025,
+ "licit": 6026,
+ "Ġcontr": 6027,
+ "Ġprimary": 6028,
+ "AA": 6029,
+ ".WriteLine": 6030,
+ "utils": 6031,
+ "Ġbi": 6032,
+ "Red": 6033,
+ ".Linq": 6034,
+ ".object": 6035,
+ "Ġleaders": 6036,
+ "unities": 6037,
+ "Ġgun": 6038,
+ "onth": 6039,
+ "ĠDev": 6040,
+ "FILE": 6041,
+ "Ġcomments": 6042,
+ "_len": 6043,
+ "arrow": 6044,
+ "amount": 6045,
+ "Range": 6046,
+ "sert": 6047,
+ "GridView": 6048,
+ "Ġupdated": 6049,
+ "ĠMo": 6050,
+ "Ġinform": 6051,
+ "ociety": 6052,
+ "ala": 6053,
+ "Access": 6054,
+ "Ġhab": 6055,
+ "Ġcreat": 6056,
+ "_arg": 6057,
+ "ĠJanuary": 6058,
+ "ĠDay": 6059,
+ "\")čĊ": 6060,
+ "uple": 6061,
+ "document": 6062,
+ "gorith": 6063,
+ "menu": 6064,
+ "ĠOver": 6065,
+ "bb": 6066,
+ ".title": 6067,
+ "_out": 6068,
+ "Ġled": 6069,
+ "uri": 6070,
+ "Ġ?>": 6071,
+ "gl": 6072,
+ "Ġbank": 6073,
+ "ayment": 6074,
+ "ĉprintf": 6075,
+ "MD": 6076,
+ "Ġsample": 6077,
+ "Ġhands": 6078,
+ "ĠVersion": 6079,
+ "uario": 6080,
+ "Ġoffers": 6081,
+ "ityEngine": 6082,
+ "Ġshape": 6083,
+ "Ġsleep": 6084,
+ "_point": 6085,
+ "Settings": 6086,
+ "Ġachie": 6087,
+ "Ġsold": 6088,
+ "ota": 6089,
+ ".bind": 6090,
+ "Am": 6091,
+ "Ġsafe": 6092,
+ "Store": 6093,
+ "Ġshared": 6094,
+ "Ġpriv": 6095,
+ "_VAL": 6096,
+ "Ġsens": 6097,
+ "){": 6098,
+ "Ġremember": 6099,
+ "shared": 6100,
+ "element": 6101,
+ "Ġshoot": 6102,
+ "Vert": 6103,
+ "cout": 6104,
+ "Ġenv": 6105,
+ "_label": 6106,
+ "Ġ>Ċ": 6107,
+ "run": 6108,
+ "Ġscene": 6109,
+ "(array": 6110,
+ "device": 6111,
+ "_title": 6112,
+ "agon": 6113,
+ "]čĊ": 6114,
+ "aby": 6115,
+ "Ġbecame": 6116,
+ "boolean": 6117,
+ "Ġpark": 6118,
+ "ĠCode": 6119,
+ "upload": 6120,
+ "riday": 6121,
+ "ĠSeptember": 6122,
+ "Fe": 6123,
+ "Ġsen": 6124,
+ "cing": 6125,
+ "FL": 6126,
+ "Col": 6127,
+ "uts": 6128,
+ "_page": 6129,
+ "inn": 6130,
+ "Ġimplied": 6131,
+ "aling": 6132,
+ "Ġyourself": 6133,
+ ".Count": 6134,
+ "conf": 6135,
+ "Ġaud": 6136,
+ "_init": 6137,
+ ".)": 6138,
+ "Ġwrote": 6139,
+ "NG": 6140,
+ ".Error": 6141,
+ "ä»": 6142,
+ ".for": 6143,
+ "Ġequal": 6144,
+ "ĠRequest": 6145,
+ "Ġserial": 6146,
+ "Ġallows": 6147,
+ "XX": 6148,
+ "Ġmiddle": 6149,
+ "chor": 6150,
+ "ø": 6151,
+ "erval": 6152,
+ ".Column": 6153,
+ "reading": 6154,
+ "Ġescort": 6155,
+ "ĠAugust": 6156,
+ "Ġquickly": 6157,
+ "Ġweap": 6158,
+ "ĠCG": 6159,
+ "ropri": 6160,
+ "ho": 6161,
+ "Ġcop": 6162,
+ "(struct": 6163,
+ "ĠBig": 6164,
+ "Ġvs": 6165,
+ "Ġfrequ": 6166,
+ ".Value": 6167,
+ "Ġactions": 6168,
+ "Ġproper": 6169,
+ "Ġinn": 6170,
+ "Ġobjects": 6171,
+ "Ġmatrix": 6172,
+ "avascript": 6173,
+ "Ġones": 6174,
+ ".group": 6175,
+ "Ġgreen": 6176,
+ "Ġpaint": 6177,
+ "ools": 6178,
+ "ycl": 6179,
+ "encode": 6180,
+ "olt": 6181,
+ "comment": 6182,
+ ".api": 6183,
+ "Dir": 6184,
+ "Ġune": 6185,
+ "izont": 6186,
+ ".position": 6187,
+ "Ġdesigned": 6188,
+ "_val": 6189,
+ "avi": 6190,
+ "iring": 6191,
+ "tab": 6192,
+ "Ġlayer": 6193,
+ "Ġviews": 6194,
+ "Ġreve": 6195,
+ "rael": 6196,
+ "ĠON": 6197,
+ "rics": 6198,
+ "np": 6199,
+ "Ġcore": 6200,
+ "());čĊ": 6201,
+ "Main": 6202,
+ "Ġexpert": 6203,
+ "ĉĉčĊ": 6204,
+ "_en": 6205,
+ "Ġ/>": 6206,
+ "utter": 6207,
+ "IAL": 6208,
+ "ails": 6209,
+ "ĠKing": 6210,
+ "*/ĊĊ": 6211,
+ "ĠMet": 6212,
+ "_end": 6213,
+ "addr": 6214,
+ "ora": 6215,
+ "Ġir": 6216,
+ "Min": 6217,
+ "Ġsurpr": 6218,
+ "Ġrepe": 6219,
+ "Ġdirectory": 6220,
+ "PUT": 6221,
+ "-S": 6222,
+ "Ġelection": 6223,
+ "haps": 6224,
+ ".pre": 6225,
+ "cm": 6226,
+ "Values": 6227,
+ "Ġ\"Ċ": 6228,
+ "column": 6229,
+ "ivil": 6230,
+ "Login": 6231,
+ "inue": 6232,
+ "Ġbeautiful": 6233,
+ "Ġsecret": 6234,
+ "(event": 6235,
+ "Ġchat": 6236,
+ "ums": 6237,
+ "Ġorigin": 6238,
+ "Ġeffects": 6239,
+ "Ġmanagement": 6240,
+ "illa": 6241,
+ "tk": 6242,
+ "Ġsetting": 6243,
+ "ĠCour": 6244,
+ "Ġmassage": 6245,
+ "ĉend": 6246,
+ "Ġhappy": 6247,
+ "Ġfinish": 6248,
+ "Ġcamera": 6249,
+ "ĠVer": 6250,
+ "ĠDemocr": 6251,
+ "ĠHer": 6252,
+ "(Q": 6253,
+ "cons": 6254,
+ "ita": 6255,
+ "Ġ'.": 6256,
+ "{}": 6257,
+ "ĉC": 6258,
+ "Ġstuff": 6259,
+ "Ġ:Ċ": 6260,
+ "ĠAR": 6261,
+ "Task": 6262,
+ "hidden": 6263,
+ "eros": 6264,
+ "IGN": 6265,
+ "atio": 6266,
+ "ĠHealth": 6267,
+ "olute": 6268,
+ "Enter": 6269,
+ "'>": 6270,
+ "ĠTwitter": 6271,
+ "ĠCounty": 6272,
+ "scribe": 6273,
+ "Ġ=>Ċ": 6274,
+ "Ġhy": 6275,
+ "fit": 6276,
+ "Ġmilitary": 6277,
+ "Ġsale": 6278,
+ "required": 6279,
+ "non": 6280,
+ "bootstrap": 6281,
+ "hold": 6282,
+ "rim": 6283,
+ "-old": 6284,
+ "ĠDown": 6285,
+ "Ġmention": 6286,
+ "contact": 6287,
+ "_group": 6288,
+ "oday": 6289,
+ "Ġtown": 6290,
+ "Ġsolution": 6291,
+ "uate": 6292,
+ "elling": 6293,
+ "]->": 6294,
+ "otes": 6295,
+ "ental": 6296,
+ "omen": 6297,
+ "ospital": 6298,
+ "ĠSup": 6299,
+ "_EN": 6300,
+ "Ġslow": 6301,
+ "SESSION": 6302,
+ "Ġblue": 6303,
+ "ago": 6304,
+ "Ġlives": 6305,
+ "Ġ^": 6306,
+ ".un": 6307,
+ "inst": 6308,
+ "enge": 6309,
+ "Ġcustomers": 6310,
+ "Ġcast": 6311,
+ "udget": 6312,
+ "ï¼ģ": 6313,
+ "icens": 6314,
+ "Ġdetermin": 6315,
+ "Selected": 6316,
+ "_pl": 6317,
+ "ueue": 6318,
+ "Ġdark": 6319,
+ "//ĊĊ": 6320,
+ "si": 6321,
+ "thern": 6322,
+ "ĠJapan": 6323,
+ "/w": 6324,
+ "PU": 6325,
+ "ĠEast": 6326,
+ "ovie": 6327,
+ "Ġpackage": 6328,
+ "Ġnor": 6329,
+ "Ġapi": 6330,
+ "bot": 6331,
+ "\"];Ċ": 6332,
+ "_post": 6333,
+ "ulate": 6334,
+ "Ġclub": 6335,
+ "'));Ċ": 6336,
+ "Ġloop": 6337,
+ "PIO": 6338,
+ "ione": 6339,
+ "shot": 6340,
+ "Initial": 6341,
+ "Ġplayed": 6342,
+ "register": 6343,
+ "rought": 6344,
+ "_max": 6345,
+ "acement": 6346,
+ "match": 6347,
+ "raphics": 6348,
+ "AST": 6349,
+ "Ġexisting": 6350,
+ "Ġcomplex": 6351,
+ "DA": 6352,
+ ".Ch": 6353,
+ ".common": 6354,
+ "mo": 6355,
+ "Ġ'../../": 6356,
+ "ito": 6357,
+ "Ġanalysis": 6358,
+ "Ġdeliver": 6359,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ": 6360,
+ "idx": 6361,
+ "Ãł": 6362,
+ "ongo": 6363,
+ "ĠEnglish": 6364,
+ "Ċ": 9992,
+ "_default": 9993,
+ "ĠDatabase": 9994,
+ "rep": 9995,
+ "ESS": 9996,
+ "nergy": 9997,
+ ".Find": 9998,
+ "_mask": 9999,
+ "Ġrise": 10000,
+ "Ġkernel": 10001,
+ "::$": 10002,
+ ".Q": 10003,
+ "Ġoffering": 10004,
+ "decl": 10005,
+ "ĠCS": 10006,
+ "Ġlisted": 10007,
+ "Ġmostly": 10008,
+ "enger": 10009,
+ "Ġblocks": 10010,
+ "olo": 10011,
+ "Ġgoverning": 10012,
+ "\\F": 10013,
+ "Ġconcent": 10014,
+ ".getText": 10015,
+ "Ġmb": 10016,
+ "Ġoccurred": 10017,
+ "Ġchanging": 10018,
+ "Scene": 10019,
+ "_CODE": 10020,
+ "Beh": 10021,
+ "\"The": 10022,
+ "Ġtile": 10023,
+ "ĠAssociation": 10024,
+ "ĉP": 10025,
+ "alty": 10026,
+ "_ad": 10027,
+ "odies": 10028,
+ "iated": 10029,
+ "Ġprepared": 10030,
+ "possible": 10031,
+ "Ġmort": 10032,
+ "TEST": 10033,
+ "Ġignore": 10034,
+ "Ġcalc": 10035,
+ "Ġrs": 10036,
+ "ĠassertEquals": 10037,
+ "Ġsz": 10038,
+ "ĠTHIS": 10039,
+ ".\"Ċ": 10040,
+ "Ġcanvas": 10041,
+ "java": 10042,
+ "Ġdut": 10043,
+ "VALID": 10044,
+ ".sql": 10045,
+ ".input": 10046,
+ "Ġaux": 10047,
+ "Sup": 10048,
+ "Ġartist": 10049,
+ "Vec": 10050,
+ "_TIME": 10051,
+ ".stringify": 10052,
+ "etween": 10053,
+ "ĠCategory": 10054,
+ "Ġ[-": 10055,
+ "ĠDevExpress": 10056,
+ "ĠJul": 10057,
+ "Ġring": 10058,
+ ".ed": 10059,
+ "YY": 10060,
+ "Let": 10061,
+ "TextField": 10062,
+ "Ġflat": 10063,
+ "_print": 10064,
+ "ĠOTHER": 10065,
+ "adian": 10066,
+ "Ġchecked": 10067,
+ "ele": 10068,
+ "Align": 10069,
+ "standing": 10070,
+ "Ġ[],": 10071,
+ "Ġlab": 10072,
+ "ucky": 10073,
+ "ĠChristmas": 10074,
+ "(image": 10075,
+ ".module": 10076,
+ "Ġlots": 10077,
+ "Ġslightly": 10078,
+ "(final": 10079,
+ "erge": 10080,
+ "è¿": 10081,
+ "ĠPolice": 10082,
+ "ĠRight": 10083,
+ "Ġaward": 10084,
+ "ĠOS": 10085,
+ "Ġ{}ĊĊ": 10086,
+ "Ġptr": 10087,
+ "oves": 10088,
+ "icated": 10089,
+ "ем": 10090,
+ "Ġmanage": 10091,
+ "oliday": 10092,
+ "Amount": 10093,
+ "oolStrip": 10094,
+ "tbody": 10095,
+ "Nav": 10096,
+ "wrap": 10097,
+ "BB": 10098,
+ "Ġwatching": 10099,
+ "arios": 10100,
+ "Ġoptional": 10101,
+ "_K": 10102,
+ "ĠLicensed": 10103,
+ ".Map": 10104,
+ "Timer": 10105,
+ "ĠAP": 10106,
+ "ĠRev": 10107,
+ "(o": 10108,
+ ",c": 10109,
+ "umin": 10110,
+ "etailed": 10111,
+ "ĠHy": 10112,
+ "Ġblank": 10113,
+ "agger": 10114,
+ "ĠSelf": 10115,
+ "()[": 10116,
+ ".make": 10117,
+ "earn": 10118,
+ "channel": 10119,
+ ";Ċ": 10133,
+ "World": 10134,
+ "Ġpython": 10135,
+ "Ġlif": 10136,
+ "Ġtrav": 10137,
+ "Ġconven": 10138,
+ "company": 10139,
+ "ĠClub": 10140,
+ "Ver": 10141,
+ "Btn": 10142,
+ "Ġzone": 10143,
+ "products": 10144,
+ "ĠEduc": 10145,
+ "Ġverify": 10146,
+ "ĠMil": 10147,
+ "ono": 10148,
+ "]);ĊĊ": 10149,
+ "ENCE": 10150,
+ "Ġpacket": 10151,
+ "Ġcer": 10152,
+ "Ġenumer": 10153,
+ "Ġpars": 10154,
+ "formed": 10155,
+ "Ġoccup": 10156,
+ "tre": 10157,
+ "Ġexercise": 10158,
+ "Day": 10159,
+ "_sum": 10160,
+ "Ġasking": 10161,
+ "aption": 10162,
+ "Ġorders": 10163,
+ "Ġspending": 10164,
+ "ĠERR": 10165,
+ ".Dis": 10166,
+ "ĠUtil": 10167,
+ "âĢľI": 10168,
+ "\\'": 10169,
+ "?)": 10170,
+ "/>Ċ": 10171,
+ "Ġemot": 10172,
+ "Ġinfluence": 10173,
+ "ĠAfrica": 10174,
+ "atters": 10175,
+ "Ùħ": 10176,
+ ".session": 10177,
+ "Ġchief": 10178,
+ "ĉĉĉĉĉĉĉĉĉĉĉ": 10179,
+ "Ġtom": 10180,
+ "cluded": 10181,
+ "serial": 10182,
+ "_handler": 10183,
+ ".Type": 10184,
+ "aped": 10185,
+ "Ġpolicies": 10186,
+ "-ex": 10187,
+ "-tr": 10188,
+ "blank": 10189,
+ "merce": 10190,
+ "Ġcoverage": 10191,
+ "Ġrc": 10192,
+ "_matrix": 10193,
+ "_box": 10194,
+ "Ġcharges": 10195,
+ "ĠBoston": 10196,
+ "Pe": 10197,
+ "Ġcircum": 10198,
+ "Ġfilled": 10199,
+ "Ġnorth": 10200,
+ "ictureBox": 10201,
+ "ĉres": 10202,
+ "è®": 10203,
+ "Ġtermin": 10204,
+ "Ġ[â̦": 10205,
+ "IRECT": 10206,
+ "Ġber": 10207,
+ "Ġ\"../../": 10208,
+ "retch": 10209,
+ ".code": 10210,
+ "_col": 10211,
+ "ĠGovernment": 10212,
+ "Ġargv": 10213,
+ "ĠLord": 10214,
+ "asi": 10215,
+ "Exec": 10216,
+ "ĉlet": 10217,
+ "vertis": 10218,
+ "Ġdiscussion": 10219,
+ "enance": 10220,
+ "outube": 10221,
+ "typeof": 10222,
+ "Ġserved": 10223,
+ "ĠPut": 10224,
+ "ĉx": 10225,
+ "Ġsweet": 10226,
+ "Before": 10227,
+ "ategy": 10228,
+ ".of": 10229,
+ "ĠMaterial": 10230,
+ "Sort": 10231,
+ "ONT": 10232,
+ "igital": 10233,
+ "Why": 10234,
+ "Ġsust": 10235,
+ "Ġç": 10236,
+ "abet": 10237,
+ "Ġsegment": 10238,
+ "Ġ[],Ċ": 10239,
+ "ĠMuslim": 10240,
+ "ĠfindViewById": 10241,
+ "cut": 10242,
+ "_TEXT": 10243,
+ "ĠMary": 10244,
+ "Ġloved": 10245,
+ "Ġlie": 10246,
+ "ĠJO": 10247,
+ "Ġisset": 10248,
+ "month": 10249,
+ "Ġprime": 10250,
+ "ti": 10251,
+ "ĠCarol": 10252,
+ "Use": 10253,
+ "ĠPop": 10254,
+ "ĠSave": 10255,
+ "Interval": 10256,
+ "execute": 10257,
+ "dy": 10258,
+ "ĠIran": 10259,
+ "_cont": 10260,
+ "ĉT": 10261,
+ "Ġphase": 10262,
+ "checkbox": 10263,
+ "week": 10264,
+ "Ġhide": 10265,
+ "Ġtil": 10266,
+ "Ġju": 10267,
+ "Custom": 10268,
+ "burg": 10269,
+ "/M": 10270,
+ "TON": 10271,
+ "Ġquant": 10272,
+ "Ġrub": 10273,
+ "ixels": 10274,
+ "Ġinstalled": 10275,
+ "Ġdump": 10276,
+ "Ġproperly": 10277,
+ "(List": 10278,
+ "Ġdecide": 10279,
+ "apply": 10280,
+ "Has": 10281,
+ "Ġkeeping": 10282,
+ "Ġcitizens": 10283,
+ "Ġjoint": 10284,
+ "pool": 10285,
+ "Socket": 10286,
+ "_op": 10287,
+ "Ġweapon": 10288,
+ "gnore": 10289,
+ "ĠExec": 10290,
+ "otten": 10291,
+ "ĠMS": 10292,
+ "Ġ(-": 10293,
+ "ĠReview": 10294,
+ "Ġexamples": 10295,
+ "Ġtight": 10296,
+ "!(": 10297,
+ "DP": 10298,
+ "ĠMessageBox": 10299,
+ "Ġphotograph": 10300,
+ "URI": 10301,
+ "ét": 10302,
+ "low": 10303,
+ "ĠGrand": 10304,
+ ".persistence": 10305,
+ "Ġmaintain": 10306,
+ "Ġnums": 10307,
+ "Ġzip": 10308,
+ "ials": 10309,
+ "ĠGets": 10310,
+ "peg": 10311,
+ "ĠBuffer": 10312,
+ "~~~~": 10313,
+ "rastructure": 10314,
+ "ĠPL": 10315,
+ "uen": 10316,
+ "obby": 10317,
+ "sizeof": 10318,
+ "Ġpic": 10319,
+ "Ġseed": 10320,
+ "Ġexperienced": 10321,
+ "Ġodd": 10322,
+ "Ġkick": 10323,
+ "Ġprocedure": 10324,
+ "avigator": 10325,
+ "-on": 10326,
+ ",j": 10327,
+ "ĠAlthough": 10328,
+ "ĠuserId": 10329,
+ "accept": 10330,
+ "Blue": 10331,
+ "IColor": 10332,
+ "layer": 10333,
+ "available": 10334,
+ "Ġends": 10335,
+ ".table": 10336,
+ "Ġdataset": 10337,
+ "bus": 10338,
+ "Ġexplain": 10339,
+ "(pro": 10340,
+ "ĠCommittee": 10341,
+ "Ġnoted": 10342,
+ "]:Ċ": 10343,
+ "Dim": 10344,
+ "stdio": 10345,
+ ".\",Ċ": 10346,
+ "_source": 10347,
+ "ĠWeek": 10348,
+ "ĠEdge": 10349,
+ "Ġoperating": 10350,
+ "Ġeste": 10351,
+ "ipl": 10352,
+ "agination": 10353,
+ "Ġproceed": 10354,
+ "Ġanimation": 10355,
+ ".Models": 10356,
+ "ĠWatch": 10357,
+ "iat": 10358,
+ "Ġoppon": 10359,
+ "/A": 10360,
+ "Report": 10361,
+ "Ġsounds": 10362,
+ "_buf": 10363,
+ "IELD": 10364,
+ "Ġbund": 10365,
+ "ĉget": 10366,
+ ".pr": 10367,
+ "(tmp": 10368,
+ "Ġkid": 10369,
+ ">ĊĊĊ": 10370,
+ "Ġyang": 10371,
+ "NotFound": 10372,
+ "ÑĨ": 10373,
+ "math": 10374,
+ "@gmail": 10375,
+ "ĠLIMIT": 10376,
+ "redients": 10377,
+ "Ġvent": 10378,
+ "avigate": 10379,
+ "Look": 10380,
+ "Ġreligious": 10381,
+ "Ġrand": 10382,
+ "rio": 10383,
+ "(GL": 10384,
+ "_ip": 10385,
+ "uan": 10386,
+ "iciency": 10387,
+ "ĠChange": 10388,
+ ">čĊčĊ": 10389,
+ "ĠEntity": 10390,
+ "Ġrencontre": 10391,
+ "ĠRet": 10392,
+ "plan": 10393,
+ "én": 10394,
+ "BOOL": 10395,
+ "uries": 10396,
+ "train": 10397,
+ "Definition": 10398,
+ "============": 10399,
+ "zz": 10400,
+ "Animation": 10401,
+ "ĠOK": 10402,
+ "_menu": 10403,
+ ".bl": 10404,
+ "_score": 10405,
+ "Ġacad": 10406,
+ "(System": 10407,
+ "Ġrefresh": 10408,
+ "'=>$": 10409,
+ ".Graphics": 10410,
+ "amento": 10411,
+ "pid": 10412,
+ "tc": 10413,
+ "Ġtips": 10414,
+ "Ġhomes": 10415,
+ "Ġfuel": 10416,
+ "âĸ": 10417,
+ "_helper": 10418,
+ "ĠĠčĊ": 10419,
+ "ĠRoom": 10420,
+ ".Close": 10421,
+ "_attr": 10422,
+ "ĠMount": 10423,
+ "ĠEv": 10424,
+ "arser": 10425,
+ "_top": 10426,
+ "eah": 10427,
+ "ĠDelete": 10428,
+ "ãĢį": 10429,
+ "uke": 10430,
+ "Ġusage": 10431,
+ "aria": 10432,
+ "_dev": 10433,
+ "Ġtexture": 10434,
+ "Ġconversation": 10435,
+ "eper": 10436,
+ "Bean": 10437,
+ "done": 10438,
+ "nonatomic": 10439,
+ "ĠSecond": 10440,
+ "Ġshooting": 10441,
+ "_pre": 10442,
+ "Components": 10443,
+ "Ġ]ĊĊ": 10444,
+ "__,": 10445,
+ "stitution": 10446,
+ ".Char": 10447,
+ ">();ĊĊ": 10448,
+ "Ġpresented": 10449,
+ "Ġwa": 10450,
+ "oker": 10451,
+ "-ĊĊ": 10452,
+ "iner": 10453,
+ "Ġbecoming": 10454,
+ "Ġincident": 10455,
+ "Att": 10456,
+ "Ġrevealed": 10457,
+ "forc": 10458,
+ "Ġboot": 10459,
+ ".page": 10460,
+ "Enumerator": 10461,
+ "_->": 10462,
+ "Photo": 10463,
+ "Ġspring": 10464,
+ ".\",": 10465,
+ "ĠDictionary": 10466,
+ "BJECT": 10467,
+ "Ġlocations": 10468,
+ "Ġsamples": 10469,
+ "InputStream": 10470,
+ "ĠBrown": 10471,
+ "Ġstats": 10472,
+ "quality": 10473,
+ "Ñħ": 10474,
+ "-dis": 10475,
+ "Ġhelping": 10476,
+ "Ġped": 10477,
+ "(se": 10478,
+ "ĠWho": 10479,
+ "alian": 10480,
+ "internal": 10481,
+ "Ġft": 10482,
+ ">().": 10483,
+ "->{": 10484,
+ "Ġmine": 10485,
+ "Ġsector": 10486,
+ "Ġgro": 10487,
+ "Ġopportunities": 10488,
+ "Ġü": 10489,
+ "Ġmp": 10490,
+ "Ġalleged": 10491,
+ "Ġdoubt": 10492,
+ "Mouse": 10493,
+ "About": 10494,
+ "_part": 10495,
+ "Ġchair": 10496,
+ "Ġstopped": 10497,
+ "loop": 10498,
+ "entities": 10499,
+ "Ġapps": 10500,
+ "ansion": 10501,
+ "Ġmental": 10502,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 10503,
+ "FR": 10504,
+ "Ġdefend": 10505,
+ "care": 10506,
+ "Ġideal": 10507,
+ "/api": 10508,
+ "urface": 10509,
+ "Ġele": 10510,
+ "ulator": 10511,
+ "ĠRights": 10512,
+ "anguages": 10513,
+ "Ġfunds": 10514,
+ "Ġadapt": 10515,
+ "Attributes": 10516,
+ "Ġdeploy": 10517,
+ "opts": 10518,
+ "Ġvalidation": 10519,
+ "Ġconcerns": 10520,
+ "uce": 10521,
+ ".num": 10522,
+ "ulture": 10523,
+ "ila": 10524,
+ "Ġcup": 10525,
+ "Ġpure": 10526,
+ ".Fore": 10527,
+ "ĠHashMap": 10528,
+ ".valueOf": 10529,
+ "asm": 10530,
+ "MO": 10531,
+ "Ġcs": 10532,
+ "Ġstores": 10533,
+ "Ġ************************************************************************": 10534,
+ "Ġcommunication": 10535,
+ "mem": 10536,
+ ".EventHandler": 10537,
+ ".Status": 10538,
+ "_right": 10539,
+ ".setOn": 10540,
+ "Sheet": 10541,
+ "Ġidentify": 10542,
+ "enerated": 10543,
+ "ordered": 10544,
+ "Ġ\"[": 10545,
+ "Ġswe": 10546,
+ "Condition": 10547,
+ "ĠAccording": 10548,
+ "Ġprepare": 10549,
+ "Ġrob": 10550,
+ "Pool": 10551,
+ "Ġsport": 10552,
+ "rv": 10553,
+ "ĠRouter": 10554,
+ "Ġalternative": 10555,
+ "([]": 10556,
+ "ĠChicago": 10557,
+ "ipher": 10558,
+ "ische": 10559,
+ "ĠDirector": 10560,
+ "kl": 10561,
+ "ĠWil": 10562,
+ "keys": 10563,
+ "Ġmysql": 10564,
+ "Ġwelcome": 10565,
+ "king": 10566,
+ "ĠManager": 10567,
+ "Ġcaught": 10568,
+ ")}Ċ": 10569,
+ "Score": 10570,
+ "_PR": 10571,
+ "Ġsurvey": 10572,
+ "hab": 10573,
+ "Headers": 10574,
+ "ADER": 10575,
+ "Ġdecor": 10576,
+ "Ġturns": 10577,
+ "Ġradius": 10578,
+ "errupt": 10579,
+ "Cor": 10580,
+ "Ġmel": 10581,
+ "Ġintr": 10582,
+ "(q": 10583,
+ "ĠAC": 10584,
+ "amos": 10585,
+ "MAX": 10586,
+ "ĠGrid": 10587,
+ "ĠJesus": 10588,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 10589,
+ ".DE": 10590,
+ "Ġts": 10591,
+ "Ġlinked": 10592,
+ "free": 10593,
+ "ĠQt": 10594,
+ "Ġ/**čĊ": 10595,
+ "Ġfaster": 10596,
+ "ctr": 10597,
+ "_J": 10598,
+ "DT": 10599,
+ ".Check": 10600,
+ "Ġcombination": 10601,
+ "Ġintended": 10602,
+ "-the": 10603,
+ "-type": 10604,
+ "ectors": 10605,
+ "ami": 10606,
+ "uting": 10607,
+ "Ġuma": 10608,
+ "XML": 10609,
+ "UCT": 10610,
+ "Ap": 10611,
+ "ĠRandom": 10612,
+ "Ġran": 10613,
+ ".sort": 10614,
+ "Ġsorted": 10615,
+ ".Un": 10616,
+ "_PER": 10617,
+ "itory": 10618,
+ "Ġpriority": 10619,
+ "ĠGal": 10620,
+ "ĠOld": 10621,
+ "hot": 10622,
+ "ĠDisplay": 10623,
+ "(sub": 10624,
+ "_TH": 10625,
+ "_Y": 10626,
+ "ĠCare": 10627,
+ "loading": 10628,
+ "Kind": 10629,
+ "_handle": 10630,
+ ",,": 10631,
+ "rase": 10632,
+ "_replace": 10633,
+ ".addEventListener": 10634,
+ "ĠRT": 10635,
+ "Ġentered": 10636,
+ "gers": 10637,
+ "Ġich": 10638,
+ "(start": 10639,
+ "/app": 10640,
+ "Ġbrother": 10641,
+ "Memory": 10642,
+ "Outlet": 10643,
+ "Ġutf": 10644,
+ "prec": 10645,
+ "Ġnavigation": 10646,
+ "ORK": 10647,
+ "Ġdst": 10648,
+ "Detail": 10649,
+ "Ġaudience": 10650,
+ "Ġdur": 10651,
+ "Ġcluster": 10652,
+ "unched": 10653,
+ "Ġ],": 10654,
+ "Ġcomfortable": 10655,
+ ".values": 10656,
+ "ĠTotal": 10657,
+ "Ġsnap": 10658,
+ "Ġstandards": 10659,
+ "Ġperformed": 10660,
+ "hand": 10661,
+ "(\"@": 10662,
+ "åŃ": 10663,
+ "Ġphil": 10664,
+ "ibr": 10665,
+ "trim": 10666,
+ "Ġforget": 10667,
+ "Ġdoctor": 10668,
+ ".TextBox": 10669,
+ "icons": 10670,
+ ",s": 10671,
+ "ĠOp": 10672,
+ "Sm": 10673,
+ "Stop": 10674,
+ "ĉList": 10675,
+ "ĉu": 10676,
+ "Comment": 10677,
+ "_VERSION": 10678,
+ ".Xtra": 10679,
+ "Person": 10680,
+ "rb": 10681,
+ "LOB": 10682,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ": 10683,
+ "ĠCentral": 10684,
+ "ICK": 10685,
+ "raq": 10686,
+ "Ġputting": 10687,
+ "Ġmd": 10688,
+ "ĠLove": 10689,
+ "Program": 10690,
+ "Border": 10691,
+ "oor": 10692,
+ "Ġallowing": 10693,
+ "after": 10694,
+ "Ġentries": 10695,
+ "ĠMaybe": 10696,
+ "]).": 10697,
+ "ĠShort": 10698,
+ ")\\": 10699,
+ ".now": 10700,
+ "friend": 10701,
+ "Ġprefer": 10702,
+ "ĠGPIO": 10703,
+ "osis": 10704,
+ "ĠGameObject": 10705,
+ "Ġskip": 10706,
+ "Ġcompetition": 10707,
+ "_match": 10708,
+ "lications": 10709,
+ "_CONT": 10710,
+ ".groupBox": 10711,
+ "Ġals": 10712,
+ "\"We": 10713,
+ "_eq": 10714,
+ "lan": 10715,
+ "_search": 10716,
+ "ĠMusic": 10717,
+ "asis": 10718,
+ "Ġbind": 10719,
+ "ĠIsland": 10720,
+ "rum": 10721,
+ "(E": 10722,
+ "Ġseat": 10723,
+ "Video": 10724,
+ "Ġack": 10725,
+ "reek": 10726,
+ "={()": 10727,
+ "Ġrating": 10728,
+ "Ġrestaurant": 10729,
+ "DEX": 10730,
+ "(buf": 10731,
+ "pping": 10732,
+ "uality": 10733,
+ "Ġleague": 10734,
+ "Ġfocused": 10735,
+ "apon": 10736,
+ "$data": 10737,
+ "CLUD": 10738,
+ "CLUDING": 10739,
+ "Ġabsolute": 10740,
+ "(query": 10741,
+ "Ġtells": 10742,
+ "Ang": 10743,
+ "Ġcommunities": 10744,
+ "Ġhonest": 10745,
+ "oking": 10746,
+ "Ġapart": 10747,
+ "arity": 10748,
+ "/$": 10749,
+ "_module": 10750,
+ "ĠEnc": 10751,
+ ".an": 10752,
+ ".Config": 10753,
+ "Cre": 10754,
+ "Ġshock": 10755,
+ "ĠArab": 10756,
+ "IENT": 10757,
+ "/re": 10758,
+ "Ġretrie": 10759,
+ "ycler": 10760,
+ "isa": 10761,
+ "ĠOrgan": 10762,
+ ".graph": 10763,
+ "Ġí": 10764,
+ "ĠBAS": 10765,
+ "Enum": 10766,
+ "Ġpossibly": 10767,
+ "ÑĢаÐ": 10768,
+ "ĠJapanese": 10769,
+ "Ġcraft": 10770,
+ "ĠPlace": 10771,
+ "Ġtalent": 10772,
+ "Ġfunding": 10773,
+ "Ġconfirmed": 10774,
+ "Ġcycle": 10775,
+ "/x": 10776,
+ "GE": 10777,
+ "Ġhearing": 10778,
+ "Ġplants": 10779,
+ "Ġmouth": 10780,
+ "pages": 10781,
+ "oria": 10782,
+ "ĠRemove": 10783,
+ "_total": 10784,
+ "Ġod": 10785,
+ "ollapse": 10786,
+ "door": 10787,
+ "Ġbought": 10788,
+ "Ġaddr": 10789,
+ "ARCH": 10790,
+ "_dim": 10791,
+ "dden": 10792,
+ "Ġdecades": 10793,
+ "REQUEST": 10794,
+ "Ġversions": 10795,
+ "fire": 10796,
+ "Ġmoves": 10797,
+ "fb": 10798,
+ "Ġcoffee": 10799,
+ ".connect": 10800,
+ "ĠRow": 10801,
+ "Ġschema": 10802,
+ "Scope": 10803,
+ "-Type": 10804,
+ "Ġfighting": 10805,
+ "Ġretail": 10806,
+ "Ġmodified": 10807,
+ "TF": 10808,
+ "Files": 10809,
+ "nie": 10810,
+ "_command": 10811,
+ "stone": 10812,
+ "ĠÑĤ": 10813,
+ "_thread": 10814,
+ "Ġbond": 10815,
+ "ĠDevelopment": 10816,
+ "Ġpt": 10817,
+ "FORM": 10818,
+ "plet": 10819,
+ "Ġidentified": 10820,
+ "cpp": 10821,
+ "Ġcoding": 10822,
+ "oked": 10823,
+ "ĠMaster": 10824,
+ "IDTH": 10825,
+ "Ġresidents": 10826,
+ "redit": 10827,
+ "ĠPhoto": 10828,
+ "=-": 10829,
+ "unte": 10830,
+ "ateur": 10831,
+ "_STATE": 10832,
+ "ĠSing": 10833,
+ "Ġsheet": 10834,
+ ".val": 10835,
+ "orse": 10836,
+ "Ġhers": 10837,
+ "Ġdetermined": 10838,
+ "Common": 10839,
+ "Ġwed": 10840,
+ "_queue": 10841,
+ "PH": 10842,
+ "ĠAtl": 10843,
+ "cred": 10844,
+ "/LICENSE": 10845,
+ "Ġmes": 10846,
+ "Ġadvanced": 10847,
+ ".java": 10848,
+ ".Sh": 10849,
+ "Go": 10850,
+ "kill": 10851,
+ "fp": 10852,
+ "_settings": 10853,
+ "Ġpal": 10854,
+ "Ġtruck": 10855,
+ "Ġcombined": 10856,
+ "Ġ\"${": 10857,
+ "ĠCorpor": 10858,
+ "Ġjoined": 10859,
+ "ĠJose": 10860,
+ "ĠCup": 10861,
+ "uns": 10862,
+ "estival": 10863,
+ "levision": 10864,
+ "Ġbroken": 10865,
+ "Ġmarriage": 10866,
+ "ĠWestern": 10867,
+ "Ġrepresents": 10868,
+ "ĠTitle": 10869,
+ "Ġss": 10870,
+ ".Ass": 10871,
+ "ongoose": 10872,
+ "iento": 10873,
+ "<>();Ċ": 10874,
+ "Ġabsolutely": 10875,
+ "Ġsmooth": 10876,
+ "TERN": 10877,
+ "ĠUnless": 10878,
+ "Word": 10879,
+ "Ġmerge": 10880,
+ "igan": 10881,
+ "ĠVol": 10882,
+ "Ġnn": 10883,
+ ".getId": 10884,
+ "Ġз": 10885,
+ "Ġsexy": 10886,
+ "Ġseeking": 10887,
+ "Single": 10888,
+ ".this": 10889,
+ "Ġkom": 10890,
+ "bound": 10891,
+ ";\"": 10892,
+ "ĠfontSize": 10893,
+ "_df": 10894,
+ "Ġinjury": 10895,
+ "(H": 10896,
+ "Ġissued": 10897,
+ "_END": 10898,
+ ":self": 10899,
+ "Ġpatch": 10900,
+ "Ġleaves": 10901,
+ "Ġadopt": 10902,
+ "FileName": 10903,
+ "ãĢIJ": 10904,
+ "Ġexecutive": 10905,
+ "ĠByte": 10906,
+ "]))Ċ": 10907,
+ "Ġnu": 10908,
+ "outing": 10909,
+ "cluding": 10910,
+ "-R": 10911,
+ ".options": 10912,
+ "Ġsubstant": 10913,
+ "avax": 10914,
+ "ĠBUT": 10915,
+ "Ġtechnical": 10916,
+ "Ġtwice": 10917,
+ "Ġmás": 10918,
+ "Ġunivers": 10919,
+ "yr": 10920,
+ "Ġdrag": 10921,
+ "ĠDC": 10922,
+ "Ġsed": 10923,
+ "Ġbot": 10924,
+ "ĠPal": 10925,
+ "ĠHall": 10926,
+ "forcement": 10927,
+ "Ġauch": 10928,
+ ".mod": 10929,
+ "notation": 10930,
+ "_files": 10931,
+ ".line": 10932,
+ "_flag": 10933,
+ "[name": 10934,
+ "Ġresolution": 10935,
+ "Ġbott": 10936,
+ "(\"[": 10937,
+ "ende": 10938,
+ "(arr": 10939,
+ "Free": 10940,
+ "(@\"": 10941,
+ "ĠDistrict": 10942,
+ "PEC": 10943,
+ ":-": 10944,
+ "Picker": 10945,
+ "ĠJo": 10946,
+ "ĠĠĠĠĠĊ": 10947,
+ "ĠRiver": 10948,
+ "_rows": 10949,
+ "Ġhelpful": 10950,
+ "Ġmassive": 10951,
+ "---Ċ": 10952,
+ "Ġmeasures": 10953,
+ "ĠRuntime": 10954,
+ "Ġworry": 10955,
+ "ĠSpec": 10956,
+ "ĉD": 10957,
+ "ãĢij": 10958,
+ "Ġ){Ċ": 10959,
+ "Ġworse": 10960,
+ "(filename": 10961,
+ "Ġlay": 10962,
+ "Ġmagic": 10963,
+ "ĠTheir": 10964,
+ "oul": 10965,
+ "stroy": 10966,
+ "ĠWhere": 10967,
+ "Ġsudden": 10968,
+ "Ġdefe": 10969,
+ "Ġbinding": 10970,
+ "Ġflight": 10971,
+ "ĠOnInit": 10972,
+ "ĠWomen": 10973,
+ "ĠPolicy": 10974,
+ "Ġdrugs": 10975,
+ "ishing": 10976,
+ "('../": 10977,
+ "ĠMel": 10978,
+ "peat": 10979,
+ "tor": 10980,
+ "Ġproposed": 10981,
+ "Ġstated": 10982,
+ "_RES": 10983,
+ "Ġeast": 10984,
+ "ĠCONDITION": 10985,
+ "_desc": 10986,
+ "Ġwinning": 10987,
+ "folio": 10988,
+ "Mapper": 10989,
+ "ĠPan": 10990,
+ "ĠAnge": 10991,
+ ".servlet": 10992,
+ "Ġcopies": 10993,
+ "LM": 10994,
+ "Ġvm": 10995,
+ "åį": 10996,
+ "Ġdictionary": 10997,
+ "Seg": 10998,
+ "elines": 10999,
+ "ĠSend": 11000,
+ "Ġiron": 11001,
+ "ĠFort": 11002,
+ ".domain": 11003,
+ "Ġdebate": 11004,
+ "NotNull": 11005,
+ "eq": 11006,
+ "acher": 11007,
+ "lf": 11008,
+ "ĉfmt": 11009,
+ "Ġlawy": 11010,
+ "ÄŁ": 11011,
+ "ĠMen": 11012,
+ "Ġtrim": 11013,
+ "(NULL": 11014,
+ "Ġ!!": 11015,
+ "Ġpad": 11016,
+ "Ġfollows": 11017,
+ "\"][\"": 11018,
+ "requ": 11019,
+ "ĠEp": 11020,
+ ".github": 11021,
+ "(img": 11022,
+ "eto": 11023,
+ "('\\": 11024,
+ "Services": 11025,
+ "umbnail": 11026,
+ "_main": 11027,
+ "pleted": 11028,
+ "fortunately": 11029,
+ "Ġwindows": 11030,
+ "Ġplane": 11031,
+ "ĠConnection": 11032,
+ ".local": 11033,
+ "uard": 11034,
+ "}\\": 11035,
+ "==\"": 11036,
+ "andon": 11037,
+ "ĠRoy": 11038,
+ "west": 11039,
+ "iginal": 11040,
+ "emies": 11041,
+ "itz": 11042,
+ "'):Ċ": 11043,
+ "ĠPeter": 11044,
+ "Ġtough": 11045,
+ "Ġreduced": 11046,
+ "Ġcalculate": 11047,
+ "Ġrapid": 11048,
+ "customer": 11049,
+ "Ġefficient": 11050,
+ "Ġmedium": 11051,
+ "Ġfell": 11052,
+ ".ref": 11053,
+ "ĠCas": 11054,
+ "Ġfeedback": 11055,
+ "Speed": 11056,
+ "(output": 11057,
+ "aje": 11058,
+ "Ġcategories": 11059,
+ "Ġfee": 11060,
+ "};": 11061,
+ "Ġdeleted": 11062,
+ "reh": 11063,
+ "Ġproof": 11064,
+ "Desc": 11065,
+ "Build": 11066,
+ "Ġsides": 11067,
+ ".ArrayList": 11068,
+ "-%": 11069,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 11070,
+ "ر": 11071,
+ ".match": 11072,
+ "ли": 11073,
+ "Ġfeels": 11074,
+ "Ġachieve": 11075,
+ "Ġclim": 11076,
+ "_ON": 11077,
+ "ĠCD": 11078,
+ "Ġteacher": 11079,
+ "_current": 11080,
+ "bn": 11081,
+ "_PL": 11082,
+ "isting": 11083,
+ "Enable": 11084,
+ "GEN": 11085,
+ "Ġtv": 11086,
+ "Ġsock": 11087,
+ "Ġplays": 11088,
+ "Ġdiscount": 11089,
+ "ĠKE": 11090,
+ "ĠDebug": 11091,
+ "Fore": 11092,
+ "ĠIraq": 11093,
+ "Ġappearance": 11094,
+ "Mon": 11095,
+ "Ġstyled": 11096,
+ "ĠHuman": 11097,
+ "iot": 11098,
+ "ĠHistory": 11099,
+ "Ġsac": 11100,
+ "ĠCollection": 11101,
+ "Ġrecommended": 11102,
+ ".Selected": 11103,
+ "Ġorganizations": 11104,
+ "Ġdiscovered": 11105,
+ "cohol": 11106,
+ "adas": 11107,
+ "ĠThomas": 11108,
+ "May": 11109,
+ "Ġconserv": 11110,
+ "Ġdomin": 11111,
+ "ĠFollow": 11112,
+ "ĠSection": 11113,
+ "ĠThanks": 11114,
+ "Username": 11115,
+ "Ġrecipe": 11116,
+ "Ġwonderful": 11117,
+ ".sleep": 11118,
+ "_if": 11119,
+ "ĉĊĉĊ": 11120,
+ "orno": 11121,
+ "Ġru": 11122,
+ "_target": 11123,
+ ".\"\"": 11124,
+ "à¦": 11125,
+ "EventArgs": 11126,
+ "Ġinputs": 11127,
+ "Ġfif": 11128,
+ "Ġvision": 11129,
+ "cy": 11130,
+ "ĠSeries": 11131,
+ ")(((": 11132,
+ "Ġtrading": 11133,
+ "Ġmarker": 11134,
+ "Begin": 11135,
+ "Ġtypically": 11136,
+ "Ġcauses": 11137,
+ "dropdown": 11138,
+ "_DEBUG": 11139,
+ "Ġdetect": 11140,
+ "country": 11141,
+ "!\");Ċ": 11142,
+ "ĉR": 11143,
+ "appy": 11144,
+ "Ġcref": 11145,
+ "('<": 11146,
+ "\"=>": 11147,
+ "ĠLE": 11148,
+ "reader": 11149,
+ "Ġadministr": 11150,
+ "õ": 11151,
+ "ucket": 11152,
+ "Ġfashion": 11153,
+ ".char": 11154,
+ "izar": 11155,
+ "Ġdisable": 11156,
+ "Ġsuc": 11157,
+ "ĠLive": 11158,
+ "issue": 11159,
+ "Ġmetadata": 11160,
+ "flags": 11161,
+ "ĠðŁ": 11162,
+ "Ġcommitted": 11163,
+ "Ġva": 11164,
+ "Ġrough": 11165,
+ "Ġ'''Ċ": 11166,
+ "Ġhighlight": 11167,
+ "_vars": 11168,
+ "VO": 11169,
+ "Ġencoding": 11170,
+ "-Z": 11171,
+ "_sign": 11172,
+ "$(\"#": 11173,
+ "Ġrain": 11174,
+ "reatest": 11175,
+ "ĠEND": 11176,
+ "Selection": 11177,
+ "Ġcandidates": 11178,
+ "Ġsav": 11179,
+ ".Empty": 11180,
+ "Ġdecisions": 11181,
+ "Ġcollabor": 11182,
+ "ridge": 11183,
+ "feed": 11184,
+ "ression": 11185,
+ "Ġpersons": 11186,
+ "VM": 11187,
+ "ega": 11188,
+ "_BIT": 11189,
+ "According": 11190,
+ "acked": 11191,
+ "Ġdollars": 11192,
+ "_loss": 11193,
+ "ĠCost": 11194,
+ "}\"Ċ": 11195,
+ "Notification": 11196,
+ "Ġprostit": 11197,
+ "Ġauthority": 11198,
+ ".rec": 11199,
+ "Ġspokes": 11200,
+ "ĠToday": 11201,
+ "istant": 11202,
+ "ĠHead": 11203,
+ "âĢĿ.": 11204,
+ "ertainment": 11205,
+ "cean": 11206,
+ "culate": 11207,
+ "Ġven": 11208,
+ "However": 11209,
+ "_arr": 11210,
+ "Ġtokens": 11211,
+ "Graph": 11212,
+ "ĠJud": 11213,
+ "ĠVirgin": 11214,
+ "ĠSerial": 11215,
+ "unning": 11216,
+ "Mutable": 11217,
+ "agers": 11218,
+ ".csv": 11219,
+ "Ġdeveloping": 11220,
+ "Ġinstructions": 11221,
+ "Ġpromise": 11222,
+ "Ġrequested": 11223,
+ "_encode": 11224,
+ "/\"": 11225,
+ "ĠIcon": 11226,
+ "uilt": 11227,
+ "-day": 11228,
+ "Ġintelligence": 11229,
+ ".IS": 11230,
+ "ĠObservable": 11231,
+ "ĠHard": 11232,
+ "Bool": 11233,
+ "idential": 11234,
+ ".Anchor": 11235,
+ "Ġselling": 11236,
+ "CI": 11237,
+ "AGES": 11238,
+ "tle": 11239,
+ "bur": 11240,
+ "UFFER": 11241,
+ "RY": 11242,
+ "Ġbigger": 11243,
+ "Ġrat": 11244,
+ "Ġfamous": 11245,
+ "Ġtypename": 11246,
+ "Ġexplained": 11247,
+ "}}Ċ": 11248,
+ "Ġnuclear": 11249,
+ "-N": 11250,
+ "Ġcrisis": 11251,
+ "ĠEnter": 11252,
+ "Ġanswers": 11253,
+ "/${": 11254,
+ "/pl": 11255,
+ "Ġsequ": 11256,
+ "_next": 11257,
+ "mask": 11258,
+ "Ġstanding": 11259,
+ "Ġplenty": 11260,
+ "ĠCross": 11261,
+ "ĉret": 11262,
+ "dro": 11263,
+ "ĠCast": 11264,
+ "=true": 11265,
+ "ĠChris": 11266,
+ "icio": 11267,
+ "ĠMike": 11268,
+ "Decimal": 11269,
+ "addComponent": 11270,
+ "Len": 11271,
+ "Ġcock": 11272,
+ "Ġ#{": 11273,
+ "URN": 11274,
+ "": 11403,
+ "Ġ*=": 11404,
+ "ĠPS": 11405,
+ "Ġdangerous": 11406,
+ "[p": 11407,
+ "OME": 11408,
+ "Other": 11409,
+ "ĠStringBuilder": 11410,
+ "Points": 11411,
+ "heading": 11412,
+ "Ġcurrency": 11413,
+ "Ġpercentage": 11414,
+ "_API": 11415,
+ "Ġclassic": 11416,
+ "thead": 11417,
+ "ĠMO": 11418,
+ "FE": 11419,
+ "Idx": 11420,
+ "await": 11421,
+ "Ġè": 11422,
+ "Ġaccident": 11423,
+ "Ġvariant": 11424,
+ "Ġmyst": 11425,
+ "ĠLand": 11426,
+ "ĠBre": 11427,
+ "Ġharm": 11428,
+ "ĠAcc": 11429,
+ "Ġcharged": 11430,
+ "iones": 11431,
+ "Visibility": 11432,
+ "arry": 11433,
+ "ĠLanguage": 11434,
+ "Ġwalking": 11435,
+ "\".ĊĊ": 11436,
+ "ifer": 11437,
+ "Ġleadership": 11438,
+ ".From": 11439,
+ "ynam": 11440,
+ "Ġtimestamp": 11441,
+ "ipt": 11442,
+ "ĠHas": 11443,
+ "REFER": 11444,
+ "ĠIts": 11445,
+ "Ġlistener": 11446,
+ "UTE": 11447,
+ "_description": 11448,
+ "Ġexperiences": 11449,
+ "Ġcreates": 11450,
+ "RS": 11451,
+ "cart": 11452,
+ "black": 11453,
+ "Ġchoices": 11454,
+ "war": 11455,
+ "Ġ'''": 11456,
+ "Ġordered": 11457,
+ "Ġevening": 11458,
+ "Ġpil": 11459,
+ "Ġtun": 11460,
+ "ĠBad": 11461,
+ "(app": 11462,
+ "random": 11463,
+ "Ġexplicit": 11464,
+ "Ġarrived": 11465,
+ "Ġfly": 11466,
+ "Ġeconom": 11467,
+ "-mail": 11468,
+ "Ġlists": 11469,
+ "Ġarchitect": 11470,
+ "ĠPay": 11471,
+ "Ġds": 11472,
+ "ĠSol": 11473,
+ "Ġvehicles": 11474,
+ "Hz": 11475,
+ "-com": 11476,
+ "Ġking": 11477,
+ "_equal": 11478,
+ "ĠHelp": 11479,
+ "Ġabuse": 11480,
+ "--;Ċ": 11481,
+ "Ġextr": 11482,
+ "Ġchemical": 11483,
+ "ä¿": 11484,
+ "Ġorient": 11485,
+ "Ġbreath": 11486,
+ "ĠSpace": 11487,
+ "(element": 11488,
+ "wait": 11489,
+ "DED": 11490,
+ "igma": 11491,
+ "Ġentr": 11492,
+ "Ġsob": 11493,
+ "-name": 11494,
+ "Ġaffected": 11495,
+ "ika": 11496,
+ "Ġcoal": 11497,
+ "_work": 11498,
+ "Ġhundreds": 11499,
+ "Ġpolitics": 11500,
+ "subject": 11501,
+ "Ġconsumer": 11502,
+ "ANGE": 11503,
+ "Ġrepeated": 11504,
+ "Send": 11505,
+ "Ġ#[": 11506,
+ "Ġprotocol": 11507,
+ "Ġleads": 11508,
+ "useum": 11509,
+ "Every": 11510,
+ "Import": 11511,
+ "(count": 11512,
+ "Ġchallenges": 11513,
+ "Ġnovel": 11514,
+ "Ġdepart": 11515,
+ "bits": 11516,
+ ".Current": 11517,
+ "Ġ`${": 11518,
+ "oting": 11519,
+ "(\\": 11520,
+ "Ġcreative": 11521,
+ "Ġbuff": 11522,
+ "Ġintroduced": 11523,
+ "usic": 11524,
+ "modules": 11525,
+ "Are": 11526,
+ "-doc": 11527,
+ "language": 11528,
+ "_cache": 11529,
+ "Ġtod": 11530,
+ "?>": 11531,
+ "omething": 11532,
+ "Ġhun": 11533,
+ "åº": 11534,
+ "aters": 11535,
+ "Intent": 11536,
+ "Ġimplemented": 11537,
+ "ĠCase": 11538,
+ "Children": 11539,
+ "Ġnotification": 11540,
+ "Renderer": 11541,
+ "Wrapper": 11542,
+ "Objects": 11543,
+ "tl": 11544,
+ ".Contains": 11545,
+ "Plugin": 11546,
+ ".row": 11547,
+ "Ġforg": 11548,
+ "Ġpermit": 11549,
+ "Ġtargets": 11550,
+ "ĠIF": 11551,
+ "Ġtip": 11552,
+ "sex": 11553,
+ "Ġsupports": 11554,
+ "Ġfold": 11555,
+ "photo": 11556,
+ "},čĊ": 11557,
+ "Ġgoogle": 11558,
+ "$('#": 11559,
+ "Ġsharing": 11560,
+ "Ġgoods": 11561,
+ "vs": 11562,
+ "ĠDan": 11563,
+ "Rate": 11564,
+ "ĠMartin": 11565,
+ "Ġmanner": 11566,
+ "lie": 11567,
+ ".The": 11568,
+ "Internal": 11569,
+ "ĠCONTR": 11570,
+ "Mock": 11571,
+ "RIGHT": 11572,
+ "Ġ'{": 11573,
+ "Ġcontrols": 11574,
+ "Mat": 11575,
+ "Ġmand": 11576,
+ "Ġextended": 11577,
+ "Ok": 11578,
+ "Ġembed": 11579,
+ "Ġplanet": 11580,
+ "ĠNon": 11581,
+ "-ch": 11582,
+ ")\",": 11583,
+ "epar": 11584,
+ "Ġbelieved": 11585,
+ "ĠEnvironment": 11586,
+ "ĠFriend": 11587,
+ "-res": 11588,
+ "Ġhandling": 11589,
+ "nic": 11590,
+ "-level": 11591,
+ "scri": 11592,
+ "Xml": 11593,
+ "BE": 11594,
+ "ungen": 11595,
+ "Ġalter": 11596,
+ "[idx": 11597,
+ "Pop": 11598,
+ "cam": 11599,
+ "Ġ(((": 11600,
+ "Ġshipping": 11601,
+ "Ġbattery": 11602,
+ "iddleware": 11603,
+ "MC": 11604,
+ "Ġimpl": 11605,
+ "otation": 11606,
+ "ĠLab": 11607,
+ "