diff --git "a/source/eval/outputs/3b_benchmark_results.txt" "b/source/eval/outputs/3b_benchmark_results.txt"
new file mode 100644--- /dev/null
+++ "b/source/eval/outputs/3b_benchmark_results.txt"
@@ -0,0 +1,393 @@
+2026-03-05:01:45:40 INFO     [_cli.run:376] Selected Tasks: ['belebele_kor_Hang', 'global_mmlu_full_ko']
+2026-03-05:01:45:40 INFO     [evaluator:211] Setting random seed to 0 | Setting numpy seed to 1234 | Setting torch manual seed to 1234 | Setting fewshot manual seed to 1234
+2026-03-05:01:45:40 INFO     [evaluator:236] Initializing hf model, with arguments: {'pretrained': '/PROJECT/0325120031_A/ghong/taketimes/llm-bang/eval/outputs/hf_3b_base', 'dtype': 'bfloat16'}
+2026-03-05:01:45:42 INFO     [models.huggingface:161] Using device 'cuda:2'
+2026-03-05:01:45:43 INFO     [models.huggingface:423] Model parallel was set to False, max memory was not set, and device map was set to {'': 'cuda:2'}
+
Loading weights:   0%|          | 0/171 [00:00<?, ?it/s]
Loading weights:   1%|          | 1/171 [00:00<00:00, 11037.64it/s, Materializing param=lm_head.weight]
Loading weights:   1%|          | 1/171 [00:00<00:00, 6967.28it/s, Materializing param=lm_head.weight] 
Loading weights:   1%|          | 2/171 [00:00<01:14,  2.28it/s, Materializing param=lm_head.weight]  
Loading weights:   1%|          | 2/171 [00:00<01:14,  2.28it/s, Materializing param=model.embed_tokens.weight]
Loading weights:   1%|          | 2/171 [00:00<01:14,  2.28it/s, Materializing param=model.embed_tokens.weight]
Loading weights:   2%|▏         | 3/171 [00:00<01:13,  2.28it/s, Materializing param=model.layers.0.input_layernorm.weight]
Loading weights:   2%|▏         | 3/171 [00:00<01:13,  2.28it/s, Materializing param=model.layers.0.input_layernorm.weight]
Loading weights:   2%|▏         | 4/171 [00:00<01:13,  2.28it/s, Materializing param=model.layers.0.mlp.down_proj.weight]  
Loading weights:   2%|▏         | 4/171 [00:00<01:13,  2.28it/s, Materializing param=model.layers.0.mlp.down_proj.weight]
Loading weights:   3%|▎         | 5/171 [00:00<01:12,  2.28it/s, Materializing param=model.layers.0.mlp.gate_proj.weight]
Loading weights:   3%|▎         | 5/171 [00:00<01:12,  2.28it/s, Materializing param=model.layers.0.mlp.gate_proj.weight]
Loading weights:   4%|▎         | 6/171 [00:00<01:12,  2.28it/s, Materializing param=model.layers.0.mlp.up_proj.weight]  
Loading weights:   4%|▎         | 6/171 [00:00<01:12,  2.28it/s, Materializing param=model.layers.0.mlp.up_proj.weight]
Loading weights:   4%|▍         | 7/171 [00:00<01:11,  2.28it/s, Materializing param=model.layers.0.post_attention_layernorm.weight]
Loading weights:   4%|▍         | 7/171 [00:00<01:11,  2.28it/s, Materializing param=model.layers.0.post_attention_layernorm.weight]
Loading weights:   5%|▍         | 8/171 [00:00<01:11,  2.28it/s, Materializing param=model.layers.0.self_attn.o_proj.weight]        
Loading weights:   5%|▍         | 8/171 [00:00<01:11,  2.28it/s, Materializing param=model.layers.0.self_attn.o_proj.weight]
Loading weights:   5%|▌         | 9/171 [00:00<01:11,  2.28it/s, Materializing param=model.layers.1.input_layernorm.weight] 
Loading weights:   5%|▌         | 9/171 [00:00<01:11,  2.28it/s, Materializing param=model.layers.1.input_layernorm.weight]
Loading weights:   6%|▌         | 10/171 [00:00<01:10,  2.28it/s, Materializing param=model.layers.1.mlp.down_proj.weight] 
Loading weights:   6%|▌         | 10/171 [00:00<01:10,  2.28it/s, Materializing param=model.layers.1.mlp.down_proj.weight]
Loading weights:   6%|▋         | 11/171 [00:00<01:10,  2.28it/s, Materializing param=model.layers.1.mlp.gate_proj.weight]
Loading weights:   6%|▋         | 11/171 [00:00<01:10,  2.28it/s, Materializing param=model.layers.1.mlp.gate_proj.weight]
Loading weights:   7%|▋         | 12/171 [00:00<01:09,  2.28it/s, Materializing param=model.layers.1.mlp.up_proj.weight]  
Loading weights:   7%|▋         | 12/171 [00:00<01:09,  2.28it/s, Materializing param=model.layers.1.mlp.up_proj.weight]
Loading weights:   8%|▊         | 13/171 [00:00<01:09,  2.28it/s, Materializing param=model.layers.1.post_attention_layernorm.weight]
Loading weights:   8%|▊         | 13/171 [00:00<01:09,  2.28it/s, Materializing param=model.layers.1.post_attention_layernorm.weight]
Loading weights:   8%|▊         | 14/171 [00:00<01:08,  2.28it/s, Materializing param=model.layers.1.self_attn.o_proj.weight]        
Loading weights:   8%|▊         | 14/171 [00:00<01:08,  2.28it/s, Materializing param=model.layers.1.self_attn.o_proj.weight]
Loading weights:   9%|▉         | 15/171 [00:01<00:08, 19.46it/s, Materializing param=model.layers.1.self_attn.o_proj.weight]
Loading weights:   9%|▉         | 15/171 [00:01<00:08, 19.46it/s, Materializing param=model.layers.2.input_layernorm.weight] 
Loading weights:   9%|▉         | 15/171 [00:01<00:08, 19.46it/s, Materializing param=model.layers.2.input_layernorm.weight]
Loading weights:   9%|▉         | 16/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.2.mlp.down_proj.weight]  
Loading weights:   9%|▉         | 16/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.2.mlp.down_proj.weight]
Loading weights:  10%|▉         | 17/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.2.mlp.gate_proj.weight]
Loading weights:  10%|▉         | 17/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.2.mlp.gate_proj.weight]
Loading weights:  11%|█         | 18/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.2.mlp.up_proj.weight]  
Loading weights:  11%|█         | 18/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.2.mlp.up_proj.weight]
Loading weights:  11%|█         | 19/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.2.post_attention_layernorm.weight]
Loading weights:  11%|█         | 19/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.2.post_attention_layernorm.weight]
Loading weights:  12%|█▏        | 20/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.2.self_attn.o_proj.weight]        
Loading weights:  12%|█▏        | 20/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.2.self_attn.o_proj.weight]
Loading weights:  12%|█▏        | 21/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.3.input_layernorm.weight] 
Loading weights:  12%|█▏        | 21/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.3.input_layernorm.weight]
Loading weights:  13%|█▎        | 22/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.3.mlp.down_proj.weight]  
Loading weights:  13%|█▎        | 22/171 [00:01<00:07, 19.46it/s, Materializing param=model.layers.3.mlp.down_proj.weight]
Loading weights:  13%|█▎        | 23/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.3.mlp.down_proj.weight]
Loading weights:  13%|█▎        | 23/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.3.mlp.gate_proj.weight]
Loading weights:  13%|█▎        | 23/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.3.mlp.gate_proj.weight]
Loading weights:  14%|█▍        | 24/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.3.mlp.up_proj.weight]  
Loading weights:  14%|█▍        | 24/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.3.mlp.up_proj.weight]
Loading weights:  15%|█▍        | 25/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.3.post_attention_layernorm.weight]
Loading weights:  15%|█▍        | 25/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.3.post_attention_layernorm.weight]
Loading weights:  15%|█▌        | 26/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.3.self_attn.o_proj.weight]        
Loading weights:  15%|█▌        | 26/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.3.self_attn.o_proj.weight]
Loading weights:  16%|█▌        | 27/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.4.input_layernorm.weight] 
Loading weights:  16%|█▌        | 27/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.4.input_layernorm.weight]
Loading weights:  16%|█▋        | 28/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.4.mlp.down_proj.weight]  
Loading weights:  16%|█▋        | 28/171 [00:01<00:05, 27.36it/s, Materializing param=model.layers.4.mlp.down_proj.weight]
Loading weights:  17%|█▋        | 29/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.4.mlp.down_proj.weight]
Loading weights:  17%|█▋        | 29/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.4.mlp.gate_proj.weight]
Loading weights:  17%|█▋        | 29/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.4.mlp.gate_proj.weight]
Loading weights:  18%|█▊        | 30/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.4.mlp.up_proj.weight]  
Loading weights:  18%|█▊        | 30/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.4.mlp.up_proj.weight]
Loading weights:  18%|█▊        | 31/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.4.post_attention_layernorm.weight]
Loading weights:  18%|█▊        | 31/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.4.post_attention_layernorm.weight]
Loading weights:  19%|█▊        | 32/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.4.self_attn.o_proj.weight]        
Loading weights:  19%|█▊        | 32/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.4.self_attn.o_proj.weight]
Loading weights:  19%|█▉        | 33/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.5.input_layernorm.weight] 
Loading weights:  19%|█▉        | 33/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.5.input_layernorm.weight]
Loading weights:  20%|█▉        | 34/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.5.mlp.down_proj.weight]  
Loading weights:  20%|█▉        | 34/171 [00:01<00:04, 32.77it/s, Materializing param=model.layers.5.mlp.down_proj.weight]
Loading weights:  20%|██        | 35/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.5.mlp.down_proj.weight]
Loading weights:  20%|██        | 35/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.5.mlp.gate_proj.weight]
Loading weights:  20%|██        | 35/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.5.mlp.gate_proj.weight]
Loading weights:  21%|██        | 36/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.5.mlp.up_proj.weight]  
Loading weights:  21%|██        | 36/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.5.mlp.up_proj.weight]
Loading weights:  22%|██▏       | 37/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.5.post_attention_layernorm.weight]
Loading weights:  22%|██▏       | 37/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.5.post_attention_layernorm.weight]
Loading weights:  22%|██▏       | 38/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.5.self_attn.o_proj.weight]        
Loading weights:  22%|██▏       | 38/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.5.self_attn.o_proj.weight]
Loading weights:  23%|██▎       | 39/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.6.input_layernorm.weight] 
Loading weights:  23%|██▎       | 39/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.6.input_layernorm.weight]
Loading weights:  23%|██▎       | 40/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.6.mlp.down_proj.weight]  
Loading weights:  23%|██▎       | 40/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.6.mlp.down_proj.weight]
Loading weights:  24%|██▍       | 41/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.6.mlp.gate_proj.weight]
Loading weights:  24%|██▍       | 41/171 [00:01<00:03, 35.32it/s, Materializing param=model.layers.6.mlp.gate_proj.weight]
Loading weights:  25%|██▍       | 42/171 [00:01<00:03, 42.36it/s, Materializing param=model.layers.6.mlp.gate_proj.weight]
Loading weights:  25%|██▍       | 42/171 [00:01<00:03, 42.36it/s, Materializing param=model.layers.6.mlp.up_proj.weight]  
Loading weights:  25%|██▍       | 42/171 [00:01<00:03, 42.36it/s, Materializing param=model.layers.6.mlp.up_proj.weight]
Loading weights:  25%|██▌       | 43/171 [00:01<00:03, 42.36it/s, Materializing param=model.layers.6.post_attention_layernorm.weight]
Loading weights:  25%|██▌       | 43/171 [00:01<00:03, 42.36it/s, Materializing param=model.layers.6.post_attention_layernorm.weight]
Loading weights:  26%|██▌       | 44/171 [00:01<00:02, 42.36it/s, Materializing param=model.layers.6.self_attn.o_proj.weight]        
Loading weights:  26%|██▌       | 44/171 [00:01<00:02, 42.36it/s, Materializing param=model.layers.6.self_attn.o_proj.weight]
Loading weights:  26%|██▋       | 45/171 [00:01<00:02, 42.36it/s, Materializing param=model.layers.7.input_layernorm.weight] 
Loading weights:  26%|██▋       | 45/171 [00:01<00:02, 42.36it/s, Materializing param=model.layers.7.input_layernorm.weight]
Loading weights:  27%|██▋       | 46/171 [00:01<00:02, 42.36it/s, Materializing param=model.layers.7.mlp.down_proj.weight]  
Loading weights:  27%|██▋       | 46/171 [00:01<00:02, 42.36it/s, Materializing param=model.layers.7.mlp.down_proj.weight]
Loading weights:  27%|██▋       | 47/171 [00:01<00:02, 42.36it/s, Materializing param=model.layers.7.mlp.gate_proj.weight]
Loading weights:  27%|██▋       | 47/171 [00:01<00:02, 42.36it/s, Materializing param=model.layers.7.mlp.gate_proj.weight]
Loading weights:  28%|██▊       | 48/171 [00:01<00:02, 42.36it/s, Materializing param=model.layers.7.mlp.up_proj.weight]  
Loading weights:  28%|██▊       | 48/171 [00:01<00:02, 42.36it/s, Materializing param=model.layers.7.mlp.up_proj.weight]
Loading weights:  29%|██▊       | 49/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.7.mlp.up_proj.weight]
Loading weights:  29%|██▊       | 49/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.7.post_attention_layernorm.weight]
Loading weights:  29%|██▊       | 49/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.7.post_attention_layernorm.weight]
Loading weights:  29%|██▉       | 50/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.7.self_attn.o_proj.weight]        
Loading weights:  29%|██▉       | 50/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.7.self_attn.o_proj.weight]
Loading weights:  30%|██▉       | 51/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.8.input_layernorm.weight] 
Loading weights:  30%|██▉       | 51/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.8.input_layernorm.weight]
Loading weights:  30%|███       | 52/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.8.mlp.down_proj.weight]  
Loading weights:  30%|███       | 52/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.8.mlp.down_proj.weight]
Loading weights:  31%|███       | 53/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.8.mlp.gate_proj.weight]
Loading weights:  31%|███       | 53/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.8.mlp.gate_proj.weight]
Loading weights:  32%|███▏      | 54/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.8.mlp.up_proj.weight]  
Loading weights:  32%|███▏      | 54/171 [00:01<00:02, 45.35it/s, Materializing param=model.layers.8.mlp.up_proj.weight]
Loading weights:  32%|███▏      | 55/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.8.mlp.up_proj.weight]
Loading weights:  32%|███▏      | 55/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.8.post_attention_layernorm.weight]
Loading weights:  32%|███▏      | 55/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.8.post_attention_layernorm.weight]
Loading weights:  33%|███▎      | 56/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.8.self_attn.o_proj.weight]        
Loading weights:  33%|███▎      | 56/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.8.self_attn.o_proj.weight]
Loading weights:  33%|███▎      | 57/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.input_layernorm.weight] 
Loading weights:  33%|███▎      | 57/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.input_layernorm.weight]
Loading weights:  34%|███▍      | 58/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.mlp.down_proj.weight]  
Loading weights:  34%|███▍      | 58/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.mlp.down_proj.weight]
Loading weights:  35%|███▍      | 59/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.mlp.gate_proj.weight]
Loading weights:  35%|███▍      | 59/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.mlp.gate_proj.weight]
Loading weights:  35%|███▌      | 60/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.mlp.up_proj.weight]  
Loading weights:  35%|███▌      | 60/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.mlp.up_proj.weight]
Loading weights:  36%|███▌      | 61/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.post_attention_layernorm.weight]
Loading weights:  36%|███▌      | 61/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.post_attention_layernorm.weight]
Loading weights:  36%|███▋      | 62/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.self_attn.o_proj.weight]        
Loading weights:  36%|███▋      | 62/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.9.self_attn.o_proj.weight]
Loading weights:  37%|███▋      | 63/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.10.input_layernorm.weight]
Loading weights:  37%|███▋      | 63/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.10.input_layernorm.weight]
Loading weights:  37%|███▋      | 64/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.10.mlp.down_proj.weight]  
Loading weights:  37%|███▋      | 64/171 [00:01<00:02, 46.46it/s, Materializing param=model.layers.10.mlp.down_proj.weight]
Loading weights:  38%|███▊      | 65/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.10.mlp.down_proj.weight]
Loading weights:  38%|███▊      | 65/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.10.mlp.gate_proj.weight]
Loading weights:  38%|███▊      | 65/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.10.mlp.gate_proj.weight]
Loading weights:  39%|███▊      | 66/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.10.mlp.up_proj.weight]  
Loading weights:  39%|███▊      | 66/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.10.mlp.up_proj.weight]
Loading weights:  39%|███▉      | 67/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.10.post_attention_layernorm.weight]
Loading weights:  39%|███▉      | 67/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.10.post_attention_layernorm.weight]
Loading weights:  40%|███▉      | 68/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.10.self_attn.o_proj.weight]        
Loading weights:  40%|███▉      | 68/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.10.self_attn.o_proj.weight]
Loading weights:  40%|████      | 69/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.11.input_layernorm.weight] 
Loading weights:  40%|████      | 69/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.11.input_layernorm.weight]
Loading weights:  41%|████      | 70/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.11.mlp.down_proj.weight]  
Loading weights:  41%|████      | 70/171 [00:01<00:02, 47.43it/s, Materializing param=model.layers.11.mlp.down_proj.weight]
Loading weights:  42%|████▏     | 71/171 [00:02<00:02, 47.43it/s, Materializing param=model.layers.11.mlp.gate_proj.weight]
Loading weights:  42%|████▏     | 71/171 [00:02<00:02, 47.43it/s, Materializing param=model.layers.11.mlp.gate_proj.weight]
Loading weights:  42%|████▏     | 72/171 [00:02<00:02, 47.43it/s, Materializing param=model.layers.11.mlp.up_proj.weight]  
Loading weights:  42%|████▏     | 72/171 [00:02<00:02, 47.43it/s, Materializing param=model.layers.11.mlp.up_proj.weight]
Loading weights:  43%|████▎     | 73/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.11.mlp.up_proj.weight]
Loading weights:  43%|████▎     | 73/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.11.post_attention_layernorm.weight]
Loading weights:  43%|████▎     | 73/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.11.post_attention_layernorm.weight]
Loading weights:  43%|████▎     | 74/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.11.self_attn.o_proj.weight]        
Loading weights:  43%|████▎     | 74/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.11.self_attn.o_proj.weight]
Loading weights:  44%|████▍     | 75/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.input_layernorm.weight] 
Loading weights:  44%|████▍     | 75/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.input_layernorm.weight]
Loading weights:  44%|████▍     | 76/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.mlp.down_proj.weight]  
Loading weights:  44%|████▍     | 76/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.mlp.down_proj.weight]
Loading weights:  45%|████▌     | 77/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.mlp.gate_proj.weight]
Loading weights:  45%|████▌     | 77/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.mlp.gate_proj.weight]
Loading weights:  46%|████▌     | 78/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.mlp.up_proj.weight]  
Loading weights:  46%|████▌     | 78/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.mlp.up_proj.weight]
Loading weights:  46%|████▌     | 79/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.post_attention_layernorm.weight]
Loading weights:  46%|████▌     | 79/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.post_attention_layernorm.weight]
Loading weights:  47%|████▋     | 80/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.self_attn.o_proj.weight]        
Loading weights:  47%|████▋     | 80/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.12.self_attn.o_proj.weight]
Loading weights:  47%|████▋     | 81/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.13.input_layernorm.weight] 
Loading weights:  47%|████▋     | 81/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.13.input_layernorm.weight]
Loading weights:  48%|████▊     | 82/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.13.mlp.down_proj.weight]  
Loading weights:  48%|████▊     | 82/171 [00:02<00:01, 51.37it/s, Materializing param=model.layers.13.mlp.down_proj.weight]
Loading weights:  49%|████▊     | 83/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.13.mlp.down_proj.weight]
Loading weights:  49%|████▊     | 83/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.13.mlp.gate_proj.weight]
Loading weights:  49%|████▊     | 83/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.13.mlp.gate_proj.weight]
Loading weights:  49%|████▉     | 84/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.13.mlp.up_proj.weight]  
Loading weights:  49%|████▉     | 84/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.13.mlp.up_proj.weight]
Loading weights:  50%|████▉     | 85/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.13.post_attention_layernorm.weight]
Loading weights:  50%|████▉     | 85/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.13.post_attention_layernorm.weight]
Loading weights:  50%|█████     | 86/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.13.self_attn.o_proj.weight]        
Loading weights:  50%|█████     | 86/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.13.self_attn.o_proj.weight]
Loading weights:  51%|█████     | 87/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.14.input_layernorm.weight] 
Loading weights:  51%|█████     | 87/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.14.input_layernorm.weight]
Loading weights:  51%|█████▏    | 88/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.14.mlp.down_proj.weight]  
Loading weights:  51%|█████▏    | 88/171 [00:02<00:01, 53.60it/s, Materializing param=model.layers.14.mlp.down_proj.weight]
Loading weights:  52%|█████▏    | 89/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.14.mlp.down_proj.weight]
Loading weights:  52%|█████��    | 89/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.14.mlp.gate_proj.weight]
Loading weights:  52%|█████▏    | 89/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.14.mlp.gate_proj.weight]
Loading weights:  53%|█████▎    | 90/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.14.mlp.up_proj.weight]  
Loading weights:  53%|█████▎    | 90/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.14.mlp.up_proj.weight]
Loading weights:  53%|█████▎    | 91/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.14.post_attention_layernorm.weight]
Loading weights:  53%|█████▎    | 91/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.14.post_attention_layernorm.weight]
Loading weights:  54%|█████▍    | 92/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.14.self_attn.o_proj.weight]        
Loading weights:  54%|█████▍    | 92/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.14.self_attn.o_proj.weight]
Loading weights:  54%|█████▍    | 93/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.15.input_layernorm.weight] 
Loading weights:  54%|█████▍    | 93/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.15.input_layernorm.weight]
Loading weights:  55%|█████▍    | 94/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.15.mlp.down_proj.weight]  
Loading weights:  55%|█████▍    | 94/171 [00:02<00:01, 54.33it/s, Materializing param=model.layers.15.mlp.down_proj.weight]
Loading weights:  56%|█████▌    | 95/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.15.mlp.down_proj.weight]
Loading weights:  56%|█████▌    | 95/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.15.mlp.gate_proj.weight]
Loading weights:  56%|█████▌    | 95/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.15.mlp.gate_proj.weight]
Loading weights:  56%|█████▌    | 96/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.15.mlp.up_proj.weight]  
Loading weights:  56%|█████▌    | 96/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.15.mlp.up_proj.weight]
Loading weights:  57%|█████▋    | 97/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.15.post_attention_layernorm.weight]
Loading weights:  57%|█████▋    | 97/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.15.post_attention_layernorm.weight]
Loading weights:  57%|█████▋    | 98/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.15.self_attn.o_proj.weight]        
Loading weights:  57%|█████▋    | 98/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.15.self_attn.o_proj.weight]
Loading weights:  58%|█████▊    | 99/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.16.input_layernorm.weight] 
Loading weights:  58%|█████▊    | 99/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.16.input_layernorm.weight]
Loading weights:  58%|█████▊    | 100/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.16.mlp.down_proj.weight] 
Loading weights:  58%|█████▊    | 100/171 [00:02<00:01, 55.09it/s, Materializing param=model.layers.16.mlp.down_proj.weight]
Loading weights:  59%|█████▉    | 101/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.16.mlp.down_proj.weight]
Loading weights:  59%|█████▉    | 101/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.16.mlp.gate_proj.weight]
Loading weights:  59%|█████▉    | 101/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.16.mlp.gate_proj.weight]
Loading weights:  60%|█████▉    | 102/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.16.mlp.up_proj.weight]  
Loading weights:  60%|█████▉    | 102/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.16.mlp.up_proj.weight]
Loading weights:  60%|██████    | 103/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.16.post_attention_layernorm.weight]
Loading weights:  60%|██████    | 103/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.16.post_attention_layernorm.weight]
Loading weights:  61%|██████    | 104/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.16.self_attn.o_proj.weight]        
Loading weights:  61%|██████    | 104/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.16.self_attn.o_proj.weight]
Loading weights:  61%|██████▏   | 105/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.17.input_layernorm.weight] 
Loading weights:  61%|██████▏   | 105/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.17.input_layernorm.weight]
Loading weights:  62%|██████▏   | 106/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.17.mlp.down_proj.weight]  
Loading weights:  62%|██████▏   | 106/171 [00:02<00:01, 53.99it/s, Materializing param=model.layers.17.mlp.down_proj.weight]
Loading weights:  63%|██████▎   | 107/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.17.mlp.down_proj.weight]
Loading weights:  63%|██████▎   | 107/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.17.mlp.gate_proj.weight]
Loading weights:  63%|██████▎   | 107/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.17.mlp.gate_proj.weight]
Loading weights:  63%|██████▎   | 108/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.17.mlp.up_proj.weight]  
Loading weights:  63%|██████▎   | 108/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.17.mlp.up_proj.weight]
Loading weights:  64%|██████▎   | 109/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.17.post_attention_layernorm.weight]
Loading weights:  64%|██████▎   | 109/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.17.post_attention_layernorm.weight]
Loading weights:  64%|██████▍   | 110/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.17.self_attn.o_proj.weight]        
Loading weights:  64%|██████▍   | 110/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.17.self_attn.o_proj.weight]
Loading weights:  65%|██████▍   | 111/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.18.input_layernorm.weight] 
Loading weights:  65%|██████▍   | 111/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.18.input_layernorm.weight]
Loading weights:  65%|██████▌   | 112/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.18.mlp.down_proj.weight]  
Loading weights:  65%|██████▌   | 112/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.18.mlp.down_proj.weight]
Loading weights:  66%|██████▌   | 113/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.18.mlp.gate_proj.weight]
Loading weights:  66%|██████▌   | 113/171 [00:02<00:01, 48.06it/s, Materializing param=model.layers.18.mlp.gate_proj.weight]
Loading weights:  67%|██████▋   | 114/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.18.mlp.gate_proj.weight]
Loading weights:  67%|██████▋   | 114/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.18.mlp.up_proj.weight]  
Loading weights:  67%|██████▋   | 114/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.18.mlp.up_proj.weight]
Loading weights:  67%|██████▋   | 115/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.18.post_attention_layernorm.weight]
Loading weights:  67%|██████▋   | 115/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.18.post_attention_layernorm.weight]
Loading weights:  68%|██████▊   | 116/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.18.self_attn.o_proj.weight]        
Loading weights:  68%|██████▊   | 116/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.18.self_attn.o_proj.weight]
Loading weights:  68%|██████▊   | 117/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.19.input_layernorm.weight] 
Loading weights:  68%|██████▊   | 117/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.19.input_layernorm.weight]
Loading weights:  69%|██████▉   | 118/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.19.mlp.down_proj.weight]  
Loading weights:  69%|██████▉   | 118/171 [00:02<00:01, 52.74it/s, Materializing param=model.layers.19.mlp.down_proj.weight]
Loading weights:  70%|██████▉   | 119/171 [00:02<00:00, 52.74it/s, Materializing param=model.layers.19.mlp.gate_proj.weight]
Loading weights:  70%|██████▉   | 119/171 [00:02<00:00, 52.74it/s, Materializing param=model.layers.19.mlp.gate_proj.weight]
Loading weights:  70%|███████   | 120/171 [00:02<00:00, 52.74it/s, Materializing param=model.layers.19.mlp.up_proj.weight]  
Loading weights:  70%|███████   | 120/171 [00:02<00:00, 52.74it/s, Materializing param=model.layers.19.mlp.up_proj.weight]
Loading weights:  71%|███████   | 121/171 [00:02<00:00, 55.69it/s, Materializing param=model.layers.19.mlp.up_proj.weight]
Loading weights:  71%|███████   | 121/171 [00:02<00:00, 55.69it/s, Materializing param=model.layers.19.post_attention_layernorm.weight]
Loading weights:  71%|███████   | 121/171 [00:02<00:00, 55.69it/s, Materializing param=model.layers.19.post_attention_layernorm.weight]
Loading weights:  71%|███████▏  | 122/171 [00:02<00:00, 55.69it/s, Materializing param=model.layers.19.self_attn.o_proj.weight]        
Loading weights:  71%|███████▏  | 122/171 [00:02<00:00, 55.69it/s, Materializing param=model.layers.19.self_attn.o_proj.weight]
Loading weights:  72%|███████▏  | 123/171 [00:02<00:00, 55.69it/s, Materializing param=model.layers.20.input_layernorm.weight] 
Loading weights:  72%|███████▏  | 123/171 [00:02<00:00, 55.69it/s, Materializing param=model.layers.20.input_layernorm.weight]
Loading weights:  73%|███████▎  | 124/171 [00:02<00:00, 55.69it/s, Materializing param=model.layers.20.mlp.down_proj.weight]  
Loading weights:  73%|███████▎  | 124/171 [00:02<00:00, 55.69it/s, Materializing param=model.layers.20.mlp.down_proj.weight]
Loading weights:  73%|███████▎  | 125/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.20.mlp.gate_proj.weight]
Loading weights:  73%|███████▎  | 125/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.20.mlp.gate_proj.weight]
Loading weights:  74%|███████▎  | 126/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.20.mlp.up_proj.weight]  
Loading weights:  74%|███████▎  | 126/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.20.mlp.up_proj.weight]
Loading weights:  74%|███████▍  | 127/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.20.post_attention_layernorm.weight]
Loading weights:  74%|███████▍  | 127/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.20.post_attention_layernorm.weight]
Loading weights:  75%|███████▍  | 128/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.20.self_attn.o_proj.weight]        
Loading weights:  75%|███████▍  | 128/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.20.self_attn.o_proj.weight]
Loading weights:  75%|███████▌  | 129/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.21.input_layernorm.weight] 
Loading weights:  75%|███████▌  | 129/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.21.input_layernorm.weight]
Loading weights:  76%|███████▌  | 130/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.21.mlp.down_proj.weight]  
Loading weights:  76%|███████▌  | 130/171 [00:03<00:00, 55.69it/s, Materializing param=model.layers.21.mlp.down_proj.weight]
Loading weights:  77%|███████▋  | 131/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.21.mlp.down_proj.weight]
Loading weights:  77%|███████▋  | 131/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.21.mlp.gate_proj.weight]
Loading weights:  77%|███████▋  | 131/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.21.mlp.gate_proj.weight]
Loading weights:  77%|███████▋  | 132/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.21.mlp.up_proj.weight]  
Loading weights:  77%|███████▋  | 132/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.21.mlp.up_proj.weight]
Loading weights:  78%|███████▊  | 133/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.21.post_attention_layernorm.weight]
Loading weights:  78%|███████▊  | 133/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.21.post_attention_layernorm.weight]
Loading weights:  78%|███████▊  | 134/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.21.self_attn.o_proj.weight]        
Loading weights:  78%|███████▊  | 134/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.21.self_attn.o_proj.weight]
Loading weights:  79%|███████▉  | 135/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.22.input_layernorm.weight] 
Loading weights:  79%|███████▉  | 135/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.22.input_layernorm.weight]
Loading weights:  80%|███████▉  | 136/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.22.mlp.down_proj.weight]  
Loading weights:  80%|███████▉  | 136/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.22.mlp.down_proj.weight]
Loading weights:  80%|████████  | 137/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.22.mlp.gate_proj.weight]
Loading weights:  80%|████████  | 137/171 [00:03<00:00, 52.17it/s, Materializing param=model.layers.22.mlp.gate_proj.weight]
Loading weights:  81%|████████  | 138/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.22.mlp.gate_proj.weight]
Loading weights:  81%|████████  | 138/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.22.mlp.up_proj.weight]  
Loading weights:  81%|████████  | 138/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.22.mlp.up_proj.weight]
Loading weights:  81%|████████▏ | 139/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.22.post_attention_layernorm.weight]
Loading weights:  81%|████████▏ | 139/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.22.post_attention_layernorm.weight]
Loading weights:  82%|████████▏ | 140/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.22.self_attn.o_proj.weight]        
Loading weights:  82%|████████▏ | 140/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.22.self_attn.o_proj.weight]
Loading weights:  82%|████████▏ | 141/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.23.input_layernorm.weight] 
Loading weights:  82%|████████▏ | 141/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.23.input_layernorm.weight]
Loading weights:  83%|████████▎ | 142/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.23.mlp.down_proj.weight]  
Loading weights:  83%|████████▎ | 142/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.23.mlp.down_proj.weight]
Loading weights:  84%|████████▎ | 143/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.23.mlp.gate_proj.weight]
Loading weights:  84%|████████▎ | 143/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.23.mlp.gate_proj.weight]
Loading weights:  84%|████████▍ | 144/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.23.mlp.up_proj.weight]  
Loading weights:  84%|████████▍ | 144/171 [00:03<00:00, 55.54it/s, Materializing param=model.layers.23.mlp.up_proj.weight]
Loading weights:  85%|████████▍ | 145/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.23.mlp.up_proj.weight]
Loading weights:  85%|████████▍ | 145/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.23.post_attention_layernorm.weight]
Loading weights:  85%|████████▍ | 145/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.23.post_attention_layernorm.weight]
Loading weights:  85%|████████▌ | 146/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.23.self_attn.o_proj.weight]        
Loading weights:  85%|████████▌ | 146/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.23.self_attn.o_proj.weight]
Loading weights:  86%|████████▌ | 147/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.input_layernorm.weight] 
Loading weights:  86%|████████▌ | 147/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.input_layernorm.weight]
Loading weights:  87%|████████▋ | 148/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.mlp.down_proj.weight]  
Loading weights:  87%|████████▋ | 148/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.mlp.down_proj.weight]
Loading weights:  87%|████████▋ | 149/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.mlp.gate_proj.weight]
Loading weights:  87%|████████▋ | 149/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.mlp.gate_proj.weight]
Loading weights:  88%|████████▊ | 150/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.mlp.up_proj.weight]  
Loading weights:  88%|████████▊ | 150/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.mlp.up_proj.weight]
Loading weights:  88%|████████▊ | 151/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.post_attention_layernorm.weight]
Loading weights:  88%|████████▊ | 151/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.post_attention_layernorm.weight]
Loading weights:  89%|████████▉ | 152/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.self_attn.o_proj.weight]        
Loading weights:  89%|████████▉ | 152/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.24.self_attn.o_proj.weight]
Loading weights:  89%|████████▉ | 153/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.25.input_layernorm.weight] 
Loading weights:  89%|████████▉ | 153/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.25.input_layernorm.weight]
Loading weights:  90%|█████████ | 154/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.25.mlp.down_proj.weight]  
Loading weights:  90%|█████████ | 154/171 [00:03<00:00, 51.56it/s, Materializing param=model.layers.25.mlp.down_proj.weight]
Loading weights:  91%|█████████ | 155/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.25.mlp.down_proj.weight]
Loading weights:  91%|█████████ | 155/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.25.mlp.gate_proj.weight]
Loading weights:  91%|█████████ | 155/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.25.mlp.gate_proj.weight]
Loading weights:  91%|█████████ | 156/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.25.mlp.up_proj.weight]  
Loading weights:  91%|█████████ | 156/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.25.mlp.up_proj.weight]
Loading weights:  92%|█████████▏| 157/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.25.post_attention_layernorm.weight]
Loading weights:  92%|█████████▏| 157/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.25.post_attention_layernorm.weight]
Loading weights:  92%|█████████▏| 158/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.25.self_attn.o_proj.weight]        
Loading weights:  92%|█████████▏| 158/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.25.self_attn.o_proj.weight]
Loading weights:  93%|█████████▎| 159/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.26.input_layernorm.weight] 
Loading weights:  93%|█████████▎| 159/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.26.input_layernorm.weight]
Loading weights:  94%|█████████▎| 160/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.26.mlp.down_proj.weight]  
Loading weights:  94%|█████████▎| 160/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.26.mlp.down_proj.weight]
Loading weights:  94%|█████████▍| 161/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.26.mlp.gate_proj.weight]
Loading weights:  94%|█████████▍| 161/171 [00:03<00:00, 52.13it/s, Materializing param=model.layers.26.mlp.gate_proj.weight]
Loading weights:  95%|█████████▍| 162/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.26.mlp.gate_proj.weight]
Loading weights:  95%|█████████▍| 162/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.26.mlp.up_proj.weight]  
Loading weights:  95%|█████████▍| 162/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.26.mlp.up_proj.weight]
Loading weights:  95%|█████████▌| 163/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.26.post_attention_layernorm.weight]
Loading weights:  95%|█████████▌| 163/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.26.post_attention_layernorm.weight]
Loading weights:  96%|█████████▌| 164/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.26.self_attn.o_proj.weight]        
Loading weights:  96%|█████████▌| 164/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.26.self_attn.o_proj.weight]
Loading weights:  96%|█████████▋| 165/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.input_layernorm.weight] 
Loading weights:  96%|█████████▋| 165/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.input_layernorm.weight]
Loading weights:  97%|█████████▋| 166/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.mlp.down_proj.weight]  
Loading weights:  97%|█████████▋| 166/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.mlp.down_proj.weight]
Loading weights:  98%|█████████▊| 167/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.mlp.gate_proj.weight]
Loading weights:  98%|█████████▊| 167/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.mlp.gate_proj.weight]
Loading weights:  98%|█████████▊| 168/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.mlp.up_proj.weight]  
Loading weights:  98%|█████████▊| 168/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.mlp.up_proj.weight]
Loading weights:  99%|█████████▉| 169/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.post_attention_layernorm.weight]
Loading weights:  99%|█████████▉| 169/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.post_attention_layernorm.weight]
Loading weights:  99%|█████████▉| 170/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.self_attn.o_proj.weight]        
Loading weights:  99%|█████████▉| 170/171 [00:03<00:00, 55.22it/s, Materializing param=model.layers.27.self_attn.o_proj.weight]
Loading weights: 100%|██████████| 171/171 [00:03<00:00, 55.22it/s, Materializing param=model.norm.weight]                      
Loading weights: 100%|██████████| 171/171 [00:03<00:00, 55.22it/s, Materializing param=model.norm.weight]
Loading weights: 100%|██████████| 171/171 [00:03<00:00, 44.86it/s, Materializing param=model.norm.weight]
+The tied weights mapping and config for this model specifies to tie model.embed_tokens.weight to lm_head.weight, but both are present in the checkpoints, so we will NOT tie them. You should update the config with `tie_word_embeddings=False` to silence this warning
+LlamaForCausalLM LOAD REPORT from: /PROJECT/0325120031_A/ghong/taketimes/llm-bang/eval/outputs/hf_3b_base
+Key                                           | Status  | 
+----------------------------------------------+---------+-
+model.layers.{0...27}.self_attn.q_proj.weight | MISSING | 
+model.layers.{0...27}.self_attn.v_proj.weight | MISSING | 
+model.layers.{0...27}.self_attn.k_proj.weight | MISSING | 
+
+Notes:
+- MISSING	:those params were newly initialized because missing from the checkpoint. Consider training on your downstream task.
+Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
+
Generating test split:   0%|          | 0/900 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 900/900 [00:00<00:00, 178793.80 examples/s]
+
Generating test split:   0%|          | 0/14042 [00:00<?, ? examples/s]
Generating test split: 100%|██████████| 14042/14042 [00:00<00:00, 785484.55 examples/s]
+
Generating dev split:   0%|          | 0/285 [00:00<?, ? examples/s]
Generating dev split: 100%|██████████| 285/285 [00:00<00:00, 120453.11 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 48144.37 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 112478.49 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 111205.05 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 60907.81 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111323.89 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110428.16 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 65886.38 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 109308.51 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 108639.32 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 62591.72 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 112723.85 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 111911.01 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 65604.34 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 113015.18 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 112324.43 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 63364.78 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 112597.25 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 111818.59 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 61408.44 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 112774.87 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 112018.08 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 61782.96 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 112190.41 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 111453.58 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 58425.06 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 110765.31 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110063.31 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 57121.26 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111143.70 examples/s]
Filter: 100%|��█████████| 14042/14042 [00:00<00:00, 110434.78 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 60178.04 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 110261.08 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 109508.51 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 62759.31 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111994.45 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110993.38 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 60311.64 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111576.60 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110818.99 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 61544.39 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111468.37 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110296.63 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 65196.44 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 109774.35 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 108936.31 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 65737.83 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111618.17 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110485.33 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 58772.64 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 109977.03 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 108169.03 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 59990.80 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111575.86 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110240.89 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 67061.80 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111448.63 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110664.49 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 67303.45 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111353.70 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110545.68 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 58333.82 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 110372.55 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 109775.08 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 64545.17 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111225.49 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110424.02 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 59715.09 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 109399.98 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 108948.40 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 63334.57 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111402.50 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110625.83 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 58827.59 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111825.49 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 111037.74 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 64054.05 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 112605.82 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 111475.09 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 64828.71 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111729.43 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110601.94 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 62914.56 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111393.62 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110595.50 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 60845.80 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111298.79 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110502.33 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 65356.84 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111734.63 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110976.23 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 61601.48 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111176.85 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110279.08 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 63381.58 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111445.42 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110702.76 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 56263.61 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 110895.88 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110072.98 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 59521.82 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111488.12 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110697.98 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 63814.68 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111674.89 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110903.72 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 62934.43 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111610.25 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110767.51 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|█���████████| 285/285 [00:00<00:00, 66024.67 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111281.31 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110496.94 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 60275.14 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111207.06 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110354.71 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 65111.21 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111713.55 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110931.92 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 62604.83 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 108212.64 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 107652.21 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 58445.05 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111081.15 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110315.64 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 60982.38 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111409.89 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110515.81 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 62418.50 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 109277.89 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 108826.81 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 64426.90 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 110805.79 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110026.71 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 61943.03 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 107998.56 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 107156.68 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 63479.19 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 110504.13 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 109694.53 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 60329.90 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111334.98 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110584.91 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 61257.39 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 110938.17 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110219.85 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 59080.54 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111107.64 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110281.35 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 60470.29 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111578.83 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110718.58 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 64277.93 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111285.74 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110500.05 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 65792.10 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111430.37 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110545.68 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 66554.01 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111931.18 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 111083.82 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 60966.83 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 110460.48 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 109693.50 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 63773.83 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111404.72 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110578.68 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 64146.85 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 111702.15 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 110885.97 examples/s]
+
Filter:   0%|          | 0/285 [00:00<?, ? examples/s]
Filter: 100%|██████████| 285/285 [00:00<00:00, 57809.10 examples/s]
+
Filter:   0%|          | 0/14042 [00:00<?, ? examples/s]
Filter:  85%|████████▌ | 12000/14042 [00:00<00:00, 110140.44 examples/s]
Filter: 100%|██████████| 14042/14042 [00:00<00:00, 109313.39 examples/s]
+2026-03-05:01:48:08 INFO     [tasks:700] Selected tasks:
+2026-03-05:01:48:08 INFO     [tasks:703] Group: global_mmlu_full_ko
+2026-03-05:01:48:08 INFO     [tasks:711]   Subgroup: global_mmlu_full_ko_stem
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_abstract_algebra (global_mmlu/full/ko/global_mmlu_full_ko_abstract_algebra.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_anatomy (global_mmlu/full/ko/global_mmlu_full_ko_anatomy.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_astronomy (global_mmlu/full/ko/global_mmlu_full_ko_astronomy.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_college_biology (global_mmlu/full/ko/global_mmlu_full_ko_college_biology.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_college_chemistry (global_mmlu/full/ko/global_mmlu_full_ko_college_chemistry.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_college_computer_science (global_mmlu/full/ko/global_mmlu_full_ko_college_computer_science.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_college_mathematics (global_mmlu/full/ko/global_mmlu_full_ko_college_mathematics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_college_physics (global_mmlu/full/ko/global_mmlu_full_ko_college_physics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_computer_security (global_mmlu/full/ko/global_mmlu_full_ko_computer_security.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_conceptual_physics (global_mmlu/full/ko/global_mmlu_full_ko_conceptual_physics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_electrical_engineering (global_mmlu/full/ko/global_mmlu_full_ko_electrical_engineering.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_elementary_mathematics (global_mmlu/full/ko/global_mmlu_full_ko_elementary_mathematics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_biology (global_mmlu/full/ko/global_mmlu_full_ko_high_school_biology.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_chemistry (global_mmlu/full/ko/global_mmlu_full_ko_high_school_chemistry.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_computer_science (global_mmlu/full/ko/global_mmlu_full_ko_high_school_computer_science.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_mathematics (global_mmlu/full/ko/global_mmlu_full_ko_high_school_mathematics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_physics (global_mmlu/full/ko/global_mmlu_full_ko_high_school_physics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_statistics (global_mmlu/full/ko/global_mmlu_full_ko_high_school_statistics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_machine_learning (global_mmlu/full/ko/global_mmlu_full_ko_machine_learning.yaml)
+2026-03-05:01:48:08 INFO     [tasks:711]   Subgroup: global_mmlu_full_ko_other
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_business_ethics (global_mmlu/full/ko/global_mmlu_full_ko_business_ethics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_clinical_knowledge (global_mmlu/full/ko/global_mmlu_full_ko_clinical_knowledge.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_college_medicine (global_mmlu/full/ko/global_mmlu_full_ko_college_medicine.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_global_facts (global_mmlu/full/ko/global_mmlu_full_ko_global_facts.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_human_aging (global_mmlu/full/ko/global_mmlu_full_ko_human_aging.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_management (global_mmlu/full/ko/global_mmlu_full_ko_management.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_marketing (global_mmlu/full/ko/global_mmlu_full_ko_marketing.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_medical_genetics (global_mmlu/full/ko/global_mmlu_full_ko_medical_genetics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_miscellaneous (global_mmlu/full/ko/global_mmlu_full_ko_miscellaneous.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_nutrition (global_mmlu/full/ko/global_mmlu_full_ko_nutrition.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_professional_accounting (global_mmlu/full/ko/global_mmlu_full_ko_professional_accounting.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_professional_medicine (global_mmlu/full/ko/global_mmlu_full_ko_professional_medicine.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_virology (global_mmlu/full/ko/global_mmlu_full_ko_virology.yaml)
+2026-03-05:01:48:08 INFO     [tasks:711]   Subgroup: global_mmlu_full_ko_social_sciences
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_econometrics (global_mmlu/full/ko/global_mmlu_full_ko_econometrics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_geography (global_mmlu/full/ko/global_mmlu_full_ko_high_school_geography.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_government_and_politics (global_mmlu/full/ko/global_mmlu_full_ko_high_school_government_and_politics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_macroeconomics (global_mmlu/full/ko/global_mmlu_full_ko_high_school_macroeconomics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_microeconomics (global_mmlu/full/ko/global_mmlu_full_ko_high_school_microeconomics.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_psychology (global_mmlu/full/ko/global_mmlu_full_ko_high_school_psychology.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_human_sexuality (global_mmlu/full/ko/global_mmlu_full_ko_human_sexuality.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_professional_psychology (global_mmlu/full/ko/global_mmlu_full_ko_professional_psychology.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_public_relations (global_mmlu/full/ko/global_mmlu_full_ko_public_relations.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_security_studies (global_mmlu/full/ko/global_mmlu_full_ko_security_studies.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_sociology (global_mmlu/full/ko/global_mmlu_full_ko_sociology.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_us_foreign_policy (global_mmlu/full/ko/global_mmlu_full_ko_us_foreign_policy.yaml)
+2026-03-05:01:48:08 INFO     [tasks:711]   Subgroup: global_mmlu_full_ko_humanities
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_formal_logic (global_mmlu/full/ko/global_mmlu_full_ko_formal_logic.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_european_history (global_mmlu/full/ko/global_mmlu_full_ko_high_school_european_history.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_us_history (global_mmlu/full/ko/global_mmlu_full_ko_high_school_us_history.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_high_school_world_history (global_mmlu/full/ko/global_mmlu_full_ko_high_school_world_history.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_international_law (global_mmlu/full/ko/global_mmlu_full_ko_international_law.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_jurisprudence (global_mmlu/full/ko/global_mmlu_full_ko_jurisprudence.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_logical_fallacies (global_mmlu/full/ko/global_mmlu_full_ko_logical_fallacies.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_moral_disputes (global_mmlu/full/ko/global_mmlu_full_ko_moral_disputes.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_moral_scenarios (global_mmlu/full/ko/global_mmlu_full_ko_moral_scenarios.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_philosophy (global_mmlu/full/ko/global_mmlu_full_ko_philosophy.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_prehistory (global_mmlu/full/ko/global_mmlu_full_ko_prehistory.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_professional_law (global_mmlu/full/ko/global_mmlu_full_ko_professional_law.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691]     Task: global_mmlu_full_ko_world_religions (global_mmlu/full/ko/global_mmlu_full_ko_world_religions.yaml)
+2026-03-05:01:48:08 INFO     [tasks:691] Task: belebele_kor_Hang (belebele/belebele_kor_Hang.yaml)
+2026-03-05:01:48:08 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_abstract_algebra on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
100%|██████████| 100/100 [00:00<00:00, 2567.98it/s]
+2026-03-05:01:48:08 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_anatomy on rank 0...
+
  0%|          | 0/135 [00:00<?, ?it/s]
100%|██████████| 135/135 [00:00<00:00, 2610.31it/s]
+2026-03-05:01:48:08 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_astronomy on rank 0...
+
  0%|          | 0/152 [00:00<?, ?it/s]
100%|██████████| 152/152 [00:00<00:00, 2645.68it/s]
+2026-03-05:01:48:08 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_college_biology on rank 0...
+
  0%|          | 0/144 [00:00<?, ?it/s]
100%|██████████| 144/144 [00:00<00:00, 2652.07it/s]
+2026-03-05:01:48:08 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_college_chemistry on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
100%|██████████| 100/100 [00:00<00:00, 2622.73it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_college_computer_science on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
100%|██████████| 100/100 [00:00<00:00, 2591.04it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_college_mathematics on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
100%|██████████| 100/100 [00:00<00:00, 2609.23it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_college_physics on rank 0...
+
  0%|          | 0/102 [00:00<?, ?it/s]
100%|██████████| 102/102 [00:00<00:00, 2601.85it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_computer_security on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
100%|██████████| 100/100 [00:00<00:00, 2626.59it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_conceptual_physics on rank 0...
+
  0%|          | 0/235 [00:00<?, ?it/s]
100%|██████████| 235/235 [00:00<00:00, 2642.05it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_electrical_engineering on rank 0...
+
  0%|          | 0/145 [00:00<?, ?it/s]
100%|██████████| 145/145 [00:00<00:00, 2634.73it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_elementary_mathematics on rank 0...
+
  0%|          | 0/378 [00:00<?, ?it/s]
 71%|███████   | 268/378 [00:00<00:00, 2678.07it/s]
100%|██████████| 378/378 [00:00<00:00, 2685.30it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_biology on rank 0...
+
  0%|          | 0/310 [00:00<?, ?it/s]
 87%|████████▋ | 269/310 [00:00<00:00, 2686.67it/s]
100%|██████████| 310/310 [00:00<00:00, 2682.42it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_chemistry on rank 0...
+
  0%|          | 0/203 [00:00<?, ?it/s]
100%|██████████| 203/203 [00:00<00:00, 2653.44it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_computer_science on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
100%|██████████| 100/100 [00:00<00:00, 2633.93it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_mathematics on rank 0...
+
  0%|          | 0/270 [00:00<?, ?it/s]
 99%|█████████▉| 267/270 [00:00<00:00, 2661.86it/s]
100%|██████████| 270/270 [00:00<00:00, 2659.17it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_physics on rank 0...
+
  0%|          | 0/151 [00:00<?, ?it/s]
100%|██████████| 151/151 [00:00<00:00, 2633.09it/s]
+2026-03-05:01:48:09 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_statistics on rank 0...
+
  0%|          | 0/216 [00:00<?, ?it/s]
100%|██████████| 216/216 [00:00<00:00, 2570.98it/s]
+2026-03-05:01:48:10 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_machine_learning on rank 0...
+
  0%|          | 0/112 [00:00<?, ?it/s]
100%|██████████| 112/112 [00:00<00:00, 2615.28it/s]
+2026-03-05:01:48:10 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_business_ethics on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
100%|██████████| 100/100 [00:00<00:00, 2617.87it/s]
+2026-03-05:01:48:10 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_clinical_knowledge on rank 0...
+
  0%|          | 0/265 [00:00<?, ?it/s]
100%|██████████| 265/265 [00:00<00:00, 2649.04it/s]
100%|██████████| 265/265 [00:00<00:00, 2647.16it/s]
+2026-03-05:01:48:10 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_college_medicine on rank 0...
+
  0%|          | 0/173 [00:00<?, ?it/s]
100%|██████████| 173/173 [00:00<00:00, 2643.10it/s]
+2026-03-05:01:48:10 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_global_facts on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
100%|██████████| 100/100 [00:00<00:00, 2626.84it/s]
+2026-03-05:01:48:10 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_human_aging on rank 0...
+
  0%|          | 0/223 [00:00<?, ?it/s]
100%|██████████| 223/223 [00:00<00:00, 2658.30it/s]
+2026-03-05:01:48:10 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_management on rank 0...
+
  0%|          | 0/103 [00:00<?, ?it/s]
100%|██████████| 103/103 [00:00<00:00, 2635.03it/s]
+2026-03-05:01:48:10 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_marketing on rank 0...
+
  0%|          | 0/234 [00:00<?, ?it/s]
100%|██████████| 234/234 [00:00<00:00, 2645.74it/s]
+2026-03-05:01:48:10 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_medical_genetics on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
100%|██████████| 100/100 [00:00<00:00, 2616.70it/s]
+2026-03-05:01:48:10 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_miscellaneous on rank 0...
+
  0%|          | 0/783 [00:00<?, ?it/s]
 34%|███▍      | 266/783 [00:00<00:00, 2657.08it/s]
 68%|██████▊   | 534/783 [00:00<00:00, 2665.01it/s]
100%|██████████| 783/783 [00:00<00:00, 2668.39it/s]
+2026-03-05:01:48:11 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_nutrition on rank 0...
+
  0%|          | 0/306 [00:00<?, ?it/s]
 87%|████████▋ | 265/306 [00:00<00:00, 2648.78it/s]
100%|██████████| 306/306 [00:00<00:00, 2642.07it/s]
+2026-03-05:01:48:11 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_professional_accounting on rank 0...
+
  0%|          | 0/282 [00:00<?, ?it/s]
 95%|█████████▍| 267/282 [00:00<00:00, 2663.42it/s]
100%|██████████| 282/282 [00:00<00:00, 2662.57it/s]
+2026-03-05:01:48:11 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_professional_medicine on rank 0...
+
  0%|          | 0/272 [00:00<?, ?it/s]
 98%|█████████▊| 266/272 [00:00<00:00, 2659.76it/s]
100%|██████████| 272/272 [00:00<00:00, 2658.26it/s]
+2026-03-05:01:48:11 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_virology on rank 0...
+
  0%|          | 0/166 [00:00<?, ?it/s]
100%|██████████| 166/166 [00:00<00:00, 2659.39it/s]
+2026-03-05:01:48:11 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_econometrics on rank 0...
+
  0%|          | 0/114 [00:00<?, ?it/s]
100%|██████████| 114/114 [00:00<00:00, 2651.84it/s]
+2026-03-05:01:48:11 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_geography on rank 0...
+
  0%|          | 0/198 [00:00<?, ?it/s]
100%|██████████| 198/198 [00:00<00:00, 2631.49it/s]
+2026-03-05:01:48:11 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_government_and_politics on rank 0...
+
  0%|          | 0/193 [00:00<?, ?it/s]
100%|██████████| 193/193 [00:00<00:00, 2658.01it/s]
+2026-03-05:01:48:11 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_macroeconomics on rank 0...
+
  0%|          | 0/390 [00:00<?, ?it/s]
 68%|██████▊   | 266/390 [00:00<00:00, 2658.97it/s]
100%|██████████| 390/390 [00:00<00:00, 2662.06it/s]
+2026-03-05:01:48:11 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_microeconomics on rank 0...
+
  0%|          | 0/238 [00:00<?, ?it/s]
100%|██████████| 238/238 [00:00<00:00, 2673.17it/s]
+2026-03-05:01:48:12 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_psychology on rank 0...
+
  0%|          | 0/545 [00:00<?, ?it/s]
 49%|████▉     | 266/545 [00:00<00:00, 2659.89it/s]
 98%|█████████▊| 533/545 [00:00<00:00, 2664.62it/s]
100%|██████████| 545/545 [00:00<00:00, 2663.47it/s]
+2026-03-05:01:48:12 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_human_sexuality on rank 0...
+
  0%|          | 0/131 [00:00<?, ?it/s]
100%|██████████| 131/131 [00:00<00:00, 2643.12it/s]
+2026-03-05:01:48:12 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_professional_psychology on rank 0...
+
  0%|          | 0/612 [00:00<?, ?it/s]
 44%|████▎     | 267/612 [00:00<00:00, 2664.93it/s]
 88%|████████▊ | 537/612 [00:00<00:00, 2685.39it/s]
100%|██████████| 612/612 [00:00<00:00, 2681.13it/s]
+2026-03-05:01:48:12 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_public_relations on rank 0...
+
  0%|          | 0/110 [00:00<?, ?it/s]
100%|██████████| 110/110 [00:00<00:00, 2629.52it/s]
+2026-03-05:01:48:12 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_security_studies on rank 0...
+
  0%|          | 0/245 [00:00<?, ?it/s]
100%|██████████| 245/245 [00:00<00:00, 2660.38it/s]
+2026-03-05:01:48:12 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_sociology on rank 0...
+
  0%|          | 0/201 [00:00<?, ?it/s]
100%|██████████| 201/201 [00:00<00:00, 2659.71it/s]
+2026-03-05:01:48:12 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_us_foreign_policy on rank 0...
+
  0%|          | 0/100 [00:00<?, ?it/s]
100%|██████████| 100/100 [00:00<00:00, 2654.17it/s]
+2026-03-05:01:48:12 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_formal_logic on rank 0...
+
  0%|          | 0/126 [00:00<?, ?it/s]
100%|██████████| 126/126 [00:00<00:00, 2607.54it/s]
+2026-03-05:01:48:12 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_european_history on rank 0...
+
  0%|          | 0/165 [00:00<?, ?it/s]
100%|██████████| 165/165 [00:00<00:00, 2635.66it/s]
+2026-03-05:01:48:13 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_us_history on rank 0...
+
  0%|          | 0/204 [00:00<?, ?it/s]
100%|██████████| 204/204 [00:00<00:00, 2639.29it/s]
+2026-03-05:01:48:13 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_high_school_world_history on rank 0...
+
  0%|          | 0/237 [00:00<?, ?it/s]
100%|██████████| 237/237 [00:00<00:00, 2654.20it/s]
+2026-03-05:01:48:13 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_international_law on rank 0...
+
  0%|          | 0/121 [00:00<?, ?it/s]
100%|██████████| 121/121 [00:00<00:00, 2629.51it/s]
+2026-03-05:01:48:13 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_jurisprudence on rank 0...
+
  0%|          | 0/108 [00:00<?, ?it/s]
100%|██████████| 108/108 [00:00<00:00, 2619.73it/s]
+2026-03-05:01:48:13 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_logical_fallacies on rank 0...
+
  0%|          | 0/163 [00:00<?, ?it/s]
100%|██████████| 163/163 [00:00<00:00, 2664.95it/s]
+2026-03-05:01:48:13 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_moral_disputes on rank 0...
+
  0%|          | 0/346 [00:00<?, ?it/s]
 77%|███████▋  | 267/346 [00:00<00:00, 2663.62it/s]
100%|██████████| 346/346 [00:00<00:00, 2667.41it/s]
+2026-03-05:01:48:13 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_moral_scenarios on rank 0...
+
  0%|          | 0/895 [00:00<?, ?it/s]
 30%|██▉       | 267/895 [00:00<00:00, 2667.60it/s]
 60%|█████▉    | 535/895 [00:00<00:00, 2674.03it/s]
 90%|████████▉ | 803/895 [00:00<00:00, 2673.21it/s]
100%|██████████| 895/895 [00:00<00:00, 2672.90it/s]
+2026-03-05:01:48:13 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_philosophy on rank 0...
+
  0%|          | 0/311 [00:00<?, ?it/s]
 86%|████████▌ | 266/311 [00:00<00:00, 2650.97it/s]
100%|██████████| 311/311 [00:00<00:00, 2649.39it/s]
+2026-03-05:01:48:14 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_prehistory on rank 0...
+
  0%|          | 0/324 [00:00<?, ?it/s]
 83%|████████▎ | 269/324 [00:00<00:00, 2682.21it/s]
100%|██████████| 324/324 [00:00<00:00, 2676.35it/s]
+2026-03-05:01:48:14 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_professional_law on rank 0...
+
  0%|          | 0/1534 [00:00<?, ?it/s]
 18%|█▊        | 269/1534 [00:00<00:00, 2683.96it/s]
 35%|███▌      | 539/1534 [00:00<00:00, 2688.33it/s]
 53%|█████▎    | 808/1534 [00:00<00:00, 2685.68it/s]
 70%|███████   | 1077/1534 [00:00<00:00, 2683.27it/s]
 88%|████████▊ | 1346/1534 [00:00<00:00, 2681.83it/s]
100%|██████████| 1534/1534 [00:00<00:00, 2683.84it/s]
+2026-03-05:01:48:14 INFO     [api.task:311] Building contexts for global_mmlu_full_ko_world_religions on rank 0...
+
  0%|          | 0/171 [00:00<?, ?it/s]
100%|██████████| 171/171 [00:00<00:00, 2544.60it/s]
+2026-03-05:01:48:14 INFO     [api.task:311] Building contexts for belebele_kor_Hang on rank 0...
+
  0%|          | 0/900 [00:00<?, ?it/s]
 11%|█         | 100/900 [00:00<00:02, 270.80it/s]
 25%|██▌       | 226/900 [00:00<00:01, 546.15it/s]
 39%|███▉      | 354/900 [00:00<00:00, 753.67it/s]
 54%|█████▍    | 485/900 [00:00<00:00, 913.87it/s]
 69%|██████▊   | 617/900 [00:00<00:00, 1030.62it/s]
 83%|████████▎ | 749/900 [00:00<00:00, 1114.43it/s]
 98%|█████████▊| 881/900 [00:00<00:00, 1174.47it/s]
100%|██████████| 900/900 [00:00<00:00, 912.40it/s] 
+2026-03-05:01:48:15 INFO     [evaluator:584] Running loglikelihood requests
+
Running loglikelihood requests:   0%|          | 0/59768 [00:00<?, ?it/s]
Running loglikelihood requests:   0%|          | 1/59768 [00:01<17:47:28,  1.07s/it]
Running loglikelihood requests:   0%|          | 65/59768 [00:01<13:57, 71.28it/s]  
Running loglikelihood requests:   0%|          | 129/59768 [00:01<07:02, 141.04it/s]
Running loglikelihood requests:   0%|          | 193/59768 [00:01<04:47, 207.33it/s]
Running loglikelihood requests:   0%|          | 257/59768 [00:01<03:42, 267.98it/s]
Running loglikelihood requests:   1%|          | 321/59768 [00:01<03:05, 319.97it/s]
Running loglikelihood requests:   1%|          | 385/59768 [00:01<02:43, 362.69it/s]
Running loglikelihood requests:   1%|          | 449/59768 [00:02<02:28, 399.14it/s]
Running loglikelihood requests:   1%|          | 513/59768 [00:02<02:17, 432.35it/s]
Running loglikelihood requests:   1%|          | 577/59768 [00:02<02:09, 457.33it/s]
Running loglikelihood requests:   1%|          | 641/59768 [00:02<02:03, 477.92it/s]
Running loglikelihood requests:   1%|          | 705/59768 [00:02<02:00, 491.75it/s]
Running loglikelihood requests:   1%|▏         | 769/59768 [00:02<01:58, 497.98it/s]
Running loglikelihood requests:   1%|▏         | 833/59768 [00:02<01:55, 509.62it/s]
Running loglikelihood requests:   2%|▏         | 897/59768 [00:02<01:53, 520.43it/s]
Running loglikelihood requests:   2%|▏         | 961/59768 [00:02<01:51, 528.98it/s]
Running loglikelihood requests:   2%|▏         | 1025/59768 [00:03<01:49, 536.65it/s]
Running loglikelihood requests:   2%|▏         | 1089/59768 [00:03<01:48, 540.03it/s]
Running loglikelihood requests:   2%|▏         | 1153/59768 [00:03<01:47, 544.61it/s]
Running loglikelihood requests:   2%|▏         | 1217/59768 [00:03<01:47, 545.19it/s]
Running loglikelihood requests:   2%|▏         | 1281/59768 [00:03<01:46, 547.38it/s]
Running loglikelihood requests:   2%|▏         | 1345/59768 [00:03<01:46, 547.67it/s]
Running loglikelihood requests:   2%|▏         | 1409/59768 [00:03<01:45, 552.93it/s]
Running loglikelihood requests:   2%|▏         | 1473/59768 [00:03<01:44, 557.23it/s]
Running loglikelihood requests:   3%|▎         | 1537/59768 [00:04<01:43, 560.44it/s]
Running loglikelihood requests:   3%|▎         | 1601/59768 [00:04<01:43, 563.55it/s]
Running loglikelihood requests:   3%|▎         | 1697/59768 [00:04<01:36, 601.76it/s]
Running loglikelihood requests:   3%|▎         | 1761/59768 [00:04<01:38, 589.81it/s]
Running loglikelihood requests:   3%|▎         | 1825/59768 [00:04<01:38, 585.57it/s]
Running loglikelihood requests:   3%|▎         | 1889/59768 [00:04<01:39, 581.46it/s]
Running loglikelihood requests:   3%|▎         | 1953/59768 [00:04<01:39, 580.57it/s]
Running loglikelihood requests:   3%|▎         | 2017/59768 [00:04<01:39, 579.90it/s]
Running loglikelihood requests:   3%|▎         | 2081/59768 [00:04<01:39, 576.95it/s]
Running loglikelihood requests:   4%|▎         | 2177/59768 [00:05<01:32, 620.28it/s]
Running loglikelihood requests:   4%|▍         | 2273/59768 [00:05<01:28, 649.23it/s]
Running loglikelihood requests:   4%|▍         | 2338/59768 [00:05<01:30, 633.97it/s]
Running loglikelihood requests:   4%|▍         | 2401/59768 [00:05<01:33, 615.79it/s]
Running loglikelihood requests:   4%|▍         | 2497/59768 [00:05<01:28, 647.91it/s]
Running loglikelihood requests:   4%|▍         | 2562/59768 [00:05<01:30, 633.16it/s]
Running loglikelihood requests:   4%|▍         | 2657/59768 [00:05<01:26, 657.08it/s]
Running loglikelihood requests:   5%|▍         | 2753/59768 [00:05<01:25, 667.85it/s]
Running loglikelihood requests:   5%|▍         | 2849/59768 [00:06<01:23, 681.08it/s]
Running loglikelihood requests:   5%|▍         | 2945/59768 [00:06<01:16, 745.98it/s]
Running loglikelihood requests:   5%|▌         | 3021/59768 [00:06<01:17, 734.44it/s]
Running loglikelihood requests:   5%|▌         | 3105/59768 [00:06<01:19, 708.38it/s]
Running loglikelihood requests:   5%|▌         | 3201/59768 [00:06<01:18, 718.32it/s]
Running loglikelihood requests:   5%|▌         | 3274/59768 [00:06<01:19, 709.88it/s]
Running loglikelihood requests:   6%|▌         | 3346/59768 [00:06<01:20, 701.16it/s]
Running loglikelihood requests:   6%|▌         | 3439/59768 [00:06<01:13, 763.32it/s]
Running loglikelihood requests:   6%|▌         | 3516/59768 [00:06<01:14, 753.14it/s]
Running loglikelihood requests:   6%|▌         | 3592/59768 [00:07<01:26, 649.36it/s]
Running loglikelihood requests:   6%|▌         | 3660/59768 [00:07<01:27, 641.62it/s]
Running loglikelihood requests:   6%|▌         | 3726/59768 [00:07<01:27, 638.74it/s]
Running loglikelihood requests:   6%|▋         | 3809/59768 [00:07<01:27, 641.65it/s]
Running loglikelihood requests:   6%|▋         | 3874/59768 [00:07<01:28, 633.56it/s]
Running loglikelihood requests:   7%|▋         | 3938/59768 [00:07<01:29, 627.29it/s]
Running loglikelihood requests:   7%|▋         | 4033/59768 [00:07<01:23, 664.99it/s]
Running loglikelihood requests:   7%|▋         | 4100/59768 [00:07<01:24, 658.68it/s]
Running loglikelihood requests:   7%|▋         | 4166/59768 [00:08<03:29, 265.15it/s]
Running loglikelihood requests:   7%|▋         | 4257/59768 [00:08<02:43, 338.77it/s]
Running loglikelihood requests:   7%|▋         | 4385/59768 [00:08<01:57, 472.68it/s]
Running loglikelihood requests:   7%|▋         | 4456/59768 [00:08<01:48, 511.86it/s]
Running loglikelihood requests:   8%|▊         | 4577/59768 [00:09<01:32, 598.18it/s]
Running loglikelihood requests:   8%|▊         | 4673/59768 [00:09<01:25, 642.01it/s]
Running loglikelihood requests:   8%|▊         | 4801/59768 [00:09<01:12, 754.33it/s]
Running loglikelihood requests:   8%|▊         | 4897/59768 [00:09<01:12, 759.55it/s]
Running loglikelihood requests:   8%|▊         | 4993/59768 [00:09<01:11, 763.96it/s]
Running loglikelihood requests:   9%|▊         | 5089/59768 [00:09<01:11, 766.65it/s]
Running loglikelihood requests:   9%|▊         | 5185/59768 [00:09<01:10, 770.94it/s]
Running loglikelihood requests:   9%|▉         | 5281/59768 [00:09<01:10, 772.20it/s]
Running loglikelihood requests:   9%|▉         | 5409/59768 [00:10<01:07, 809.45it/s]
Running loglikelihood requests:   9%|▉         | 5505/59768 [00:10<01:07, 804.83it/s]
Running loglikelihood requests:   9%|▉         | 5633/59768 [00:10<01:04, 836.62it/s]
Running loglikelihood requests:  10%|▉         | 5761/59768 [00:10<00:59, 914.22it/s]
Running loglikelihood requests:  10%|▉         | 5857/59768 [00:10<01:01, 881.05it/s]
Running loglikelihood requests:  10%|▉         | 5953/59768 [00:10<01:02, 856.00it/s]
Running loglikelihood requests:  10%|█         | 6081/59768 [00:10<01:02, 859.71it/s]
Running loglikelihood requests:  10%|█         | 6177/59768 [00:10<01:03, 842.27it/s]
Running loglikelihood requests:  11%|█         | 6305/59768 [00:11<01:02, 859.95it/s]
Running loglikelihood requests:  11%|█         | 6401/59768 [00:11<01:03, 839.65it/s]
Running loglikelihood requests:  11%|█         | 6529/59768 [00:11<01:01, 862.20it/s]
Running loglikelihood requests:  11%|█         | 6625/59768 [00:11<01:02, 846.44it/s]
Running loglikelihood requests:  11%|█▏        | 6753/59768 [00:11<00:57, 928.42it/s]
Running loglikelihood requests:  11%|█▏        | 6849/59768 [00:11<00:59, 892.47it/s]
Running loglikelihood requests:  12%|█▏        | 6977/59768 [00:11<00:54, 965.67it/s]
Running loglikelihood requests:  12%|█▏        | 7075/59768 [00:11<00:57, 916.76it/s]
Running loglikelihood requests:  12%|█▏        | 7201/59768 [00:12<00:53, 983.12it/s]
Running loglikelihood requests:  12%|█▏        | 7301/59768 [00:12<00:55, 941.50it/s]
Running loglikelihood requests:  12%|█▏        | 7425/59768 [00:12<00:52, 997.32it/s]
Running loglikelihood requests:  13%|█▎        | 7526/59768 [00:12<00:54, 957.40it/s]
Running loglikelihood requests:  13%|█▎        | 7649/59768 [00:12<00:55, 938.56it/s]
Running loglikelihood requests:  13%|█▎        | 7745/59768 [00:12<00:57, 907.68it/s]
Running loglikelihood requests:  13%|█▎        | 7873/59768 [00:12<00:52, 990.43it/s]
Running loglikelihood requests:  13%|█▎        | 8001/59768 [00:12<00:52, 978.38it/s]
Running loglikelihood requests:  14%|█▎        | 8100/59768 [00:13<00:54, 944.36it/s]
Running loglikelihood requests:  14%|█▍        | 8225/59768 [00:13<00:50, 1014.96it/s]
Running loglikelihood requests:  14%|█▍        | 8328/59768 [00:13<00:52, 982.17it/s] 
Running loglikelihood requests:  14%|█▍        | 8449/59768 [00:13<00:49, 1035.46it/s]
Running loglikelihood requests:  14%|█▍        | 8554/59768 [00:13<00:51, 998.57it/s] 
Running loglikelihood requests:  15%|█▍        | 8673/59768 [00:13<00:49, 1041.60it/s]
Running loglikelihood requests:  15%|█▍        | 8778/59768 [00:13<00:50, 1003.16it/s]
Running loglikelihood requests:  15%|█▍        | 8897/59768 [00:13<00:48, 1049.57it/s]
Running loglikelihood requests:  15%|█▌        | 9025/59768 [00:13<00:49, 1021.88it/s]
Running loglikelihood requests:  15%|█▌        | 9153/59768 [00:13<00:46, 1086.25it/s]
Running loglikelihood requests:  16%|█▌        | 9281/59768 [00:14<00:48, 1048.91it/s]
Running loglikelihood requests:  16%|█▌        | 9409/59768 [00:14<00:49, 1022.28it/s]
Running loglikelihood requests:  16%|█▌        | 9537/59768 [00:14<00:46, 1082.12it/s]
Running loglikelihood requests:  16%|█▌        | 9647/59768 [00:14<00:47, 1049.42it/s]
Running loglikelihood requests:  16%|█▋        | 9761/59768 [00:14<00:46, 1069.64it/s]
Running loglikelihood requests:  17%|█▋        | 9889/59768 [00:14<00:48, 1037.16it/s]
Running loglikelihood requests:  17%|█▋        | 10017/59768 [00:14<00:45, 1100.28it/s]
Running loglikelihood requests:  17%|█▋        | 10145/59768 [00:14<00:46, 1057.98it/s]
Running loglikelihood requests:  17%|█▋        | 10273/59768 [00:15<00:44, 1116.09it/s]
Running loglikelihood requests:  17%|█▋        | 10401/59768 [00:15<00:42, 1160.34it/s]
Running loglikelihood requests:  18%|█▊        | 10529/59768 [00:15<00:41, 1183.73it/s]
Running loglikelihood requests:  18%|█▊        | 10657/59768 [00:15<00:40, 1210.24it/s]
Running loglikelihood requests:  18%|█▊        | 10785/59768 [00:15<00:40, 1222.55it/s]
Running loglikelihood requests:  18%|█▊        | 10913/59768 [00:15<00:42, 1147.99it/s]
Running loglikelihood requests:  19%|█▊        | 11072/59768 [00:15<00:38, 1270.87it/s]
Running loglikelihood requests:  19%|█▊        | 11202/59768 [00:15<00:39, 1235.21it/s]
Running loglikelihood requests:  19%|█▉        | 11329/59768 [00:15<00:41, 1160.73it/s]
Running loglikelihood requests:  19%|█▉        | 11481/59768 [00:16<00:38, 1258.75it/s]
Running loglikelihood requests:  19%|█▉        | 11617/59768 [00:16<00:41, 1152.39it/s]
Running loglikelihood requests:  20%|█▉        | 11767/59768 [00:16<00:38, 1244.02it/s]
Running loglikelihood requests:  20%|█▉        | 11905/59768 [00:16<00:41, 1147.11it/s]
Running loglikelihood requests:  20%|██        | 12065/59768 [00:16<00:38, 1225.61it/s]
Running loglikelihood requests:  20%|██        | 12193/59768 [00:16<00:41, 1156.83it/s]
Running loglikelihood requests:  21%|██        | 12353/59768 [00:16<00:38, 1234.77it/s]
Running loglikelihood requests:  21%|██        | 12513/59768 [00:16<00:39, 1209.16it/s]
Running loglikelihood requests:  21%|██        | 12673/59768 [00:16<00:36, 1280.82it/s]
Running loglikelihood requests:  21%|██▏       | 12833/59768 [00:17<00:35, 1334.22it/s]
Running loglikelihood requests:  22%|██▏       | 12969/59768 [00:17<00:36, 1265.85it/s]
Running loglikelihood requests:  22%|██▏       | 13121/59768 [00:17<00:35, 1305.33it/s]
Running loglikelihood requests:  22%|██▏       | 13281/59768 [00:17<00:34, 1354.29it/s]
Running loglikelihood requests:  22%|██▏       | 13441/59768 [00:17<00:33, 1381.03it/s]
Running loglikelihood requests:  23%|██▎       | 13601/59768 [00:17<00:32, 1399.59it/s]
Running loglikelihood requests:  23%|██▎       | 13761/59768 [00:17<00:32, 1421.31it/s]
Running loglikelihood requests:  23%|██▎       | 13921/59768 [00:17<00:31, 1441.94it/s]
Running loglikelihood requests:  24%|██▎       | 14081/59768 [00:17<00:31, 1456.01it/s]
Running loglikelihood requests:  24%|██▍       | 14241/59768 [00:18<00:31, 1465.09it/s]
Running loglikelihood requests:  24%|██▍       | 14401/59768 [00:18<00:30, 1475.65it/s]
Running loglikelihood requests:  24%|██▍       | 14549/59768 [00:18<00:45, 986.18it/s] 
Running loglikelihood requests:  25%|██▍       | 14724/59768 [00:18<00:39, 1148.91it/s]
Running loglikelihood requests:  25%|██▍       | 14885/59768 [00:18<00:38, 1168.49it/s]
Running loglikelihood requests:  25%|██▌       | 15077/59768 [00:18<00:34, 1309.41it/s]
Running loglikelihood requests:  26%|██▌       | 15269/59768 [00:18<00:31, 1421.68it/s]
Running loglikelihood requests:  26%|██▌       | 15461/59768 [00:19<00:29, 1504.98it/s]
Running loglikelihood requests:  26%|██▌       | 15653/59768 [00:19<00:29, 1487.40it/s]
Running loglikelihood requests:  27%|██▋       | 15881/59768 [00:19<00:27, 1591.05it/s]
Running loglikelihood requests:  27%|██▋       | 16073/59768 [00:19<00:26, 1636.27it/s]
Running loglikelihood requests:  27%|██▋       | 16265/59768 [00:19<00:26, 1671.51it/s]
Running loglikelihood requests:  28%|██▊       | 16489/59768 [00:19<00:23, 1811.15it/s]
Running loglikelihood requests:  28%|██▊       | 16681/59768 [00:19<00:24, 1793.01it/s]
Running loglikelihood requests:  28%|██▊       | 16873/59768 [00:19<00:24, 1777.88it/s]
Running loglikelihood requests:  29%|██▊       | 17065/59768 [00:19<00:24, 1763.29it/s]
Running loglikelihood requests:  29%|██▉       | 17289/59768 [00:20<00:22, 1885.80it/s]
Running loglikelihood requests:  29%|██▉       | 17481/59768 [00:20<00:22, 1846.74it/s]
Running loglikelihood requests:  30%|██▉       | 17673/59768 [00:20<00:23, 1814.78it/s]
Running loglikelihood requests:  30%|██▉       | 17897/59768 [00:20<00:21, 1919.97it/s]
Running loglikelihood requests:  30%|███       | 18091/59768 [00:20<00:22, 1862.80it/s]
Running loglikelihood requests:  31%|███       | 18313/59768 [00:20<00:21, 1953.55it/s]
Running loglikelihood requests:  31%|███       | 18510/59768 [00:20<00:21, 1909.57it/s]
Running loglikelihood requests:  31%|███▏      | 18729/59768 [00:20<00:20, 1984.56it/s]
Running loglikelihood requests:  32%|███▏      | 18929/59768 [00:20<00:21, 1940.76it/s]
Running loglikelihood requests:  32%|███▏      | 19145/59768 [00:21<00:20, 1999.89it/s]
Running loglikelihood requests:  32%|███▏      | 19346/59768 [00:21<00:20, 1956.71it/s]
Running loglikelihood requests:  33%|███▎      | 19593/59768 [00:21<00:19, 2034.69it/s]
Running loglikelihood requests:  33%|███▎      | 19797/59768 [00:21<00:20, 1998.18it/s]
Running loglikelihood requests:  34%|███▎      | 20041/59768 [00:21<00:19, 2054.28it/s]
Running loglikelihood requests:  34%|███▍      | 20247/59768 [00:21<00:19, 2015.94it/s]
Running loglikelihood requests:  34%|███▍      | 20489/59768 [00:21<00:19, 2064.11it/s]
Running loglikelihood requests:  35%|███▍      | 20696/59768 [00:21<00:19, 2029.58it/s]
Running loglikelihood requests:  35%|███▌      | 20941/59768 [00:21<00:18, 2082.15it/s]
Running loglikelihood requests:  35%|███▌      | 21149/59768 [00:21<00:18, 2034.06it/s]
Running loglikelihood requests:  36%|███▌      | 21379/59768 [00:22<00:18, 2108.88it/s]
Running loglikelihood requests:  36%|███▌      | 21608/59768 [00:22<00:17, 2160.72it/s]
Running loglikelihood requests:  37%|███▋      | 21825/59768 [00:22<00:18, 2034.83it/s]
Running loglikelihood requests:  37%|███▋      | 22061/59768 [00:22<00:18, 2058.17it/s]
Running loglikelihood requests:  37%|███▋      | 22268/59768 [00:22<00:18, 2033.52it/s]
Running loglikelihood requests:  38%|███▊      | 22477/59768 [00:22<00:19, 1955.51it/s]
Running loglikelihood requests:  38%|███▊      | 22733/59768 [00:22<00:17, 2064.24it/s]
Running loglikelihood requests:  38%|███▊      | 22941/59768 [00:22<00:18, 2037.89it/s]
Running loglikelihood requests:  39%|███▉      | 23181/59768 [00:22<00:17, 2084.48it/s]
Running loglikelihood requests:  39%|███▉      | 23390/59768 [00:23<00:17, 2060.48it/s]
Running loglikelihood requests:  39%|███▉      | 23597/59768 [00:23<00:18, 1973.77it/s]
Running loglikelihood requests:  40%|███▉      | 23821/59768 [00:23<00:18, 1955.54it/s]
Running loglikelihood requests:  40%|████      | 24081/59768 [00:23<00:18, 1949.08it/s]
Running loglikelihood requests:  41%|████      | 24309/59768 [00:23<00:18, 1959.03it/s]
Running loglikelihood requests:  41%|████      | 24565/59768 [00:23<00:17, 1972.86it/s]
Running loglikelihood requests:  42%|████▏     | 24821/59768 [00:23<00:16, 2090.66it/s]
Running loglikelihood requests:  42%|████▏     | 25032/59768 [00:23<00:16, 2082.99it/s]
Running loglikelihood requests:  42%|████▏     | 25242/59768 [00:24<00:17, 2014.01it/s]
Running loglikelihood requests:  43%|████▎     | 25444/59768 [00:24<00:17, 2012.27it/s]
Running loglikelihood requests:  43%|████▎     | 25689/59768 [00:24<00:16, 2108.75it/s]
Running loglikelihood requests:  43%|████▎     | 25901/59768 [00:24<00:16, 2097.64it/s]
Running loglikelihood requests:  44%|████▎     | 26112/59768 [00:24<00:16, 2031.35it/s]
Running loglikelihood requests:  44%|████▍     | 26333/59768 [00:24<00:16, 2010.59it/s]
Running loglikelihood requests:  44%|████▍     | 26589/59768 [00:24<00:15, 2136.08it/s]
Running loglikelihood requests:  45%|████▍     | 26804/59768 [00:24<00:15, 2119.64it/s]
Running loglikelihood requests:  45%|████▌     | 27037/59768 [00:24<00:15, 2160.59it/s]
Running loglikelihood requests:  46%|████▌     | 27254/59768 [00:24<00:15, 2157.90it/s]
Running loglikelihood requests:  46%|████▌     | 27471/59768 [00:25<00:15, 2083.85it/s]
Running loglikelihood requests:  46%|████▋     | 27709/59768 [00:25<00:14, 2144.78it/s]
Running loglikelihood requests:  47%|████▋     | 27924/59768 [00:25<00:14, 2144.80it/s]
Running loglikelihood requests:  47%|████▋     | 28139/59768 [00:25<00:15, 2078.06it/s]
Running loglikelihood requests:  47%|████▋     | 28385/59768 [00:25<00:14, 2171.92it/s]
Running loglikelihood requests:  48%|████▊     | 28603/59768 [00:25<00:14, 2173.40it/s]
Running loglikelihood requests:  48%|████▊     | 28821/59768 [00:25<00:14, 2110.24it/s]
Running loglikelihood requests:  49%|████▊     | 29061/59768 [00:25<00:14, 2180.66it/s]
Running loglikelihood requests:  49%|████▉     | 29289/59768 [00:25<00:14, 2124.42it/s]
Running loglikelihood requests:  49%|████▉     | 29549/59768 [00:26<00:13, 2239.19it/s]
Running loglikelihood requests:  50%|████▉     | 29774/59768 [00:26<00:13, 2165.13it/s]
Running loglikelihood requests:  50%|█████     | 30033/59768 [00:26<00:13, 2268.71it/s]
Running loglikelihood requests:  51%|█████     | 30261/59768 [00:26<00:13, 2205.44it/s]
Running loglikelihood requests:  51%|█████     | 30517/59768 [00:26<00:12, 2301.63it/s]
Running loglikelihood requests:  51%|█████▏    | 30749/59768 [00:26<00:12, 2240.87it/s]
Running loglikelihood requests:  52%|█████▏    | 31001/59768 [00:26<00:12, 2318.16it/s]
Running loglikelihood requests:  52%|█████▏    | 31234/59768 [00:26<00:12, 2253.07it/s]
Running loglikelihood requests:  53%|█████▎    | 31485/59768 [00:26<00:12, 2319.07it/s]
Running loglikelihood requests:  53%|█████▎    | 31741/59768 [00:26<00:12, 2230.55it/s]
Running loglikelihood requests:  54%|█████▎    | 31997/59768 [00:27<00:12, 2313.13it/s]
Running loglikelihood requests:  54%|█████▍    | 32230/59768 [00:27<00:12, 2255.69it/s]
Running loglikelihood requests:  54%|█████▍    | 32477/59768 [00:27<00:11, 2309.72it/s]
Running loglikelihood requests:  55%|█████▍    | 32710/59768 [00:27<00:12, 2253.70it/s]
Running loglikelihood requests:  55%|█████▌    | 32965/59768 [00:27<00:11, 2337.83it/s]
Running loglikelihood requests:  56%|█████▌    | 33200/59768 [00:27<00:11, 2273.80it/s]
Running loglikelihood requests:  56%|█████▌    | 33477/59768 [00:27<00:11, 2350.80it/s]
Running loglikelihood requests:  56%|█████▋    | 33713/59768 [00:27<00:11, 2303.39it/s]
Running loglikelihood requests:  57%|█████▋    | 33989/59768 [00:27<00:10, 2371.59it/s]
Running loglikelihood requests:  57%|█████▋    | 34227/59768 [00:28<00:11, 2320.07it/s]
Running loglikelihood requests:  58%|█████▊    | 34505/59768 [00:28<00:10, 2387.05it/s]
Running loglikelihood requests:  58%|█████▊    | 34744/59768 [00:28<00:10, 2338.92it/s]
Running loglikelihood requests:  59%|█████▊    | 35021/59768 [00:28<00:10, 2399.66it/s]
Running loglikelihood requests:  59%|█████▉    | 35261/59768 [00:28<00:10, 2345.90it/s]
Running loglikelihood requests:  59%|█████▉    | 35537/59768 [00:28<00:10, 2402.51it/s]
Running loglikelihood requests:  60%|█████▉    | 35778/59768 [00:28<00:10, 2351.13it/s]
Running loglikelihood requests:  60%|██████    | 36053/59768 [00:28<00:09, 2409.38it/s]
Running loglikelihood requests:  61%|██████    | 36294/59768 [00:28<00:09, 2366.61it/s]
Running loglikelihood requests:  61%|██████    | 36573/59768 [00:29<00:09, 2442.55it/s]
Running loglikelihood requests:  62%|██████▏   | 36818/59768 [00:29<00:09, 2405.64it/s]
Running loglikelihood requests:  62%|██████▏   | 37093/59768 [00:29<00:09, 2452.44it/s]
Running loglikelihood requests:  62%|██████▏   | 37353/59768 [00:29<00:09, 2309.70it/s]
Running loglikelihood requests:  63%|██████▎   | 37645/59768 [00:29<00:09, 2451.32it/s]
Running loglikelihood requests:  63%|██████▎   | 37945/59768 [00:29<00:08, 2573.27it/s]
Running loglikelihood requests:  64%|██████▍   | 38205/59768 [00:29<00:08, 2483.71it/s]
Running loglikelihood requests:  64%|██████▍   | 38489/59768 [00:29<00:08, 2563.17it/s]
Running loglikelihood requests:  65%|██████▍   | 38747/59768 [00:29<00:08, 2472.33it/s]
Running loglikelihood requests:  65%|██████▌   | 39033/59768 [00:30<00:08, 2559.94it/s]
Running loglikelihood requests:  66%|██████▌   | 39291/59768 [00:30<00:08, 2540.17it/s]
Running loglikelihood requests:  66%|██████▌   | 39557/59768 [00:30<00:07, 2556.79it/s]
Running loglikelihood requests:  67%|██████▋   | 39849/59768 [00:30<00:08, 2480.88it/s]
Running loglikelihood requests:  67%|██████▋   | 40141/59768 [00:30<00:07, 2579.46it/s]
Running loglikelihood requests:  68%|██████▊   | 40429/59768 [00:30<00:07, 2641.23it/s]
Running loglikelihood requests:  68%|██████▊   | 40695/59768 [00:30<00:07, 2546.91it/s]
Running loglikelihood requests:  69%|██████▊   | 40977/59768 [00:30<00:07, 2605.59it/s]
Running loglikelihood requests:  69%|██████▉   | 41239/59768 [00:30<00:07, 2514.22it/s]
Running loglikelihood requests:  69%|██████▉   | 41525/59768 [00:30<00:07, 2591.59it/s]
Running loglikelihood requests:  70%|██████▉   | 41786/59768 [00:31<00:07, 2568.59it/s]
Running loglikelihood requests:  70%|███████   | 42045/59768 [00:31<00:06, 2567.99it/s]
Running loglikelihood requests:  71%|███████   | 42341/59768 [00:31<00:06, 2657.81it/s]
Running loglikelihood requests:  71%|███████▏  | 42608/59768 [00:31<00:06, 2555.35it/s]
Running loglikelihood requests:  72%|███████▏  | 42889/59768 [00:31<00:06, 2602.73it/s]
Running loglikelihood requests:  72%|███████▏  | 43151/59768 [00:31<00:06, 2503.72it/s]
Running loglikelihood requests:  73%|███████▎  | 43437/59768 [00:31<00:06, 2593.96it/s]
Running loglikelihood requests:  73%|███████▎  | 43729/59768 [00:31<00:05, 2674.28it/s]
Running loglikelihood requests:  74%|███████▎  | 43998/59768 [00:31<00:06, 2575.38it/s]
Running loglikelihood requests:  74%|███████▍  | 44304/59768 [00:32<00:05, 2712.94it/s]
Running loglikelihood requests:  75%|███████▍  | 44577/59768 [00:32<00:05, 2571.52it/s]
Running loglikelihood requests:  75%|███████▌  | 44881/59768 [00:32<00:05, 2703.13it/s]
Running loglikelihood requests:  76%|███████▌  | 45185/59768 [00:32<00:05, 2799.28it/s]
Running loglikelihood requests:  76%|███████▌  | 45468/59768 [00:32<00:05, 2646.06it/s]
Running loglikelihood requests:  77%|███████▋  | 45757/59768 [00:32<00:05, 2713.99it/s]
Running loglikelihood requests:  77%|███████▋  | 46031/59768 [00:32<00:05, 2635.21it/s]
Running loglikelihood requests:  77%|███████▋  | 46310/59768 [00:32<00:05, 2678.89it/s]
Running loglikelihood requests:  78%|███████▊  | 46593/59768 [00:32<00:04, 2719.27it/s]
Running loglikelihood requests:  78%|███████▊  | 46867/59768 [00:32<00:04, 2640.95it/s]
Running loglikelihood requests:  79%|███████▉  | 47167/59768 [00:33<00:04, 2744.04it/s]
Running loglikelihood requests:  79%|███████▉  | 47443/59768 [00:33<00:04, 2588.54it/s]
Running loglikelihood requests:  80%|███████▉  | 47747/59768 [00:33<00:04, 2715.21it/s]
Running loglikelihood requests:  80%|████████  | 48040/59768 [00:33<00:04, 2776.38it/s]
Running loglikelihood requests:  81%|████████  | 48320/59768 [00:33<00:04, 2615.01it/s]
Running loglikelihood requests:  81%|████████▏ | 48621/59768 [00:33<00:04, 2725.68it/s]
Running loglikelihood requests:  82%|████████▏ | 48917/59768 [00:33<00:03, 2726.55it/s]
Running loglikelihood requests:  82%|████████▏ | 49192/59768 [00:33<00:03, 2650.86it/s]
Running loglikelihood requests:  83%|████████▎ | 49497/59768 [00:33<00:03, 2763.70it/s]
Running loglikelihood requests:  83%|████████▎ | 49805/59768 [00:34<00:03, 2782.02it/s]
Running loglikelihood requests:  84%|████████▍ | 50085/59768 [00:34<00:03, 2700.76it/s]
Running loglikelihood requests:  84%|████████▍ | 50389/59768 [00:34<00:03, 2729.05it/s]
Running loglikelihood requests:  85%|████████▍ | 50691/59768 [00:34<00:03, 2811.31it/s]
Running loglikelihood requests:  85%|████████▌ | 50974/59768 [00:34<00:03, 2661.77it/s]
Running loglikelihood requests:  86%|████████▌ | 51279/59768 [00:34<00:03, 2770.33it/s]
Running loglikelihood requests:  86%|████████▋ | 51559/59768 [00:34<00:03, 2626.47it/s]
Running loglikelihood requests:  87%|████████▋ | 51866/59768 [00:34<00:02, 2749.65it/s]
Running loglikelihood requests:  87%|████████▋ | 52173/59768 [00:34<00:02, 2772.04it/s]
Running loglikelihood requests:  88%|████████▊ | 52453/59768 [00:35<00:02, 2685.94it/s]
Running loglikelihood requests:  88%|████████▊ | 52749/59768 [00:35<00:02, 2697.31it/s]
Running loglikelihood requests:  89%|████████▉ | 53054/59768 [00:35<00:02, 2796.60it/s]
Running loglikelihood requests:  89%|████████▉ | 53336/59768 [00:35<00:02, 2640.28it/s]
Running loglikelihood requests:  90%|████████▉ | 53640/59768 [00:35<00:02, 2751.65it/s]
Running loglikelihood requests:  90%|█████████ | 53918/59768 [00:35<00:02, 2609.55it/s]
Running loglikelihood requests:  91%|█████████ | 54226/59768 [00:35<00:02, 2740.05it/s]
Running loglikelihood requests:  91%|█████████ | 54521/59768 [00:35<00:01, 2738.03it/s]
Running loglikelihood requests:  92%|█████████▏| 54797/59768 [00:35<00:01, 2633.91it/s]
Running loglikelihood requests:  92%|█████████▏| 55101/59768 [00:36<00:01, 2689.30it/s]
Running loglikelihood requests:  93%|█████████▎| 55393/59768 [00:36<00:01, 2584.45it/s]
Running loglikelihood requests:  93%|█████████▎| 55702/59768 [00:36<00:01, 2722.86it/s]
Running loglikelihood requests:  94%|█████████▎| 55977/59768 [00:36<00:01, 2589.36it/s]
Running loglikelihood requests:  94%|█████████▍| 56289/59768 [00:36<00:01, 2735.19it/s]
Running loglikelihood requests:  95%|█████████▍| 56581/59768 [00:36<00:01, 2727.67it/s]
Running loglikelihood requests:  95%|█████████▌| 56856/59768 [00:36<00:01, 2655.19it/s]
Running loglikelihood requests:  96%|█████████▌| 57165/59768 [00:36<00:00, 2710.18it/s]
Running loglikelihood requests:  96%|█████████▌| 57438/59768 [00:36<00:00, 2629.19it/s]
Running loglikelihood requests:  97%|█████████▋| 57743/59768 [00:37<00:00, 2747.08it/s]
Running loglikelihood requests:  97%|█████████▋| 58020/59768 [00:37<00:00, 2607.93it/s]
Running loglikelihood requests:  98%|█████████▊| 58329/59768 [00:37<00:00, 2528.72it/s]
Running loglikelihood requests:  98%|█████████▊| 58640/59768 [00:37<00:00, 2684.70it/s]
Running loglikelihood requests:  99%|█████████▊| 58912/59768 [00:37<00:00, 2532.74it/s]
Running loglikelihood requests:  99%|█████████▉| 59169/59768 [00:37<00:00, 2465.79it/s]
Running loglikelihood requests:  99%|█████████▉| 59421/59768 [00:37<00:00, 2422.87it/s]
Running loglikelihood requests: 100%|█████████▉| 59665/59768 [00:37<00:00, 2297.83it/s]
Running loglikelihood requests: 100%|██████████| 59768/59768 [00:37<00:00, 1576.92it/s]
+2026-03-05:01:49:09 INFO     [loggers.evaluation_tracker:247] Saving results aggregated
+hf ({'pretrained': '/PROJECT/0325120031_A/ghong/taketimes/llm-bang/eval/outputs/hf_3b_base', 'dtype': 'bfloat16'}), gen_kwargs: ({}), limit: None, num_fewshot: None, batch_size: 8
+|                           Tasks                           |Version|Filter|n-shot| Metric |   |Value |   |Stderr|
+|-----------------------------------------------------------|------:|------|-----:|--------|---|-----:|---|-----:|
+|belebele_kor_Hang                                          |      0|none  |     0|acc     |↑  |0.2189|±  |0.0138|
+|                                                           |       |none  |     0|acc_norm|↑  |0.2189|±  |0.0138|
+|global_mmlu_full_ko                                        |      0|none  |      |acc     |↑  |0.2339|±  |0.0036|
+| - global_mmlu_full_ko_humanities                          |      0|none  |      |acc     |↑  |0.2389|±  |0.0062|
+|  - global_mmlu_full_ko_formal_logic                       |      0|none  |     0|acc     |↑  |0.2857|±  |0.0404|
+|  - global_mmlu_full_ko_high_school_european_history       |      0|none  |     0|acc     |↑  |0.2000|±  |0.0312|
+|  - global_mmlu_full_ko_high_school_us_history             |      0|none  |     0|acc     |↑  |0.2500|±  |0.0304|
+|  - global_mmlu_full_ko_high_school_world_history          |      0|none  |     0|acc     |↑  |0.2447|±  |0.0280|
+|  - global_mmlu_full_ko_international_law                  |      0|none  |     0|acc     |↑  |0.1653|±  |0.0339|
+|  - global_mmlu_full_ko_jurisprudence                      |      0|none  |     0|acc     |↑  |0.2130|±  |0.0396|
+|  - global_mmlu_full_ko_logical_fallacies                  |      0|none  |     0|acc     |↑  |0.2147|±  |0.0323|
+|  - global_mmlu_full_ko_moral_disputes                     |      0|none  |     0|acc     |↑  |0.2312|±  |0.0227|
+|  - global_mmlu_full_ko_moral_scenarios                    |      0|none  |     0|acc     |↑  |0.2536|±  |0.0146|
+|  - global_mmlu_full_ko_philosophy                         |      0|none  |     0|acc     |↑  |0.2219|±  |0.0236|
+|  - global_mmlu_full_ko_prehistory                         |      0|none  |     0|acc     |↑  |0.2500|±  |0.0241|
+|  - global_mmlu_full_ko_professional_law                   |      0|none  |     0|acc     |↑  |0.2373|±  |0.0109|
+|  - global_mmlu_full_ko_world_religions                    |      0|none  |     0|acc     |↑  |0.2749|±  |0.0342|
+| - global_mmlu_full_ko_other                               |      0|none  |      |acc     |↑  |0.2327|±  |0.0076|
+|  - global_mmlu_full_ko_business_ethics                    |      0|none  |     0|acc     |↑  |0.3000|±  |0.0461|
+|  - global_mmlu_full_ko_clinical_knowledge                 |      0|none  |     0|acc     |↑  |0.2340|±  |0.0261|
+|  - global_mmlu_full_ko_college_medicine                   |      0|none  |     0|acc     |↑  |0.2543|±  |0.0332|
+|  - global_mmlu_full_ko_global_facts                       |      0|none  |     0|acc     |↑  |0.2200|±  |0.0416|
+|  - global_mmlu_full_ko_human_aging                        |      0|none  |     0|acc     |↑  |0.2511|±  |0.0291|
+|  - global_mmlu_full_ko_management                         |      0|none  |     0|acc     |↑  |0.2718|±  |0.0441|
+|  - global_mmlu_full_ko_marketing                          |      0|none  |     0|acc     |↑  |0.2991|±  |0.0300|
+|  - global_mmlu_full_ko_medical_genetics                   |      0|none  |     0|acc     |↑  |0.2500|±  |0.0435|
+|  - global_mmlu_full_ko_miscellaneous                      |      0|none  |     0|acc     |↑  |0.2031|±  |0.0144|
+|  - global_mmlu_full_ko_nutrition                          |      0|none  |     0|acc     |↑  |0.2222|±  |0.0238|
+|  - global_mmlu_full_ko_professional_accounting            |      0|none  |     0|acc     |↑  |0.2057|±  |0.0241|
+|  - global_mmlu_full_ko_professional_medicine              |      0|none  |     0|acc     |↑  |0.2059|±  |0.0246|
+|  - global_mmlu_full_ko_virology                           |      0|none  |     0|acc     |↑  |0.2711|±  |0.0346|
+| - global_mmlu_full_ko_social_sciences                     |      0|none  |      |acc     |↑  |0.2301|±  |0.0076|
+|  - global_mmlu_full_ko_econometrics                       |      0|none  |     0|acc     |↑  |0.2456|±  |0.0405|
+|  - global_mmlu_full_ko_high_school_geography              |      0|none  |     0|acc     |↑  |0.2828|±  |0.0321|
+|  - global_mmlu_full_ko_high_school_government_and_politics|      0|none  |     0|acc     |↑  |0.1762|±  |0.0275|
+|  - global_mmlu_full_ko_high_school_macroeconomics         |      0|none  |     0|acc     |↑  |0.2128|±  |0.0208|
+|  - global_mmlu_full_ko_high_school_microeconomics         |      0|none  |     0|acc     |↑  |0.2269|±  |0.0272|
+|  - global_mmlu_full_ko_high_school_psychology             |      0|none  |     0|acc     |↑  |0.2440|±  |0.0184|
+|  - global_mmlu_full_ko_human_sexuality                    |      0|none  |     0|acc     |↑  |0.1832|±  |0.0339|
+|  - global_mmlu_full_ko_professional_psychology            |      0|none  |     0|acc     |↑  |0.2337|±  |0.0171|
+|  - global_mmlu_full_ko_public_relations                   |      0|none  |     0|acc     |↑  |0.2636|±  |0.0422|
+|  - global_mmlu_full_ko_security_studies                   |      0|none  |     0|acc     |↑  |0.2000|±  |0.0256|
+|  - global_mmlu_full_ko_sociology                          |      0|none  |     0|acc     |↑  |0.2289|±  |0.0297|
+|  - global_mmlu_full_ko_us_foreign_policy                  |      0|none  |     0|acc     |↑  |0.2900|±  |0.0456|
+| - global_mmlu_full_ko_stem                                |      0|none  |      |acc     |↑  |0.2312|±  |0.0075|
+|  - global_mmlu_full_ko_abstract_algebra                   |      0|none  |     0|acc     |↑  |0.2100|±  |0.0409|
+|  - global_mmlu_full_ko_anatomy                            |      0|none  |     0|acc     |↑  |0.2296|±  |0.0363|
+|  - global_mmlu_full_ko_astronomy                          |      0|none  |     0|acc     |↑  |0.2237|±  |0.0339|
+|  - global_mmlu_full_ko_college_biology                    |      0|none  |     0|acc     |↑  |0.2569|±  |0.0365|
+|  - global_mmlu_full_ko_college_chemistry                  |      0|none  |     0|acc     |↑  |0.2100|±  |0.0409|
+|  - global_mmlu_full_ko_college_computer_science           |      0|none  |     0|acc     |↑  |0.2100|±  |0.0409|
+|  - global_mmlu_full_ko_college_mathematics                |      0|none  |     0|acc     |↑  |0.2700|±  |0.0446|
+|  - global_mmlu_full_ko_college_physics                    |      0|none  |     0|acc     |↑  |0.2451|±  |0.0428|
+|  - global_mmlu_full_ko_computer_security                  |      0|none  |     0|acc     |↑  |0.3100|±  |0.0465|
+|  - global_mmlu_full_ko_conceptual_physics                 |      0|none  |     0|acc     |↑  |0.2511|±  |0.0283|
+|  - global_mmlu_full_ko_electrical_engineering             |      0|none  |     0|acc     |↑  |0.2138|±  |0.0342|
+|  - global_mmlu_full_ko_elementary_mathematics             |      0|none  |     0|acc     |↑  |0.2090|±  |0.0209|
+|  - global_mmlu_full_ko_high_school_biology                |      0|none  |     0|acc     |↑  |0.2258|±  |0.0238|
+|  - global_mmlu_full_ko_high_school_chemistry              |      0|none  |     0|acc     |↑  |0.1823|±  |0.0272|
+|  - global_mmlu_full_ko_high_school_computer_science       |      0|none  |     0|acc     |↑  |0.2300|±  |0.0423|
+|  - global_mmlu_full_ko_high_school_mathematics            |      0|none  |     0|acc     |↑  |0.2593|±  |0.0267|
+|  - global_mmlu_full_ko_high_school_physics                |      0|none  |     0|acc     |↑  |0.2384|±  |0.0348|
+|  - global_mmlu_full_ko_high_school_statistics             |      0|none  |     0|acc     |↑  |0.1898|±  |0.0267|
+|  - global_mmlu_full_ko_machine_learning                   |      0|none  |     0|acc     |↑  |0.3125|±  |0.0440|
+
+|                Groups                |Version|Filter|n-shot|Metric|   |Value |   |Stderr|
+|--------------------------------------|------:|------|------|------|---|-----:|---|-----:|
+|global_mmlu_full_ko                   |      0|none  |      |acc   |↑  |0.2339|±  |0.0036|
+| - global_mmlu_full_ko_humanities     |      0|none  |      |acc   |↑  |0.2389|±  |0.0062|
+| - global_mmlu_full_ko_other          |      0|none  |      |acc   |↑  |0.2327|±  |0.0076|
+| - global_mmlu_full_ko_social_sciences|      0|none  |      |acc   |↑  |0.2301|±  |0.0076|
+| - global_mmlu_full_ko_stem           |      0|none  |      |acc   |↑  |0.2312|±  |0.0075|
+