evaluations: arc_challenge_poly_pt_acc: 0.3034188034188034 arc_challenge_poly_pt_acc_norm: 0.32564102564102565 arc_challenge_poly_pt_acc_norm_stderr: 0.013705909887142202 arc_challenge_poly_pt_acc_stderr: 0.013446209191343464 arc_challenge_poly_pt_alias: arc_challenge_poly_pt assin2_rte_acc,all: 0.49550653594771243 assin2_rte_acc_stderr,all: 0.007161632447060184 assin2_rte_alias: assin2_rte assin2_rte_f1_macro,all: 0.39225947373298437 assin2_rte_f1_macro_stderr,all: 0.006052346943009515 assin2_sts_alias: assin2_sts assin2_sts_mse,all: 2.660600490196079 assin2_sts_mse_stderr,all: N/A assin2_sts_pearson,all: 0.2148458953200812 assin2_sts_pearson_stderr,all: 0.013898569061894066 assin_entailment_acc: 0.5035 assin_entailment_acc_stderr: 0.007906488831422602 assin_entailment_alias: assin_entailment assin_paraphrase_acc: 0.6565 assin_paraphrase_acc_stderr: 0.007509396464351494 assin_paraphrase_alias: assin_paraphrase belebele_por_Latn_acc: 0.2733333333333333 belebele_por_Latn_acc_norm: 0.2733333333333333 belebele_por_Latn_acc_norm_stderr: 0.014863944409417502 belebele_por_Latn_acc_stderr: 0.014863944409417502 belebele_por_Latn_alias: belebele_por_Latn bluex_acc,all: 0.23783031988873435 bluex_acc,exam_id__UNICAMP_2018: 0.24074074074074073 bluex_acc,exam_id__UNICAMP_2019: 0.34 bluex_acc,exam_id__UNICAMP_2020: 0.2727272727272727 bluex_acc,exam_id__UNICAMP_2021_1: 0.10869565217391304 bluex_acc,exam_id__UNICAMP_2021_2: 0.3137254901960784 bluex_acc,exam_id__UNICAMP_2022: 0.20512820512820512 bluex_acc,exam_id__UNICAMP_2023: 0.3953488372093023 bluex_acc,exam_id__UNICAMP_2024: 0.28888888888888886 bluex_acc,exam_id__USP_2018: 0.18518518518518517 bluex_acc,exam_id__USP_2019: 0.2 bluex_acc,exam_id__USP_2020: 0.19642857142857142 bluex_acc,exam_id__USP_2021: 0.28846153846153844 bluex_acc,exam_id__USP_2022: 0.14285714285714285 bluex_acc,exam_id__USP_2023: 0.22727272727272727 bluex_acc,exam_id__USP_2024: 0.14634146341463414 bluex_acc_stderr,all: 0.009180502079837468 bluex_acc_stderr,exam_id__UNICAMP_2018: 0.03357236031259842 bluex_acc_stderr,exam_id__UNICAMP_2019: 0.03871868636296179 bluex_acc_stderr,exam_id__UNICAMP_2020: 0.034624246043716285 bluex_acc_stderr,exam_id__UNICAMP_2021_1: 0.02653632676838462 bluex_acc_stderr,exam_id__UNICAMP_2021_2: 0.03758901266085253 bluex_acc_stderr,exam_id__UNICAMP_2022: 0.03733550664917063 bluex_acc_stderr,exam_id__UNICAMP_2023: 0.04286792084312051 bluex_acc_stderr,exam_id__UNICAMP_2024: 0.03898294466911919 bluex_acc_stderr,exam_id__USP_2018: 0.03047487904733902 bluex_acc_stderr,exam_id__USP_2019: 0.03655034763250411 bluex_acc_stderr,exam_id__USP_2020: 0.030710584372441754 bluex_acc_stderr,exam_id__USP_2021: 0.03641695834309908 bluex_acc_stderr,exam_id__USP_2022: 0.028940786261222796 bluex_acc_stderr,exam_id__USP_2023: 0.036448438128444414 bluex_acc_stderr,exam_id__USP_2024: 0.03183638867903787 bluex_alias: bluex calame_pt_acc: 0.5611753371868978 calame_pt_acc_stderr: 0.010893959376702221 calame_pt_alias: calame_pt calame_pt_perplexity: 11.081119797769698 calame_pt_perplexity_stderr: 0.8507255221732782 enem_challenge_acc,all: 0.23023093072078377 enem_challenge_acc,exam_id__2009: 0.21739130434782608 enem_challenge_acc,exam_id__2010: 0.20512820512820512 enem_challenge_acc,exam_id__2011: 0.21367521367521367 enem_challenge_acc,exam_id__2012: 0.19827586206896552 enem_challenge_acc,exam_id__2013: 0.3055555555555556 enem_challenge_acc,exam_id__2014: 0.1743119266055046 enem_challenge_acc,exam_id__2015: 0.31932773109243695 enem_challenge_acc,exam_id__2016: 0.19008264462809918 enem_challenge_acc,exam_id__2016_2: 0.25203252032520324 enem_challenge_acc,exam_id__2017: 0.21551724137931033 enem_challenge_acc,exam_id__2022: 0.19548872180451127 enem_challenge_acc,exam_id__2023: 0.2740740740740741 enem_challenge_acc_stderr,all: 0.006452584805075603 enem_challenge_acc_stderr,exam_id__2009: 0.022200124485817433 enem_challenge_acc_stderr,exam_id__2010: 0.021505478091911806 enem_challenge_acc_stderr,exam_id__2011: 0.02191035348313933 enem_challenge_acc_stderr,exam_id__2012: 0.0213731581186071 enem_challenge_acc_stderr,exam_id__2013: 0.02558129315646034 enem_challenge_acc_stderr,exam_id__2014: 0.020945564199407546 enem_challenge_acc_stderr,exam_id__2015: 0.024705706231707848 enem_challenge_acc_stderr,exam_id__2016: 0.0205641096196576 enem_challenge_acc_stderr,exam_id__2016_2: 0.022527285910525656 enem_challenge_acc_stderr,exam_id__2017: 0.02202285771241114 enem_challenge_acc_stderr,exam_id__2022: 0.01978570355082206 enem_challenge_acc_stderr,exam_id__2023: 0.022151269839012434 enem_challenge_alias: enem faquad_nli_acc,all: 0.7846153846153846 faquad_nli_acc_stderr,all: 0.011396120309131366 faquad_nli_alias: faquad_nli faquad_nli_f1_macro,all: 0.4396551724137931 faquad_nli_f1_macro_stderr,all: 0.00357969847290883 global_piqa_completions_por_latn_braz_acc: 0.73 global_piqa_completions_por_latn_braz_acc_bytes: 0.73 global_piqa_completions_por_latn_braz_acc_bytes_stderr: 0.04461960433384737 global_piqa_completions_por_latn_braz_acc_norm: 0.72 global_piqa_completions_por_latn_braz_acc_norm_stderr: 0.045126085985421296 global_piqa_completions_por_latn_braz_acc_stderr: 0.04461960433384737 global_piqa_completions_por_latn_braz_alias: global_piqa_completions_por_latn_braz hatebr_offensive_acc,all: 0.535 hatebr_offensive_acc_stderr,all: 0.00946933415805255 hatebr_offensive_alias: hatebr_offensive_binary hatebr_offensive_f1_macro,all: 0.4986442932419297 hatebr_offensive_f1_macro_stderr,all: 0.009472946693655268 hellaswag_poly_pt_acc: 0.35193412070646873 hellaswag_poly_pt_acc_norm: 0.43926752627586957 hellaswag_poly_pt_acc_norm_stderr: 0.005166406931353431 hellaswag_poly_pt_acc_stderr: 0.004971488882718327 hellaswag_poly_pt_alias: hellaswag_poly_pt lambada_poly_pt_acc: 0.3388317484960217 lambada_poly_pt_acc_stderr: 0.006594169660148477 lambada_poly_pt_alias: lambada_poly_pt lambada_poly_pt_perplexity: 33.6673393665592 lambada_poly_pt_perplexity_stderr: 1.3109466933901732 mmlu_poly_pt_acc: 0.2695887120984689 mmlu_poly_pt_acc_stderr: 0.0038444443276646535 mmlu_poly_pt_alias: mmlu_poly_pt oab_exams_acc,all: 0.2592255125284738 oab_exams_acc,exam_id__2010-01: 0.2823529411764706 oab_exams_acc,exam_id__2010-02: 0.21 oab_exams_acc,exam_id__2011-03: 0.25252525252525254 oab_exams_acc,exam_id__2011-04: 0.275 oab_exams_acc,exam_id__2011-05: 0.2375 oab_exams_acc,exam_id__2012-06: 0.2625 oab_exams_acc,exam_id__2012-06a: 0.2375 oab_exams_acc,exam_id__2012-07: 0.3625 oab_exams_acc,exam_id__2012-08: 0.2625 oab_exams_acc,exam_id__2012-09: 0.2597402597402597 oab_exams_acc,exam_id__2013-10: 0.1625 oab_exams_acc,exam_id__2013-11: 0.3 oab_exams_acc,exam_id__2013-12: 0.25 oab_exams_acc,exam_id__2014-13: 0.2625 oab_exams_acc,exam_id__2014-14: 0.375 oab_exams_acc,exam_id__2014-15: 0.2948717948717949 oab_exams_acc,exam_id__2015-16: 0.2125 oab_exams_acc,exam_id__2015-17: 0.2564102564102564 oab_exams_acc,exam_id__2015-18: 0.2125 oab_exams_acc,exam_id__2016-19: 0.2692307692307692 oab_exams_acc,exam_id__2016-20: 0.2 oab_exams_acc,exam_id__2016-20a: 0.2 oab_exams_acc,exam_id__2016-21: 0.3 oab_exams_acc,exam_id__2017-22: 0.2375 oab_exams_acc,exam_id__2017-23: 0.25 oab_exams_acc,exam_id__2017-24: 0.2625 oab_exams_acc,exam_id__2018-25: 0.325 oab_exams_acc_stderr,all: 0.005416528564580985 oab_exams_acc_stderr,exam_id__2010-01: 0.028174121226509266 oab_exams_acc_stderr,exam_id__2010-02: 0.023512908105304108 oab_exams_acc_stderr,exam_id__2011-03: 0.025205143471018352 oab_exams_acc_stderr,exam_id__2011-04: 0.02876970081837964 oab_exams_acc_stderr,exam_id__2011-05: 0.027412150990991712 oab_exams_acc_stderr,exam_id__2012-06: 0.028332826332428174 oab_exams_acc_stderr,exam_id__2012-06a: 0.027372775452882293 oab_exams_acc_stderr,exam_id__2012-07: 0.031197782924965372 oab_exams_acc_stderr,exam_id__2012-08: 0.028266423544104552 oab_exams_acc_stderr,exam_id__2012-09: 0.028914985129938155 oab_exams_acc_stderr,exam_id__2013-10: 0.023844917945490186 oab_exams_acc_stderr,exam_id__2013-11: 0.02958436668685766 oab_exams_acc_stderr,exam_id__2013-12: 0.028007197131130823 oab_exams_acc_stderr,exam_id__2014-13: 0.028352135845047022 oab_exams_acc_stderr,exam_id__2014-14: 0.03119210115334702 oab_exams_acc_stderr,exam_id__2014-15: 0.029792777263781055 oab_exams_acc_stderr,exam_id__2015-16: 0.02648742610607097 oab_exams_acc_stderr,exam_id__2015-17: 0.028572134705007562 oab_exams_acc_stderr,exam_id__2015-18: 0.026310452589845126 oab_exams_acc_stderr,exam_id__2016-19: 0.02891678275950995 oab_exams_acc_stderr,exam_id__2016-20: 0.025714242907505878 oab_exams_acc_stderr,exam_id__2016-20a: 0.025887233921239393 oab_exams_acc_stderr,exam_id__2016-21: 0.029513857628589144 oab_exams_acc_stderr,exam_id__2017-22: 0.02752892017641432 oab_exams_acc_stderr,exam_id__2017-23: 0.027899920647540464 oab_exams_acc_stderr,exam_id__2017-24: 0.028391598033779684 oab_exams_acc_stderr,exam_id__2018-25: 0.03026049626006088 oab_exams_alias: oab_exams portuguese_hate_speech_acc,all: 0.6392479435957696 portuguese_hate_speech_acc_stderr,all: 0.011617512312881497 portuguese_hate_speech_alias: portuguese_hate_speech_binary portuguese_hate_speech_f1_macro,all: 0.5458019314852965 portuguese_hate_speech_f1_macro_stderr,all: 0.012551468345788727 tweetsentbr_acc,all: 0.3034825870646766 tweetsentbr_acc_stderr,all: 0.007247347058112842 tweetsentbr_alias: tweetsentbr tweetsentbr_f1_macro,all: 0.26402315517625063 tweetsentbr_f1_macro_stderr,all: 0.00564226151141287 step: 22000