data/results/MT - en-xx - 0-shot.csv

model,prompt,fra_Latn,aeb_Arab,afr_Latn,aka_Latn,amh_Ethi,ary_Arab,arz_Arab,bam_Latn,bem_Latn,cjk_Latn,dik_Latn,dyu_Latn,ewe_Latn,fon_Latn,fuv_Latn,gaz_Latn,hau_Latn,ibo_Latn,kab_Latn,kam_Latn,kbp_Latn,kea_Latn,kik_Latn,kin_Latn,kmb_Latn,knc_Arab,knc_Latn,kon_Latn,lin_Latn,lua_Latn,lug_Latn,luo_Latn,mos_Latn,nso_Latn,nus_Latn,nya_Latn,plt_Latn,run_Latn,sag_Latn,sna_Latn,som_Latn,sot_Latn,ssw_Latn,swh_Latn,taq_Latn,taq_Tfng,tir_Ethi,tsn_Latn,tso_Latn,tum_Latn,twi_Latn,tzm_Tfng,umb_Latn,wol_Latn,xho_Latn,yor_Latn,zul_Latn,afr_Latn,amh_Ethi,bem_Latn,ewe_Latn,hau_Latn,ibo_Latn,kin_Latn,mlg_Latn,nde_Latn,nso_Latn,nya_Latn,orm_Ethi,sna_Latn,som_Latn,ssw_Latn,swa_Latn,tir_Ethi,tsn_Latn,ven_Latn,wol_Latn,xho_Latn,yor_Latn,zul_Latn,eng-ach,eng-lgg,eng-lug,eng-nyn,eng-swa,eng-teo,en_amh,en_hau,en_ibo,en_kin,en_lug,en_luo,en_nya,en_pcm,en_sna,en_swa,en_tsn,en_twi,en_xho,en_yor,en_zul,fr_bam,fr_bbj,fr_ewe,fr_fon,fr_mos,fr_wol,avg,,,eng-ibo
gemma-1.1-7b-it,prompt_1,26.15,7.06,34.71,8.37,1.78,7.82,5.38,5.69,7.96,4.4,3.3,4.44,5.87,2.6,3.65,4.03,21.72,16.46,2.43,9.32,2.78,8.12,7.3,8.18,5.01,2.52,6.53,10.56,10.52,5.17,6.88,2.96,4.7,15.29,1.92,7.38,10.09,11.03,4.86,11.76,18.42,13.32,5.32,15.23,3.06,0.1,0.93,15.28,8.04,5.66,5.37,0.1,9.36,12.12,9.15,7.49,11.23,28.6,2.1,6.9,6.5,25.3,20.8,7.0,24.1,8.2,16.0,16.1,4.6,5.1,17.5,5.3,15.9,1.2,12.6,2.0,12.2,8.6,7.22,10.71,3.04,2.34,0.96,10.46,11.79,3.18,8.3,30.7,25.7,25.6,23.4,18.9,27.1,43.2,27.2,47.7,26.8,20.7,26.8,17.6,27.2,11.9,4.5,14.4,7.5,8.2,13.45,11.3,,,12.86
gemma-1.1-7b-it,prompt_2,1.39,9.62,17.62,11.18,2.15,4.58,1.64,9.02,3.74,3.61,6.32,5.82,6.21,4.4,11.21,7.74,23.29,19.3,0.96,10.76,6.31,11.13,5.94,4.67,6.87,0.62,4.96,13.74,13.13,1.85,3.91,7.45,6.48,14.97,4.15,3.62,25.11,14.95,8.74,13.32,15.95,8.17,6.17,4.56,4.66,0.16,1.32,8.09,9.27,4.92,11.84,0.07,9.25,12.1,12.23,10.04,17.71,15.9,2.5,5.0,7.2,24.8,24.0,4.3,15.6,11.3,14.7,12.4,9.1,7.7,16.8,6.0,4.3,1.5,5.5,11.4,12.7,10.6,9.36,16.79,3.6,0.35,0.23,10.3,2.24,3.68,10.8,32.3,24.3,31.5,26.5,18.0,32.8,41.1,29.9,49.7,32.5,25.3,31.8,19.7,31.6,10.8,4.9,16.1,8.7,9.1,13.51,11.7,,,13.85
gemma-1.1-7b-it,prompt_3,1.46,5.58,20.75,7.79,2.14,2.35,1.17,9.52,2.36,1.1,6.26,4.86,7.29,6.13,11.09,7.74,20.55,19.28,1.85,7.28,6.44,11.27,3.77,5,4.05,0.66,4.76,7.25,6.99,1.38,2.59,6.68,6.52,8.96,5,2.76,24.46,12.5,8.34,7.68,13.57,4.24,8.02,5.05,6.48,0.25,1.37,10.04,6.56,3.5,7.53,0.13,8.92,8.47,12.73,9.76,16.49,18.8,2.5,2.3,8.8,21.7,24.0,4.7,10.4,10.6,8.0,7.7,9.0,6.6,14.3,8.5,3.4,1.5,5.8,10.5,9.3,10.8,9.1,16.25,4.83,0.41,0.31,10.57,2.12,2.85,7.3,28.7,23.3,27.6,24.5,18.8,28.3,39.2,28.6,45.3,27.7,21.7,27.8,18.1,26.7,10.6,4.3,14.7,6.7,7.5,12.79,10.4,,,14.05
Llama-2-7b-chat-hf,prompt_1,51.97,15.34,35.5,8.21,3.63,15.84,17.44,9.31,6.56,6.8,3.92,9.01,11.66,2.53,8.16,6.47,6.47,8.65,3.31,9.79,6.01,22.01,8.44,10.1,6.16,6.4,3.43,10.35,10.82,9.03,11.08,6.74,7.22,10.2,3.75,10.33,10.07,8.82,11.2,10.14,10.66,11.36,10.65,13.77,4.03,4.64,2.77,9.39,7.72,8.23,12.9,6.42,6.04,8.37,12.18,6.79,11.25,37.9,4.0,9.0,14.9,8.1,9.8,10.9,12.1,7.4,12.0,12.8,7.9,9.1,12.8,10.9,16.3,3.0,11.1,7.3,10.9,13.1,6.66,11.99,4,3.83,9.16,7.81,14.14,3.63,2.7,13.4,10.8,15.8,15.4,16.3,19.3,28.6,16.8,15.8,16.3,14.9,15.1,7.5,16.3,11.9,5.7,14.8,7.9,11.3,13.28,10.5,,,6.13
Llama-2-7b-chat-hf,prompt_2,21.19,22.1,3.5,10.87,4.66,21.72,22.51,3.56,9.06,1.16,0.27,4.16,6.14,4.14,8.65,1.67,8.44,8.09,0.09,8.79,5.67,7.74,8.01,3.2,0.32,1.06,2.54,7.53,6.73,1.25,0.34,2.83,2.12,4.27,0.05,12.27,6.09,1.47,6.58,0.66,6.91,5.83,0.83,6.14,0.14,8.2,3.32,1.03,2.16,4.71,11.15,8.88,1.08,4.83,2.68,7.14,1.25,2.9,4.5,12.0,7.9,9.3,9.7,3.6,5.6,1.5,6.5,15.1,2.3,0.3,8.2,0.9,5.7,3.3,2.7,1.0,6.0,3.5,7.22,2.27,2.72,0.52,0.02,0.37,1.38,0.37,4.7,16.3,14.4,18.8,17.3,18.5,20.9,36.7,19.2,20.6,19.0,17.4,19.1,10.9,19.7,13.5,6.0,15.6,8.2,11.7,14.37,7.3,,,5.32
Llama-2-7b-chat-hf,prompt_3,30.79,22.77,1.65,9.99,4.68,22.32,22.91,3.36,4.1,3.62,1.53,5.04,2.71,4.59,1.95,7.48,1.11,6.57,0.44,4.37,5.58,7.65,5.42,8.89,0.45,1.16,1.66,3.1,3.49,1.67,0.7,1.68,2.15,0.81,0.35,6.08,0.49,1.19,6.73,1.05,3.15,1.98,0.58,3,0.1,9.01,3.49,0.28,0.46,5.86,10.08,8.62,1.35,5.29,0.65,6.14,0.46,1.4,4.5,7.1,5.2,2.5,8.3,10.1,1.0,0.4,2.3,15.5,8.4,0.4,5.3,0.7,4.0,3.4,0.2,0.3,7.2,1.2,6.93,0.66,3.46,0.17,0.49,0.59,2.77,0.21,1.7,12.8,11.0,15.3,15.2,14.6,15.3,24.7,15.7,15.5,16.3,15.1,15.8,7.9,16.1,12.4,6.3,15.8,8.1,11.1,13.13,6.0,,,4.89
Meta-Llama-3-8B-Instruct,prompt_1,63.91,18.46,58.51,7.76,3.6,18.7,24.53,5.15,11.83,13.02,10.05,5.22,12.97,9.42,6.31,5.09,26.41,12.72,7.14,13.32,5.03,23.09,14.2,12.68,9.87,1.95,2.8,10.76,11.39,13.13,10.62,13.69,11.65,13.85,7.91,11.55,16.63,11.68,12.79,12.04,16.34,12.62,16.14,36.79,4,0.85,3.05,11.94,11.88,11.52,9.37,0.39,10.67,8.94,11.9,6.94,12.29,58.2,3.1,15.7,16.2,27.8,14.9,13.7,19.5,12.4,13.7,13.8,3.6,15.8,16.5,17.9,40.7,2.7,12.8,15.6,9.1,13.3,8.26,15.59,3.51,0.92,10.78,8.28,42.54,4.92,9.3,27.4,19.5,25.9,21.5,16.0,20.4,45.1,22.1,39.1,27.0,22.4,20.2,13.8,22.9,11.4,6.2,14.6,7.3,9.0,13.31,14.6,,,11.62
Meta-Llama-3-8B-Instruct,prompt_2,66.14,25.99,61.56,11.02,4.21,22.07,29.6,4.53,10.9,6.39,3.65,2.72,7.02,3.51,5.61,12.7,28.81,14.85,8.78,12.86,4.65,24.72,13.3,14.34,8.68,1.34,3.38,10.85,11.94,5.88,11.98,6.49,2.72,12.17,2.79,13.03,22.06,11.96,5.14,13.24,17.64,14.09,5.47,42.34,5.45,1.66,3.46,13.58,10.58,10.81,12.07,0.39,7.87,8.23,13.26,9.81,14.44,60.9,3.7,12.8,6.0,29.7,15.7,15.2,22.5,10.3,12.0,14.8,12.5,13.7,18.4,3.7,44.0,3.3,13.7,12.0,9.3,13.5,10.34,14.35,5.16,4.24,10.59,8.55,43.67,4.74,13.4,33.5,35.4,36.1,32.7,21.4,29.0,52.2,28.8,50.1,36.0,31.1,31.7,19.2,33.7,15.5,7.0,23.8,9.5,12.1,18.01,15.9,,,12.73
Meta-Llama-3-8B-Instruct,prompt_3,66.09,27.9,61.65,10.35,4.14,25.82,31.68,4.86,10.68,6.43,3.24,2.46,3.69,4.64,4.97,12.47,28.75,13.73,8.51,11.56,3.9,26.67,11.47,14.12,8.72,0.51,2.88,10.96,11.86,7.19,11.5,5.58,2.65,12.56,2.62,12.89,21.54,11.43,2.62,13.32,17.58,13.72,11.79,42.53,3.24,0.67,3.38,13.34,10.27,10.49,11.96,0.31,7.51,8.01,12.88,9.74,13.88,61.0,3.8,12.1,4.8,30.1,15.7,15.3,23.0,10.5,12.2,15.1,12.3,12.6,18.3,13.1,44.3,3.2,14.2,11.7,9.4,12.5,10.02,14.34,5.71,3.11,10.28,8.46,42.65,3.8,8.5,27.2,23.5,26.7,23.0,17.4,23.1,42.8,24.4,33.6,26.4,24.5,22.9,16.0,24.3,12.4,4.6,16.7,7.1,7.5,13.44,14.6,,,11.35
LLaMAX3-8B-Alpaca,prompt_1,64.64,35.77,64.19,15.89,19.12,32.05,38,7.11,16.36,16.78,4.7,8.45,16.46,11.21,10.6,14.65,43.3,33.24,8.42,22.14,12.02,25.2,19.73,17.83,10.7,4.23,11.22,14.4,16.64,11.7,17.73,9.65,9.94,17.57,5.79,38,26.95,14.43,13.24,37.71,37.7,16.66,18.87,52.49,5.69,0.48,5.71,16.28,18.56,24.47,8.22,0.38,16.77,14.78,36.96,19.64,37.85,63.4,13.6,17.7,19.4,45.0,37.1,20.6,28.7,28.6,16.6,40.8,15.0,36.2,40.6,11.1,55.3,5.4,17.3,17.7,16.8,36.4,20,37.24,8.78,4.01,20.44,14.86,56.41,11.1,9.5,29.3,27.3,25.4,24.9,19.1,32.9,27.9,29.3,37.0,24.6,19.9,29.7,15.3,30.3,13.8,9.7,19.0,10.2,13.3,16.43,21.8,,,32.64
LLaMAX3-8B-Alpaca,prompt_2,65.59,37.87,65.28,8.37,21.25,34.98,39.11,8.11,16.87,16.19,5.41,5.91,6.98,7.8,9.99,16.79,44.41,35.26,9.56,23.71,12.56,24.11,20.05,19.86,13.25,4.97,14.13,15.93,20.64,12.69,19.73,18.72,6.56,18.95,5.91,40.61,30.05,13.18,13.36,38.89,39.64,17.87,26.97,53.69,5.78,0.36,7.92,18.21,20.99,25.56,8.86,0.39,16.93,16.41,38.36,20.28,39.06,64.3,14.9,18.4,8.9,46.8,39.3,22.8,30.5,31.4,17.2,42.4,17.2,39.3,43.0,27.9,56.2,7.7,18.7,15.3,18.3,37.9,20.67,39.25,6.02,3.28,20.62,15.2,56.2,9.24,11.6,31.8,30.1,27.8,26.2,19.0,34.9,26.6,31.8,42.1,26.1,19.6,30.9,16.0,31.7,15.0,6.3,17.0,8.5,12.5,17.51,22.7,,,32.66
LLaMAX3-8B-Alpaca,prompt_3,65.45,37.18,65.27,6.82,20.86,32.92,38.73,7.2,17.01,15.36,5.7,5.81,6.67,10.55,9.81,16.55,44.53,35.08,9.62,23.57,12.38,27.62,19.09,19.84,13.44,3.33,12.91,15.49,20.27,12.84,19.93,11.97,5.33,19.2,4.32,40.29,29.24,14.83,11.8,39.06,39.44,17.81,28.77,53.6,4.32,0.33,8.05,17.6,20.28,25.16,8.92,0.38,16.26,16.42,38.31,20.16,39,64.2,14.7,18.7,7.9,47.2,39.4,23.0,30.6,31.8,17.2,42.2,17.0,39.0,43.0,29.4,56.4,7.7,18.6,14.9,18.3,38.2,20.6,39.29,5.19,3.11,20.81,14.99,56.79,8.6,1.5,17.8,15.1,18.3,18.6,16.7,20.6,23.8,19.1,21.5,18.8,15.7,19.1,9.3,19.9,12.3,7.5,14.9,8.9,12.4,13.97,21.1,,,32.83
Llama-3.1-8B-Instruct,prompt_1,66.36,33.7,59.77,13.33,6.89,31.78,36.63,8.3,15.1,12.1,5.05,6.7,11.15,7.38,10.7,3.35,24.02,15.6,9.55,18.28,8.02,23.91,16.72,15.03,14.44,5.43,4.66,14.96,16.07,13.35,11.88,10.9,9.12,12.56,5.97,15.8,16.85,15.097,9.8,15.48,16.91,11.71,13.058,42.88,7.07,1.91,4.29,12.66,12.33,15.96,13.69,0.58,11.55,10.08,13.77,7.15,13.04,60.4,6.2,16.6,15.6,26.3,18.5,17.3,19.9,12.6,14.2,19.8,2.9,16.7,18.6,14.9,43.1,4.1,14.2,14.7,12.4,14.7,8.24,15,4.88,1.3,10.1,7.79,44.96,4.79,3.8,22.4,22.7,24.5,23.5,20.7,24.7,43.9,23.4,39.2,24.0,22.6,21.3,14.9,24.7,13.8,8.2,16.8,9.8,12.9,16.45,16.1,,,15.04
Llama-3.1-8B-Instruct,prompt_2,68.05,33.26,62.93,14.35,8,31.29,36.71,7.26,15.02,9.41,2.87,5.39,7.65,4.43,10.03,11.31,28.59,18.01,9.878,22.49,5.45,21.01,17.56,16.16,11.73,6.98,5.19,12.92,15.64,9.22,12.59,16.56,5.85,13.23,3.62,16.1,21.37,14.53,6,15.27,19.78,12.63,14.47,46.28,7.45,2.54,5.09,13.65,9.8,15.01,14.88,0.45,8.48,8.97,15.1,8.15,15.02,62.6,7.1,17.4,8.8,30.9,21.2,18.1,22.7,13.9,13.6,20.5,11.3,16.2,21.1,16.7,48.5,4.9,15.0,10.9,10.2,15.4,9.36,15.87,4.04,3.14,10.1,7.49,45.1,4.77,2.8,23.1,21.0,23.9,22.0,17.2,21.4,42.3,22.3,34.2,24.0,22.9,22.2,14.3,24.9,14.3,4.6,18.2,7.8,12.8,16.11,16.3,,,14.53
Llama-3.1-8B-Instruct,prompt_3,68.06,31.66,62.94,13.72,7.99,29.06,36.21,6.47,13.46,8.14,1.84,4.77,6.76,3.56,9.78,11.38,28.37,16.89,9.45,21.54,5.41,23.17,16.61,15.33,10.87,3.71,3.17,12.56,13.19,9.16,11.84,15.52,5.76,12.8,2.96,15.05,21.22,14.57,6.23,13.89,19.5,11.97,13.32,46.25,6.72,0.75,5.01,13.16,9.13,14.09,14.44,0.35,8.3,8.64,14.24,8.08,14.64,62.7,7.1,15.6,7.7,30.0,20.0,17.7,22.7,13.3,13.5,20.0,11.4,15.4,20.7,14.6,48.3,4.8,14.4,10.4,9.8,14.6,9.66,15.17,4.4,3.15,10.34,7.43,44.13,4.98,2.8,24.8,19.3,24.5,24.3,20.8,25.0,40.1,25.7,31.8,25.2,22.8,21.7,13.5,23.6,14.5,7.0,18.9,9.7,13.3,16.85,15.9,,,14.05
AfroLlama-V1,prompt_1,9.69,0.14,5.45,4.31,0.15,0.15,0.12,4.48,7.4,5.38,5.26,4.92,3.73,2.89,6.75,4.04,11.06,3.9,4.93,5.7,2.76,5.05,6.73,7.05,6.54,0.3,6.22,6.52,6.72,5.84,7.76,6.27,6.01,6.72,3.54,8.18,5.52,6.29,5.26,6.12,6.6,7.55,6.72,13.4,5.21,0.5,0.14,6.13,6.98,7.17,6.07,0.3,5.72,4.67,12.11,6.06,13.24,5.3,0.4,6.9,3.9,10.3,4.0,6.8,5.2,8.5,6.3,6.6,3.6,6.2,6.0,6.7,12.8,0.2,6.7,5.4,4.6,10.8,5.63,11.04,1.41,1.68,1.76,1.6,5.11,1.69,0.3,14.5,4.9,13.2,9.1,11.1,15.3,10.8,12.9,15.9,10.5,9.6,17.4,9.5,17.3,8.0,2.9,7.7,5.4,6.5,8.67,6.3,,,1.19
AfroLlama-V1,prompt_2,15.66,0.19,6.52,6.05,0.2,0.17,0.16,4.72,10.03,8.13,8.81,5.14,3.8,3.84,8.91,7.45,13.99,4.23,7.07,6.1,4.37,6.39,11.89,13.65,14.27,0.39,9.06,11.55,8.92,16.51,12.43,10.2,5.41,8.58,5.24,9.41,11.58,10.62,4.94,9.66,9.07,9.99,10.08,15.48,6.29,0.51,0.16,7.81,7.68,8.53,8.05,0.72,10.87,5.79,13.81,6.44,17.04,6.2,0.5,8.8,4.0,12.4,4.5,10.9,6.4,13.0,8.1,9.3,5.9,11.9,7.8,9.2,14.5,0.2,7.9,7.6,5.4,12.1,6.23,13.34,1.63,1.82,2.09,3.84,5.38,2.36,0.3,12.5,4.4,11.5,7.5,9.7,13.6,10.3,10.9,13.2,8.8,9.2,14.8,8.2,15.0,6.8,2.7,6.4,4.7,5.6,7.08,7.7,,,1.18
AfroLlama-V1,prompt_3,21.1,0.17,8.96,5.42,0.24,0.12,0.14,5.33,10.75,6.63,8.92,5.26,3.8,3.4,9.05,8.56,13.87,4.2,9.04,6.01,4.56,7.45,13.83,17.3,12.24,0.49,15.46,10.44,10.69,15.27,17.07,13.03,4.75,8.69,6.04,10.25,14.41,14.4,6.54,11.97,9.19,9.64,14.46,17.91,6.6,0.58,0.23,7.57,6.82,11.85,10.91,0.55,10,4.78,13.63,7.39,17.07,7.6,0.6,9.3,4.0,12.6,4.3,14.6,8.5,11.1,8.0,12.8,7.0,18.6,7.5,12.4,16.1,0.3,8.9,7.3,4.8,12.3,6.5,14.25,1.9,1.89,2.2,3.42,5.5,2.39,0.3,12.5,5.4,12.6,8.6,11.1,13.7,12.7,12.7,12.1,10.4,9.8,14.2,6.9,13.0,7.7,2.9,8.0,5.0,6.2,8.43,8.4,,,1.14
gemma-2-9b-it,prompt_1,64.48,51.95,70.13,29.04,42.63,49.13,53.92,20.05,29.23,22.11,19.53,19.62,20.00,18.32,20.69,23.72,45.34,36.09,21.31,23.62,21.88,51.24,25.72,38.24,22.42,14.16,20.51,25.96,27.07,24.18,29.66,20.61,18.57,34.86,17.62,34.62,38.52,30.91,20.80,37.05,38.68,37.17,32.36,59.15,21.25,16.55,26.93,33.95,28.51,29.03,30.88,18.02,21.35,22.86,42.77,30.22,42.39,58.4,9.4,16.1,12.7,38.8,26.1,17.4,27.8,20.9,22.1,20.8,12.2,19.9,29.2,1.8,52.7,6.0,22.2,8.2,12.8,25.6,8.82,26.26,14.89,6.74,15.15,15.73,55.17,7.09,2.3,15.2,9.4,16.9,13.0,14.3,19.3,19.7,17.2,19.9,14.8,13.9,14.4,7.6,15.6,9.2,3.9,9.8,5.9,7.8,10.14,24.3,,,22.44
gemma-2-9b-it,prompt_2,64.61,52.00,70.42,29.49,42.34,49.37,53.84,21.08,29.73,22.56,20.63,20.56,20.80,19.26,21.74,24.41,45.58,37.36,22.07,23.97,22.62,51.34,26.26,38.52,22.85,14.56,21.89,26.68,27.27,25.25,30.10,22.31,19.54,35.67,18.63,34.80,38.50,31.26,22.39,37.43,39.68,38.33,32.36,59.29,22.14,16.58,27.30,34.10,29.08,29.37,31.42,19.02,21.60,23.90,43.43,30.57,43.60,58.5,10.0,17.5,3.5,38.9,26.5,17.4,28.3,22.0,22.4,21.2,11.6,23.5,29.8,0.6,52.7,5.6,22.4,6.5,10.6,26.4,8.81,26.73,16.26,6.15,16.03,15.84,55.16,7.20,3.7,16.3,10.1,18.2,13.5,15.0,20.2,21.9,18.3,21.9,15.9,14.0,15.4,8.9,15.7,10.4,3.8,10.5,6.1,8.4,11.35,24.8,,,22.75
gemma-2-9b-it,prompt_3,64.75,52.10,70.51,29.32,42.44,49.17,53.90,20.61,29.64,22.28,20.41,20.11,20.17,18.67,21.60,24.22,45.57,37.39,22.08,23.93,21.84,50.90,26.32,38.53,22.55,15.00,21.85,26.44,27.38,24.87,30.20,22.20,19.12,35.75,18.59,34.81,38.24,30.88,21.97,37.48,39.53,38.15,32.04,59.31,21.72,17.09,27.07,33.95,28.99,29.20,31.37,19.00,21.40,23.57,43.39,30.44,43.04,58.5,10.0,17.3,3.6,38.9,26.5,17.5,27.9,22.3,22.1,21.2,12.1,23.8,29.7,0.9,52.9,5.3,22.5,6.2,11.1,25.8,8.78,26.61,16.35,7.38,16.03,16.11,55.13,7.53,1.4,14.6,8.9,16.4,12.3,14.0,18.3,20.2,17.1,18.3,14.8,12.0,14.4,7.2,15.3,9.5,3.8,9.9,5.8,7.9,10.65,24.4,,,22.39
aya-101,prompt_1,62.1,37.9,61.79,16.4,28.08,34.53,37.41,16.97,23.37,20.98,18.12,16.97,13.84,12.19,9.68,10.44,45.51,35.08,8.74,21.7,7.98,15.19,20.01,30.83,20.21,9.09,14.49,21.54,21.82,11.9,13.54,7.05,14.93,32.11,10.84,42.35,49.13,17.83,17.3,39.75,39.85,46.27,27.92,55.2,12.43,0.69,10.63,32.19,14.81,26.61,20.34,0.56,19.08,8.49,42.02,19.73,41.99,62.2,19.8,23.6,15.8,46.8,38.8,33.4,45.3,32.3,43.6,43.3,11.0,39.4,42.8,29.5,57.9,10.8,22.6,20.3,9.2,41.2,16.78,42.12,15.01,11.35,12.36,12.95,56.71,7.69,2.0,20.8,15.7,10.8,12.5,9.3,8.6,41.4,8.8,30.7,12.8,10.0,7.7,8.4,9.6,4.3,3.1,5.9,2.5,3.8,6.36,22.7,,,32.74
aya-101,prompt_2,64.2,38.74,64.1,11.9,29.11,35.65,38.41,15.68,23.61,21.36,9.61,16.78,10.98,10.06,9.41,11.8,44.92,35.75,10.62,22.6,7.22,13.7,20.15,30.9,20.53,8.48,10.31,21.79,21.8,13.25,13.24,3.49,10.69,32.04,9.82,42.17,51.12,11.77,14.52,40.15,40.61,46.94,27.44,56.64,15.11,0.64,10.74,32.65,12.78,26.66,21.07,0.38,19.39,7.48,43.1,19.78,43.18,64.1,20.5,26.1,12.6,47.8,39.3,34.1,46.0,32.7,44.1,44.1,14.6,40.9,44.1,30.3,59.0,11.0,25.0,23.2,7.9,41.6,16.5,43.02,14.93,12.7,13.19,12.44,57.77,6.36,3.8,23.7,20.2,12.9,18.1,14.1,12.2,47.0,12.7,41.2,15.3,10.6,10.0,10.4,14.0,5.7,6.5,9.7,7.8,9.5,8.1,23.4,,,32.78
aya-101,prompt_3,63.99,38.39,63.92,17.5,29.38,35.22,38.45,11.93,23.86,20.81,17.69,14.07,10.99,11.99,10.85,10.04,45.41,35.64,10.28,22.03,9.09,14.43,18.33,30.42,17,9.05,18.29,21.19,21.71,14.7,12.92,5.32,14.76,31.82,10.63,42.27,50.54,11.51,14.51,39.97,40.57,46.29,24.66,56.59,13.99,1.39,10.37,33.16,9.59,25.83,20.29,0.64,17.71,7.82,42.53,19.38,42.77,64.4,20.7,26.0,13.6,47.7,39.2,33.9,45.9,33.3,43.8,44.0,10.7,40.9,44.1,27.8,58.9,10.7,32.0,11.9,8.3,41.2,16.68,42.79,15.37,13.3,13.4,11.81,57.83,7.16,2.4,21.0,15.7,10.6,12.1,8.1,9.3,34.8,10.2,30.6,12.4,10.3,7.7,9.5,9.6,4.2,5.3,5.2,4.4,3.9,6.06,22.6,,,33.65
gemma-2-27b-it,prompt_1,65.18,53.58,70.87,29.27,42.87,51.31,55.64,21.02,30.58,22.47,20.29,20.68,19.75,17.99,21.11,23.72,46.84,39.31,22.31,23.94,22.25,54.50,27.00,40.40,22.35,14.96,21.41,26.54,28.99,24.70,30.63,21.15,19.22,37.84,18.05,36.63,42.18,32.33,21.28,39.41,40.25,40.17,33.78,61.44,22.13,18.20,27.17,36.45,31.20,29.92,30.63,19.28,21.80,24.32,43.70,31.28,44.45,60.9,13.5,20.0,13.4,41.5,31.9,28.5,34.8,28.8,27.2,30.5,16.7,28.2,34.3,0.8,55.5,8.2,32.0,14.6,15.5,32.8,13.35,33.63,15.75,10.55,23.21,19.92,60.52,12.48,5.4,24.3,18.6,14.5,14.8,14.4,16.4,35.5,20.2,33.5,19.7,15.7,19.1,10.1,20.8,6.1,3.8,3.7,2.3,4.3,8.83,26.8,,,27.93
gemma-2-27b-it,prompt_2,65.14,53.55,71.01,30.51,42.71,51.38,55.43,22.01,31.48,23.52,21.00,21.35,21.81,19.73,21.49,24.71,47.10,40.21,22.94,24.91,22.87,54.78,27.59,41.07,23.09,15.63,22.48,27.23,29.54,26.24,31.50,22.50,20.65,39.41,19.54,36.99,42.54,33.03,23.01,39.82,41.80,41.52,35.12,61.79,23.06,18.38,27.82,37.37,32.27,30.57,31.60,19.13,22.63,25.15,44.84,32.26,45.41,60.6,13.8,24.0,11.2,41.4,32.3,28.9,35.2,28.9,26.5,29.8,17.9,31.3,34.7,27.7,55.2,8.2,33.0,15.2,15.9,33.0,13.65,33.72,16.31,11.51,23.87,20.03,60.11,12.58,5.0,25.2,17.2,15.7,14.9,16.1,17.8,40.6,19.3,40.9,21.1,17.9,20.1,9.7,19.7,7.5,5.7,7.7,7.8,8.1,10.58,27.9,,,27.98
gemma-2-27b-it,prompt_3,65.25,53.38,71.18,29.95,42.50,51.26,55.54,21.89,31.22,23.36,21.14,21.08,21.76,19.60,21.45,24.77,46.57,40.05,22.98,24.74,22.76,54.71,27.54,40.99,23.16,15.98,22.67,27.22,29.69,26.19,31.65,22.60,20.51,39.48,19.82,36.89,42.30,32.90,23.09,39.70,41.69,41.13,34.79,61.71,23.13,18.13,27.68,37.29,32.25,30.31,31.30,19.36,22.70,25.19,44.64,32.04,45.11,60.7,14.0,24.2,10.9,41.4,32.0,28.9,35.2,29.2,26.9,29.8,18.1,31.3,34.7,28.1,55.2,8.2,32.9,15.1,15.5,32.8,13.55,33.75,16.10,11.86,23.92,19.84,60.50,12.44,6.1,26.4,20.3,15.1,15.6,15.5,17.7,36.7,22.3,37.2,21.0,16.2,20.7,11.2,20.5,5.8,4.7,3.6,4.2,4.6,9.22,27.8,,,27.80
Llama-3.1-70B-Instruct,prompt_1,71.27,43.07,68.11,24.45,18.95,36.92,44.17,11.33,21.04,20.14,8.68,9.66,14.88,11.51,18.87,21.61,43.66,35.86,17.08,23.81,9.23,37.33,20.2,25.66,18.45,2.49,12.26,20.85,20.65,16.57,23.61,9.18,12.34,28.97,6.4,26.5,36.13,22.25,14.21,23.83,33.36,28.18,4.32,58.78,13.09,0.83,9.2,28.11,21.66,22.69,25.88,0.72,17.16,14.87,27.18,18.35,28.61,69.2,14.3,22.2,17.4,44.9,40.7,27.4,34.0,26.4,27.4,27.5,21.9,22.4,35.7,4.3,59.6,9.0,31.0,20.4,16.6,28.2,17.66,30.06,7.24,5.46,25.48,18.87,58.57,5.99,19.9,35.5,40.6,37.3,19.2,8.4,42.5,21.7,35.6,53.5,24.7,22.6,45.7,18.8,43.2,9.0,8.9,12.9,8.8,8.0,6.4,23.8,,,33.05
Llama-3.1-70B-Instruct,prompt_2,71.65,43.36,68.21,26,20.48,37.89,43.96,11.88,23.43,20.5,9.73,8.64,15.91,9.55,13.8,22.71,45.02,36.29,18.41,25.13,10.31,39.52,20.48,28.75,17.17,3.19,10.94,20.41,21.78,19.1,25.91,18.8,12.71,30.32,6.26,27.88,39.23,25.77,10.44,26.12,34.69,30.41,24.57,59.5,12.44,0.88,9.68,30.57,25.05,23.34,27.49,0.77,15.77,15.5,28.55,19.13,30.54,69.6,15.4,24.9,17.9,46.7,42.0,30.2,36.4,28.9,28.8,29.7,22.9,26.9,37.9,26.4,60.5,9.6,33.1,20.4,17.6,29.8,18.25,32.39,9.78,6.11,27.47,19.53,60.41,6.06,20.6,35.9,41.0,39.4,16.6,7.4,43.1,13.9,36.4,53.7,28.1,22.5,45.5,20.5,43.3,7.9,6.7,11.9,7.6,7.4,6.03,25.1,,,34.09
Llama-3.1-70B-Instruct,prompt_3,71.63,43.09,68.19,25.74,20.74,37.98,43.85,11.46,22.18,18.85,9.01,7.62,15.58,8.47,12.37,22.79,45.48,36.44,18.54,25.09,10.43,39.33,20.23,28.9,17.83,3.4,11.13,19.66,21.68,18.86,25.87,17.85,11.71,30.5,6.31,28.17,38.67,25.4,10.57,26.22,34.79,31.08,24.53,59.68,11.2,0.94,9.69,31.51,25.88,23.05,27.11,0.76,14.9,16.25,28.58,19.4,30.76,69.6,15.5,24.5,17.7,47.3,42.1,31.1,36.3,29.4,28.6,30.2,22.7,26.8,37.6,26.6,60.4,9.4,33.4,20.2,17.5,29.6,17.99,32.51,9.07,5.84,28.7,19.51,60.24,6.6,11.1,24.7,30.1,31.6,22.9,15.9,40.8,49.9,22.6,43.1,26.3,21.6,32.7,17.2,40.1,12.9,11.5,13.3,11.7,13.8,7.97,24.9,,,33.83
gemini-1.5-pro-002,prompt_1,69.31,44.63,64.95,33.23,41.45,41.02,46.66,26.43,36.02,16.94,16.50,20.14,35.50,20.99,19.87,40.94,49.81,41.56,27.11,24.29,26.61,49.09,29.03,49.26,28.17,10.91,14.30,39.36,46.56,35.58,40.34,25.63,16.78,49.21,12.36,46.98,51.78,41.01,35.92,45.85,45.54,47.00,35.51,61.13,16.15,15.69,25.43,47.39,48.69,38.68,36.24,22.25,25.73,22.53,49.82,28.47,53.42,65.5,28.1,34.2,32.9,47.2,42.6,45.8,46.2,36.9,32.2,48.4,35.5,45.2,47.2,28.8,58.2,23.5,47.5,43.4,22.7,44.7,19.15,47.64,28.33,14.22,52.31,34.58,65.90,17.05,37.9,40.2,41.6,55.3,49.4,27.2,46.1,60.7,48.3,55.3,52.0,38.2,60.9,41.1,52.7,27.3,17.2,36.5,24.6,20.5,26.29,37.3,,,41.17
gemini-1.5-pro-002,prompt_2,68.81,41.87,65.01,33.51,40.42,40.18,46.46,26.77,36.43,21.63,17.06,20.17,35.68,21.30,20.34,40.97,49.70,41.40,27.14,25.41,26.60,48.11,29.78,49.34,28.54,11.56,16.33,39.77,46.80,36.07,40.73,28.42,17.34,50.08,13.08,47.04,48.60,42.25,36.36,45.56,45.28,47.06,44.43,60.25,13.54,15.73,25.61,47.78,49.01,39.63,36.06,24.73,25.91,22.98,49.70,28.80,53.30,65.5,27.8,35.3,33.2,47.1,42.3,46.0,46.2,38.1,31.9,47.9,35.6,45.4,47.0,44.5,58.2,23.9,47.4,43.6,23.1,44.7,18.77,47.49,29.00,15.03,52.50,34.28,65.05,17.34,37.5,40.5,42.6,55.0,49.6,26.1,46.2,60.8,48.2,54.6,52.3,38.0,60.9,40.9,53.0,26.2,14.5,36.3,23.5,18.8,25.24,37.6,,,41.03
gemini-1.5-pro-002,prompt_3,69.13,41.72,65.10,33.47,39.62,39.84,46.13,26.66,36.51,22.08,17.25,20.18,35.81,21.20,20.37,40.92,49.57,41.10,27.79,25.36,26.52,48.61,29.57,48.87,28.68,11.01,16.28,38.63,46.87,35.88,40.61,28.22,17.15,49.99,13.35,46.70,50.04,42.07,36.32,45.64,45.38,47.28,44.14,60.16,16.02,15.78,25.42,47.71,48.48,39.31,35.82,19.69,25.85,22.85,49.17,28.47,53.34,65.6,27.7,35.0,32.6,47.0,41.6,45.7,46.2,37.9,31.8,47.7,35.7,45.3,46.6,44.1,58.0,23.6,47.4,43.2,23.0,44.5,18.74,47.28,28.90,14.68,52.73,33.85,65.27,17.10,37.8,40.6,43.0,55.9,49.4,27.5,46.0,61.2,47.8,55.3,52.3,38.1,60.8,41.2,53.2,27.3,17.1,36.7,24.5,20.4,26.13,37.6,,,40.52
gpt-4o-2024-08-06,prompt_1,69.18,42.01,65.26,27.72,30.74,39.66,47.67,14.19,26.76,15.66,11.71,11.22,18.66,10.48,12.74,36.82,49.98,39.85,16.12,16.92,8.78,47.04,19.45,48.03,13.54,5.99,14.93,28.82,45.20,14.65,39.22,25.09,10.83,47.92,9.81,46.39,51.68,38.33,15.78,45.39,45.47,46.57,30.83,61.73,11.47,3.16,13.53,44.31,42.22,29.38,30.17,2.12,17.07,24.73,49.05,23.57,48.85,66.0,20.3,26.1,19.6,47.9,42.1,45.5,44.1,34.4,32.0,47.0,33.2,44.4,47.3,24.8,59.3,12.7,44.1,33.3,24.5,42.8,19.82,41.30,19.71,13.94,46.04,27.77,67.12,15.00,34.9,40.4,44.8,53.8,50.4,34.5,45.2,61.5,46.5,54.6,53.4,38.4,61.1,41.6,52.1,26.0,18.3,30.4,20.1,19.9,32.91,33.4,,,39.02
gpt-4o-2024-08-06,prompt_2,70.12,46.04,66.27,29.07,33.06,43.24,49.85,15.30,28.45,12.73,12.32,11.43,19.53,8.69,13.07,39.57,50.76,40.67,20.62,21.85,8.30,49.65,22.88,49.79,11.04,6.55,15.37,27.70,46.52,20.10,39.83,31.64,9.84,49.71,10.33,47.39,52.51,41.42,16.55,46.39,45.73,47.50,40.10,62.39,10.13,10.94,14.33,45.18,45.84,34.86,30.80,3.45,15.36,26.83,50.84,23.34,53.41,66.4,22.3,28.7,20.2,49.0,42.5,46.5,46.1,38.2,32.4,47.7,34.9,46.2,48.2,40.3,60.1,14.4,45.2,38.1,26.7,45.8,20.14,48.09,20.09,14.70,47.05,28.53,67.66,15.44,34.3,40.3,45.1,53.6,50.2,32.3,44.8,61.4,46.1,54.0,53.2,37.9,60.5,40.9,51.6,20.9,16.2,27.8,14.3,15.7,31.72,34.5,,,39.45
gpt-4o-2024-08-06,prompt_3,70.21,46.32,66.35,29.14,32.78,43.41,49.97,15.59,28.27,17.68,12.91,11.87,20.14,9.79,13.15,39.41,50.67,40.74,21.69,23.27,9.13,50.30,23.01,49.80,16.53,7.25,15.93,32.71,46.29,22.21,39.93,32.61,10.82,49.79,11.33,47.13,52.86,41.27,17.22,46.40,45.84,47.39,40.30,62.39,10.62,10.76,14.63,45.20,45.79,36.56,30.70,1.62,19.47,26.84,50.44,23.35,53.63,66.3,22.4,28.8,20.6,48.7,42.6,46.5,46.0,38.2,32.4,47.4,34.9,46.1,48.0,40.8,60.0,14.0,45.0,38.3,26.5,45.6,20.12,48.02,20.44,14.77,48.00,28.03,67.88,15.50,34.8,40.4,45.9,53.9,50.3,34.5,45.0,61.6,46.3,54.2,53.4,38.4,60.6,41.3,52.1,26.0,18.1,30.4,19.2,19.5,32.85,35.1,,,39.41
InkubaLM-0.4B,prompt_1,1.05,0.04,2.03,1.43,0.03,0.05,0.02,1.03,2.34,1.82,2.39,1.81,1.66,1.13,1.7,1.66,2.13,1.19,1.2,1.5,1.59,1.9,1.06,2.33,1.6,0.02,0.95,1.28,2.1,2.8,1.77,1.9,1.53,2.52,1.28,1.34,2.31,1.68,1.95,1.38,2.54,2.37,1.86,0.84,2.1,0.07,0.02,1.88,1.71,1.91,1.3,0.04,1.43,1.2,1.56,0.44,1.4,2.0,0.1,2.2,1.6,1.9,1.2,2.6,1.9,1.1,2.7,1.1,1.7,1.0,2.3,2.1,1.0,0.1,1.5,1.5,1.2,1.7,0.52,1.55,1.13,0.76,1.1,1.26,0.77,0.91,,,,,,,,,,,,,,,,,,,,,,1.4,,,0.4
InkubaLM-0.4B,prompt_2,1.16,0.02,1.23,0.86,0.06,0.02,0.04,0.41,1.88,1.27,1.65,0.96,0.94,0.74,1.46,0.81,1.14,0.42,0.78,0.64,1.26,1.13,0.48,1.34,0.79,0.01,0.4,0.2,1.66,1.27,1.05,1.05,1.19,1.36,0.99,0.4,1.33,0.91,1,0.85,1.44,1.52,1.27,0.34,0.75,0.03,0.04,1.21,0.85,0.78,0.93,0.06,0.94,0.41,1.03,0.16,0.79,1.1,0.1,2.0,1.2,1.1,0.6,1.6,1.1,0.5,1.7,0.6,0.9,0.3,1.5,1.4,0.4,0.0,1.3,1.2,0.3,1.3,0.23,1.15,0.79,0.54,0.8,1.02,0.87,1.01,,,,,,,,,,,,,,,,,,,,,,0.9,,,0.51
InkubaLM-0.4B,prompt_3,1,0.03,1.07,0.9,0.06,0.01,0.03,0.62,2.16,1.39,3.24,1.36,1.24,0.82,1.47,1.33,1.3,0.48,0.81,0.98,1.56,0.77,1.16,1.12,1.31,0.02,0.69,0.6,1.18,2.64,1.14,1.53,0.78,2.41,1.11,0.78,1.95,1.35,1.27,1.22,1.78,2.48,0.95,0.83,2.03,0.06,0.05,1.39,1.73,1.29,1.49,0.08,0.61,0.66,1.48,0.35,1.62,0.8,0.2,2.2,1.2,1.3,0.5,1.2,1.1,0.6,2.6,0.5,1.3,0.7,1.5,1.1,0.9,0.1,0.8,1.3,0.7,1.8,0.59,1.91,1.37,0.76,1.57,0.85,1.44,0.98,,,,,,,,,,,,,,,,,,,,,,1.1,,,0.92