Spaces:
Runtime error
Runtime error
| { | |
| "task": "QA", | |
| "subtasks": { | |
| "XQA": { | |
| "datasets": { | |
| "afriqa": { | |
| "languages": [ | |
| "bem", | |
| "fon", | |
| "hau", | |
| "ibo", | |
| "kin", | |
| "twi", | |
| "yor", | |
| "zul" | |
| ], | |
| "scores": { | |
| "AfroLLaMa 8B": [ | |
| 15.4, | |
| 8.7, | |
| 38.9, | |
| 11.4, | |
| 17.4, | |
| 9.2, | |
| 37.7, | |
| 35.6 | |
| ], | |
| "LLaMAX3 8B": [ | |
| 1.4, | |
| 4.6, | |
| 1.7, | |
| 2.3, | |
| 2.4, | |
| 1.5, | |
| 1.4, | |
| 1.9 | |
| ], | |
| "LLaMa2 7b": [ | |
| 14.5, | |
| 14.1, | |
| 10.8, | |
| 18.3, | |
| 16.6, | |
| 11.7, | |
| 12.4, | |
| 11.2 | |
| ], | |
| "LLaMa3.1 8B": [ | |
| 15.4, | |
| 14.8, | |
| 24.6, | |
| 41.7, | |
| 31.2, | |
| 8.9, | |
| 19.8, | |
| 17.6 | |
| ], | |
| "LLaMa3 8B": [ | |
| 8.2, | |
| 14.9, | |
| 12.2, | |
| 22.8, | |
| 14.8, | |
| 8.8, | |
| 9.6, | |
| 9.8 | |
| ], | |
| "Aya-101 13B": [ | |
| 45.2, | |
| 29.2, | |
| 78.2, | |
| 84.4, | |
| 70.7, | |
| 42.7, | |
| 69.6, | |
| 80.1 | |
| ], | |
| "Gemma1.1 7b": [ | |
| 13.6, | |
| 17.4, | |
| 22.1, | |
| 24.6, | |
| 19.2, | |
| 11.0, | |
| 10.6, | |
| 20.5 | |
| ], | |
| "Gemma2 27b": [ | |
| 29.4, | |
| 28.2, | |
| 57.8, | |
| 75.6, | |
| 64.8, | |
| 34.1, | |
| 51.2, | |
| 63.0 | |
| ], | |
| "Gemma2 9b": [ | |
| 24.0, | |
| 21.2, | |
| 52.2, | |
| 66.0, | |
| 62.6, | |
| 34.6, | |
| 46.3, | |
| 60.6 | |
| ], | |
| "LLaMa3.1 70B": [ | |
| 29.8, | |
| 26.0, | |
| 57.7, | |
| 70.1, | |
| 48.0, | |
| 27.2, | |
| 42.1, | |
| 51.2 | |
| ], | |
| "Gemini 1.5 pro": [ | |
| 35.1, | |
| 20.5, | |
| 47.0, | |
| 61.1, | |
| 47.5, | |
| 45.1, | |
| 29.0, | |
| 38.3 | |
| ], | |
| "GPT-4o (Aug)": [ | |
| 35.7, | |
| 12.7, | |
| 50.0, | |
| 61.6, | |
| 50.4, | |
| 52.5, | |
| 37.0, | |
| 47.4 | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| "RC": { | |
| "datasets": { | |
| "naijarc": { | |
| "languages": [ | |
| "hau", | |
| "ibo", | |
| "yor" | |
| ], | |
| "scores": { | |
| "AfroLLaMa 8B": [ | |
| 19.5, | |
| 26.2, | |
| 33.0 | |
| ], | |
| "LLaMAX3 8B": [ | |
| 31.7, | |
| 32.1, | |
| 31.9 | |
| ], | |
| "LLaMa2 7b": [ | |
| 20.7, | |
| 32.1, | |
| 30.9 | |
| ], | |
| "LLaMa3.1 8B": [ | |
| 61.0, | |
| 45.2, | |
| 38.2 | |
| ], | |
| "LLaMa3.1 70B": [ | |
| 64.6, | |
| 52.4, | |
| 41.9 | |
| ], | |
| "LLaMa3 8B": [ | |
| 26.8, | |
| 39.3, | |
| 29.8 | |
| ], | |
| "Aya-101 13B": [ | |
| 70.7, | |
| 71.4, | |
| 49.2 | |
| ], | |
| "Gemma1.1 7b": [ | |
| 48.8, | |
| 41.7, | |
| 33.5 | |
| ], | |
| "Gemma2 27b": [ | |
| 67.1, | |
| 53.6, | |
| 48.2 | |
| ], | |
| "Gemma2 9b": [ | |
| 72.0, | |
| 56.0, | |
| 38.7 | |
| ], | |
| "Gemini 1.5 pro": [ | |
| 53.7, | |
| 59.5, | |
| 45.6 | |
| ], | |
| "GPT-4o (Aug)": [ | |
| 80.5, | |
| 73.8, | |
| 66.5 | |
| ] | |
| } | |
| }, | |
| "belebele": { | |
| "languages": [ | |
| "afr", | |
| "amh", | |
| "ary", | |
| "arz", | |
| "bam", | |
| "hau", | |
| "ibo", | |
| "lin", | |
| "por", | |
| "sna", | |
| "swa", | |
| "tir", | |
| "tsn", | |
| "tso", | |
| "wol", | |
| "xho", | |
| "yor", | |
| "zul", | |
| "fuv", | |
| "gaz", | |
| "kea", | |
| "kin", | |
| "lug", | |
| "luo", | |
| "nya", | |
| "plt", | |
| "som", | |
| "sot", | |
| "ssw" | |
| ], | |
| "scores": { | |
| "AfroLLaMa 8B": [ | |
| 24.1, | |
| 22.7, | |
| 23.8, | |
| 25.0, | |
| 22.0, | |
| 25.9, | |
| 24.6, | |
| 25.6, | |
| 24.2, | |
| 24.7, | |
| 23.3, | |
| 22.6, | |
| 26.8, | |
| 24.0, | |
| 27.0, | |
| 24.3, | |
| 27.0, | |
| 28.0, | |
| 22.6, | |
| 22.9, | |
| 23.0, | |
| 22.9, | |
| 22.9, | |
| 23.0, | |
| 22.7, | |
| 22.9, | |
| 22.9, | |
| 22.6, | |
| 22.9 | |
| ], | |
| "LLaMAX3 8B": [ | |
| 37.0, | |
| 23.9, | |
| 31.2, | |
| 33.6, | |
| 25.3, | |
| 30.7, | |
| 26.8, | |
| 27.3, | |
| 33.8, | |
| 32.2, | |
| 35.4, | |
| 23.1, | |
| 26.9, | |
| 26.4, | |
| 25.8, | |
| 28.4, | |
| 25.0, | |
| 27.4, | |
| 26.6, | |
| 30.0, | |
| 34.9, | |
| 34.3, | |
| 28.1, | |
| 25.0, | |
| 30.1, | |
| 35.0, | |
| 31.4, | |
| 30.8, | |
| 29.3 | |
| ], | |
| "LLaMa2 7b": [ | |
| 27.4, | |
| 22.9, | |
| 22.9, | |
| 23.1, | |
| 23.0, | |
| 23.3, | |
| 23.3, | |
| 23.4, | |
| 34.0, | |
| 23.0, | |
| 23.7, | |
| 22.9, | |
| 23.1, | |
| 23.9, | |
| 24.2, | |
| 22.8, | |
| 23.2, | |
| 23.9, | |
| 24.1, | |
| 23.3, | |
| 23.7, | |
| 24.1, | |
| 23.7, | |
| 23.6, | |
| 23.8, | |
| 23.4, | |
| 23.8, | |
| 24.0, | |
| 22.3 | |
| ], | |
| "LLaMa3.1 8B": [ | |
| 66.9, | |
| 33.2, | |
| 50.8, | |
| 55.0, | |
| 35.4, | |
| 39.2, | |
| 36.7, | |
| 33.9, | |
| 68.7, | |
| 37.2, | |
| 53.2, | |
| 28.7, | |
| 32.3, | |
| 37.0, | |
| 30.6, | |
| 35.1, | |
| 31.7, | |
| 35.3, | |
| 29.4, | |
| 30.8, | |
| 45.7, | |
| 37.4, | |
| 33.2, | |
| 32.0, | |
| 31.9, | |
| 38.8, | |
| 32.9, | |
| 33.7, | |
| 32.3 | |
| ], | |
| "LLaMa3.1 70B": [ | |
| 88.9, | |
| 39.4, | |
| 60.2, | |
| 75.1, | |
| 36.3, | |
| 54.2, | |
| 45.8, | |
| 39.7, | |
| 91.3, | |
| 43.4, | |
| 75.4, | |
| 30.9, | |
| 41.4, | |
| 44.0, | |
| 31.4, | |
| 41.3, | |
| 36.1, | |
| 42.9, | |
| 29.0, | |
| 38.8, | |
| 59.1, | |
| 53.6, | |
| 41.4, | |
| 36.2, | |
| 44.1, | |
| 68.6, | |
| 49.4, | |
| 51.3, | |
| 42.9 | |
| ], | |
| "LLaMa3 8B": [ | |
| 40.9, | |
| 23.2, | |
| 29.3, | |
| 34.1, | |
| 24.4, | |
| 28.2, | |
| 26.9, | |
| 25.4, | |
| 43.3, | |
| 26.1, | |
| 35.1, | |
| 23.6, | |
| 24.2, | |
| 24.6, | |
| 25.0, | |
| 24.7, | |
| 24.8, | |
| 25.6, | |
| 23.6, | |
| 23.2, | |
| 30.1, | |
| 26.1, | |
| 24.8, | |
| 24.9, | |
| 25.7, | |
| 26.2, | |
| 24.8, | |
| 24.6, | |
| 24.7 | |
| ], | |
| "Aya-101 13B": [ | |
| 81.7, | |
| 73.8, | |
| 69.6, | |
| 75.2, | |
| 45.0, | |
| 66.9, | |
| 56.2, | |
| 50.4, | |
| 82.0, | |
| 66.7, | |
| 76.9, | |
| 63.8, | |
| 62.8, | |
| 54.8, | |
| 32.9, | |
| 65.9, | |
| 52.7, | |
| 64.9, | |
| 32.2, | |
| 52.8, | |
| 58.3, | |
| 67.9, | |
| 46.9, | |
| 39.6, | |
| 60.9, | |
| 75.1, | |
| 62.9, | |
| 61.7, | |
| 57.6 | |
| ], | |
| "Gemma1.1 7b": [ | |
| 68.6, | |
| 33.1, | |
| 43.4, | |
| 52.2, | |
| 35.1, | |
| 34.9, | |
| 32.0, | |
| 32.8, | |
| 78.8, | |
| 36.9, | |
| 52.9, | |
| 31.8, | |
| 33.1, | |
| 36.3, | |
| 30.0, | |
| 31.4, | |
| 29.3, | |
| 34.4, | |
| 27.4, | |
| 31.0, | |
| 42.7, | |
| 33.6, | |
| 32.0, | |
| 33.1, | |
| 31.9, | |
| 35.9, | |
| 33.0, | |
| 33.0, | |
| 33.4 | |
| ], | |
| "Gemma2 27b": [ | |
| 91.0, | |
| 60.0, | |
| 70.1, | |
| 81.2, | |
| 31.9, | |
| 69.6, | |
| 49.0, | |
| 37.6, | |
| 91.1, | |
| 61.3, | |
| 87.4, | |
| 36.1, | |
| 48.7, | |
| 46.2, | |
| 31.4, | |
| 56.2, | |
| 39.4, | |
| 57.2, | |
| 26.8, | |
| 33.4, | |
| 59.6, | |
| 58.0, | |
| 37.7, | |
| 30.3, | |
| 51.1, | |
| 67.7, | |
| 50.9, | |
| 50.6, | |
| 43.7 | |
| ], | |
| "Gemma2 9b": [ | |
| 89.0, | |
| 55.3, | |
| 65.8, | |
| 79.9, | |
| 34.9, | |
| 65.8, | |
| 45.7, | |
| 35.1, | |
| 90.4, | |
| 54.2, | |
| 85.0, | |
| 35.0, | |
| 46.0, | |
| 44.2, | |
| 34.1, | |
| 55.1, | |
| 37.9, | |
| 53.1, | |
| 26.7, | |
| 34.4, | |
| 56.9, | |
| 53.0, | |
| 36.9, | |
| 33.2, | |
| 47.2, | |
| 61.4, | |
| 48.2, | |
| 44.8, | |
| 41.0 | |
| ], | |
| "Gemini 1.5 pro": [ | |
| 65.8, | |
| 68.9, | |
| 55.1, | |
| 61.6, | |
| 32.8, | |
| 41.9, | |
| 43.6, | |
| 51.7, | |
| 32.8, | |
| 53.1, | |
| 81.4, | |
| 49.1, | |
| 49.2, | |
| 51.8, | |
| 30.6, | |
| 64.2, | |
| 46.6, | |
| 61.6, | |
| 28.4, | |
| 47.0, | |
| 57.4, | |
| 68.9, | |
| 48.7, | |
| 32.2, | |
| 57.2, | |
| 70.8, | |
| 71.7, | |
| 54.2, | |
| 59.2 | |
| ], | |
| "GPT-4o (Aug)": [ | |
| 93.7, | |
| 75.1, | |
| 82.4, | |
| 87.2, | |
| 36.6, | |
| 67.1, | |
| 59.4, | |
| 64.7, | |
| 75.6, | |
| 78.6, | |
| 90.6, | |
| 42.3, | |
| 64.2, | |
| 66.2, | |
| 41.8, | |
| 78.9, | |
| 61.2, | |
| 75.8, | |
| 29.3, | |
| 69.0, | |
| 83.2, | |
| 80.6, | |
| 66.4, | |
| 46.6, | |
| 68.9, | |
| 89.7, | |
| 80.3, | |
| 75.2, | |
| 74.2 | |
| ] | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } |