data/leaderboard_json/qa.json

{
    "task": "QA",
    "subtasks": {
        "XQA": {
            "datasets": {
                "afriqa": {
                    "languages": [
                        "bem",
                        "fon",
                        "hau",
                        "ibo",
                        "kin",
                        "twi",
                        "yor",
                        "zul"
                    ],
                    "scores": {
                        "AfroLLaMa 8B": [
                            15.4,
                            8.7,
                            38.9,
                            11.4,
                            17.4,
                            9.2,
                            37.7,
                            35.6
                        ],
                        "LLaMAX3 8B": [
                            1.4,
                            4.6,
                            1.7,
                            2.3,
                            2.4,
                            1.5,
                            1.4,
                            1.9
                        ],
                        "LLaMa2 7b": [
                            14.5,
                            14.1,
                            10.8,
                            18.3,
                            16.6,
                            11.7,
                            12.4,
                            11.2
                        ],
                        "LLaMa3.1 8B": [
                            15.4,
                            14.8,
                            24.6,
                            41.7,
                            31.2,
                            8.9,
                            19.8,
                            17.6
                        ],
                        "LLaMa3 8B": [
                            8.2,
                            14.9,
                            12.2,
                            22.8,
                            14.8,
                            8.8,
                            9.6,
                            9.8
                        ],
                        "Aya-101 13B": [
                            45.2,
                            29.2,
                            78.2,
                            84.4,
                            70.7,
                            42.7,
                            69.6,
                            80.1
                        ],
                        "Gemma1.1 7b": [
                            13.6,
                            17.4,
                            22.1,
                            24.6,
                            19.2,
                            11.0,
                            10.6,
                            20.5
                        ],
                        "Gemma2 27b": [
                            29.4,
                            28.2,
                            57.8,
                            75.6,
                            64.8,
                            34.1,
                            51.2,
                            63.0
                        ],
                        "Gemma2 9b": [
                            24.0,
                            21.2,
                            52.2,
                            66.0,
                            62.6,
                            34.6,
                            46.3,
                            60.6
                        ],
                        "LLaMa3.1 70B": [
                            29.8,
                            26.0,
                            57.7,
                            70.1,
                            48.0,
                            27.2,
                            42.1,
                            51.2
                        ],
                        "Gemini 1.5 pro": [
                            35.1,
                            20.5,
                            47.0,
                            61.1,
                            47.5,
                            45.1,
                            29.0,
                            38.3
                        ],
                        "GPT-4o (Aug)": [
                            35.7,
                            12.7,
                            50.0,
                            61.6,
                            50.4,
                            52.5,
                            37.0,
                            47.4
                        ]
                    }
                }
            }
        },
        "RC": {
            "datasets": {
                "naijarc": {
                    "languages": [
                        "hau",
                        "ibo",
                        "yor"
                    ],
                    "scores": {
                        "AfroLLaMa 8B": [
                            19.5,
                            26.2,
                            33.0
                        ],
                        "LLaMAX3 8B": [
                            31.7,
                            32.1,
                            31.9
                        ],
                        "LLaMa2 7b": [
                            20.7,
                            32.1,
                            30.9
                        ],
                        "LLaMa3.1 8B": [
                            61.0,
                            45.2,
                            38.2
                        ],
                        "LLaMa3.1 70B": [
                            64.6,
                            52.4,
                            41.9
                        ],
                        "LLaMa3 8B": [
                            26.8,
                            39.3,
                            29.8
                        ],
                        "Aya-101 13B": [
                            70.7,
                            71.4,
                            49.2
                        ],
                        "Gemma1.1 7b": [
                            48.8,
                            41.7,
                            33.5
                        ],
                        "Gemma2 27b": [
                            67.1,
                            53.6,
                            48.2
                        ],
                        "Gemma2 9b": [
                            72.0,
                            56.0,
                            38.7
                        ],
                        "Gemini 1.5 pro": [
                            53.7,
                            59.5,
                            45.6
                        ],
                        "GPT-4o (Aug)": [
                            80.5,
                            73.8,
                            66.5
                        ]
                    }
                },
                "belebele": {
                    "languages": [
                        "afr",
                        "amh",
                        "ary",
                        "arz",
                        "bam",
                        "hau",
                        "ibo",
                        "lin",
                        "por",
                        "sna",
                        "swa",
                        "tir",
                        "tsn",
                        "tso",
                        "wol",
                        "xho",
                        "yor",
                        "zul",
                        "fuv",
                        "gaz",
                        "kea",
                        "kin",
                        "lug",
                        "luo",
                        "nya",
                        "plt",
                        "som",
                        "sot",
                        "ssw"
                    ],
                    "scores": {
                        "AfroLLaMa 8B": [
                            24.1,
                            22.7,
                            23.8,
                            25.0,
                            22.0,
                            25.9,
                            24.6,
                            25.6,
                            24.2,
                            24.7,
                            23.3,
                            22.6,
                            26.8,
                            24.0,
                            27.0,
                            24.3,
                            27.0,
                            28.0,
                            22.6,
                            22.9,
                            23.0,
                            22.9,
                            22.9,
                            23.0,
                            22.7,
                            22.9,
                            22.9,
                            22.6,
                            22.9
                        ],
                        "LLaMAX3 8B": [
                            37.0,
                            23.9,
                            31.2,
                            33.6,
                            25.3,
                            30.7,
                            26.8,
                            27.3,
                            33.8,
                            32.2,
                            35.4,
                            23.1,
                            26.9,
                            26.4,
                            25.8,
                            28.4,
                            25.0,
                            27.4,
                            26.6,
                            30.0,
                            34.9,
                            34.3,
                            28.1,
                            25.0,
                            30.1,
                            35.0,
                            31.4,
                            30.8,
                            29.3
                        ],
                        "LLaMa2 7b": [
                            27.4,
                            22.9,
                            22.9,
                            23.1,
                            23.0,
                            23.3,
                            23.3,
                            23.4,
                            34.0,
                            23.0,
                            23.7,
                            22.9,
                            23.1,
                            23.9,
                            24.2,
                            22.8,
                            23.2,
                            23.9,
                            24.1,
                            23.3,
                            23.7,
                            24.1,
                            23.7,
                            23.6,
                            23.8,
                            23.4,
                            23.8,
                            24.0,
                            22.3
                        ],
                        "LLaMa3.1 8B": [
                            66.9,
                            33.2,
                            50.8,
                            55.0,
                            35.4,
                            39.2,
                            36.7,
                            33.9,
                            68.7,
                            37.2,
                            53.2,
                            28.7,
                            32.3,
                            37.0,
                            30.6,
                            35.1,
                            31.7,
                            35.3,
                            29.4,
                            30.8,
                            45.7,
                            37.4,
                            33.2,
                            32.0,
                            31.9,
                            38.8,
                            32.9,
                            33.7,
                            32.3
                        ],
                        "LLaMa3.1 70B": [
                            88.9,
                            39.4,
                            60.2,
                            75.1,
                            36.3,
                            54.2,
                            45.8,
                            39.7,
                            91.3,
                            43.4,
                            75.4,
                            30.9,
                            41.4,
                            44.0,
                            31.4,
                            41.3,
                            36.1,
                            42.9,
                            29.0,
                            38.8,
                            59.1,
                            53.6,
                            41.4,
                            36.2,
                            44.1,
                            68.6,
                            49.4,
                            51.3,
                            42.9
                        ],
                        "LLaMa3 8B": [
                            40.9,
                            23.2,
                            29.3,
                            34.1,
                            24.4,
                            28.2,
                            26.9,
                            25.4,
                            43.3,
                            26.1,
                            35.1,
                            23.6,
                            24.2,
                            24.6,
                            25.0,
                            24.7,
                            24.8,
                            25.6,
                            23.6,
                            23.2,
                            30.1,
                            26.1,
                            24.8,
                            24.9,
                            25.7,
                            26.2,
                            24.8,
                            24.6,
                            24.7
                        ],
                        "Aya-101 13B": [
                            81.7,
                            73.8,
                            69.6,
                            75.2,
                            45.0,
                            66.9,
                            56.2,
                            50.4,
                            82.0,
                            66.7,
                            76.9,
                            63.8,
                            62.8,
                            54.8,
                            32.9,
                            65.9,
                            52.7,
                            64.9,
                            32.2,
                            52.8,
                            58.3,
                            67.9,
                            46.9,
                            39.6,
                            60.9,
                            75.1,
                            62.9,
                            61.7,
                            57.6
                        ],
                        "Gemma1.1 7b": [
                            68.6,
                            33.1,
                            43.4,
                            52.2,
                            35.1,
                            34.9,
                            32.0,
                            32.8,
                            78.8,
                            36.9,
                            52.9,
                            31.8,
                            33.1,
                            36.3,
                            30.0,
                            31.4,
                            29.3,
                            34.4,
                            27.4,
                            31.0,
                            42.7,
                            33.6,
                            32.0,
                            33.1,
                            31.9,
                            35.9,
                            33.0,
                            33.0,
                            33.4
                        ],
                        "Gemma2 27b": [
                            91.0,
                            60.0,
                            70.1,
                            81.2,
                            31.9,
                            69.6,
                            49.0,
                            37.6,
                            91.1,
                            61.3,
                            87.4,
                            36.1,
                            48.7,
                            46.2,
                            31.4,
                            56.2,
                            39.4,
                            57.2,
                            26.8,
                            33.4,
                            59.6,
                            58.0,
                            37.7,
                            30.3,
                            51.1,
                            67.7,
                            50.9,
                            50.6,
                            43.7
                        ],
                        "Gemma2 9b": [
                            89.0,
                            55.3,
                            65.8,
                            79.9,
                            34.9,
                            65.8,
                            45.7,
                            35.1,
                            90.4,
                            54.2,
                            85.0,
                            35.0,
                            46.0,
                            44.2,
                            34.1,
                            55.1,
                            37.9,
                            53.1,
                            26.7,
                            34.4,
                            56.9,
                            53.0,
                            36.9,
                            33.2,
                            47.2,
                            61.4,
                            48.2,
                            44.8,
                            41.0
                        ],
                        "Gemini 1.5 pro": [
                            65.8,
                            68.9,
                            55.1,
                            61.6,
                            32.8,
                            41.9,
                            43.6,
                            51.7,
                            32.8,
                            53.1,
                            81.4,
                            49.1,
                            49.2,
                            51.8,
                            30.6,
                            64.2,
                            46.6,
                            61.6,
                            28.4,
                            47.0,
                            57.4,
                            68.9,
                            48.7,
                            32.2,
                            57.2,
                            70.8,
                            71.7,
                            54.2,
                            59.2
                        ],
                        "GPT-4o (Aug)": [
                            93.7,
                            75.1,
                            82.4,
                            87.2,
                            36.6,
                            67.1,
                            59.4,
                            64.7,
                            75.6,
                            78.6,
                            90.6,
                            42.3,
                            64.2,
                            66.2,
                            41.8,
                            78.9,
                            61.2,
                            75.8,
                            29.3,
                            69.0,
                            83.2,
                            80.6,
                            66.4,
                            46.6,
                            68.9,
                            89.7,
                            80.3,
                            75.2,
                            74.2
                        ]
                    }
                }
            }
        }
    }
}