{
  "random_baseline": 0.1789,
  "n_questions": 14850,
  "subjects": [
    "biology",
    "english",
    "geography",
    "history_of_kazakhstan",
    "human_society_rights",
    "kazakh_and_literature",
    "world_history"
  ],
  "models": {
    "KazBERT": {
      "model_id": "Eraly-ml/KazBERT",
      "pll_overall": 0.2228,
      "pll_by_subject": {
        "biology": 0.2246,
        "english": 0.198,
        "geography": 0.2293,
        "history_of_kazakhstan": 0.2267,
        "human_society_rights": 0.2511,
        "kazakh_and_literature": 0.2267,
        "world_history": 0.2248
      },
      "emb_overall": 0.2126,
      "emb_by_subject": {
        "biology": 0.2122,
        "english": 0.1793,
        "geography": 0.2119,
        "history_of_kazakhstan": 0.2101,
        "human_society_rights": 0.3094,
        "kazakh_and_literature": 0.2223,
        "world_history": 0.2256
      },
      "vocab_size": 32000,
      "seconds": 801.8
    },
    "mBERT": {
      "model_id": "google-bert/bert-base-multilingual-cased",
      "pll_overall": 0.2205,
      "pll_by_subject": {
        "biology": 0.2318,
        "english": 0.2663,
        "geography": 0.209,
        "history_of_kazakhstan": 0.2064,
        "human_society_rights": 0.2601,
        "kazakh_and_literature": 0.2063,
        "world_history": 0.2248
      },
      "vocab_size": 119547,
      "emb_overall": 0.2332,
      "emb_by_subject": {
        "biology": 0.222,
        "english": 0.2112,
        "geography": 0.2392,
        "history_of_kazakhstan": 0.2264,
        "human_society_rights": 0.296,
        "kazakh_and_literature": 0.2436,
        "world_history": 0.2451
      },
      "seconds": 2631.3
    },
    "XLM-R base": {
      "model_id": "FacebookAI/xlm-roberta-base",
      "pll_overall": 0.2188,
      "pll_by_subject": {
        "biology": 0.2161,
        "english": 0.2599,
        "geography": 0.2084,
        "history_of_kazakhstan": 0.1856,
        "human_society_rights": 0.2511,
        "kazakh_and_literature": 0.2248,
        "world_history": 0.2223
      },
      "vocab_size": 250002,
      "emb_overall": 0.2087,
      "emb_by_subject": {
        "biology": 0.224,
        "english": 0.203,
        "geography": 0.2078,
        "history_of_kazakhstan": 0.1663,
        "human_society_rights": 0.1883,
        "kazakh_and_literature": 0.2317,
        "world_history": 0.2296
      },
      "seconds": 2867.7
    },
    "kaz-roberta": {
      "model_id": "kz-transformers/kaz-roberta-conversational",
      "pll_overall": 0.2273,
      "pll_by_subject": {
        "biology": 0.1927,
        "english": 0.2117,
        "geography": 0.2276,
        "history_of_kazakhstan": 0.2433,
        "human_society_rights": 0.2915,
        "kazakh_and_literature": 0.2321,
        "world_history": 0.2264
      },
      "emb_overall": 0.2424,
      "emb_by_subject": {
        "biology": 0.2057,
        "english": 0.208,
        "geography": 0.2544,
        "history_of_kazakhstan": 0.2696,
        "human_society_rights": 0.3229,
        "kazakh_and_literature": 0.2457,
        "world_history": 0.2337
      },
      "vocab_size": 52000,
      "seconds": 680.2
    },
    "KazakhBERTmulti": {
      "model_id": "amandyk/KazakhBERTmulti",
      "pll_overall": 0.207,
      "pll_by_subject": {
        "biology": 0.1882,
        "english": 0.2212,
        "geography": 0.2165,
        "history_of_kazakhstan": 0.1758,
        "human_society_rights": 0.1928,
        "kazakh_and_literature": 0.2255,
        "world_history": 0.2068
      },
      "emb_overall": 0.2247,
      "emb_by_subject": {
        "biology": 0.2057,
        "english": 0.2076,
        "geography": 0.2357,
        "history_of_kazakhstan": 0.2215,
        "human_society_rights": 0.3453,
        "kazakh_and_literature": 0.2321,
        "world_history": 0.2223
      },
      "vocab_size": 100000,
      "seconds": 1173.4
    }
  }
}