data/exp_wqk_spectral/gqa_comparison.json

{
  "results": [
    {
      "model": "Qwen/Qwen2.5-0.5B",
      "short": "Qwen2.5-0.5B",
      "theta": 1000000,
      "T_train": 8192,
      "n_heads": 14,
      "n_kv": 2,
      "d_head": 64,
      "n_layers": 24,
      "k_dead": 18,
      "n_pairs": 32,
      "gamma_rand": 0.919,
      "gamma_text": 1.028,
      "snr_global": 0.9032163289605386,
      "snr_corrected": 0.9032163289605386,
      "layer_snr_mean": 0.8835472645906415,
      "mean_band_energies": [
        3.953065952777519,
        3.4384359982509523,
        4.1422359481491595,
        4.268165305996948,
        4.632941612358489,
        5.441545099392218,
        5.980114131083787,
        7.1059112606126495,
        7.826006958083158,
        8.531433813431297,
        8.994599044934718,
        8.738993894586562,
        9.026195495278385,
        9.391076977074082,
        11.239765205132842,
        14.578905901453771,
        4.019639411852872,
        3.547412737584139,
        4.14686690299469,
        4.096405410445768,
        4.715394478028094,
        6.559043221502086,
        5.506484901626362,
        8.051860459712595,
        6.994483263806611,
        7.43585016604469,
        7.714063071480374,
        8.919990325941168,
        8.248087209461895,
        9.285694310590683,
        14.386049072365926,
        11.456127235124493
      ],
      "layer_snrs": [
        0.9793584804934982,
        0.8763143955445245,
        0.8415403359194379,
        0.8880745215498504,
        0.8936372075488783,
        0.8988820321930456,
        0.8455459279067038,
        0.9174679267556224,
        0.8857812187230721,
        0.8881125626920003,
        0.8804814500507204,
        0.8686871886734093,
        0.8695513110118298,
        0.8388153746485172,
        0.8515448048796498,
        0.862703839852447,
        0.8196578026448774,
        0.8623651498475126,
        0.9022014077477166,
        0.9215660790087216,
        0.8654169316595541,
        0.9131334971786473,
        0.8981551244665501,
        0.9361397791786076
      ]
    },
    {
      "model": "Qwen/Qwen2.5-3B",
      "short": "Qwen2.5-3B",
      "theta": 1000000,
      "T_train": 8192,
      "n_heads": 16,
      "n_kv": 2,
      "d_head": 128,
      "n_layers": 36,
      "k_dead": 36,
      "n_pairs": 64,
      "gamma_rand": 0.964,
      "gamma_text": 0.772,
      "snr_global": 0.8806074396662721,
      "snr_corrected": 0.8806074396662721,
      "layer_snr_mean": 0.8779535893465105,
      "mean_band_energies": [
        8.258374764617118,
        8.530581014978711,
        9.0921296208914,
        9.74542850871339,
        10.371327689217063,
        11.261751124148692,
        11.71900305074188,
        12.581573111282864,
        12.948883425581554,
        13.546673103479739,
        14.056310061908352,
        15.298073682638128,
        15.733690931548734,
        16.799295038545257,
        17.732245044596723,
        18.440038135491903,
        19.06599694553063,
        19.94782929088912,
        21.022966521857015,
        21.640077361154017,
        22.013505968683077,
        23.746046001497056,
        23.814673884287956,
        24.382287336727906,
        24.793978182098225,
        24.047625035665675,
        25.73469192577065,
        25.090471547866112,
        25.08550830014289,
        23.762961901493114,
        23.346760990650456,
        23.34272611501102,
        8.114272662849654,
        8.503591329382498,
        9.29552959483764,
        10.02337132649432,
        10.755936938695564,
        11.43039146612689,
        11.35037791111829,
        12.920628461773065,
        12.937383417807848,
        14.074980798794932,
        13.948108792425352,
        15.41086812106611,
        15.696963239944687,
        16.754686477587907,
        17.442182821265224,
        18.050304410411844,
        19.205301634052635,
        20.025494927580738,
        20.888260829595882,
        21.874271581998087,
        22.33561786918985,
        23.59895934718369,
        23.608095262430194,
        24.358027585563537,
        24.255542115442992,
        24.385263006623724,
        25.257572677137315,
        25.08432064967099,
        24.191150739594768,
        24.077838579577005,
        23.47843496437321,
        23.92530407936351
      ],
      "layer_snrs": [
        0.8956451721489964,
        0.9294661349376346,
        0.9374797848466799,
        0.8784490078780575,
        0.8986553151908518,
        0.9111804532089779,
        0.8444166244245765,
        0.9019454843726556,
        0.9013127858909121,
        0.9266468106212808,
        0.8988197360501505,
        0.8886783439886468,
        0.8459588558832479,
        0.8605977438740229,
        0.8742748593403651,
        0.872370043078339,
        0.8812653405373329,
        0.8655064910830728,
        0.8935768831052082,
        0.8400306349943036,
        0.8228604450323329,
        0.8409152894854571,
        0.841861232954458,
        0.8817057334009857,
        0.8694221842535114,
        0.8249895147391799,
        0.8843177330393549,
        0.8490698111475532,
        0.8856938116930246,
        0.8692375161200255,
        0.8678331376707393,
        0.8974064675935306,
        0.859287875029756,
        0.8356666953169407,
        0.9136962459148364,
        0.916089017627379
      ]
    }
  ]
}