Spaces:
Running
Running
File size: 5,320 Bytes
535348a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 | {
"results": [
{
"model": "Qwen/Qwen2.5-0.5B",
"short": "Qwen2.5-0.5B",
"theta": 1000000,
"T_train": 8192,
"n_heads": 14,
"n_kv": 2,
"d_head": 64,
"n_layers": 24,
"k_dead": 18,
"n_pairs": 32,
"gamma_rand": 0.919,
"gamma_text": 1.028,
"snr_global": 0.9032163289605386,
"snr_corrected": 0.9032163289605386,
"layer_snr_mean": 0.8835472645906415,
"mean_band_energies": [
3.953065952777519,
3.4384359982509523,
4.1422359481491595,
4.268165305996948,
4.632941612358489,
5.441545099392218,
5.980114131083787,
7.1059112606126495,
7.826006958083158,
8.531433813431297,
8.994599044934718,
8.738993894586562,
9.026195495278385,
9.391076977074082,
11.239765205132842,
14.578905901453771,
4.019639411852872,
3.547412737584139,
4.14686690299469,
4.096405410445768,
4.715394478028094,
6.559043221502086,
5.506484901626362,
8.051860459712595,
6.994483263806611,
7.43585016604469,
7.714063071480374,
8.919990325941168,
8.248087209461895,
9.285694310590683,
14.386049072365926,
11.456127235124493
],
"layer_snrs": [
0.9793584804934982,
0.8763143955445245,
0.8415403359194379,
0.8880745215498504,
0.8936372075488783,
0.8988820321930456,
0.8455459279067038,
0.9174679267556224,
0.8857812187230721,
0.8881125626920003,
0.8804814500507204,
0.8686871886734093,
0.8695513110118298,
0.8388153746485172,
0.8515448048796498,
0.862703839852447,
0.8196578026448774,
0.8623651498475126,
0.9022014077477166,
0.9215660790087216,
0.8654169316595541,
0.9131334971786473,
0.8981551244665501,
0.9361397791786076
]
},
{
"model": "Qwen/Qwen2.5-3B",
"short": "Qwen2.5-3B",
"theta": 1000000,
"T_train": 8192,
"n_heads": 16,
"n_kv": 2,
"d_head": 128,
"n_layers": 36,
"k_dead": 36,
"n_pairs": 64,
"gamma_rand": 0.964,
"gamma_text": 0.772,
"snr_global": 0.8806074396662721,
"snr_corrected": 0.8806074396662721,
"layer_snr_mean": 0.8779535893465105,
"mean_band_energies": [
8.258374764617118,
8.530581014978711,
9.0921296208914,
9.74542850871339,
10.371327689217063,
11.261751124148692,
11.71900305074188,
12.581573111282864,
12.948883425581554,
13.546673103479739,
14.056310061908352,
15.298073682638128,
15.733690931548734,
16.799295038545257,
17.732245044596723,
18.440038135491903,
19.06599694553063,
19.94782929088912,
21.022966521857015,
21.640077361154017,
22.013505968683077,
23.746046001497056,
23.814673884287956,
24.382287336727906,
24.793978182098225,
24.047625035665675,
25.73469192577065,
25.090471547866112,
25.08550830014289,
23.762961901493114,
23.346760990650456,
23.34272611501102,
8.114272662849654,
8.503591329382498,
9.29552959483764,
10.02337132649432,
10.755936938695564,
11.43039146612689,
11.35037791111829,
12.920628461773065,
12.937383417807848,
14.074980798794932,
13.948108792425352,
15.41086812106611,
15.696963239944687,
16.754686477587907,
17.442182821265224,
18.050304410411844,
19.205301634052635,
20.025494927580738,
20.888260829595882,
21.874271581998087,
22.33561786918985,
23.59895934718369,
23.608095262430194,
24.358027585563537,
24.255542115442992,
24.385263006623724,
25.257572677137315,
25.08432064967099,
24.191150739594768,
24.077838579577005,
23.47843496437321,
23.92530407936351
],
"layer_snrs": [
0.8956451721489964,
0.9294661349376346,
0.9374797848466799,
0.8784490078780575,
0.8986553151908518,
0.9111804532089779,
0.8444166244245765,
0.9019454843726556,
0.9013127858909121,
0.9266468106212808,
0.8988197360501505,
0.8886783439886468,
0.8459588558832479,
0.8605977438740229,
0.8742748593403651,
0.872370043078339,
0.8812653405373329,
0.8655064910830728,
0.8935768831052082,
0.8400306349943036,
0.8228604450323329,
0.8409152894854571,
0.841861232954458,
0.8817057334009857,
0.8694221842535114,
0.8249895147391799,
0.8843177330393549,
0.8490698111475532,
0.8856938116930246,
0.8692375161200255,
0.8678331376707393,
0.8974064675935306,
0.859287875029756,
0.8356666953169407,
0.9136962459148364,
0.916089017627379
]
}
]
} |