| -- Results | |
| idx | parts | d. bits | d. KL-div | |
| ---------|----------------------------------------------------------------------|----------------------|----------- | |
| - | | | 0.07912 | |
| 0 1 | 0 self_attn q | 62,914,560 | -0.00014 | |
| 0 2 | 0 self_attn q | 125,829,120 | -0.00009 | |
| 0 3 | 0 self_attn q | 188,743,680 | +0.00003 | |
| 0 4 | 0 self_attn q | 314,572,800 | -0.00010 | |
| 1 1 | 0 self_attn [k, v] | 10,485,760 | -0.00005 | |
| 1 2 | 0 self_attn [k, v] | 20,971,520 | -0.00006 | |
| 1 3 | 0 self_attn [k, v] | 31,457,280 | +0.00004 | |
| 1 4 | 0 self_attn [k, v] | 52,428,800 | -0.00007 | |
| 2 1 | 0 self_attn o | 62,914,560 | -0.00040 | |
| 2 2 | 0 self_attn o | 125,829,120 | -0.00022 | |
| 2 3 | 0 self_attn o | 188,743,680 | -0.00015 | |
| 2 4 | 0 self_attn o | 314,572,800 | -0.00023 | |
| 3 1 | 0 mlp [gate, up] | 125,829,120 | -0.00044 | |
| 3 2 | 0 mlp [gate, up] | 251,658,240 | -0.00052 | |
| 3 3 | 0 mlp [gate, up] | 377,487,360 | -0.00053 | |
| 3 4 | 0 mlp [gate, up] | 629,145,600 | -0.00054 | |
| 4 1 | 0 mlp down | 62,914,560 | -0.00036 | |
| 4 2 | 0 mlp down | 125,829,120 | -0.00018 | |
| 4 3 | 0 mlp down | 188,743,680 | -0.00024 | |
| 4 4 | 0 mlp down | 314,572,800 | -0.00028 | |
| 5 1 | 1 self_attn q | 62,914,560 | -0.00008 | |
| 5 2 | 1 self_attn q | 125,829,120 | +0.00000 | |
| 5 3 | 1 self_attn q | 188,743,680 | +0.00008 | |
| 5 4 | 1 self_attn q | 314,572,800 | -0.00003 | |
| 6 1 | 1 self_attn [k, v] | 10,485,760 | -0.00012 | |
| 6 2 | 1 self_attn [k, v] | 20,971,520 | -0.00034 | |
| 6 3 | 1 self_attn [k, v] | 31,457,280 | -0.00022 | |
| 6 4 | 1 self_attn [k, v] | 52,428,800 | -0.00012 | |
| 7 1 | 1 self_attn o | 62,914,560 | +0.00001 | |
| 7 2 | 1 self_attn o | 125,829,120 | +0.00007 | |
| 7 3 | 1 self_attn o | 188,743,680 | -0.00012 | |
| 7 4 | 1 self_attn o | 314,572,800 | +0.00000 | |
| 8 1 | 1 mlp [gate, up] | 125,829,120 | -0.00024 | |
| 8 2 | 1 mlp [gate, up] | 251,658,240 | -0.00037 | |
| 8 3 | 1 mlp [gate, up] | 377,487,360 | -0.00039 | |
| 8 4 | 1 mlp [gate, up] | 629,145,600 | -0.00040 | |
| 9 1 | 1 mlp down | 62,914,560 | +0.00003 | |
| 9 2 | 1 mlp down | 125,829,120 | -0.00005 | |
| 9 3 | 1 mlp down | 188,743,680 | -0.00005 | |
| 9 4 | 1 mlp down | 314,572,800 | +0.00004 | |
| 10 1 | 2 self_attn q | 62,914,560 | +0.00001 | |
| 10 2 | 2 self_attn q | 125,829,120 | -0.00007 | |
| 10 3 | 2 self_attn q | 188,743,680 | -0.00009 | |
| 10 4 | 2 self_attn q | 314,572,800 | -0.00010 | |
| 11 1 | 2 self_attn [k, v] | 10,485,760 | -0.00062 | |
| 11 2 | 2 self_attn [k, v] | 20,971,520 | -0.00047 | |
| 11 3 | 2 self_attn [k, v] | 31,457,280 | -0.00038 | |
| 11 4 | 2 self_attn [k, v] | 52,428,800 | -0.00046 | |
| 12 1 | 2 self_attn o | 62,914,560 | -0.00006 | |
| 12 2 | 2 self_attn o | 125,829,120 | +0.00002 | |
| 12 3 | 2 self_attn o | 188,743,680 | +0.00014 | |
| 12 4 | 2 self_attn o | 314,572,800 | -0.00012 | |
| 13 1 | 2 mlp [gate, up] | 125,829,120 | -0.00008 | |
| 13 2 | 2 mlp [gate, up] | 251,658,240 | -0.00058 | |
| 13 3 | 2 mlp [gate, up] | 377,487,360 | -0.00064 | |
| 13 4 | 2 mlp [gate, up] | 629,145,600 | -0.00061 | |
| 14 1 | 2 mlp down | 62,914,560 | -0.00062 | |
| 14 2 | 2 mlp down | 125,829,120 | -0.00037 | |
| 14 3 | 2 mlp down | 188,743,680 | -0.00028 | |
| 14 4 | 2 mlp down | 314,572,800 | -0.00035 | |
| 15 1 | 3 self_attn q | 62,914,560 | -0.00025 | |
| 15 2 | 3 self_attn q | 125,829,120 | -0.00012 | |
| 15 3 | 3 self_attn q | 188,743,680 | -0.00027 | |
| 15 4 | 3 self_attn q | 314,572,800 | -0.00017 | |
| 16 1 | 3 self_attn [k, v] | 10,485,760 | -0.00060 | |
| 16 2 | 3 self_attn [k, v] | 20,971,520 | -0.00071 | |
| 16 3 | 3 self_attn [k, v] | 31,457,280 | -0.00056 | |
| 16 4 | 3 self_attn [k, v] | 52,428,800 | -0.00047 | |
| 17 1 | 3 self_attn o | 62,914,560 | -0.00006 | |
| 17 2 | 3 self_attn o | 125,829,120 | -0.00007 | |
| 17 3 | 3 self_attn o | 188,743,680 | +0.00000 | |
| 17 4 | 3 self_attn o | 314,572,800 | +0.00003 | |
| 18 1 | 3 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00031 | |
| 18 2 | 3 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00011 | |
| 18 3 | 3 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00016 | |
| 18 4 | 3 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00015 | |
| 19 1 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00042 | |
| 19 2 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00027 | |
| 19 3 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00030 | |
| 19 4 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00032 | |
| 20 1 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00019 | |
| 20 2 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00020 | |
| 20 3 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00028 | |
| 20 4 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00020 | |
| 21 1 | 4 self_attn q | 62,914,560 | +0.00006 | |
| 21 2 | 4 self_attn q | 125,829,120 | +0.00013 | |
| 21 3 | 4 self_attn q | 188,743,680 | -0.00003 | |
| 21 4 | 4 self_attn q | 314,572,800 | +0.00012 | |
| 22 1 | 4 self_attn [k, v] | 10,485,760 | +0.00022 | |
| 22 2 | 4 self_attn [k, v] | 20,971,520 | +0.00011 | |
| 22 3 | 4 self_attn [k, v] | 31,457,280 | +0.00009 | |
| 22 4 | 4 self_attn [k, v] | 52,428,800 | +0.00014 | |
| 23 1 | 4 self_attn o | 62,914,560 | -0.00001 | |
| 23 2 | 4 self_attn o | 125,829,120 | +0.00003 | |
| 23 3 | 4 self_attn o | 188,743,680 | +0.00015 | |
| 23 4 | 4 self_attn o | 314,572,800 | +0.00009 | |
| 24 1 | 4 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 | |
| 24 2 | 4 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00036 | |
| 24 3 | 4 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00022 | |
| 24 4 | 4 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00040 | |
| 25 1 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00018 | |
| 25 2 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00031 | |
| 25 3 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00043 | |
| 25 4 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00032 | |
| 26 1 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00026 | |
| 26 2 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00026 | |
| 26 3 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00040 | |
| 26 4 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00030 | |
| 27 1 | 5 self_attn q | 62,914,560 | -0.00009 | |
| 27 2 | 5 self_attn q | 125,829,120 | -0.00008 | |
| 27 3 | 5 self_attn q | 188,743,680 | -0.00020 | |
| 27 4 | 5 self_attn q | 314,572,800 | -0.00015 | |
| 28 1 | 5 self_attn [k, v] | 10,485,760 | +0.00007 | |
| 28 2 | 5 self_attn [k, v] | 20,971,520 | +0.00015 | |
| 28 3 | 5 self_attn [k, v] | 31,457,280 | +0.00006 | |
| 28 4 | 5 self_attn [k, v] | 52,428,800 | +0.00007 | |
| 29 1 | 5 self_attn o | 62,914,560 | -0.00026 | |
| 29 2 | 5 self_attn o | 125,829,120 | -0.00013 | |
| 29 3 | 5 self_attn o | 188,743,680 | -0.00028 | |
| 29 4 | 5 self_attn o | 314,572,800 | -0.00020 | |
| 30 1 | 5 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00048 | |
| 30 2 | 5 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00034 | |
| 30 3 | 5 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00034 | |
| 30 4 | 5 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00049 | |
| 31 1 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00020 | |
| 31 2 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00015 | |
| 31 3 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00013 | |
| 31 4 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00024 | |
| 32 1 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00021 | |
| 32 2 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00025 | |
| 32 3 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00022 | |
| 32 4 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00030 | |
| 33 1 | 6 self_attn q | 62,914,560 | +0.00010 | |
| 33 2 | 6 self_attn q | 125,829,120 | +0.00016 | |
| 33 3 | 6 self_attn q | 188,743,680 | +0.00016 | |
| 33 4 | 6 self_attn q | 314,572,800 | +0.00015 | |
| 34 1 | 6 self_attn [k, v] | 10,485,760 | -0.00018 | |
| 34 2 | 6 self_attn [k, v] | 20,971,520 | -0.00023 | |
| 34 3 | 6 self_attn [k, v] | 31,457,280 | -0.00020 | |
| 34 4 | 6 self_attn [k, v] | 52,428,800 | -0.00012 | |
| 35 1 | 6 self_attn o | 62,914,560 | -0.00030 | |
| 35 2 | 6 self_attn o | 125,829,120 | -0.00031 | |
| 35 3 | 6 self_attn o | 188,743,680 | -0.00030 | |
| 35 4 | 6 self_attn o | 314,572,800 | -0.00038 | |
| 36 1 | 6 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009 | |
| 36 2 | 6 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022 | |
| 36 3 | 6 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00037 | |
| 36 4 | 6 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00031 | |
| 37 1 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00027 | |
| 37 2 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00024 | |
| 37 3 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00036 | |
| 37 4 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00023 | |
| 38 1 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00013 | |
| 38 2 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00043 | |
| 38 3 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00035 | |
| 38 4 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00035 | |
| 39 1 | 7 self_attn q | 62,914,560 | -0.00029 | |
| 39 2 | 7 self_attn q | 125,829,120 | -0.00034 | |
| 39 3 | 7 self_attn q | 188,743,680 | -0.00032 | |
| 39 4 | 7 self_attn q | 314,572,800 | -0.00030 | |
| 40 1 | 7 self_attn [k, v] | 10,485,760 | -0.00028 | |
| 40 2 | 7 self_attn [k, v] | 20,971,520 | -0.00004 | |
| 40 3 | 7 self_attn [k, v] | 31,457,280 | -0.00013 | |
| 40 4 | 7 self_attn [k, v] | 52,428,800 | -0.00014 | |
| 41 1 | 7 self_attn o | 62,914,560 | -0.00011 | |
| 41 2 | 7 self_attn o | 125,829,120 | -0.00023 | |
| 41 3 | 7 self_attn o | 188,743,680 | -0.00024 | |
| 41 4 | 7 self_attn o | 314,572,800 | -0.00031 | |
| 42 1 | 7 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00010 | |
| 42 2 | 7 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00045 | |
| 42 3 | 7 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00028 | |
| 42 4 | 7 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00032 | |
| 43 1 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00028 | |
| 43 2 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00033 | |
| 43 3 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00046 | |
| 43 4 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00049 | |
| 44 1 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00040 | |
| 44 2 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00039 | |
| 44 3 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00035 | |
| 44 4 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00036 | |
| 45 1 | 8 self_attn q | 62,914,560 | -0.00017 | |
| 45 2 | 8 self_attn q | 125,829,120 | -0.00026 | |
| 45 3 | 8 self_attn q | 188,743,680 | -0.00027 | |
| 45 4 | 8 self_attn q | 314,572,800 | -0.00019 | |
| 46 1 | 8 self_attn [k, v] | 10,485,760 | +0.00086 | |
| 46 2 | 8 self_attn [k, v] | 20,971,520 | +0.00031 | |
| 46 3 | 8 self_attn [k, v] | 31,457,280 | +0.00027 | |
| 46 4 | 8 self_attn [k, v] | 52,428,800 | +0.00030 | |
| 47 1 | 8 self_attn o | 62,914,560 | -0.00128 | |
| 47 2 | 8 self_attn o | 125,829,120 | -0.00105 | |
| 47 3 | 8 self_attn o | 188,743,680 | -0.00109 | |
| 47 4 | 8 self_attn o | 314,572,800 | -0.00101 | |
| 48 1 | 8 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00019 | |
| 48 2 | 8 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00013 | |
| 48 3 | 8 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00005 | |
| 48 4 | 8 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00013 | |
| 49 1 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00037 | |
| 49 2 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00023 | |
| 49 3 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00036 | |
| 49 4 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00028 | |
| 50 1 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00026 | |
| 50 2 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00030 | |
| 50 3 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00031 | |
| 50 4 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00032 | |
| 51 1 | 9 self_attn q | 62,914,560 | -0.00022 | |
| 51 2 | 9 self_attn q | 125,829,120 | -0.00017 | |
| 51 3 | 9 self_attn q | 188,743,680 | -0.00029 | |
| 51 4 | 9 self_attn q | 314,572,800 | -0.00033 | |
| 52 1 | 9 self_attn [k, v] | 10,485,760 | -0.00070 | |
| 52 2 | 9 self_attn [k, v] | 20,971,520 | -0.00023 | |
| 52 3 | 9 self_attn [k, v] | 31,457,280 | -0.00047 | |
| 52 4 | 9 self_attn [k, v] | 52,428,800 | -0.00052 | |
| 53 1 | 9 self_attn o | 62,914,560 | -0.00030 | |
| 53 2 | 9 self_attn o | 125,829,120 | -0.00026 | |
| 53 3 | 9 self_attn o | 188,743,680 | -0.00032 | |
| 53 4 | 9 self_attn o | 314,572,800 | -0.00034 | |
| 54 1 | 9 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 | |
| 54 2 | 9 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00007 | |
| 54 3 | 9 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00011 | |
| 54 4 | 9 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00012 | |
| 55 1 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00047 | |
| 55 2 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00037 | |
| 55 3 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00044 | |
| 55 4 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00039 | |
| 56 1 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00025 | |
| 56 2 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00032 | |
| 56 3 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00047 | |
| 56 4 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00036 | |
| 57 1 | 10 self_attn q | 62,914,560 | -0.00030 | |
| 57 2 | 10 self_attn q | 125,829,120 | -0.00026 | |
| 57 3 | 10 self_attn q | 188,743,680 | -0.00017 | |
| 57 4 | 10 self_attn q | 314,572,800 | -0.00021 | |
| 58 1 | 10 self_attn [k, v] | 10,485,760 | -0.00006 | |
| 58 2 | 10 self_attn [k, v] | 20,971,520 | +0.00030 | |
| 58 3 | 10 self_attn [k, v] | 31,457,280 | +0.00009 | |
| 58 4 | 10 self_attn [k, v] | 52,428,800 | +0.00008 | |
| 59 1 | 10 self_attn o | 62,914,560 | +0.00033 | |
| 59 2 | 10 self_attn o | 125,829,120 | +0.00003 | |
| 59 3 | 10 self_attn o | 188,743,680 | +0.00007 | |
| 59 4 | 10 self_attn o | 314,572,800 | +0.00018 | |
| 60 1 | 10 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006 | |
| 60 2 | 10 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00009 | |
| 60 3 | 10 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00005 | |
| 60 4 | 10 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00007 | |
| 61 1 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00020 | |
| 61 2 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00021 | |
| 61 3 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00020 | |
| 61 4 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00029 | |
| 62 1 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 | |
| 62 2 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00037 | |
| 62 3 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00024 | |
| 62 4 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00024 | |
| 63 1 | 11 self_attn q | 62,914,560 | +0.00010 | |
| 63 2 | 11 self_attn q | 125,829,120 | -0.00010 | |
| 63 3 | 11 self_attn q | 188,743,680 | +0.00005 | |
| 63 4 | 11 self_attn q | 314,572,800 | +0.00003 | |
| 64 1 | 11 self_attn [k, v] | 10,485,760 | +0.00122 | |
| 64 2 | 11 self_attn [k, v] | 20,971,520 | +0.00123 | |
| 64 3 | 11 self_attn [k, v] | 31,457,280 | +0.00114 | |
| 64 4 | 11 self_attn [k, v] | 52,428,800 | +0.00095 | |
| 65 1 | 11 self_attn o | 62,914,560 | -0.00109 | |
| 65 2 | 11 self_attn o | 125,829,120 | -0.00174 | |
| 65 3 | 11 self_attn o | 188,743,680 | -0.00166 | |
| 65 4 | 11 self_attn o | 314,572,800 | -0.00153 | |
| 66 1 | 11 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00046 | |
| 66 2 | 11 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00044 | |
| 66 3 | 11 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00031 | |
| 66 4 | 11 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00026 | |
| 67 1 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00042 | |
| 67 2 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00054 | |
| 67 3 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00069 | |
| 67 4 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00069 | |
| 68 1 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00057 | |
| 68 2 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00058 | |
| 68 3 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00064 | |
| 68 4 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00057 | |
| 69 1 | 12 self_attn q | 62,914,560 | +0.00001 | |
| 69 2 | 12 self_attn q | 125,829,120 | +0.00001 | |
| 69 3 | 12 self_attn q | 188,743,680 | +0.00005 | |
| 69 4 | 12 self_attn q | 314,572,800 | +0.00002 | |
| 70 1 | 12 self_attn [k, v] | 10,485,760 | -0.00015 | |
| 70 2 | 12 self_attn [k, v] | 20,971,520 | +0.00026 | |
| 70 3 | 12 self_attn [k, v] | 31,457,280 | +0.00029 | |
| 70 4 | 12 self_attn [k, v] | 52,428,800 | +0.00014 | |
| 71 1 | 12 self_attn o | 62,914,560 | -0.00105 | |
| 71 2 | 12 self_attn o | 125,829,120 | -0.00077 | |
| 71 3 | 12 self_attn o | 188,743,680 | -0.00082 | |
| 71 4 | 12 self_attn o | 314,572,800 | -0.00077 | |
| 72 1 | 12 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00042 | |
| 72 2 | 12 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00010 | |
| 72 3 | 12 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00007 | |
| 72 4 | 12 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00006 | |
| 73 1 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 | |
| 73 2 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00084 | |
| 73 3 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00083 | |
| 73 4 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00079 | |
| 74 1 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00072 | |
| 74 2 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00077 | |
| 74 3 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00086 | |
| 74 4 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00088 | |
| 75 1 | 13 self_attn q | 62,914,560 | -0.00023 | |
| 75 2 | 13 self_attn q | 125,829,120 | -0.00033 | |
| 75 3 | 13 self_attn q | 188,743,680 | -0.00046 | |
| 75 4 | 13 self_attn q | 314,572,800 | -0.00058 | |
| 76 1 | 13 self_attn [k, v] | 10,485,760 | -0.00156 | |
| 76 2 | 13 self_attn [k, v] | 20,971,520 | -0.00126 | |
| 76 3 | 13 self_attn [k, v] | 31,457,280 | -0.00116 | |
| 76 4 | 13 self_attn [k, v] | 52,428,800 | -0.00111 | |
| 77 1 | 13 self_attn o | 62,914,560 | -0.00087 | |
| 77 2 | 13 self_attn o | 125,829,120 | -0.00080 | |
| 77 3 | 13 self_attn o | 188,743,680 | -0.00085 | |
| 77 4 | 13 self_attn o | 314,572,800 | -0.00084 | |
| 78 1 | 13 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00054 | |
| 78 2 | 13 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00032 | |
| 78 3 | 13 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00029 | |
| 78 4 | 13 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00022 | |
| 79 1 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00031 | |
| 79 2 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00051 | |
| 79 3 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00064 | |
| 79 4 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00062 | |
| 80 1 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 | |
| 80 2 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00064 | |
| 80 3 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00066 | |
| 80 4 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00049 | |
| 81 1 | 14 self_attn q | 62,914,560 | -0.00018 | |
| 81 2 | 14 self_attn q | 125,829,120 | -0.00004 | |
| 81 3 | 14 self_attn q | 188,743,680 | -0.00011 | |
| 81 4 | 14 self_attn q | 314,572,800 | -0.00026 | |
| 82 1 | 14 self_attn [k, v] | 10,485,760 | -0.00025 | |
| 82 2 | 14 self_attn [k, v] | 20,971,520 | +0.00013 | |
| 82 3 | 14 self_attn [k, v] | 31,457,280 | +0.00003 | |
| 82 4 | 14 self_attn [k, v] | 52,428,800 | +0.00016 | |
| 83 1 | 14 self_attn o | 62,914,560 | -0.00037 | |
| 83 2 | 14 self_attn o | 125,829,120 | -0.00087 | |
| 83 3 | 14 self_attn o | 188,743,680 | -0.00077 | |
| 83 4 | 14 self_attn o | 314,572,800 | -0.00084 | |
| 84 1 | 14 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004 | |
| 84 2 | 14 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00006 | |
| 84 3 | 14 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00007 | |
| 84 4 | 14 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00019 | |
| 85 1 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 | |
| 85 2 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00096 | |
| 85 3 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00089 | |
| 85 4 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00090 | |
| 86 1 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00038 | |
| 86 2 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00067 | |
| 86 3 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00060 | |
| 86 4 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00075 | |
| 87 1 | 15 self_attn q | 62,914,560 | +0.00008 | |
| 87 2 | 15 self_attn q | 125,829,120 | +0.00005 | |
| 87 3 | 15 self_attn q | 188,743,680 | +0.00004 | |
| 87 4 | 15 self_attn q | 314,572,800 | +0.00005 | |
| 88 1 | 15 self_attn [k, v] | 10,485,760 | +0.00054 | |
| 88 2 | 15 self_attn [k, v] | 20,971,520 | +0.00031 | |
| 88 3 | 15 self_attn [k, v] | 31,457,280 | +0.00037 | |
| 88 4 | 15 self_attn [k, v] | 52,428,800 | +0.00052 | |
| 89 1 | 15 self_attn o | 62,914,560 | -0.00013 | |
| 89 2 | 15 self_attn o | 125,829,120 | -0.00033 | |
| 89 3 | 15 self_attn o | 188,743,680 | -0.00029 | |
| 89 4 | 15 self_attn o | 314,572,800 | -0.00034 | |
| 90 1 | 15 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00017 | |
| 90 2 | 15 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00006 | |
| 90 3 | 15 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00010 | |
| 90 4 | 15 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00013 | |
| 91 1 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00103 | |
| 91 2 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00092 | |
| 91 3 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00110 | |
| 91 4 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00112 | |
| 92 1 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00069 | |
| 92 2 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00066 | |
| 92 3 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00072 | |
| 92 4 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00074 | |
| 93 1 | 16 self_attn q | 62,914,560 | -0.00042 | |
| 93 2 | 16 self_attn q | 125,829,120 | -0.00002 | |
| 93 3 | 16 self_attn q | 188,743,680 | -0.00014 | |
| 93 4 | 16 self_attn q | 314,572,800 | -0.00007 | |
| 94 1 | 16 self_attn [k, v] | 10,485,760 | +0.00047 | |
| 94 2 | 16 self_attn [k, v] | 20,971,520 | +0.00032 | |
| 94 3 | 16 self_attn [k, v] | 31,457,280 | +0.00040 | |
| 94 4 | 16 self_attn [k, v] | 52,428,800 | +0.00027 | |
| 95 1 | 16 self_attn o | 62,914,560 | -0.00011 | |
| 95 2 | 16 self_attn o | 125,829,120 | -0.00021 | |
| 95 3 | 16 self_attn o | 188,743,680 | -0.00017 | |
| 95 4 | 16 self_attn o | 314,572,800 | -0.00013 | |
| 96 1 | 16 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 | |
| 96 2 | 16 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00015 | |
| 96 3 | 16 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00007 | |
| 96 4 | 16 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00008 | |
| 97 1 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00091 | |
| 97 2 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00137 | |
| 97 3 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00142 | |
| 97 4 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00138 | |
| 98 1 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00065 | |
| 98 2 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00103 | |
| 98 3 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00087 | |
| 98 4 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00086 | |
| 99 1 | 17 self_attn q | 62,914,560 | +0.00015 | |
| 99 2 | 17 self_attn q | 125,829,120 | +0.00013 | |
| 99 3 | 17 self_attn q | 188,743,680 | +0.00017 | |
| 99 4 | 17 self_attn q | 314,572,800 | +0.00007 | |
| 100 1 | 17 self_attn [k, v] | 10,485,760 | -0.00075 | |
| 100 2 | 17 self_attn [k, v] | 20,971,520 | -0.00078 | |
| 100 3 | 17 self_attn [k, v] | 31,457,280 | -0.00063 | |
| 100 4 | 17 self_attn [k, v] | 52,428,800 | -0.00062 | |
| 101 1 | 17 self_attn o | 62,914,560 | -0.00036 | |
| 101 2 | 17 self_attn o | 125,829,120 | -0.00032 | |
| 101 3 | 17 self_attn o | 188,743,680 | -0.00036 | |
| 101 4 | 17 self_attn o | 314,572,800 | -0.00032 | |
| 102 1 | 17 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00048 | |
| 102 2 | 17 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00038 | |
| 102 3 | 17 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00039 | |
| 102 4 | 17 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00037 | |
| 103 1 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00100 | |
| 103 2 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00095 | |
| 103 3 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00107 | |
| 103 4 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00115 | |
| 104 1 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00070 | |
| 104 2 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00094 | |
| 104 3 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00094 | |
| 104 4 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00101 | |
| 105 1 | 18 self_attn q | 62,914,560 | -0.00003 | |
| 105 2 | 18 self_attn q | 125,829,120 | +0.00023 | |
| 105 3 | 18 self_attn q | 188,743,680 | +0.00014 | |
| 105 4 | 18 self_attn q | 314,572,800 | +0.00022 | |
| 106 1 | 18 self_attn [k, v] | 10,485,760 | -0.00102 | |
| 106 2 | 18 self_attn [k, v] | 20,971,520 | -0.00091 | |
| 106 3 | 18 self_attn [k, v] | 31,457,280 | -0.00100 | |
| 106 4 | 18 self_attn [k, v] | 52,428,800 | -0.00109 | |
| 107 1 | 18 self_attn o | 62,914,560 | -0.00018 | |
| 107 2 | 18 self_attn o | 125,829,120 | -0.00020 | |
| 107 3 | 18 self_attn o | 188,743,680 | -0.00069 | |
| 107 4 | 18 self_attn o | 314,572,800 | -0.00056 | |
| 108 1 | 18 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00039 | |
| 108 2 | 18 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00037 | |
| 108 3 | 18 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00003 | |
| 108 4 | 18 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00003 | |
| 109 1 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00090 | |
| 109 2 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00097 | |
| 109 3 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00105 | |
| 109 4 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00103 | |
| 110 1 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 | |
| 110 2 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00040 | |
| 110 3 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00053 | |
| 110 4 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00049 | |
| 111 1 | 19 self_attn q | 62,914,560 | +0.00021 | |
| 111 2 | 19 self_attn q | 125,829,120 | -0.00004 | |
| 111 3 | 19 self_attn q | 188,743,680 | +0.00002 | |
| 111 4 | 19 self_attn q | 314,572,800 | -0.00004 | |
| 112 1 | 19 self_attn [k, v] | 10,485,760 | +0.00059 | |
| 112 2 | 19 self_attn [k, v] | 20,971,520 | +0.00079 | |
| 112 3 | 19 self_attn [k, v] | 31,457,280 | +0.00056 | |
| 112 4 | 19 self_attn [k, v] | 52,428,800 | +0.00043 | |
| 113 1 | 19 self_attn o | 62,914,560 | -0.00079 | |
| 113 2 | 19 self_attn o | 125,829,120 | -0.00089 | |
| 113 3 | 19 self_attn o | 188,743,680 | -0.00094 | |
| 113 4 | 19 self_attn o | 314,572,800 | -0.00093 | |
| 114 1 | 19 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00162 | |
| 114 2 | 19 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00098 | |
| 114 3 | 19 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00070 | |
| 114 4 | 19 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00081 | |
| 115 1 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00096 | |
| 115 2 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00105 | |
| 115 3 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00108 | |
| 115 4 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00096 | |
| 116 1 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00065 | |
| 116 2 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00072 | |
| 116 3 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00079 | |
| 116 4 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00083 | |
| 117 1 | 20 self_attn q | 62,914,560 | -0.00000 | |
| 117 2 | 20 self_attn q | 125,829,120 | +0.00016 | |
| 117 3 | 20 self_attn q | 188,743,680 | +0.00019 | |
| 117 4 | 20 self_attn q | 314,572,800 | +0.00018 | |
| 118 1 | 20 self_attn [k, v] | 10,485,760 | +0.00023 | |
| 118 2 | 20 self_attn [k, v] | 20,971,520 | -0.00038 | |
| 118 3 | 20 self_attn [k, v] | 31,457,280 | +0.00006 | |
| 118 4 | 20 self_attn [k, v] | 52,428,800 | +0.00003 | |
| 119 1 | 20 self_attn o | 62,914,560 | +0.00068 | |
| 119 2 | 20 self_attn o | 125,829,120 | -0.00002 | |
| 119 3 | 20 self_attn o | 188,743,680 | +0.00020 | |
| 119 4 | 20 self_attn o | 314,572,800 | +0.00019 | |
| 120 1 | 20 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00098 | |
| 120 2 | 20 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00153 | |
| 120 3 | 20 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00156 | |
| 120 4 | 20 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00159 | |
| 121 1 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00070 | |
| 121 2 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00086 | |
| 121 3 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00091 | |
| 121 4 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00095 | |
| 122 1 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00085 | |
| 122 2 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00084 | |
| 122 3 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00083 | |
| 122 4 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00087 | |
| 123 1 | 21 self_attn q | 62,914,560 | -0.00027 | |
| 123 2 | 21 self_attn q | 125,829,120 | -0.00025 | |
| 123 3 | 21 self_attn q | 188,743,680 | -0.00021 | |
| 123 4 | 21 self_attn q | 314,572,800 | -0.00012 | |
| 124 1 | 21 self_attn [k, v] | 10,485,760 | +0.00026 | |
| 124 2 | 21 self_attn [k, v] | 20,971,520 | +0.00027 | |
| 124 3 | 21 self_attn [k, v] | 31,457,280 | +0.00008 | |
| 124 4 | 21 self_attn [k, v] | 52,428,800 | +0.00019 | |
| 125 1 | 21 self_attn o | 62,914,560 | -0.00022 | |
| 125 2 | 21 self_attn o | 125,829,120 | +0.00026 | |
| 125 3 | 21 self_attn o | 188,743,680 | +0.00025 | |
| 125 4 | 21 self_attn o | 314,572,800 | +0.00029 | |
| 126 1 | 21 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011 | |
| 126 2 | 21 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00072 | |
| 126 3 | 21 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00048 | |
| 126 4 | 21 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00040 | |
| 127 1 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00056 | |
| 127 2 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00089 | |
| 127 3 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00086 | |
| 127 4 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00079 | |
| 128 1 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00017 | |
| 128 2 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00042 | |
| 128 3 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00040 | |
| 128 4 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00036 | |
| 129 1 | 22 self_attn q | 62,914,560 | -0.00027 | |
| 129 2 | 22 self_attn q | 125,829,120 | -0.00029 | |
| 129 3 | 22 self_attn q | 188,743,680 | -0.00038 | |
| 129 4 | 22 self_attn q | 314,572,800 | -0.00037 | |
| 130 1 | 22 self_attn [k, v] | 10,485,760 | +0.00042 | |
| 130 2 | 22 self_attn [k, v] | 20,971,520 | +0.00001 | |
| 130 3 | 22 self_attn [k, v] | 31,457,280 | +0.00037 | |
| 130 4 | 22 self_attn [k, v] | 52,428,800 | +0.00031 | |
| 131 1 | 22 self_attn o | 62,914,560 | -0.00048 | |
| 131 2 | 22 self_attn o | 125,829,120 | -0.00034 | |
| 131 3 | 22 self_attn o | 188,743,680 | -0.00027 | |
| 131 4 | 22 self_attn o | 314,572,800 | -0.00026 | |
| 132 1 | 22 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00094 | |
| 132 2 | 22 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00085 | |
| 132 3 | 22 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00064 | |
| 132 4 | 22 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00058 | |
| 133 1 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00045 | |
| 133 2 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00058 | |
| 133 3 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00056 | |
| 133 4 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00064 | |
| 134 1 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00023 | |
| 134 2 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00033 | |
| 134 3 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00033 | |
| 134 4 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00036 | |
| 135 1 | 23 self_attn q | 62,914,560 | -0.00067 | |
| 135 2 | 23 self_attn q | 125,829,120 | -0.00063 | |
| 135 3 | 23 self_attn q | 188,743,680 | -0.00054 | |
| 135 4 | 23 self_attn q | 314,572,800 | -0.00063 | |
| 136 1 | 23 self_attn [k, v] | 10,485,760 | -0.00013 | |
| 136 2 | 23 self_attn [k, v] | 20,971,520 | -0.00052 | |
| 136 3 | 23 self_attn [k, v] | 31,457,280 | -0.00018 | |
| 136 4 | 23 self_attn [k, v] | 52,428,800 | -0.00035 | |
| 137 1 | 23 self_attn o | 62,914,560 | -0.00050 | |
| 137 2 | 23 self_attn o | 125,829,120 | +0.00007 | |
| 137 3 | 23 self_attn o | 188,743,680 | -0.00016 | |
| 137 4 | 23 self_attn o | 314,572,800 | -0.00019 | |
| 138 1 | 23 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00094 | |
| 138 2 | 23 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00065 | |
| 138 3 | 23 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00069 | |
| 138 4 | 23 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00083 | |
| 139 1 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00082 | |
| 139 2 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00101 | |
| 139 3 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00101 | |
| 139 4 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00094 | |
| 140 1 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 | |
| 140 2 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00046 | |
| 140 3 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00044 | |
| 140 4 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00051 | |
| 141 1 | 24 self_attn q | 62,914,560 | -0.00007 | |
| 141 2 | 24 self_attn q | 125,829,120 | -0.00033 | |
| 141 3 | 24 self_attn q | 188,743,680 | -0.00039 | |
| 141 4 | 24 self_attn q | 314,572,800 | -0.00034 | |
| 142 1 | 24 self_attn [k, v] | 10,485,760 | -0.00082 | |
| 142 2 | 24 self_attn [k, v] | 20,971,520 | -0.00100 | |
| 142 3 | 24 self_attn [k, v] | 31,457,280 | -0.00122 | |
| 142 4 | 24 self_attn [k, v] | 52,428,800 | -0.00114 | |
| 143 1 | 24 self_attn o | 62,914,560 | -0.00023 | |
| 143 2 | 24 self_attn o | 125,829,120 | -0.00007 | |
| 143 3 | 24 self_attn o | 188,743,680 | -0.00037 | |
| 143 4 | 24 self_attn o | 314,572,800 | -0.00027 | |
| 144 1 | 24 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00058 | |
| 144 2 | 24 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00050 | |
| 144 3 | 24 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00073 | |
| 144 4 | 24 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00060 | |
| 145 1 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00105 | |
| 145 2 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00106 | |
| 145 3 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00105 | |
| 145 4 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00102 | |
| 146 1 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00036 | |
| 146 2 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 | |
| 146 3 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00046 | |
| 146 4 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00057 | |
| 147 1 | 25 self_attn q | 62,914,560 | +0.00065 | |
| 147 2 | 25 self_attn q | 125,829,120 | +0.00064 | |
| 147 3 | 25 self_attn q | 188,743,680 | +0.00071 | |
| 147 4 | 25 self_attn q | 314,572,800 | +0.00071 | |
| 148 1 | 25 self_attn [k, v] | 10,485,760 | +0.00041 | |
| 148 2 | 25 self_attn [k, v] | 20,971,520 | +0.00035 | |
| 148 3 | 25 self_attn [k, v] | 31,457,280 | +0.00044 | |
| 148 4 | 25 self_attn [k, v] | 52,428,800 | +0.00036 | |
| 149 1 | 25 self_attn o | 62,914,560 | -0.00089 | |
| 149 2 | 25 self_attn o | 125,829,120 | -0.00086 | |
| 149 3 | 25 self_attn o | 188,743,680 | -0.00074 | |
| 149 4 | 25 self_attn o | 314,572,800 | -0.00084 | |
| 150 1 | 25 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00115 | |
| 150 2 | 25 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00084 | |
| 150 3 | 25 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00093 | |
| 150 4 | 25 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00084 | |
| 151 1 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00041 | |
| 151 2 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00066 | |
| 151 3 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00069 | |
| 151 4 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00073 | |
| 152 1 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00043 | |
| 152 2 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00060 | |
| 152 3 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00054 | |
| 152 4 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00050 | |
| 153 1 | 26 self_attn q | 62,914,560 | -0.00035 | |
| 153 2 | 26 self_attn q | 125,829,120 | -0.00033 | |
| 153 3 | 26 self_attn q | 188,743,680 | -0.00045 | |
| 153 4 | 26 self_attn q | 314,572,800 | -0.00041 | |
| 154 1 | 26 self_attn [k, v] | 10,485,760 | +0.00018 | |
| 154 2 | 26 self_attn [k, v] | 20,971,520 | -0.00017 | |
| 154 3 | 26 self_attn [k, v] | 31,457,280 | -0.00029 | |
| 154 4 | 26 self_attn [k, v] | 52,428,800 | -0.00026 | |
| 155 1 | 26 self_attn o | 62,914,560 | -0.00035 | |
| 155 2 | 26 self_attn o | 125,829,120 | -0.00106 | |
| 155 3 | 26 self_attn o | 188,743,680 | -0.00104 | |
| 155 4 | 26 self_attn o | 314,572,800 | -0.00103 | |
| 156 1 | 26 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00090 | |
| 156 2 | 26 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00073 | |
| 156 3 | 26 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00065 | |
| 156 4 | 26 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00068 | |
| 157 1 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 | |
| 157 2 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00085 | |
| 157 3 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00077 | |
| 157 4 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00087 | |
| 158 1 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00035 | |
| 158 2 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00044 | |
| 158 3 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00043 | |
| 158 4 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00043 | |
| 159 1 | 27 self_attn q | 62,914,560 | -0.00028 | |
| 159 2 | 27 self_attn q | 125,829,120 | -0.00024 | |
| 159 3 | 27 self_attn q | 188,743,680 | -0.00022 | |
| 159 4 | 27 self_attn q | 314,572,800 | -0.00035 | |
| 160 1 | 27 self_attn [k, v] | 10,485,760 | -0.00073 | |
| 160 2 | 27 self_attn [k, v] | 20,971,520 | -0.00092 | |
| 160 3 | 27 self_attn [k, v] | 31,457,280 | -0.00095 | |
| 160 4 | 27 self_attn [k, v] | 52,428,800 | -0.00090 | |
| 161 1 | 27 self_attn o | 62,914,560 | -0.00115 | |
| 161 2 | 27 self_attn o | 125,829,120 | -0.00110 | |
| 161 3 | 27 self_attn o | 188,743,680 | -0.00145 | |
| 161 4 | 27 self_attn o | 314,572,800 | -0.00134 | |
| 162 1 | 27 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00105 | |
| 162 2 | 27 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00132 | |
| 162 3 | 27 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00127 | |
| 162 4 | 27 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00128 | |
| 163 1 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00055 | |
| 163 2 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00056 | |
| 163 3 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00055 | |
| 163 4 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00054 | |
| 164 1 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00037 | |
| 164 2 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00052 | |
| 164 3 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00056 | |
| 164 4 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00051 | |
| 165 1 | 28 self_attn q | 62,914,560 | -0.00011 | |
| 165 2 | 28 self_attn q | 125,829,120 | -0.00024 | |
| 165 3 | 28 self_attn q | 188,743,680 | -0.00023 | |
| 165 4 | 28 self_attn q | 314,572,800 | -0.00028 | |
| 166 1 | 28 self_attn [k, v] | 10,485,760 | +0.00080 | |
| 166 2 | 28 self_attn [k, v] | 20,971,520 | +0.00048 | |
| 166 3 | 28 self_attn [k, v] | 31,457,280 | +0.00031 | |
| 166 4 | 28 self_attn [k, v] | 52,428,800 | +0.00038 | |
| 167 1 | 28 self_attn o | 62,914,560 | -0.00065 | |
| 167 2 | 28 self_attn o | 125,829,120 | -0.00095 | |
| 167 3 | 28 self_attn o | 188,743,680 | -0.00106 | |
| 167 4 | 28 self_attn o | 314,572,800 | -0.00104 | |
| 168 1 | 28 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00038 | |
| 168 2 | 28 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00026 | |
| 168 3 | 28 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00003 | |
| 168 4 | 28 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00022 | |
| 169 1 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00087 | |
| 169 2 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00083 | |
| 169 3 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00094 | |
| 169 4 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00095 | |
| 170 1 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00033 | |
| 170 2 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 | |
| 170 3 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00042 | |
| 170 4 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00047 | |
| 171 1 | 29 self_attn q | 62,914,560 | -0.00006 | |
| 171 2 | 29 self_attn q | 125,829,120 | +0.00003 | |
| 171 3 | 29 self_attn q | 188,743,680 | +0.00005 | |
| 171 4 | 29 self_attn q | 314,572,800 | +0.00019 | |
| 172 1 | 29 self_attn [k, v] | 10,485,760 | -0.00055 | |
| 172 2 | 29 self_attn [k, v] | 20,971,520 | -0.00008 | |
| 172 3 | 29 self_attn [k, v] | 31,457,280 | -0.00048 | |
| 172 4 | 29 self_attn [k, v] | 52,428,800 | -0.00042 | |
| 173 1 | 29 self_attn o | 62,914,560 | -0.00040 | |
| 173 2 | 29 self_attn o | 125,829,120 | -0.00062 | |
| 173 3 | 29 self_attn o | 188,743,680 | -0.00056 | |
| 173 4 | 29 self_attn o | 314,572,800 | -0.00057 | |
| 174 1 | 29 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00091 | |
| 174 2 | 29 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00076 | |
| 174 3 | 29 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00066 | |
| 174 4 | 29 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00072 | |
| 175 1 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00069 | |
| 175 2 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00057 | |
| 175 3 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00071 | |
| 175 4 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00076 | |
| 176 1 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00037 | |
| 176 2 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 | |
| 176 3 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00043 | |
| 176 4 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00052 | |
| 177 1 | 30 self_attn q | 62,914,560 | -0.00054 | |
| 177 2 | 30 self_attn q | 125,829,120 | -0.00042 | |
| 177 3 | 30 self_attn q | 188,743,680 | -0.00054 | |
| 177 4 | 30 self_attn q | 314,572,800 | -0.00059 | |
| 178 1 | 30 self_attn [k, v] | 10,485,760 | +0.00031 | |
| 178 2 | 30 self_attn [k, v] | 20,971,520 | +0.00004 | |
| 178 3 | 30 self_attn [k, v] | 31,457,280 | -0.00011 | |
| 178 4 | 30 self_attn [k, v] | 52,428,800 | +0.00000 | |
| 179 1 | 30 self_attn o | 62,914,560 | +0.00012 | |
| 179 2 | 30 self_attn o | 125,829,120 | +0.00010 | |
| 179 3 | 30 self_attn o | 188,743,680 | +0.00016 | |
| 179 4 | 30 self_attn o | 314,572,800 | +0.00015 | |
| 180 1 | 30 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00117 | |
| 180 2 | 30 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00137 | |
| 180 3 | 30 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00110 | |
| 180 4 | 30 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00120 | |
| 181 1 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00064 | |
| 181 2 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00072 | |
| 181 3 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00077 | |
| 181 4 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00079 | |
| 182 1 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 | |
| 182 2 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00029 | |
| 182 3 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00035 | |
| 182 4 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00034 | |
| 183 1 | 31 self_attn q | 62,914,560 | +0.00057 | |
| 183 2 | 31 self_attn q | 125,829,120 | +0.00081 | |
| 183 3 | 31 self_attn q | 188,743,680 | +0.00078 | |
| 183 4 | 31 self_attn q | 314,572,800 | +0.00087 | |
| 184 1 | 31 self_attn [k, v] | 10,485,760 | +0.00164 | |
| 184 2 | 31 self_attn [k, v] | 20,971,520 | +0.00088 | |
| 184 3 | 31 self_attn [k, v] | 31,457,280 | +0.00136 | |
| 184 4 | 31 self_attn [k, v] | 52,428,800 | +0.00115 | |
| 185 1 | 31 self_attn o | 62,914,560 | -0.00012 | |
| 185 2 | 31 self_attn o | 125,829,120 | -0.00022 | |
| 185 3 | 31 self_attn o | 188,743,680 | +0.00026 | |
| 185 4 | 31 self_attn o | 314,572,800 | +0.00032 | |
| 186 1 | 31 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00199 | |
| 186 2 | 31 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00063 | |
| 186 3 | 31 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00042 | |
| 186 4 | 31 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00051 | |
| 187 1 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00064 | |
| 187 2 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00092 | |
| 187 3 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00091 | |
| 187 4 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00098 | |
| 188 1 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007 | |
| 188 2 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00040 | |
| 188 3 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00040 | |
| 188 4 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00043 | |
| 189 1 | 32 self_attn q | 62,914,560 | +0.00057 | |
| 189 2 | 32 self_attn q | 125,829,120 | +0.00062 | |
| 189 3 | 32 self_attn q | 188,743,680 | +0.00048 | |
| 189 4 | 32 self_attn q | 314,572,800 | +0.00056 | |
| 190 1 | 32 self_attn [k, v] | 10,485,760 | -0.00001 | |
| 190 2 | 32 self_attn [k, v] | 20,971,520 | +0.00041 | |
| 190 3 | 32 self_attn [k, v] | 31,457,280 | +0.00018 | |
| 190 4 | 32 self_attn [k, v] | 52,428,800 | -0.00014 | |
| 191 1 | 32 self_attn o | 62,914,560 | -0.00070 | |
| 191 2 | 32 self_attn o | 125,829,120 | -0.00018 | |
| 191 3 | 32 self_attn o | 188,743,680 | -0.00002 | |
| 191 4 | 32 self_attn o | 314,572,800 | -0.00003 | |
| 192 1 | 32 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00119 | |
| 192 2 | 32 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00164 | |
| 192 3 | 32 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00145 | |
| 192 4 | 32 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00152 | |
| 193 1 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 | |
| 193 2 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00038 | |
| 193 3 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00036 | |
| 193 4 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00043 | |
| 194 1 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00030 | |
| 194 2 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00044 | |
| 194 3 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00049 | |
| 194 4 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00052 | |
| 195 1 | 33 self_attn q | 62,914,560 | -0.00042 | |
| 195 2 | 33 self_attn q | 125,829,120 | -0.00040 | |
| 195 3 | 33 self_attn q | 188,743,680 | -0.00017 | |
| 195 4 | 33 self_attn q | 314,572,800 | -0.00033 | |
| 196 1 | 33 self_attn [k, v] | 10,485,760 | -0.00040 | |
| 196 2 | 33 self_attn [k, v] | 20,971,520 | -0.00042 | |
| 196 3 | 33 self_attn [k, v] | 31,457,280 | -0.00037 | |
| 196 4 | 33 self_attn [k, v] | 52,428,800 | -0.00047 | |
| 197 1 | 33 self_attn o | 62,914,560 | +0.00082 | |
| 197 2 | 33 self_attn o | 125,829,120 | +0.00038 | |
| 197 3 | 33 self_attn o | 188,743,680 | +0.00074 | |
| 197 4 | 33 self_attn o | 314,572,800 | +0.00070 | |
| 198 1 | 33 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00144 | |
| 198 2 | 33 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00200 | |
| 198 3 | 33 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00163 | |
| 198 4 | 33 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00139 | |
| 199 1 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00027 | |
| 199 2 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00013 | |
| 199 3 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00003 | |
| 199 4 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | +0.00009 | |
| 200 1 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 | |
| 200 2 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00025 | |
| 200 3 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00032 | |
| 200 4 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00030 | |
| 201 1 | 34 self_attn q | 62,914,560 | +0.00033 | |
| 201 2 | 34 self_attn q | 125,829,120 | +0.00000 | |
| 201 3 | 34 self_attn q | 188,743,680 | -0.00011 | |
| 201 4 | 34 self_attn q | 314,572,800 | -0.00005 | |
| 202 1 | 34 self_attn [k, v] | 10,485,760 | +0.00022 | |
| 202 2 | 34 self_attn [k, v] | 20,971,520 | -0.00011 | |
| 202 3 | 34 self_attn [k, v] | 31,457,280 | -0.00006 | |
| 202 4 | 34 self_attn [k, v] | 52,428,800 | -0.00014 | |
| 203 1 | 34 self_attn o | 62,914,560 | -0.00080 | |
| 203 2 | 34 self_attn o | 125,829,120 | -0.00064 | |
| 203 3 | 34 self_attn o | 188,743,680 | -0.00044 | |
| 203 4 | 34 self_attn o | 314,572,800 | -0.00059 | |
| 204 1 | 34 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00241 | |
| 204 2 | 34 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00285 | |
| 204 3 | 34 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00300 | |
| 204 4 | 34 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00300 | |
| 205 1 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00084 | |
| 205 2 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00105 | |
| 205 3 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00090 | |
| 205 4 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00099 | |
| 206 1 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00040 | |
| 206 2 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00041 | |
| 206 3 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00053 | |
| 206 4 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00053 | |
| 207 1 | 35 self_attn q | 62,914,560 | -0.00040 | |
| 207 2 | 35 self_attn q | 125,829,120 | -0.00061 | |
| 207 3 | 35 self_attn q | 188,743,680 | -0.00049 | |
| 207 4 | 35 self_attn q | 314,572,800 | -0.00049 | |
| 208 1 | 35 self_attn [k, v] | 10,485,760 | -0.00090 | |
| 208 2 | 35 self_attn [k, v] | 20,971,520 | -0.00092 | |
| 208 3 | 35 self_attn [k, v] | 31,457,280 | -0.00075 | |
| 208 4 | 35 self_attn [k, v] | 52,428,800 | -0.00078 | |
| 209 1 | 35 self_attn o | 62,914,560 | -0.00054 | |
| 209 2 | 35 self_attn o | 125,829,120 | -0.00082 | |
| 209 3 | 35 self_attn o | 188,743,680 | -0.00056 | |
| 209 4 | 35 self_attn o | 314,572,800 | -0.00076 | |
| 210 1 | 35 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00015 | |
| 210 2 | 35 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016 | |
| 210 3 | 35 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00018 | |
| 210 4 | 35 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00006 | |
| 211 1 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00088 | |
| 211 2 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00108 | |
| 211 3 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00118 | |
| 211 4 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00120 | |
| 212 1 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00053 | |
| 212 2 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00071 | |
| 212 3 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00077 | |
| 212 4 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00080 | |
| 213 1 | 36 self_attn q | 62,914,560 | +0.00103 | |
| 213 2 | 36 self_attn q | 125,829,120 | +0.00090 | |
| 213 3 | 36 self_attn q | 188,743,680 | +0.00064 | |
| 213 4 | 36 self_attn q | 314,572,800 | +0.00064 | |
| 214 1 | 36 self_attn [k, v] | 10,485,760 | -0.00008 | |
| 214 2 | 36 self_attn [k, v] | 20,971,520 | -0.00176 | |
| 214 3 | 36 self_attn [k, v] | 31,457,280 | -0.00158 | |
| 214 4 | 36 self_attn [k, v] | 52,428,800 | -0.00160 | |
| 215 1 | 36 self_attn o | 62,914,560 | -0.00210 | |
| 215 2 | 36 self_attn o | 125,829,120 | -0.00219 | |
| 215 3 | 36 self_attn o | 188,743,680 | -0.00214 | |
| 215 4 | 36 self_attn o | 314,572,800 | -0.00225 | |
| 216 1 | 36 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00170 | |
| 216 2 | 36 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00136 | |
| 216 3 | 36 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00144 | |
| 216 4 | 36 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00137 | |
| 217 1 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00023 | |
| 217 2 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00053 | |
| 217 3 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00060 | |
| 217 4 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00061 | |
| 218 1 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 | |
| 218 2 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00036 | |
| 218 3 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00051 | |
| 218 4 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00051 | |
| 219 1 | 37 self_attn q | 62,914,560 | -0.00078 | |
| 219 2 | 37 self_attn q | 125,829,120 | -0.00057 | |
| 219 3 | 37 self_attn q | 188,743,680 | -0.00060 | |
| 219 4 | 37 self_attn q | 314,572,800 | -0.00062 | |
| 220 1 | 37 self_attn [k, v] | 10,485,760 | +0.00030 | |
| 220 2 | 37 self_attn [k, v] | 20,971,520 | -0.00025 | |
| 220 3 | 37 self_attn [k, v] | 31,457,280 | +0.00002 | |
| 220 4 | 37 self_attn [k, v] | 52,428,800 | -0.00029 | |
| 221 1 | 37 self_attn o | 62,914,560 | -0.00185 | |
| 221 2 | 37 self_attn o | 125,829,120 | -0.00134 | |
| 221 3 | 37 self_attn o | 188,743,680 | -0.00124 | |
| 221 4 | 37 self_attn o | 314,572,800 | -0.00116 | |
| 222 1 | 37 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00082 | |
| 222 2 | 37 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00095 | |
| 222 3 | 37 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00077 | |
| 222 4 | 37 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00073 | |
| 223 1 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00015 | |
| 223 2 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 | |
| 223 3 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00026 | |
| 223 4 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00025 | |
| 224 1 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 | |
| 224 2 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 | |
| 224 3 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00020 | |
| 224 4 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00022 | |
| 225 1 | 38 self_attn q | 62,914,560 | -0.00100 | |
| 225 2 | 38 self_attn q | 125,829,120 | -0.00120 | |
| 225 3 | 38 self_attn q | 188,743,680 | -0.00101 | |
| 225 4 | 38 self_attn q | 314,572,800 | -0.00101 | |
| 226 1 | 38 self_attn [k, v] | 10,485,760 | -0.00007 | |
| 226 2 | 38 self_attn [k, v] | 20,971,520 | -0.00013 | |
| 226 3 | 38 self_attn [k, v] | 31,457,280 | -0.00005 | |
| 226 4 | 38 self_attn [k, v] | 52,428,800 | -0.00021 | |
| 227 1 | 38 self_attn o | 62,914,560 | -0.00007 | |
| 227 2 | 38 self_attn o | 125,829,120 | -0.00017 | |
| 227 3 | 38 self_attn o | 188,743,680 | -0.00017 | |
| 227 4 | 38 self_attn o | 314,572,800 | -0.00019 | |
| 228 1 | 38 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 | |
| 228 2 | 38 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00028 | |
| 228 3 | 38 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00043 | |
| 228 4 | 38 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00045 | |
| 229 1 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00040 | |
| 229 2 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00061 | |
| 229 3 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00063 | |
| 229 4 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00060 | |
| 230 1 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 | |
| 230 2 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00034 | |
| 230 3 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00039 | |
| 230 4 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00043 | |
| 231 1 | 39 self_attn q | 62,914,560 | -0.00046 | |
| 231 2 | 39 self_attn q | 125,829,120 | -0.00052 | |
| 231 3 | 39 self_attn q | 188,743,680 | -0.00052 | |
| 231 4 | 39 self_attn q | 314,572,800 | -0.00051 | |
| 232 1 | 39 self_attn [k, v] | 10,485,760 | +0.00050 | |
| 232 2 | 39 self_attn [k, v] | 20,971,520 | -0.00047 | |
| 232 3 | 39 self_attn [k, v] | 31,457,280 | -0.00067 | |
| 232 4 | 39 self_attn [k, v] | 52,428,800 | -0.00052 | |
| 233 1 | 39 self_attn o | 62,914,560 | -0.00100 | |
| 233 2 | 39 self_attn o | 125,829,120 | -0.00038 | |
| 233 3 | 39 self_attn o | 188,743,680 | -0.00060 | |
| 233 4 | 39 self_attn o | 314,572,800 | -0.00053 | |
| 234 1 | 39 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00087 | |
| 234 2 | 39 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00101 | |
| 234 3 | 39 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00109 | |
| 234 4 | 39 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00110 | |
| 235 1 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00045 | |
| 235 2 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00057 | |
| 235 3 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00064 | |
| 235 4 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00067 | |
| 236 1 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013 | |
| 236 2 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00028 | |
| 236 3 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00031 | |
| 236 4 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00026 | |
| 237 1 | 40 self_attn q | 62,914,560 | -0.00051 | |
| 237 2 | 40 self_attn q | 125,829,120 | -0.00034 | |
| 237 3 | 40 self_attn q | 188,743,680 | -0.00038 | |
| 237 4 | 40 self_attn q | 314,572,800 | -0.00032 | |
| 238 1 | 40 self_attn [k, v] | 10,485,760 | -0.00089 | |
| 238 2 | 40 self_attn [k, v] | 20,971,520 | -0.00134 | |
| 238 3 | 40 self_attn [k, v] | 31,457,280 | -0.00095 | |
| 238 4 | 40 self_attn [k, v] | 52,428,800 | -0.00108 | |
| 239 1 | 40 self_attn o | 62,914,560 | +0.00010 | |
| 239 2 | 40 self_attn o | 125,829,120 | -0.00029 | |
| 239 3 | 40 self_attn o | 188,743,680 | -0.00035 | |
| 239 4 | 40 self_attn o | 314,572,800 | -0.00038 | |
| 240 1 | 40 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00051 | |
| 240 2 | 40 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00090 | |
| 240 3 | 40 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00088 | |
| 240 4 | 40 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00087 | |
| 241 1 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00055 | |
| 241 2 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00065 | |
| 241 3 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00059 | |
| 241 4 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00058 | |
| 242 1 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00036 | |
| 242 2 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00058 | |
| 242 3 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00055 | |
| 242 4 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00057 | |
| 243 1 | 41 self_attn q | 62,914,560 | +0.00062 | |
| 243 2 | 41 self_attn q | 125,829,120 | +0.00063 | |
| 243 3 | 41 self_attn q | 188,743,680 | +0.00064 | |
| 243 4 | 41 self_attn q | 314,572,800 | +0.00067 | |
| 244 1 | 41 self_attn [k, v] | 10,485,760 | -0.00110 | |
| 244 2 | 41 self_attn [k, v] | 20,971,520 | -0.00079 | |
| 244 3 | 41 self_attn [k, v] | 31,457,280 | -0.00100 | |
| 244 4 | 41 self_attn [k, v] | 52,428,800 | -0.00100 | |
| 245 1 | 41 self_attn o | 62,914,560 | +0.00059 | |
| 245 2 | 41 self_attn o | 125,829,120 | +0.00076 | |
| 245 3 | 41 self_attn o | 188,743,680 | +0.00074 | |
| 245 4 | 41 self_attn o | 314,572,800 | +0.00065 | |
| 246 1 | 41 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00056 | |
| 246 2 | 41 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00014 | |
| 246 3 | 41 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00039 | |
| 246 4 | 41 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00031 | |
| 247 1 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00074 | |
| 247 2 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00091 | |
| 247 3 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00080 | |
| 247 4 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00075 | |
| 248 1 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00021 | |
| 248 2 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 | |
| 248 3 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00012 | |
| 248 4 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00016 | |
| 249 1 | 42 self_attn q | 62,914,560 | -0.00035 | |
| 249 2 | 42 self_attn q | 125,829,120 | -0.00034 | |
| 249 3 | 42 self_attn q | 188,743,680 | -0.00037 | |
| 249 4 | 42 self_attn q | 314,572,800 | -0.00037 | |
| 250 1 | 42 self_attn [k, v] | 10,485,760 | +0.00023 | |
| 250 2 | 42 self_attn [k, v] | 20,971,520 | -0.00048 | |
| 250 3 | 42 self_attn [k, v] | 31,457,280 | -0.00055 | |
| 250 4 | 42 self_attn [k, v] | 52,428,800 | -0.00053 | |
| 251 1 | 42 self_attn o | 62,914,560 | -0.00097 | |
| 251 2 | 42 self_attn o | 125,829,120 | -0.00076 | |
| 251 3 | 42 self_attn o | 188,743,680 | -0.00077 | |
| 251 4 | 42 self_attn o | 314,572,800 | -0.00077 | |
| 252 1 | 42 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00072 | |
| 252 2 | 42 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00079 | |
| 252 3 | 42 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00064 | |
| 252 4 | 42 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00065 | |
| 253 1 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00038 | |
| 253 2 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00035 | |
| 253 3 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00037 | |
| 253 4 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00043 | |
| 254 1 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 254 2 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00028 | |
| 254 3 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00032 | |
| 254 4 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00029 | |
| 255 1 | 43 self_attn q | 62,914,560 | -0.00023 | |
| 255 2 | 43 self_attn q | 125,829,120 | -0.00029 | |
| 255 3 | 43 self_attn q | 188,743,680 | -0.00034 | |
| 255 4 | 43 self_attn q | 314,572,800 | -0.00038 | |
| 256 1 | 43 self_attn [k, v] | 10,485,760 | -0.00082 | |
| 256 2 | 43 self_attn [k, v] | 20,971,520 | -0.00066 | |
| 256 3 | 43 self_attn [k, v] | 31,457,280 | -0.00075 | |
| 256 4 | 43 self_attn [k, v] | 52,428,800 | -0.00080 | |
| 257 1 | 43 self_attn o | 62,914,560 | -0.00034 | |
| 257 2 | 43 self_attn o | 125,829,120 | -0.00058 | |
| 257 3 | 43 self_attn o | 188,743,680 | -0.00067 | |
| 257 4 | 43 self_attn o | 314,572,800 | -0.00056 | |
| 258 1 | 43 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005 | |
| 258 2 | 43 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00000 | |
| 258 3 | 43 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00010 | |
| 258 4 | 43 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00002 | |
| 259 1 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00054 | |
| 259 2 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00059 | |
| 259 3 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00057 | |
| 259 4 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00059 | |
| 260 1 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00045 | |
| 260 2 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00039 | |
| 260 3 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00037 | |
| 260 4 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00040 | |
| 261 1 | 44 self_attn q | 62,914,560 | -0.00056 | |
| 261 2 | 44 self_attn q | 125,829,120 | -0.00037 | |
| 261 3 | 44 self_attn q | 188,743,680 | -0.00042 | |
| 261 4 | 44 self_attn q | 314,572,800 | -0.00042 | |
| 262 1 | 44 self_attn [k, v] | 10,485,760 | +0.00008 | |
| 262 2 | 44 self_attn [k, v] | 20,971,520 | +0.00049 | |
| 262 3 | 44 self_attn [k, v] | 31,457,280 | +0.00050 | |
| 262 4 | 44 self_attn [k, v] | 52,428,800 | +0.00044 | |
| 263 1 | 44 self_attn o | 62,914,560 | -0.00041 | |
| 263 2 | 44 self_attn o | 125,829,120 | -0.00079 | |
| 263 3 | 44 self_attn o | 188,743,680 | -0.00078 | |
| 263 4 | 44 self_attn o | 314,572,800 | -0.00083 | |
| 264 1 | 44 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00005 | |
| 264 2 | 44 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016 | |
| 264 3 | 44 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00003 | |
| 264 4 | 44 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00003 | |
| 265 1 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00030 | |
| 265 2 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00033 | |
| 265 3 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00031 | |
| 265 4 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00036 | |
| 266 1 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00033 | |
| 266 2 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00030 | |
| 266 3 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00031 | |
| 266 4 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00033 | |
| 267 1 | 45 self_attn q | 62,914,560 | -0.00047 | |
| 267 2 | 45 self_attn q | 125,829,120 | -0.00035 | |
| 267 3 | 45 self_attn q | 188,743,680 | -0.00039 | |
| 267 4 | 45 self_attn q | 314,572,800 | -0.00040 | |
| 268 1 | 45 self_attn [k, v] | 10,485,760 | -0.00117 | |
| 268 2 | 45 self_attn [k, v] | 20,971,520 | -0.00121 | |
| 268 3 | 45 self_attn [k, v] | 31,457,280 | -0.00119 | |
| 268 4 | 45 self_attn [k, v] | 52,428,800 | -0.00116 | |
| 269 1 | 45 self_attn o | 62,914,560 | -0.00021 | |
| 269 2 | 45 self_attn o | 125,829,120 | -0.00019 | |
| 269 3 | 45 self_attn o | 188,743,680 | -0.00026 | |
| 269 4 | 45 self_attn o | 314,572,800 | -0.00024 | |
| 270 1 | 45 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00032 | |
| 270 2 | 45 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 | |
| 270 3 | 45 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00034 | |
| 270 4 | 45 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00032 | |
| 271 1 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00049 | |
| 271 2 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00046 | |
| 271 3 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00042 | |
| 271 4 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00048 | |
| 272 1 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 272 2 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00023 | |
| 272 3 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00026 | |
| 272 4 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00022 | |
| 273 1 | 46 self_attn q | 62,914,560 | -0.00018 | |
| 273 2 | 46 self_attn q | 125,829,120 | -0.00060 | |
| 273 3 | 46 self_attn q | 188,743,680 | -0.00050 | |
| 273 4 | 46 self_attn q | 314,572,800 | -0.00050 | |
| 274 1 | 46 self_attn [k, v] | 10,485,760 | -0.00030 | |
| 274 2 | 46 self_attn [k, v] | 20,971,520 | -0.00036 | |
| 274 3 | 46 self_attn [k, v] | 31,457,280 | -0.00038 | |
| 274 4 | 46 self_attn [k, v] | 52,428,800 | -0.00037 | |
| 275 1 | 46 self_attn o | 62,914,560 | -0.00001 | |
| 275 2 | 46 self_attn o | 125,829,120 | -0.00011 | |
| 275 3 | 46 self_attn o | 188,743,680 | -0.00004 | |
| 275 4 | 46 self_attn o | 314,572,800 | -0.00003 | |
| 276 1 | 46 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001 | |
| 276 2 | 46 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00012 | |
| 276 3 | 46 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00023 | |
| 276 4 | 46 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00022 | |
| 277 1 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 | |
| 277 2 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00013 | |
| 277 3 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00020 | |
| 277 4 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00019 | |
| 278 1 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00030 | |
| 278 2 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00033 | |
| 278 3 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00036 | |
| 278 4 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00035 | |
| 279 1 | 47 self_attn q | 62,914,560 | -0.00030 | |
| 279 2 | 47 self_attn q | 125,829,120 | -0.00023 | |
| 279 3 | 47 self_attn q | 188,743,680 | -0.00020 | |
| 279 4 | 47 self_attn q | 314,572,800 | -0.00018 | |
| 280 1 | 47 self_attn [k, v] | 10,485,760 | -0.00078 | |
| 280 2 | 47 self_attn [k, v] | 20,971,520 | -0.00070 | |
| 280 3 | 47 self_attn [k, v] | 31,457,280 | -0.00069 | |
| 280 4 | 47 self_attn [k, v] | 52,428,800 | -0.00069 | |
| 281 1 | 47 self_attn o | 62,914,560 | -0.00022 | |
| 281 2 | 47 self_attn o | 125,829,120 | -0.00030 | |
| 281 3 | 47 self_attn o | 188,743,680 | -0.00028 | |
| 281 4 | 47 self_attn o | 314,572,800 | -0.00023 | |
| 282 1 | 47 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00021 | |
| 282 2 | 47 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00015 | |
| 282 3 | 47 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00024 | |
| 282 4 | 47 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00020 | |
| 283 1 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 | |
| 283 2 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 | |
| 283 3 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00023 | |
| 283 4 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00024 | |
| 284 1 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 284 2 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00019 | |
| 284 3 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00027 | |
| 284 4 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00025 | |
| 285 1 | 48 self_attn q | 62,914,560 | +0.00035 | |
| 285 2 | 48 self_attn q | 125,829,120 | +0.00008 | |
| 285 3 | 48 self_attn q | 188,743,680 | +0.00019 | |
| 285 4 | 48 self_attn q | 314,572,800 | +0.00018 | |
| 286 1 | 48 self_attn [k, v] | 10,485,760 | -0.00085 | |
| 286 2 | 48 self_attn [k, v] | 20,971,520 | -0.00128 | |
| 286 3 | 48 self_attn [k, v] | 31,457,280 | -0.00118 | |
| 286 4 | 48 self_attn [k, v] | 52,428,800 | -0.00117 | |
| 287 1 | 48 self_attn o | 62,914,560 | -0.00057 | |
| 287 2 | 48 self_attn o | 125,829,120 | -0.00058 | |
| 287 3 | 48 self_attn o | 188,743,680 | -0.00051 | |
| 287 4 | 48 self_attn o | 314,572,800 | -0.00048 | |
| 288 1 | 48 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00055 | |
| 288 2 | 48 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00034 | |
| 288 3 | 48 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00033 | |
| 288 4 | 48 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00039 | |
| 289 1 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00039 | |
| 289 2 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00039 | |
| 289 3 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00040 | |
| 289 4 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00040 | |
| 290 1 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00025 | |
| 290 2 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00027 | |
| 290 3 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00024 | |
| 290 4 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00025 | |
| 291 1 | 49 self_attn q | 62,914,560 | -0.00038 | |
| 291 2 | 49 self_attn q | 125,829,120 | -0.00033 | |
| 291 3 | 49 self_attn q | 188,743,680 | -0.00021 | |
| 291 4 | 49 self_attn q | 314,572,800 | -0.00032 | |
| 292 1 | 49 self_attn [k, v] | 10,485,760 | +0.00027 | |
| 292 2 | 49 self_attn [k, v] | 20,971,520 | +0.00019 | |
| 292 3 | 49 self_attn [k, v] | 31,457,280 | +0.00017 | |
| 292 4 | 49 self_attn [k, v] | 52,428,800 | +0.00020 | |
| 293 1 | 49 self_attn o | 62,914,560 | -0.00006 | |
| 293 2 | 49 self_attn o | 125,829,120 | +0.00005 | |
| 293 3 | 49 self_attn o | 188,743,680 | +0.00004 | |
| 293 4 | 49 self_attn o | 314,572,800 | +0.00001 | |
| 294 1 | 49 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003 | |
| 294 2 | 49 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022 | |
| 294 3 | 49 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00003 | |
| 294 4 | 49 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00011 | |
| 295 1 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00031 | |
| 295 2 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00033 | |
| 295 3 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00030 | |
| 295 4 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00036 | |
| 296 1 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00020 | |
| 296 2 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 | |
| 296 3 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00022 | |
| 296 4 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00019 | |
| 297 1 | 50 self_attn q | 62,914,560 | -0.00012 | |
| 297 2 | 50 self_attn q | 125,829,120 | -0.00005 | |
| 297 3 | 50 self_attn q | 188,743,680 | +0.00001 | |
| 297 4 | 50 self_attn q | 314,572,800 | -0.00003 | |
| 298 1 | 50 self_attn [k, v] | 10,485,760 | +0.00018 | |
| 298 2 | 50 self_attn [k, v] | 20,971,520 | -0.00010 | |
| 298 3 | 50 self_attn [k, v] | 31,457,280 | -0.00011 | |
| 298 4 | 50 self_attn [k, v] | 52,428,800 | -0.00015 | |
| 299 1 | 50 self_attn o | 62,914,560 | -0.00006 | |
| 299 2 | 50 self_attn o | 125,829,120 | -0.00020 | |
| 299 3 | 50 self_attn o | 188,743,680 | -0.00019 | |
| 299 4 | 50 self_attn o | 314,572,800 | -0.00020 | |
| 300 1 | 50 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 | |
| 300 2 | 50 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011 | |
| 300 3 | 50 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00003 | |
| 300 4 | 50 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00002 | |
| 301 1 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 | |
| 301 2 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 | |
| 301 3 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00029 | |
| 301 4 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00031 | |
| 302 1 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 302 2 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00016 | |
| 302 3 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00017 | |
| 302 4 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00020 | |
| 303 1 | 51 self_attn q | 62,914,560 | -0.00014 | |
| 303 2 | 51 self_attn q | 125,829,120 | -0.00012 | |
| 303 3 | 51 self_attn q | 188,743,680 | -0.00013 | |
| 303 4 | 51 self_attn q | 314,572,800 | -0.00010 | |
| 304 1 | 51 self_attn [k, v] | 10,485,760 | -0.00033 | |
| 304 2 | 51 self_attn [k, v] | 20,971,520 | -0.00033 | |
| 304 3 | 51 self_attn [k, v] | 31,457,280 | -0.00047 | |
| 304 4 | 51 self_attn [k, v] | 52,428,800 | -0.00034 | |
| 305 1 | 51 self_attn o | 62,914,560 | -0.00015 | |
| 305 2 | 51 self_attn o | 125,829,120 | +0.00000 | |
| 305 3 | 51 self_attn o | 188,743,680 | +0.00004 | |
| 305 4 | 51 self_attn o | 314,572,800 | -0.00001 | |
| 306 1 | 51 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00016 | |
| 306 2 | 51 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00031 | |
| 306 3 | 51 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00024 | |
| 306 4 | 51 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00025 | |
| 307 1 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 | |
| 307 2 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00027 | |
| 307 3 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00029 | |
| 307 4 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00029 | |
| 308 1 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00026 | |
| 308 2 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 | |
| 308 3 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00022 | |
| 308 4 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00021 | |
| 309 1 | 52 self_attn q | 62,914,560 | +0.00001 | |
| 309 2 | 52 self_attn q | 125,829,120 | +0.00010 | |
| 309 3 | 52 self_attn q | 188,743,680 | +0.00009 | |
| 309 4 | 52 self_attn q | 314,572,800 | +0.00007 | |
| 310 1 | 52 self_attn [k, v] | 10,485,760 | -0.00075 | |
| 310 2 | 52 self_attn [k, v] | 20,971,520 | -0.00049 | |
| 310 3 | 52 self_attn [k, v] | 31,457,280 | -0.00063 | |
| 310 4 | 52 self_attn [k, v] | 52,428,800 | -0.00057 | |
| 311 1 | 52 self_attn o | 62,914,560 | -0.00009 | |
| 311 2 | 52 self_attn o | 125,829,120 | -0.00008 | |
| 311 3 | 52 self_attn o | 188,743,680 | -0.00025 | |
| 311 4 | 52 self_attn o | 314,572,800 | -0.00023 | |
| 312 1 | 52 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00026 | |
| 312 2 | 52 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00021 | |
| 312 3 | 52 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00019 | |
| 312 4 | 52 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00022 | |
| 313 1 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00019 | |
| 313 2 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00025 | |
| 313 3 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00019 | |
| 313 4 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00023 | |
| 314 1 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00024 | |
| 314 2 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00025 | |
| 314 3 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00023 | |
| 314 4 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00027 | |
| 315 1 | 53 self_attn q | 62,914,560 | -0.00007 | |
| 315 2 | 53 self_attn q | 125,829,120 | -0.00017 | |
| 315 3 | 53 self_attn q | 188,743,680 | -0.00016 | |
| 315 4 | 53 self_attn q | 314,572,800 | -0.00018 | |
| 316 1 | 53 self_attn [k, v] | 10,485,760 | -0.00069 | |
| 316 2 | 53 self_attn [k, v] | 20,971,520 | -0.00067 | |
| 316 3 | 53 self_attn [k, v] | 31,457,280 | -0.00058 | |
| 316 4 | 53 self_attn [k, v] | 52,428,800 | -0.00061 | |
| 317 1 | 53 self_attn o | 62,914,560 | +0.00003 | |
| 317 2 | 53 self_attn o | 125,829,120 | +0.00002 | |
| 317 3 | 53 self_attn o | 188,743,680 | -0.00003 | |
| 317 4 | 53 self_attn o | 314,572,800 | -0.00002 | |
| 318 1 | 53 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00047 | |
| 318 2 | 53 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 | |
| 318 3 | 53 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00036 | |
| 318 4 | 53 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00031 | |
| 319 1 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00013 | |
| 319 2 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00021 | |
| 319 3 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00023 | |
| 319 4 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00024 | |
| 320 1 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 320 2 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 | |
| 320 3 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00021 | |
| 320 4 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00019 | |
| 321 1 | 54 self_attn q | 62,914,560 | +0.00008 | |
| 321 2 | 54 self_attn q | 125,829,120 | +0.00002 | |
| 321 3 | 54 self_attn q | 188,743,680 | -0.00009 | |
| 321 4 | 54 self_attn q | 314,572,800 | -0.00010 | |
| 322 1 | 54 self_attn [k, v] | 10,485,760 | +0.00048 | |
| 322 2 | 54 self_attn [k, v] | 20,971,520 | +0.00017 | |
| 322 3 | 54 self_attn [k, v] | 31,457,280 | +0.00017 | |
| 322 4 | 54 self_attn [k, v] | 52,428,800 | +0.00020 | |
| 323 1 | 54 self_attn o | 62,914,560 | -0.00054 | |
| 323 2 | 54 self_attn o | 125,829,120 | -0.00040 | |
| 323 3 | 54 self_attn o | 188,743,680 | -0.00045 | |
| 323 4 | 54 self_attn o | 314,572,800 | -0.00045 | |
| 324 1 | 54 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011 | |
| 324 2 | 54 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 | |
| 324 3 | 54 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00001 | |
| 324 4 | 54 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00004 | |
| 325 1 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 | |
| 325 2 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015 | |
| 325 3 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00018 | |
| 325 4 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00014 | |
| 326 1 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00009 | |
| 326 2 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 | |
| 326 3 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00007 | |
| 326 4 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00008 | |
| 327 1 | 55 self_attn q | 62,914,560 | -0.00018 | |
| 327 2 | 55 self_attn q | 125,829,120 | -0.00009 | |
| 327 3 | 55 self_attn q | 188,743,680 | -0.00007 | |
| 327 4 | 55 self_attn q | 314,572,800 | -0.00010 | |
| 328 1 | 55 self_attn [k, v] | 10,485,760 | -0.00012 | |
| 328 2 | 55 self_attn [k, v] | 20,971,520 | +0.00007 | |
| 328 3 | 55 self_attn [k, v] | 31,457,280 | +0.00005 | |
| 328 4 | 55 self_attn [k, v] | 52,428,800 | +0.00007 | |
| 329 1 | 55 self_attn o | 62,914,560 | -0.00044 | |
| 329 2 | 55 self_attn o | 125,829,120 | -0.00046 | |
| 329 3 | 55 self_attn o | 188,743,680 | -0.00042 | |
| 329 4 | 55 self_attn o | 314,572,800 | -0.00042 | |
| 330 1 | 55 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 | |
| 330 2 | 55 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00028 | |
| 330 3 | 55 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00024 | |
| 330 4 | 55 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00026 | |
| 331 1 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 | |
| 331 2 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 | |
| 331 3 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00014 | |
| 331 4 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00014 | |
| 332 1 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 | |
| 332 2 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 | |
| 332 3 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00010 | |
| 332 4 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00010 | |
| 333 1 | 56 self_attn q | 62,914,560 | +0.00030 | |
| 333 2 | 56 self_attn q | 125,829,120 | +0.00025 | |
| 333 3 | 56 self_attn q | 188,743,680 | +0.00024 | |
| 333 4 | 56 self_attn q | 314,572,800 | +0.00024 | |
| 334 1 | 56 self_attn [k, v] | 10,485,760 | -0.00004 | |
| 334 2 | 56 self_attn [k, v] | 20,971,520 | +0.00010 | |
| 334 3 | 56 self_attn [k, v] | 31,457,280 | +0.00012 | |
| 334 4 | 56 self_attn [k, v] | 52,428,800 | +0.00010 | |
| 335 1 | 56 self_attn o | 62,914,560 | +0.00005 | |
| 335 2 | 56 self_attn o | 125,829,120 | +0.00013 | |
| 335 3 | 56 self_attn o | 188,743,680 | +0.00012 | |
| 335 4 | 56 self_attn o | 314,572,800 | +0.00009 | |
| 336 1 | 56 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 | |
| 336 2 | 56 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00013 | |
| 336 3 | 56 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00014 | |
| 336 4 | 56 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00013 | |
| 337 1 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00017 | |
| 337 2 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 | |
| 337 3 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00022 | |
| 337 4 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00019 | |
| 338 1 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 | |
| 338 2 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 | |
| 338 3 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00017 | |
| 338 4 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00014 | |
| 339 1 | 57 self_attn q | 62,914,560 | -0.00017 | |
| 339 2 | 57 self_attn q | 125,829,120 | -0.00020 | |
| 339 3 | 57 self_attn q | 188,743,680 | -0.00019 | |
| 339 4 | 57 self_attn q | 314,572,800 | -0.00018 | |
| 340 1 | 57 self_attn [k, v] | 10,485,760 | +0.00007 | |
| 340 2 | 57 self_attn [k, v] | 20,971,520 | +0.00004 | |
| 340 3 | 57 self_attn [k, v] | 31,457,280 | +0.00002 | |
| 340 4 | 57 self_attn [k, v] | 52,428,800 | +0.00006 | |
| 341 1 | 57 self_attn o | 62,914,560 | -0.00016 | |
| 341 2 | 57 self_attn o | 125,829,120 | -0.00005 | |
| 341 3 | 57 self_attn o | 188,743,680 | -0.00009 | |
| 341 4 | 57 self_attn o | 314,572,800 | -0.00011 | |
| 342 1 | 57 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00018 | |
| 342 2 | 57 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00032 | |
| 342 3 | 57 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00028 | |
| 342 4 | 57 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00028 | |
| 343 1 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018 | |
| 343 2 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015 | |
| 343 3 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00014 | |
| 343 4 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00017 | |
| 344 1 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 | |
| 344 2 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 | |
| 344 3 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00003 | |
| 344 4 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00005 | |
| 345 1 | 58 self_attn q | 62,914,560 | +0.00013 | |
| 345 2 | 58 self_attn q | 125,829,120 | +0.00023 | |
| 345 3 | 58 self_attn q | 188,743,680 | +0.00008 | |
| 345 4 | 58 self_attn q | 314,572,800 | +0.00016 | |
| 346 1 | 58 self_attn [k, v] | 10,485,760 | -0.00017 | |
| 346 2 | 58 self_attn [k, v] | 20,971,520 | -0.00011 | |
| 346 3 | 58 self_attn [k, v] | 31,457,280 | -0.00019 | |
| 346 4 | 58 self_attn [k, v] | 52,428,800 | -0.00013 | |
| 347 1 | 58 self_attn o | 62,914,560 | +0.00002 | |
| 347 2 | 58 self_attn o | 125,829,120 | +0.00002 | |
| 347 3 | 58 self_attn o | 188,743,680 | -0.00007 | |
| 347 4 | 58 self_attn o | 314,572,800 | -0.00010 | |
| 348 1 | 58 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00021 | |
| 348 2 | 58 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 | |
| 348 3 | 58 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00001 | |
| 348 4 | 58 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00001 | |
| 349 1 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 | |
| 349 2 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00023 | |
| 349 3 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00024 | |
| 349 4 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00025 | |
| 350 1 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00024 | |
| 350 2 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00020 | |
| 350 3 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00025 | |
| 350 4 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00023 | |
| 351 1 | 59 self_attn q | 62,914,560 | -0.00017 | |
| 351 2 | 59 self_attn q | 125,829,120 | -0.00017 | |
| 351 3 | 59 self_attn q | 188,743,680 | -0.00020 | |
| 351 4 | 59 self_attn q | 314,572,800 | -0.00022 | |
| 352 1 | 59 self_attn [k, v] | 10,485,760 | -0.00011 | |
| 352 2 | 59 self_attn [k, v] | 20,971,520 | -0.00027 | |
| 352 3 | 59 self_attn [k, v] | 31,457,280 | -0.00023 | |
| 352 4 | 59 self_attn [k, v] | 52,428,800 | -0.00023 | |
| 353 1 | 59 self_attn o | 62,914,560 | +0.00001 | |
| 353 2 | 59 self_attn o | 125,829,120 | +0.00009 | |
| 353 3 | 59 self_attn o | 188,743,680 | +0.00004 | |
| 353 4 | 59 self_attn o | 314,572,800 | +0.00005 | |
| 354 1 | 59 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00018 | |
| 354 2 | 59 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00032 | |
| 354 3 | 59 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00031 | |
| 354 4 | 59 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00030 | |
| 355 1 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 | |
| 355 2 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 | |
| 355 3 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00013 | |
| 355 4 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00013 | |
| 356 1 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00015 | |
| 356 2 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00011 | |
| 356 3 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | +0.00012 | |
| 356 4 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | +0.00014 | |
| 357 1 | 60 self_attn q | 62,914,560 | +0.00002 | |
| 357 2 | 60 self_attn q | 125,829,120 | -0.00008 | |
| 357 3 | 60 self_attn q | 188,743,680 | -0.00009 | |
| 357 4 | 60 self_attn q | 314,572,800 | -0.00008 | |
| 358 1 | 60 self_attn [k, v] | 10,485,760 | -0.00019 | |
| 358 2 | 60 self_attn [k, v] | 20,971,520 | -0.00029 | |
| 358 3 | 60 self_attn [k, v] | 31,457,280 | -0.00041 | |
| 358 4 | 60 self_attn [k, v] | 52,428,800 | -0.00035 | |
| 359 1 | 60 self_attn o | 62,914,560 | -0.00003 | |
| 359 2 | 60 self_attn o | 125,829,120 | -0.00002 | |
| 359 3 | 60 self_attn o | 188,743,680 | -0.00002 | |
| 359 4 | 60 self_attn o | 314,572,800 | -0.00004 | |
| 360 1 | 60 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00029 | |
| 360 2 | 60 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00039 | |
| 360 3 | 60 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00035 | |
| 360 4 | 60 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00035 | |
| 361 1 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018 | |
| 361 2 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00020 | |
| 361 3 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00018 | |
| 361 4 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00019 | |
| 362 1 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 | |
| 362 2 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014 | |
| 362 3 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00011 | |
| 362 4 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00014 | |
| 363 1 | 61 self_attn q | 62,914,560 | -0.00022 | |
| 363 2 | 61 self_attn q | 125,829,120 | -0.00023 | |
| 363 3 | 61 self_attn q | 188,743,680 | -0.00020 | |
| 363 4 | 61 self_attn q | 314,572,800 | -0.00019 | |
| 364 1 | 61 self_attn [k, v] | 10,485,760 | -0.00025 | |
| 364 2 | 61 self_attn [k, v] | 20,971,520 | -0.00026 | |
| 364 3 | 61 self_attn [k, v] | 31,457,280 | -0.00022 | |
| 364 4 | 61 self_attn [k, v] | 52,428,800 | -0.00022 | |
| 365 1 | 61 self_attn o | 62,914,560 | -0.00017 | |
| 365 2 | 61 self_attn o | 125,829,120 | -0.00021 | |
| 365 3 | 61 self_attn o | 188,743,680 | -0.00020 | |
| 365 4 | 61 self_attn o | 314,572,800 | -0.00023 | |
| 366 1 | 61 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012 | |
| 366 2 | 61 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010 | |
| 366 3 | 61 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00016 | |
| 366 4 | 61 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00012 | |
| 367 1 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 | |
| 367 2 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00009 | |
| 367 3 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00010 | |
| 367 4 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00009 | |
| 368 1 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 | |
| 368 2 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004 | |
| 368 3 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00005 | |
| 368 4 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00007 | |
| 369 1 | 62 self_attn q | 62,914,560 | -0.00008 | |
| 369 2 | 62 self_attn q | 125,829,120 | -0.00001 | |
| 369 3 | 62 self_attn q | 188,743,680 | -0.00003 | |
| 369 4 | 62 self_attn q | 314,572,800 | +0.00001 | |
| 370 1 | 62 self_attn [k, v] | 10,485,760 | -0.00008 | |
| 370 2 | 62 self_attn [k, v] | 20,971,520 | -0.00014 | |
| 370 3 | 62 self_attn [k, v] | 31,457,280 | -0.00006 | |
| 370 4 | 62 self_attn [k, v] | 52,428,800 | -0.00013 | |
| 371 1 | 62 self_attn o | 62,914,560 | -0.00012 | |
| 371 2 | 62 self_attn o | 125,829,120 | -0.00009 | |
| 371 3 | 62 self_attn o | 188,743,680 | -0.00007 | |
| 371 4 | 62 self_attn o | 314,572,800 | -0.00007 | |
| 372 1 | 62 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006 | |
| 372 2 | 62 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 | |
| 372 3 | 62 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00002 | |
| 372 4 | 62 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00007 | |
| 373 1 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00009 | |
| 373 2 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019 | |
| 373 3 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00016 | |
| 373 4 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00018 | |
| 374 1 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 | |
| 374 2 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00019 | |
| 374 3 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00015 | |
| 374 4 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00014 | |
| 375 1 | 63 self_attn q | 62,914,560 | +0.00009 | |
| 375 2 | 63 self_attn q | 125,829,120 | -0.00000 | |
| 375 3 | 63 self_attn q | 188,743,680 | +0.00000 | |
| 375 4 | 63 self_attn q | 314,572,800 | -0.00001 | |
| 376 1 | 63 self_attn [k, v] | 10,485,760 | -0.00013 | |
| 376 2 | 63 self_attn [k, v] | 20,971,520 | -0.00010 | |
| 376 3 | 63 self_attn [k, v] | 31,457,280 | -0.00013 | |
| 376 4 | 63 self_attn [k, v] | 52,428,800 | -0.00014 | |
| 377 1 | 63 self_attn o | 62,914,560 | -0.00003 | |
| 377 2 | 63 self_attn o | 125,829,120 | -0.00005 | |
| 377 3 | 63 self_attn o | 188,743,680 | -0.00006 | |
| 377 4 | 63 self_attn o | 314,572,800 | -0.00002 | |
| 378 1 | 63 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 | |
| 378 2 | 63 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003 | |
| 378 3 | 63 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00000 | |
| 378 4 | 63 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00001 | |
| 379 1 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004 | |
| 379 2 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 | |
| 379 3 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00003 | |
| 379 4 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00003 | |
| 380 1 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 | |
| 380 2 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 | |
| 380 3 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00007 | |
| 380 4 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00007 | |
| 381 1 | 64 self_attn q | 62,914,560 | -0.00011 | |
| 381 2 | 64 self_attn q | 125,829,120 | -0.00006 | |
| 381 3 | 64 self_attn q | 188,743,680 | -0.00009 | |
| 381 4 | 64 self_attn q | 314,572,800 | -0.00008 | |
| 382 1 | 64 self_attn [k, v] | 10,485,760 | -0.00028 | |
| 382 2 | 64 self_attn [k, v] | 20,971,520 | -0.00021 | |
| 382 3 | 64 self_attn [k, v] | 31,457,280 | -0.00020 | |
| 382 4 | 64 self_attn [k, v] | 52,428,800 | -0.00019 | |
| 383 1 | 64 self_attn o | 62,914,560 | -0.00011 | |
| 383 2 | 64 self_attn o | 125,829,120 | -0.00008 | |
| 383 3 | 64 self_attn o | 188,743,680 | -0.00009 | |
| 383 4 | 64 self_attn o | 314,572,800 | -0.00008 | |
| 384 1 | 64 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00008 | |
| 384 2 | 64 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 | |
| 384 3 | 64 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00001 | |
| 384 4 | 64 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00002 | |
| 385 1 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010 | |
| 385 2 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014 | |
| 385 3 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00015 | |
| 385 4 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00015 | |
| 386 1 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 | |
| 386 2 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 386 3 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00008 | |
| 386 4 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00006 | |
| 387 1 | 65 self_attn q | 62,914,560 | -0.00006 | |
| 387 2 | 65 self_attn q | 125,829,120 | +0.00001 | |
| 387 3 | 65 self_attn q | 188,743,680 | -0.00006 | |
| 387 4 | 65 self_attn q | 314,572,800 | -0.00001 | |
| 388 1 | 65 self_attn [k, v] | 10,485,760 | -0.00026 | |
| 388 2 | 65 self_attn [k, v] | 20,971,520 | -0.00039 | |
| 388 3 | 65 self_attn [k, v] | 31,457,280 | -0.00030 | |
| 388 4 | 65 self_attn [k, v] | 52,428,800 | -0.00032 | |
| 389 1 | 65 self_attn o | 62,914,560 | -0.00008 | |
| 389 2 | 65 self_attn o | 125,829,120 | -0.00009 | |
| 389 3 | 65 self_attn o | 188,743,680 | -0.00010 | |
| 389 4 | 65 self_attn o | 314,572,800 | -0.00008 | |
| 390 1 | 65 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00023 | |
| 390 2 | 65 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023 | |
| 390 3 | 65 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00013 | |
| 390 4 | 65 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00016 | |
| 391 1 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 | |
| 391 2 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 | |
| 391 3 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00007 | |
| 391 4 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00007 | |
| 392 1 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00011 | |
| 392 2 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00009 | |
| 392 3 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00012 | |
| 392 4 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00012 | |
| 393 1 | 66 self_attn q | 62,914,560 | -0.00060 | |
| 393 2 | 66 self_attn q | 125,829,120 | -0.00062 | |
| 393 3 | 66 self_attn q | 188,743,680 | -0.00061 | |
| 393 4 | 66 self_attn q | 314,572,800 | -0.00063 | |
| 394 1 | 66 self_attn [k, v] | 10,485,760 | +0.00028 | |
| 394 2 | 66 self_attn [k, v] | 20,971,520 | +0.00035 | |
| 394 3 | 66 self_attn [k, v] | 31,457,280 | +0.00030 | |
| 394 4 | 66 self_attn [k, v] | 52,428,800 | +0.00033 | |
| 395 1 | 66 self_attn o | 62,914,560 | -0.00016 | |
| 395 2 | 66 self_attn o | 125,829,120 | -0.00019 | |
| 395 3 | 66 self_attn o | 188,743,680 | -0.00024 | |
| 395 4 | 66 self_attn o | 314,572,800 | -0.00026 | |
| 396 1 | 66 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012 | |
| 396 2 | 66 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00019 | |
| 396 3 | 66 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00017 | |
| 396 4 | 66 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00014 | |
| 397 1 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00022 | |
| 397 2 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00020 | |
| 397 3 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00021 | |
| 397 4 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00020 | |
| 398 1 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00021 | |
| 398 2 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00024 | |
| 398 3 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00024 | |
| 398 4 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00026 | |
| 399 1 | 67 self_attn q | 62,914,560 | -0.00002 | |
| 399 2 | 67 self_attn q | 125,829,120 | -0.00006 | |
| 399 3 | 67 self_attn q | 188,743,680 | -0.00009 | |
| 399 4 | 67 self_attn q | 314,572,800 | -0.00004 | |
| 400 1 | 67 self_attn [k, v] | 10,485,760 | -0.00028 | |
| 400 2 | 67 self_attn [k, v] | 20,971,520 | -0.00019 | |
| 400 3 | 67 self_attn [k, v] | 31,457,280 | -0.00021 | |
| 400 4 | 67 self_attn [k, v] | 52,428,800 | -0.00020 | |
| 401 1 | 67 self_attn o | 62,914,560 | -0.00011 | |
| 401 2 | 67 self_attn o | 125,829,120 | -0.00009 | |
| 401 3 | 67 self_attn o | 188,743,680 | -0.00018 | |
| 401 4 | 67 self_attn o | 314,572,800 | -0.00015 | |
| 402 1 | 67 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 | |
| 402 2 | 67 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008 | |
| 402 3 | 67 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00005 | |
| 402 4 | 67 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00002 | |
| 403 1 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004 | |
| 403 2 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005 | |
| 403 3 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | +0.00008 | |
| 403 4 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | +0.00007 | |
| 404 1 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00026 | |
| 404 2 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00022 | |
| 404 3 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00020 | |
| 404 4 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00019 | |
| 405 1 | 68 self_attn q | 62,914,560 | -0.00009 | |
| 405 2 | 68 self_attn q | 125,829,120 | -0.00021 | |
| 405 3 | 68 self_attn q | 188,743,680 | -0.00018 | |
| 405 4 | 68 self_attn q | 314,572,800 | -0.00020 | |
| 406 1 | 68 self_attn [k, v] | 10,485,760 | -0.00031 | |
| 406 2 | 68 self_attn [k, v] | 20,971,520 | -0.00011 | |
| 406 3 | 68 self_attn [k, v] | 31,457,280 | -0.00012 | |
| 406 4 | 68 self_attn [k, v] | 52,428,800 | -0.00010 | |
| 407 1 | 68 self_attn o | 62,914,560 | -0.00022 | |
| 407 2 | 68 self_attn o | 125,829,120 | -0.00023 | |
| 407 3 | 68 self_attn o | 188,743,680 | -0.00030 | |
| 407 4 | 68 self_attn o | 314,572,800 | -0.00027 | |
| 408 1 | 68 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00007 | |
| 408 2 | 68 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002 | |
| 408 3 | 68 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00003 | |
| 408 4 | 68 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00000 | |
| 409 1 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006 | |
| 409 2 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00013 | |
| 409 3 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00007 | |
| 409 4 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00008 | |
| 410 1 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 | |
| 410 2 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00031 | |
| 410 3 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00028 | |
| 410 4 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00032 | |
| 411 1 | 69 self_attn q | 62,914,560 | -0.00007 | |
| 411 2 | 69 self_attn q | 125,829,120 | -0.00011 | |
| 411 3 | 69 self_attn q | 188,743,680 | -0.00010 | |
| 411 4 | 69 self_attn q | 314,572,800 | -0.00014 | |
| 412 1 | 69 self_attn [k, v] | 10,485,760 | -0.00018 | |
| 412 2 | 69 self_attn [k, v] | 20,971,520 | -0.00033 | |
| 412 3 | 69 self_attn [k, v] | 31,457,280 | -0.00029 | |
| 412 4 | 69 self_attn [k, v] | 52,428,800 | -0.00030 | |
| 413 1 | 69 self_attn o | 62,914,560 | -0.00012 | |
| 413 2 | 69 self_attn o | 125,829,120 | -0.00011 | |
| 413 3 | 69 self_attn o | 188,743,680 | -0.00011 | |
| 413 4 | 69 self_attn o | 314,572,800 | -0.00013 | |
| 414 1 | 69 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 | |
| 414 2 | 69 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008 | |
| 414 3 | 69 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00007 | |
| 414 4 | 69 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00003 | |
| 415 1 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 | |
| 415 2 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00030 | |
| 415 3 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00031 | |
| 415 4 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00031 | |
| 416 1 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002 | |
| 416 2 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 | |
| 416 3 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00002 | |
| 416 4 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00004 | |
| 417 1 | 70 self_attn q | 62,914,560 | +0.00002 | |
| 417 2 | 70 self_attn q | 125,829,120 | +0.00003 | |
| 417 3 | 70 self_attn q | 188,743,680 | +0.00003 | |
| 417 4 | 70 self_attn q | 314,572,800 | -0.00003 | |
| 418 1 | 70 self_attn [k, v] | 10,485,760 | -0.00014 | |
| 418 2 | 70 self_attn [k, v] | 20,971,520 | -0.00011 | |
| 418 3 | 70 self_attn [k, v] | 31,457,280 | -0.00014 | |
| 418 4 | 70 self_attn [k, v] | 52,428,800 | -0.00016 | |
| 419 1 | 70 self_attn o | 62,914,560 | -0.00001 | |
| 419 2 | 70 self_attn o | 125,829,120 | -0.00003 | |
| 419 3 | 70 self_attn o | 188,743,680 | -0.00005 | |
| 419 4 | 70 self_attn o | 314,572,800 | -0.00007 | |
| 420 1 | 70 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00035 | |
| 420 2 | 70 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00026 | |
| 420 3 | 70 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00025 | |
| 420 4 | 70 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00029 | |
| 421 1 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 | |
| 421 2 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 | |
| 421 3 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00003 | |
| 421 4 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00005 | |
| 422 1 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 | |
| 422 2 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00008 | |
| 422 3 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00008 | |
| 422 4 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00008 | |
| 423 1 | 71 self_attn q | 62,914,560 | -0.00000 | |
| 423 2 | 71 self_attn q | 125,829,120 | +0.00009 | |
| 423 3 | 71 self_attn q | 188,743,680 | +0.00011 | |
| 423 4 | 71 self_attn q | 314,572,800 | +0.00007 | |
| 424 1 | 71 self_attn [k, v] | 10,485,760 | -0.00033 | |
| 424 2 | 71 self_attn [k, v] | 20,971,520 | -0.00033 | |
| 424 3 | 71 self_attn [k, v] | 31,457,280 | -0.00039 | |
| 424 4 | 71 self_attn [k, v] | 52,428,800 | -0.00036 | |
| 425 1 | 71 self_attn o | 62,914,560 | -0.00023 | |
| 425 2 | 71 self_attn o | 125,829,120 | -0.00036 | |
| 425 3 | 71 self_attn o | 188,743,680 | -0.00031 | |
| 425 4 | 71 self_attn o | 314,572,800 | -0.00036 | |
| 426 1 | 71 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00057 | |
| 426 2 | 71 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00040 | |
| 426 3 | 71 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00046 | |
| 426 4 | 71 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00044 | |
| 427 1 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00038 | |
| 427 2 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00045 | |
| 427 3 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00045 | |
| 427 4 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00047 | |
| 428 1 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00017 | |
| 428 2 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014 | |
| 428 3 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00013 | |
| 428 4 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00013 | |
| 429 1 | 72 self_attn q | 62,914,560 | -0.00014 | |
| 429 2 | 72 self_attn q | 125,829,120 | -0.00011 | |
| 429 3 | 72 self_attn q | 188,743,680 | -0.00010 | |
| 429 4 | 72 self_attn q | 314,572,800 | -0.00011 | |
| 430 1 | 72 self_attn [k, v] | 10,485,760 | -0.00007 | |
| 430 2 | 72 self_attn [k, v] | 20,971,520 | -0.00003 | |
| 430 3 | 72 self_attn [k, v] | 31,457,280 | -0.00002 | |
| 430 4 | 72 self_attn [k, v] | 52,428,800 | -0.00001 | |
| 431 1 | 72 self_attn o | 62,914,560 | +0.00005 | |
| 431 2 | 72 self_attn o | 125,829,120 | -0.00009 | |
| 431 3 | 72 self_attn o | 188,743,680 | -0.00007 | |
| 431 4 | 72 self_attn o | 314,572,800 | -0.00008 | |
| 432 1 | 72 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00032 | |
| 432 2 | 72 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00051 | |
| 432 3 | 72 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00041 | |
| 432 4 | 72 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00043 | |
| 433 1 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00021 | |
| 433 2 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00013 | |
| 433 3 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | +0.00013 | |
| 433 4 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | +0.00010 | |
| 434 1 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 | |
| 434 2 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00022 | |
| 434 3 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00025 | |
| 434 4 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00026 | |
| 435 1 | 73 self_attn q | 62,914,560 | -0.00007 | |
| 435 2 | 73 self_attn q | 125,829,120 | -0.00014 | |
| 435 3 | 73 self_attn q | 188,743,680 | -0.00010 | |
| 435 4 | 73 self_attn q | 314,572,800 | -0.00011 | |
| 436 1 | 73 self_attn [k, v] | 10,485,760 | -0.00005 | |
| 436 2 | 73 self_attn [k, v] | 20,971,520 | +0.00005 | |
| 436 3 | 73 self_attn [k, v] | 31,457,280 | +0.00003 | |
| 436 4 | 73 self_attn [k, v] | 52,428,800 | +0.00002 | |
| 437 1 | 73 self_attn o | 62,914,560 | -0.00025 | |
| 437 2 | 73 self_attn o | 125,829,120 | -0.00022 | |
| 437 3 | 73 self_attn o | 188,743,680 | -0.00013 | |
| 437 4 | 73 self_attn o | 314,572,800 | -0.00018 | |
| 438 1 | 73 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 | |
| 438 2 | 73 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00029 | |
| 438 3 | 73 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00030 | |
| 438 4 | 73 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00034 | |
| 439 1 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00021 | |
| 439 2 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00017 | |
| 439 3 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00018 | |
| 439 4 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00018 | |
| 440 1 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00012 | |
| 440 2 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00015 | |
| 440 3 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00013 | |
| 440 4 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00015 | |
| 441 1 | 74 self_attn q | 62,914,560 | -0.00006 | |
| 441 2 | 74 self_attn q | 125,829,120 | -0.00002 | |
| 441 3 | 74 self_attn q | 188,743,680 | -0.00002 | |
| 441 4 | 74 self_attn q | 314,572,800 | -0.00002 | |
| 442 1 | 74 self_attn [k, v] | 10,485,760 | -0.00009 | |
| 442 2 | 74 self_attn [k, v] | 20,971,520 | -0.00012 | |
| 442 3 | 74 self_attn [k, v] | 31,457,280 | -0.00008 | |
| 442 4 | 74 self_attn [k, v] | 52,428,800 | -0.00011 | |
| 443 1 | 74 self_attn o | 62,914,560 | -0.00001 | |
| 443 2 | 74 self_attn o | 125,829,120 | +0.00004 | |
| 443 3 | 74 self_attn o | 188,743,680 | +0.00007 | |
| 443 4 | 74 self_attn o | 314,572,800 | +0.00006 | |
| 444 1 | 74 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 | |
| 444 2 | 74 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010 | |
| 444 3 | 74 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00012 | |
| 444 4 | 74 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00014 | |
| 445 1 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 | |
| 445 2 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00017 | |
| 445 3 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00018 | |
| 445 4 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00017 | |
| 446 1 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013 | |
| 446 2 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00016 | |
| 446 3 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00016 | |
| 446 4 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00016 | |
| 447 1 | 75 self_attn q | 62,914,560 | +0.00004 | |
| 447 2 | 75 self_attn q | 125,829,120 | -0.00002 | |
| 447 3 | 75 self_attn q | 188,743,680 | +0.00002 | |
| 447 4 | 75 self_attn q | 314,572,800 | +0.00001 | |
| 448 1 | 75 self_attn [k, v] | 10,485,760 | -0.00007 | |
| 448 2 | 75 self_attn [k, v] | 20,971,520 | -0.00005 | |
| 448 3 | 75 self_attn [k, v] | 31,457,280 | -0.00001 | |
| 448 4 | 75 self_attn [k, v] | 52,428,800 | +0.00000 | |
| 449 1 | 75 self_attn o | 62,914,560 | -0.00022 | |
| 449 2 | 75 self_attn o | 125,829,120 | -0.00012 | |
| 449 3 | 75 self_attn o | 188,743,680 | -0.00016 | |
| 449 4 | 75 self_attn o | 314,572,800 | -0.00015 | |
| 450 1 | 75 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 | |
| 450 2 | 75 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002 | |
| 450 3 | 75 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00007 | |
| 450 4 | 75 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00009 | |
| 451 1 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 | |
| 451 2 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00035 | |
| 451 3 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00037 | |
| 451 4 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00033 | |
| 452 1 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014 | |
| 452 2 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00015 | |
| 452 3 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00010 | |
| 452 4 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00013 | |
| 453 1 | 76 self_attn q | 62,914,560 | +0.00013 | |
| 453 2 | 76 self_attn q | 125,829,120 | -0.00011 | |
| 453 3 | 76 self_attn q | 188,743,680 | -0.00011 | |
| 453 4 | 76 self_attn q | 314,572,800 | -0.00015 | |
| 454 1 | 76 self_attn [k, v] | 10,485,760 | -0.00016 | |
| 454 2 | 76 self_attn [k, v] | 20,971,520 | -0.00029 | |
| 454 3 | 76 self_attn [k, v] | 31,457,280 | -0.00032 | |
| 454 4 | 76 self_attn [k, v] | 52,428,800 | -0.00026 | |
| 455 1 | 76 self_attn o | 62,914,560 | +0.00001 | |
| 455 2 | 76 self_attn o | 125,829,120 | -0.00010 | |
| 455 3 | 76 self_attn o | 188,743,680 | -0.00014 | |
| 455 4 | 76 self_attn o | 314,572,800 | -0.00014 | |
| 456 1 | 76 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 | |
| 456 2 | 76 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 | |
| 456 3 | 76 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00005 | |
| 456 4 | 76 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00010 | |
| 457 1 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00023 | |
| 457 2 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00024 | |
| 457 3 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | +0.00027 | |
| 457 4 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | +0.00022 | |
| 458 1 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 | |
| 458 2 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 458 3 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00010 | |
| 458 4 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00012 | |
| 459 1 | 77 self_attn q | 62,914,560 | -0.00028 | |
| 459 2 | 77 self_attn q | 125,829,120 | -0.00037 | |
| 459 3 | 77 self_attn q | 188,743,680 | -0.00040 | |
| 459 4 | 77 self_attn q | 314,572,800 | -0.00040 | |
| 460 1 | 77 self_attn [k, v] | 10,485,760 | -0.00024 | |
| 460 2 | 77 self_attn [k, v] | 20,971,520 | -0.00056 | |
| 460 3 | 77 self_attn [k, v] | 31,457,280 | -0.00062 | |
| 460 4 | 77 self_attn [k, v] | 52,428,800 | -0.00057 | |
| 461 1 | 77 self_attn o | 62,914,560 | -0.00006 | |
| 461 2 | 77 self_attn o | 125,829,120 | -0.00013 | |
| 461 3 | 77 self_attn o | 188,743,680 | -0.00015 | |
| 461 4 | 77 self_attn o | 314,572,800 | -0.00009 | |
| 462 1 | 77 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 | |
| 462 2 | 77 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00024 | |
| 462 3 | 77 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00021 | |
| 462 4 | 77 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00018 | |
| 463 1 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00030 | |
| 463 2 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00038 | |
| 463 3 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00034 | |
| 463 4 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00034 | |
| 464 1 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 | |
| 464 2 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 | |
| 464 3 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00005 | |
| 464 4 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00005 | |
| 465 1 | 78 self_attn q | 62,914,560 | -0.00010 | |
| 465 2 | 78 self_attn q | 125,829,120 | -0.00001 | |
| 465 3 | 78 self_attn q | 188,743,680 | -0.00003 | |
| 465 4 | 78 self_attn q | 314,572,800 | -0.00002 | |
| 466 1 | 78 self_attn [k, v] | 10,485,760 | -0.00040 | |
| 466 2 | 78 self_attn [k, v] | 20,971,520 | -0.00047 | |
| 466 3 | 78 self_attn [k, v] | 31,457,280 | -0.00052 | |
| 466 4 | 78 self_attn [k, v] | 52,428,800 | -0.00049 | |
| 467 1 | 78 self_attn o | 62,914,560 | -0.00016 | |
| 467 2 | 78 self_attn o | 125,829,120 | -0.00015 | |
| 467 3 | 78 self_attn o | 188,743,680 | -0.00011 | |
| 467 4 | 78 self_attn o | 314,572,800 | -0.00013 | |
| 468 1 | 78 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00010 | |
| 468 2 | 78 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00015 | |
| 468 3 | 78 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00006 | |
| 468 4 | 78 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00007 | |
| 469 1 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 | |
| 469 2 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 | |
| 469 3 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00002 | |
| 469 4 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00008 | |
| 470 1 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 | |
| 470 2 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 470 3 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00013 | |
| 470 4 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00011 | |
| 471 1 | 79 self_attn q | 62,914,560 | -0.00027 | |
| 471 2 | 79 self_attn q | 125,829,120 | -0.00028 | |
| 471 3 | 79 self_attn q | 188,743,680 | -0.00032 | |
| 471 4 | 79 self_attn q | 314,572,800 | -0.00032 | |
| 472 1 | 79 self_attn [k, v] | 10,485,760 | -0.00052 | |
| 472 2 | 79 self_attn [k, v] | 20,971,520 | -0.00096 | |
| 472 3 | 79 self_attn [k, v] | 31,457,280 | -0.00106 | |
| 472 4 | 79 self_attn [k, v] | 52,428,800 | -0.00103 | |
| 473 1 | 79 self_attn o | 62,914,560 | -0.00037 | |
| 473 2 | 79 self_attn o | 125,829,120 | -0.00035 | |
| 473 3 | 79 self_attn o | 188,743,680 | -0.00030 | |
| 473 4 | 79 self_attn o | 314,572,800 | -0.00034 | |
| 474 1 | 79 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00067 | |
| 474 2 | 79 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00070 | |
| 474 3 | 79 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00069 | |
| 474 4 | 79 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00075 | |
| 475 1 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 | |
| 475 2 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00040 | |
| 475 3 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00033 | |
| 475 4 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00036 | |
| 476 1 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014 | |
| 476 2 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 476 3 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00010 | |
| 476 4 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00009 | |
| 477 1 | 80 self_attn q | 62,914,560 | +0.00083 | |
| 477 2 | 80 self_attn q | 125,829,120 | +0.00137 | |
| 477 3 | 80 self_attn q | 188,743,680 | +0.00106 | |
| 477 4 | 80 self_attn q | 314,572,800 | +0.00083 | |
| 478 1 | 80 self_attn [k, v] | 10,485,760 | -0.00136 | |
| 478 2 | 80 self_attn [k, v] | 20,971,520 | -0.00107 | |
| 478 3 | 80 self_attn [k, v] | 31,457,280 | -0.00109 | |
| 478 4 | 80 self_attn [k, v] | 52,428,800 | -0.00118 | |
| 479 1 | 80 self_attn o | 62,914,560 | -0.00045 | |
| 479 2 | 80 self_attn o | 125,829,120 | -0.00038 | |
| 479 3 | 80 self_attn o | 188,743,680 | -0.00033 | |
| 479 4 | 80 self_attn o | 314,572,800 | -0.00038 | |
| 480 1 | 80 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 | |
| 480 2 | 80 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00046 | |
| 480 3 | 80 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00048 | |
| 480 4 | 80 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00044 | |
| 481 1 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010 | |
| 481 2 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 | |
| 481 3 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00009 | |
| 481 4 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00009 | |
| 482 1 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 | |
| 482 2 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00013 | |
| 482 3 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00012 | |
| 482 4 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00016 | |
| 483 1 | 81 self_attn q | 62,914,560 | -0.00027 | |
| 483 2 | 81 self_attn q | 125,829,120 | -0.00009 | |
| 483 3 | 81 self_attn q | 188,743,680 | -0.00009 | |
| 483 4 | 81 self_attn q | 314,572,800 | -0.00016 | |
| 484 1 | 81 self_attn [k, v] | 10,485,760 | -0.00097 | |
| 484 2 | 81 self_attn [k, v] | 20,971,520 | -0.00104 | |
| 484 3 | 81 self_attn [k, v] | 31,457,280 | -0.00096 | |
| 484 4 | 81 self_attn [k, v] | 52,428,800 | -0.00096 | |
| 485 1 | 81 self_attn o | 62,914,560 | -0.00048 | |
| 485 2 | 81 self_attn o | 125,829,120 | -0.00047 | |
| 485 3 | 81 self_attn o | 188,743,680 | -0.00044 | |
| 485 4 | 81 self_attn o | 314,572,800 | -0.00047 | |
| 486 1 | 81 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 | |
| 486 2 | 81 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00037 | |
| 486 3 | 81 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00040 | |
| 486 4 | 81 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00041 | |
| 487 1 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00015 | |
| 487 2 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00016 | |
| 487 3 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00016 | |
| 487 4 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00014 | |
| 488 1 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 | |
| 488 2 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 488 3 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00010 | |
| 488 4 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00012 | |
| 489 1 | 82 self_attn q | 62,914,560 | -0.00014 | |
| 489 2 | 82 self_attn q | 125,829,120 | -0.00028 | |
| 489 3 | 82 self_attn q | 188,743,680 | -0.00020 | |
| 489 4 | 82 self_attn q | 314,572,800 | -0.00021 | |
| 490 1 | 82 self_attn [k, v] | 10,485,760 | -0.00009 | |
| 490 2 | 82 self_attn [k, v] | 20,971,520 | -0.00017 | |
| 490 3 | 82 self_attn [k, v] | 31,457,280 | -0.00016 | |
| 490 4 | 82 self_attn [k, v] | 52,428,800 | -0.00019 | |
| 491 1 | 82 self_attn o | 62,914,560 | -0.00001 | |
| 491 2 | 82 self_attn o | 125,829,120 | -0.00023 | |
| 491 3 | 82 self_attn o | 188,743,680 | -0.00020 | |
| 491 4 | 82 self_attn o | 314,572,800 | -0.00021 | |
| 492 1 | 82 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00022 | |
| 492 2 | 82 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00029 | |
| 492 3 | 82 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00021 | |
| 492 4 | 82 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00013 | |
| 493 1 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00028 | |
| 493 2 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00029 | |
| 493 3 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00029 | |
| 493 4 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00028 | |
| 494 1 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 494 2 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 | |
| 494 3 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00017 | |
| 494 4 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00017 | |
| 495 1 | 83 self_attn q | 62,914,560 | -0.00019 | |
| 495 2 | 83 self_attn q | 125,829,120 | -0.00016 | |
| 495 3 | 83 self_attn q | 188,743,680 | -0.00026 | |
| 495 4 | 83 self_attn q | 314,572,800 | -0.00025 | |
| 496 1 | 83 self_attn [k, v] | 10,485,760 | -0.00068 | |
| 496 2 | 83 self_attn [k, v] | 20,971,520 | -0.00093 | |
| 496 3 | 83 self_attn [k, v] | 31,457,280 | -0.00083 | |
| 496 4 | 83 self_attn [k, v] | 52,428,800 | -0.00079 | |
| 497 1 | 83 self_attn o | 62,914,560 | +0.00012 | |
| 497 2 | 83 self_attn o | 125,829,120 | -0.00005 | |
| 497 3 | 83 self_attn o | 188,743,680 | +0.00005 | |
| 497 4 | 83 self_attn o | 314,572,800 | +0.00001 | |
| 498 1 | 83 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00015 | |
| 498 2 | 83 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023 | |
| 498 3 | 83 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00010 | |
| 498 4 | 83 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00019 | |
| 499 1 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00020 | |
| 499 2 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019 | |
| 499 3 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00024 | |
| 499 4 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00023 | |
| 500 1 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 | |
| 500 2 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00012 | |
| 500 3 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00014 | |
| 500 4 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00013 | |
| 501 1 | 84 self_attn q | 62,914,560 | +0.00005 | |
| 501 2 | 84 self_attn q | 125,829,120 | -0.00014 | |
| 501 3 | 84 self_attn q | 188,743,680 | -0.00026 | |
| 501 4 | 84 self_attn q | 314,572,800 | -0.00014 | |
| 502 1 | 84 self_attn [k, v] | 10,485,760 | +0.00036 | |
| 502 2 | 84 self_attn [k, v] | 20,971,520 | -0.00004 | |
| 502 3 | 84 self_attn [k, v] | 31,457,280 | +0.00015 | |
| 502 4 | 84 self_attn [k, v] | 52,428,800 | +0.00021 | |
| 503 1 | 84 self_attn o | 62,914,560 | -0.00030 | |
| 503 2 | 84 self_attn o | 125,829,120 | -0.00008 | |
| 503 3 | 84 self_attn o | 188,743,680 | -0.00011 | |
| 503 4 | 84 self_attn o | 314,572,800 | -0.00005 | |
| 504 1 | 84 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00060 | |
| 504 2 | 84 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00033 | |
| 504 3 | 84 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00029 | |
| 504 4 | 84 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00037 | |
| 505 1 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00017 | |
| 505 2 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 | |
| 505 3 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00014 | |
| 505 4 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00013 | |
| 506 1 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 | |
| 506 2 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 | |
| 506 3 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00003 | |
| 506 4 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00002 | |
| 507 1 | 85 self_attn q | 62,914,560 | -0.00047 | |
| 507 2 | 85 self_attn q | 125,829,120 | -0.00052 | |
| 507 3 | 85 self_attn q | 188,743,680 | -0.00072 | |
| 507 4 | 85 self_attn q | 314,572,800 | -0.00084 | |
| 508 1 | 85 self_attn [k, v] | 10,485,760 | -0.00065 | |
| 508 2 | 85 self_attn [k, v] | 20,971,520 | -0.00086 | |
| 508 3 | 85 self_attn [k, v] | 31,457,280 | -0.00102 | |
| 508 4 | 85 self_attn [k, v] | 52,428,800 | -0.00100 | |
| 509 1 | 85 self_attn o | 62,914,560 | -0.00049 | |
| 509 2 | 85 self_attn o | 125,829,120 | -0.00066 | |
| 509 3 | 85 self_attn o | 188,743,680 | -0.00069 | |
| 509 4 | 85 self_attn o | 314,572,800 | -0.00069 | |
| 510 1 | 85 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00024 | |
| 510 2 | 85 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011 | |
| 510 3 | 85 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00012 | |
| 510 4 | 85 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00013 | |
| 511 1 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00007 | |
| 511 2 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00024 | |
| 511 3 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | +0.00013 | |
| 511 4 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | +0.00010 | |
| 512 1 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007 | |
| 512 2 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 | |
| 512 3 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00006 | |
| 512 4 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00005 | |
| 513 1 | 86 self_attn q | 62,914,560 | -0.00049 | |
| 513 2 | 86 self_attn q | 125,829,120 | -0.00040 | |
| 513 3 | 86 self_attn q | 188,743,680 | -0.00048 | |
| 513 4 | 86 self_attn q | 314,572,800 | -0.00047 | |
| 514 1 | 86 self_attn [k, v] | 10,485,760 | +0.00003 | |
| 514 2 | 86 self_attn [k, v] | 20,971,520 | -0.00036 | |
| 514 3 | 86 self_attn [k, v] | 31,457,280 | -0.00034 | |
| 514 4 | 86 self_attn [k, v] | 52,428,800 | -0.00041 | |
| 515 1 | 86 self_attn o | 62,914,560 | +0.00016 | |
| 515 2 | 86 self_attn o | 125,829,120 | -0.00005 | |
| 515 3 | 86 self_attn o | 188,743,680 | -0.00002 | |
| 515 4 | 86 self_attn o | 314,572,800 | -0.00003 | |
| 516 1 | 86 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00057 | |
| 516 2 | 86 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00053 | |
| 516 3 | 86 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00058 | |
| 516 4 | 86 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00056 | |
| 517 1 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 | |
| 517 2 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00022 | |
| 517 3 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00020 | |
| 517 4 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00019 | |
| 518 1 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 | |
| 518 2 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 | |
| 518 3 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00007 | |
| 518 4 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00009 | |
| 519 1 | 87 self_attn q | 62,914,560 | +0.00022 | |
| 519 2 | 87 self_attn q | 125,829,120 | +0.00020 | |
| 519 3 | 87 self_attn q | 188,743,680 | +0.00021 | |
| 519 4 | 87 self_attn q | 314,572,800 | +0.00016 | |
| 520 1 | 87 self_attn [k, v] | 10,485,760 | +0.00041 | |
| 520 2 | 87 self_attn [k, v] | 20,971,520 | +0.00044 | |
| 520 3 | 87 self_attn [k, v] | 31,457,280 | -0.00026 | |
| 520 4 | 87 self_attn [k, v] | 52,428,800 | -0.00008 | |
| 521 1 | 87 self_attn o | 62,914,560 | -0.00050 | |
| 521 2 | 87 self_attn o | 125,829,120 | -0.00042 | |
| 521 3 | 87 self_attn o | 188,743,680 | -0.00031 | |
| 521 4 | 87 self_attn o | 314,572,800 | -0.00036 | |
| 522 1 | 87 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 | |
| 522 2 | 87 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00033 | |
| 522 3 | 87 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00036 | |
| 522 4 | 87 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00035 | |
| 523 1 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 | |
| 523 2 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 | |
| 523 3 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00010 | |
| 523 4 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00007 | |
| 524 1 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010 | |
| 524 2 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 | |
| 524 3 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00011 | |
| 524 4 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00012 | |
| 525 1 | 88 self_attn q | 62,914,560 | -0.00005 | |
| 525 2 | 88 self_attn q | 125,829,120 | -0.00007 | |
| 525 3 | 88 self_attn q | 188,743,680 | -0.00005 | |
| 525 4 | 88 self_attn q | 314,572,800 | -0.00008 | |
| 526 1 | 88 self_attn [k, v] | 10,485,760 | -0.00073 | |
| 526 2 | 88 self_attn [k, v] | 20,971,520 | -0.00051 | |
| 526 3 | 88 self_attn [k, v] | 31,457,280 | -0.00051 | |
| 526 4 | 88 self_attn [k, v] | 52,428,800 | -0.00057 | |
| 527 1 | 88 self_attn o | 62,914,560 | -0.00012 | |
| 527 2 | 88 self_attn o | 125,829,120 | -0.00019 | |
| 527 3 | 88 self_attn o | 188,743,680 | -0.00019 | |
| 527 4 | 88 self_attn o | 314,572,800 | -0.00018 | |
| 528 1 | 88 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00008 | |
| 528 2 | 88 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00018 | |
| 528 3 | 88 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00023 | |
| 528 4 | 88 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00023 | |
| 529 1 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 | |
| 529 2 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014 | |
| 529 3 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00012 | |
| 529 4 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00013 | |
| 530 1 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00009 | |
| 530 2 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 | |
| 530 3 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00009 | |
| 530 4 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00011 | |
| 531 1 | 89 self_attn q | 62,914,560 | -0.00017 | |
| 531 2 | 89 self_attn q | 125,829,120 | -0.00023 | |
| 531 3 | 89 self_attn q | 188,743,680 | -0.00028 | |
| 531 4 | 89 self_attn q | 314,572,800 | -0.00026 | |
| 532 1 | 89 self_attn [k, v] | 10,485,760 | -0.00054 | |
| 532 2 | 89 self_attn [k, v] | 20,971,520 | -0.00070 | |
| 532 3 | 89 self_attn [k, v] | 31,457,280 | -0.00049 | |
| 532 4 | 89 self_attn [k, v] | 52,428,800 | -0.00057 | |
| 533 1 | 89 self_attn o | 62,914,560 | -0.00029 | |
| 533 2 | 89 self_attn o | 125,829,120 | -0.00023 | |
| 533 3 | 89 self_attn o | 188,743,680 | -0.00028 | |
| 533 4 | 89 self_attn o | 314,572,800 | -0.00027 | |
| 534 1 | 89 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00016 | |
| 534 2 | 89 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 | |
| 534 3 | 89 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00034 | |
| 534 4 | 89 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00029 | |
| 535 1 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 | |
| 535 2 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 | |
| 535 3 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00009 | |
| 535 4 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00009 | |
| 536 1 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 | |
| 536 2 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 | |
| 536 3 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00006 | |
| 536 4 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00006 | |
| 537 1 | 90 self_attn q | 62,914,560 | -0.00019 | |
| 537 2 | 90 self_attn q | 125,829,120 | -0.00024 | |
| 537 3 | 90 self_attn q | 188,743,680 | -0.00023 | |
| 537 4 | 90 self_attn q | 314,572,800 | -0.00022 | |
| 538 1 | 90 self_attn [k, v] | 10,485,760 | -0.00017 | |
| 538 2 | 90 self_attn [k, v] | 20,971,520 | -0.00028 | |
| 538 3 | 90 self_attn [k, v] | 31,457,280 | -0.00025 | |
| 538 4 | 90 self_attn [k, v] | 52,428,800 | -0.00022 | |
| 539 1 | 90 self_attn o | 62,914,560 | -0.00009 | |
| 539 2 | 90 self_attn o | 125,829,120 | -0.00009 | |
| 539 3 | 90 self_attn o | 188,743,680 | -0.00011 | |
| 539 4 | 90 self_attn o | 314,572,800 | -0.00014 | |
| 540 1 | 90 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00062 | |
| 540 2 | 90 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00064 | |
| 540 3 | 90 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00055 | |
| 540 4 | 90 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00060 | |
| 541 1 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 | |
| 541 2 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 | |
| 541 3 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00010 | |
| 541 4 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00010 | |
| 542 1 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010 | |
| 542 2 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 542 3 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00009 | |
| 542 4 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00009 | |
| 543 1 | 91 self_attn q | 62,914,560 | -0.00020 | |
| 543 2 | 91 self_attn q | 125,829,120 | -0.00004 | |
| 543 3 | 91 self_attn q | 188,743,680 | -0.00008 | |
| 543 4 | 91 self_attn q | 314,572,800 | -0.00004 | |
| 544 1 | 91 self_attn [k, v] | 10,485,760 | -0.00072 | |
| 544 2 | 91 self_attn [k, v] | 20,971,520 | -0.00106 | |
| 544 3 | 91 self_attn [k, v] | 31,457,280 | -0.00110 | |
| 544 4 | 91 self_attn [k, v] | 52,428,800 | -0.00118 | |
| 545 1 | 91 self_attn o | 62,914,560 | -0.00016 | |
| 545 2 | 91 self_attn o | 125,829,120 | -0.00034 | |
| 545 3 | 91 self_attn o | 188,743,680 | -0.00031 | |
| 545 4 | 91 self_attn o | 314,572,800 | -0.00034 | |
| 546 1 | 91 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00076 | |
| 546 2 | 91 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 | |
| 546 3 | 91 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00013 | |
| 546 4 | 91 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00027 | |
| 547 1 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 | |
| 547 2 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 | |
| 547 3 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00009 | |
| 547 4 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00010 | |
| 548 1 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004 | |
| 548 2 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 | |
| 548 3 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00005 | |
| 548 4 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00005 | |
| -- Writing /home/beta/AI/local_models_exl/glm-4.5-measurements-3vs4vs5vs6vs8.json | |