| -- Results | |
| idx | parts | d. bits | d. KL-div | |
| ---------|----------------------------------------------------------------------|----------------------|----------- | |
| - | | | 0.07912 | |
| 0 1 | 0 self_attn q | 62,914,560 | -0.00014 | |
| 0 2 | 0 self_attn q | 125,829,120 | -0.00009 | |
| 1 1 | 0 self_attn [k, v] | 10,485,760 | -0.00005 | |
| 1 2 | 0 self_attn [k, v] | 20,971,520 | -0.00006 | |
| 2 1 | 0 self_attn o | 62,914,560 | -0.00040 | |
| 2 2 | 0 self_attn o | 125,829,120 | -0.00022 | |
| 3 1 | 0 mlp [gate, up] | 125,829,120 | -0.00044 | |
| 3 2 | 0 mlp [gate, up] | 251,658,240 | -0.00052 | |
| 4 1 | 0 mlp down | 62,914,560 | -0.00036 | |
| 4 2 | 0 mlp down | 125,829,120 | -0.00018 | |
| 5 1 | 1 self_attn q | 62,914,560 | -0.00008 | |
| 5 2 | 1 self_attn q | 125,829,120 | +0.00000 | |
| 6 1 | 1 self_attn [k, v] | 10,485,760 | -0.00012 | |
| 6 2 | 1 self_attn [k, v] | 20,971,520 | -0.00034 | |
| 7 1 | 1 self_attn o | 62,914,560 | +0.00001 | |
| 7 2 | 1 self_attn o | 125,829,120 | +0.00007 | |
| 8 1 | 1 mlp [gate, up] | 125,829,120 | -0.00024 | |
| 8 2 | 1 mlp [gate, up] | 251,658,240 | -0.00037 | |
| 9 1 | 1 mlp down | 62,914,560 | +0.00003 | |
| 9 2 | 1 mlp down | 125,829,120 | -0.00005 | |
| 10 1 | 2 self_attn q | 62,914,560 | +0.00001 | |
| 10 2 | 2 self_attn q | 125,829,120 | -0.00007 | |
| 11 1 | 2 self_attn [k, v] | 10,485,760 | -0.00062 | |
| 11 2 | 2 self_attn [k, v] | 20,971,520 | -0.00047 | |
| 12 1 | 2 self_attn o | 62,914,560 | -0.00006 | |
| 12 2 | 2 self_attn o | 125,829,120 | +0.00002 | |
| 13 1 | 2 mlp [gate, up] | 125,829,120 | -0.00008 | |
| 13 2 | 2 mlp [gate, up] | 251,658,240 | -0.00058 | |
| 14 1 | 2 mlp down | 62,914,560 | -0.00062 | |
| 14 2 | 2 mlp down | 125,829,120 | -0.00037 | |
| 15 1 | 3 self_attn q | 62,914,560 | -0.00025 | |
| 15 2 | 3 self_attn q | 125,829,120 | -0.00012 | |
| 16 1 | 3 self_attn [k, v] | 10,485,760 | -0.00060 | |
| 16 2 | 3 self_attn [k, v] | 20,971,520 | -0.00071 | |
| 17 1 | 3 self_attn o | 62,914,560 | -0.00006 | |
| 17 2 | 3 self_attn o | 125,829,120 | -0.00007 | |
| 18 1 | 3 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00031 | |
| 18 2 | 3 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00011 | |
| 19 1 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00042 | |
| 19 2 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00027 | |
| 20 1 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00019 | |
| 20 2 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00020 | |
| 21 1 | 4 self_attn q | 62,914,560 | +0.00006 | |
| 21 2 | 4 self_attn q | 125,829,120 | +0.00013 | |
| 22 1 | 4 self_attn [k, v] | 10,485,760 | +0.00022 | |
| 22 2 | 4 self_attn [k, v] | 20,971,520 | +0.00011 | |
| 23 1 | 4 self_attn o | 62,914,560 | -0.00001 | |
| 23 2 | 4 self_attn o | 125,829,120 | +0.00003 | |
| 24 1 | 4 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 | |
| 24 2 | 4 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00036 | |
| 25 1 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00018 | |
| 25 2 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00031 | |
| 26 1 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00026 | |
| 26 2 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00026 | |
| 27 1 | 5 self_attn q | 62,914,560 | -0.00009 | |
| 27 2 | 5 self_attn q | 125,829,120 | -0.00008 | |
| 28 1 | 5 self_attn [k, v] | 10,485,760 | +0.00007 | |
| 28 2 | 5 self_attn [k, v] | 20,971,520 | +0.00015 | |
| 29 1 | 5 self_attn o | 62,914,560 | -0.00026 | |
| 29 2 | 5 self_attn o | 125,829,120 | -0.00013 | |
| 30 1 | 5 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00048 | |
| 30 2 | 5 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00034 | |
| 31 1 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00020 | |
| 31 2 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00015 | |
| 32 1 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00021 | |
| 32 2 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00025 | |
| 33 1 | 6 self_attn q | 62,914,560 | +0.00010 | |
| 33 2 | 6 self_attn q | 125,829,120 | +0.00016 | |
| 34 1 | 6 self_attn [k, v] | 10,485,760 | -0.00018 | |
| 34 2 | 6 self_attn [k, v] | 20,971,520 | -0.00023 | |
| 35 1 | 6 self_attn o | 62,914,560 | -0.00030 | |
| 35 2 | 6 self_attn o | 125,829,120 | -0.00031 | |
| 36 1 | 6 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009 | |
| 36 2 | 6 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022 | |
| 37 1 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00027 | |
| 37 2 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00024 | |
| 38 1 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00013 | |
| 38 2 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00043 | |
| 39 1 | 7 self_attn q | 62,914,560 | -0.00029 | |
| 39 2 | 7 self_attn q | 125,829,120 | -0.00034 | |
| 40 1 | 7 self_attn [k, v] | 10,485,760 | -0.00028 | |
| 40 2 | 7 self_attn [k, v] | 20,971,520 | -0.00004 | |
| 41 1 | 7 self_attn o | 62,914,560 | -0.00011 | |
| 41 2 | 7 self_attn o | 125,829,120 | -0.00023 | |
| 42 1 | 7 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00010 | |
| 42 2 | 7 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00045 | |
| 43 1 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00028 | |
| 43 2 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00033 | |
| 44 1 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00040 | |
| 44 2 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00039 | |
| 45 1 | 8 self_attn q | 62,914,560 | -0.00017 | |
| 45 2 | 8 self_attn q | 125,829,120 | -0.00026 | |
| 46 1 | 8 self_attn [k, v] | 10,485,760 | +0.00086 | |
| 46 2 | 8 self_attn [k, v] | 20,971,520 | +0.00031 | |
| 47 1 | 8 self_attn o | 62,914,560 | -0.00128 | |
| 47 2 | 8 self_attn o | 125,829,120 | -0.00105 | |
| 48 1 | 8 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00019 | |
| 48 2 | 8 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00013 | |
| 49 1 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00037 | |
| 49 2 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00023 | |
| 50 1 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00026 | |
| 50 2 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00030 | |
| 51 1 | 9 self_attn q | 62,914,560 | -0.00022 | |
| 51 2 | 9 self_attn q | 125,829,120 | -0.00017 | |
| 52 1 | 9 self_attn [k, v] | 10,485,760 | -0.00070 | |
| 52 2 | 9 self_attn [k, v] | 20,971,520 | -0.00023 | |
| 53 1 | 9 self_attn o | 62,914,560 | -0.00030 | |
| 53 2 | 9 self_attn o | 125,829,120 | -0.00026 | |
| 54 1 | 9 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 | |
| 54 2 | 9 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00007 | |
| 55 1 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00047 | |
| 55 2 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00037 | |
| 56 1 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00025 | |
| 56 2 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00032 | |
| 57 1 | 10 self_attn q | 62,914,560 | -0.00030 | |
| 57 2 | 10 self_attn q | 125,829,120 | -0.00026 | |
| 58 1 | 10 self_attn [k, v] | 10,485,760 | -0.00006 | |
| 58 2 | 10 self_attn [k, v] | 20,971,520 | +0.00030 | |
| 59 1 | 10 self_attn o | 62,914,560 | +0.00033 | |
| 59 2 | 10 self_attn o | 125,829,120 | +0.00003 | |
| 60 1 | 10 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006 | |
| 60 2 | 10 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00009 | |
| 61 1 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00020 | |
| 61 2 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00021 | |
| 62 1 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 | |
| 62 2 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00037 | |
| 63 1 | 11 self_attn q | 62,914,560 | +0.00010 | |
| 63 2 | 11 self_attn q | 125,829,120 | -0.00010 | |
| 64 1 | 11 self_attn [k, v] | 10,485,760 | +0.00122 | |
| 64 2 | 11 self_attn [k, v] | 20,971,520 | +0.00123 | |
| 65 1 | 11 self_attn o | 62,914,560 | -0.00109 | |
| 65 2 | 11 self_attn o | 125,829,120 | -0.00174 | |
| 66 1 | 11 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00046 | |
| 66 2 | 11 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00044 | |
| 67 1 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00042 | |
| 67 2 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00054 | |
| 68 1 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00057 | |
| 68 2 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00058 | |
| 69 1 | 12 self_attn q | 62,914,560 | +0.00001 | |
| 69 2 | 12 self_attn q | 125,829,120 | +0.00001 | |
| 70 1 | 12 self_attn [k, v] | 10,485,760 | -0.00015 | |
| 70 2 | 12 self_attn [k, v] | 20,971,520 | +0.00026 | |
| 71 1 | 12 self_attn o | 62,914,560 | -0.00105 | |
| 71 2 | 12 self_attn o | 125,829,120 | -0.00077 | |
| 72 1 | 12 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00042 | |
| 72 2 | 12 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00010 | |
| 73 1 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 | |
| 73 2 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00084 | |
| 74 1 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00072 | |
| 74 2 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00077 | |
| 75 1 | 13 self_attn q | 62,914,560 | -0.00023 | |
| 75 2 | 13 self_attn q | 125,829,120 | -0.00033 | |
| 76 1 | 13 self_attn [k, v] | 10,485,760 | -0.00156 | |
| 76 2 | 13 self_attn [k, v] | 20,971,520 | -0.00126 | |
| 77 1 | 13 self_attn o | 62,914,560 | -0.00087 | |
| 77 2 | 13 self_attn o | 125,829,120 | -0.00080 | |
| 78 1 | 13 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00054 | |
| 78 2 | 13 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00032 | |
| 79 1 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00031 | |
| 79 2 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00051 | |
| 80 1 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 | |
| 80 2 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00064 | |
| 81 1 | 14 self_attn q | 62,914,560 | -0.00018 | |
| 81 2 | 14 self_attn q | 125,829,120 | -0.00004 | |
| 82 1 | 14 self_attn [k, v] | 10,485,760 | -0.00025 | |
| 82 2 | 14 self_attn [k, v] | 20,971,520 | +0.00013 | |
| 83 1 | 14 self_attn o | 62,914,560 | -0.00037 | |
| 83 2 | 14 self_attn o | 125,829,120 | -0.00087 | |
| 84 1 | 14 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004 | |
| 84 2 | 14 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00006 | |
| 85 1 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 | |
| 85 2 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00096 | |
| 86 1 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00038 | |
| 86 2 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00067 | |
| 87 1 | 15 self_attn q | 62,914,560 | +0.00008 | |
| 87 2 | 15 self_attn q | 125,829,120 | +0.00005 | |
| 88 1 | 15 self_attn [k, v] | 10,485,760 | +0.00054 | |
| 88 2 | 15 self_attn [k, v] | 20,971,520 | +0.00031 | |
| 89 1 | 15 self_attn o | 62,914,560 | -0.00013 | |
| 89 2 | 15 self_attn o | 125,829,120 | -0.00033 | |
| 90 1 | 15 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00017 | |
| 90 2 | 15 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00006 | |
| 91 1 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00103 | |
| 91 2 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00092 | |
| 92 1 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00069 | |
| 92 2 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00066 | |
| 93 1 | 16 self_attn q | 62,914,560 | -0.00042 | |
| 93 2 | 16 self_attn q | 125,829,120 | -0.00002 | |
| 94 1 | 16 self_attn [k, v] | 10,485,760 | +0.00047 | |
| 94 2 | 16 self_attn [k, v] | 20,971,520 | +0.00032 | |
| 95 1 | 16 self_attn o | 62,914,560 | -0.00011 | |
| 95 2 | 16 self_attn o | 125,829,120 | -0.00021 | |
| 96 1 | 16 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 | |
| 96 2 | 16 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00015 | |
| 97 1 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00091 | |
| 97 2 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00137 | |
| 98 1 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00065 | |
| 98 2 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00103 | |
| 99 1 | 17 self_attn q | 62,914,560 | +0.00015 | |
| 99 2 | 17 self_attn q | 125,829,120 | +0.00013 | |
| 100 1 | 17 self_attn [k, v] | 10,485,760 | -0.00075 | |
| 100 2 | 17 self_attn [k, v] | 20,971,520 | -0.00078 | |
| 101 1 | 17 self_attn o | 62,914,560 | -0.00036 | |
| 101 2 | 17 self_attn o | 125,829,120 | -0.00032 | |
| 102 1 | 17 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00048 | |
| 102 2 | 17 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00038 | |
| 103 1 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00100 | |
| 103 2 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00095 | |
| 104 1 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00070 | |
| 104 2 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00094 | |
| 105 1 | 18 self_attn q | 62,914,560 | -0.00003 | |
| 105 2 | 18 self_attn q | 125,829,120 | +0.00023 | |
| 106 1 | 18 self_attn [k, v] | 10,485,760 | -0.00102 | |
| 106 2 | 18 self_attn [k, v] | 20,971,520 | -0.00091 | |
| 107 1 | 18 self_attn o | 62,914,560 | -0.00018 | |
| 107 2 | 18 self_attn o | 125,829,120 | -0.00020 | |
| 108 1 | 18 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00039 | |
| 108 2 | 18 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00037 | |
| 109 1 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00090 | |
| 109 2 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00097 | |
| 110 1 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 | |
| 110 2 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00040 | |
| 111 1 | 19 self_attn q | 62,914,560 | +0.00021 | |
| 111 2 | 19 self_attn q | 125,829,120 | -0.00004 | |
| 112 1 | 19 self_attn [k, v] | 10,485,760 | +0.00059 | |
| 112 2 | 19 self_attn [k, v] | 20,971,520 | +0.00079 | |
| 113 1 | 19 self_attn o | 62,914,560 | -0.00079 | |
| 113 2 | 19 self_attn o | 125,829,120 | -0.00089 | |
| 114 1 | 19 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00162 | |
| 114 2 | 19 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00098 | |
| 115 1 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00096 | |
| 115 2 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00105 | |
| 116 1 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00065 | |
| 116 2 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00072 | |
| 117 1 | 20 self_attn q | 62,914,560 | -0.00000 | |
| 117 2 | 20 self_attn q | 125,829,120 | +0.00016 | |
| 118 1 | 20 self_attn [k, v] | 10,485,760 | +0.00023 | |
| 118 2 | 20 self_attn [k, v] | 20,971,520 | -0.00038 | |
| 119 1 | 20 self_attn o | 62,914,560 | +0.00068 | |
| 119 2 | 20 self_attn o | 125,829,120 | -0.00002 | |
| 120 1 | 20 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00098 | |
| 120 2 | 20 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00153 | |
| 121 1 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00070 | |
| 121 2 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00086 | |
| 122 1 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00085 | |
| 122 2 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00084 | |
| 123 1 | 21 self_attn q | 62,914,560 | -0.00027 | |
| 123 2 | 21 self_attn q | 125,829,120 | -0.00025 | |
| 124 1 | 21 self_attn [k, v] | 10,485,760 | +0.00026 | |
| 124 2 | 21 self_attn [k, v] | 20,971,520 | +0.00027 | |
| 125 1 | 21 self_attn o | 62,914,560 | -0.00022 | |
| 125 2 | 21 self_attn o | 125,829,120 | +0.00026 | |
| 126 1 | 21 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011 | |
| 126 2 | 21 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00072 | |
| 127 1 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00056 | |
| 127 2 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00089 | |
| 128 1 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00017 | |
| 128 2 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00042 | |
| 129 1 | 22 self_attn q | 62,914,560 | -0.00027 | |
| 129 2 | 22 self_attn q | 125,829,120 | -0.00029 | |
| 130 1 | 22 self_attn [k, v] | 10,485,760 | +0.00042 | |
| 130 2 | 22 self_attn [k, v] | 20,971,520 | +0.00001 | |
| 131 1 | 22 self_attn o | 62,914,560 | -0.00048 | |
| 131 2 | 22 self_attn o | 125,829,120 | -0.00034 | |
| 132 1 | 22 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00094 | |
| 132 2 | 22 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00085 | |
| 133 1 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00045 | |
| 133 2 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00058 | |
| 134 1 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00023 | |
| 134 2 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00033 | |
| 135 1 | 23 self_attn q | 62,914,560 | -0.00067 | |
| 135 2 | 23 self_attn q | 125,829,120 | -0.00063 | |
| 136 1 | 23 self_attn [k, v] | 10,485,760 | -0.00013 | |
| 136 2 | 23 self_attn [k, v] | 20,971,520 | -0.00052 | |
| 137 1 | 23 self_attn o | 62,914,560 | -0.00050 | |
| 137 2 | 23 self_attn o | 125,829,120 | +0.00007 | |
| 138 1 | 23 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00094 | |
| 138 2 | 23 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00065 | |
| 139 1 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00082 | |
| 139 2 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00101 | |
| 140 1 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 | |
| 140 2 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00046 | |
| 141 1 | 24 self_attn q | 62,914,560 | -0.00007 | |
| 141 2 | 24 self_attn q | 125,829,120 | -0.00033 | |
| 142 1 | 24 self_attn [k, v] | 10,485,760 | -0.00082 | |
| 142 2 | 24 self_attn [k, v] | 20,971,520 | -0.00100 | |
| 143 1 | 24 self_attn o | 62,914,560 | -0.00023 | |
| 143 2 | 24 self_attn o | 125,829,120 | -0.00007 | |
| 144 1 | 24 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00058 | |
| 144 2 | 24 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00050 | |
| 145 1 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00105 | |
| 145 2 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00106 | |
| 146 1 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00036 | |
| 146 2 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 | |
| 147 1 | 25 self_attn q | 62,914,560 | +0.00065 | |
| 147 2 | 25 self_attn q | 125,829,120 | +0.00064 | |
| 148 1 | 25 self_attn [k, v] | 10,485,760 | +0.00041 | |
| 148 2 | 25 self_attn [k, v] | 20,971,520 | +0.00035 | |
| 149 1 | 25 self_attn o | 62,914,560 | -0.00089 | |
| 149 2 | 25 self_attn o | 125,829,120 | -0.00086 | |
| 150 1 | 25 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00115 | |
| 150 2 | 25 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00084 | |
| 151 1 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00041 | |
| 151 2 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00066 | |
| 152 1 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00043 | |
| 152 2 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00060 | |
| 153 1 | 26 self_attn q | 62,914,560 | -0.00035 | |
| 153 2 | 26 self_attn q | 125,829,120 | -0.00033 | |
| 154 1 | 26 self_attn [k, v] | 10,485,760 | +0.00018 | |
| 154 2 | 26 self_attn [k, v] | 20,971,520 | -0.00017 | |
| 155 1 | 26 self_attn o | 62,914,560 | -0.00035 | |
| 155 2 | 26 self_attn o | 125,829,120 | -0.00106 | |
| 156 1 | 26 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00090 | |
| 156 2 | 26 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00073 | |
| 157 1 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 | |
| 157 2 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00085 | |
| 158 1 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00035 | |
| 158 2 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00044 | |
| 159 1 | 27 self_attn q | 62,914,560 | -0.00028 | |
| 159 2 | 27 self_attn q | 125,829,120 | -0.00024 | |
| 160 1 | 27 self_attn [k, v] | 10,485,760 | -0.00073 | |
| 160 2 | 27 self_attn [k, v] | 20,971,520 | -0.00092 | |
| 161 1 | 27 self_attn o | 62,914,560 | -0.00115 | |
| 161 2 | 27 self_attn o | 125,829,120 | -0.00110 | |
| 162 1 | 27 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00105 | |
| 162 2 | 27 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00132 | |
| 163 1 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00055 | |
| 163 2 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00056 | |
| 164 1 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00037 | |
| 164 2 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00052 | |
| 165 1 | 28 self_attn q | 62,914,560 | -0.00011 | |
| 165 2 | 28 self_attn q | 125,829,120 | -0.00024 | |
| 166 1 | 28 self_attn [k, v] | 10,485,760 | +0.00080 | |
| 166 2 | 28 self_attn [k, v] | 20,971,520 | +0.00048 | |
| 167 1 | 28 self_attn o | 62,914,560 | -0.00065 | |
| 167 2 | 28 self_attn o | 125,829,120 | -0.00095 | |
| 168 1 | 28 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00038 | |
| 168 2 | 28 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00026 | |
| 169 1 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00087 | |
| 169 2 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00083 | |
| 170 1 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00033 | |
| 170 2 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 | |
| 171 1 | 29 self_attn q | 62,914,560 | -0.00006 | |
| 171 2 | 29 self_attn q | 125,829,120 | +0.00003 | |
| 172 1 | 29 self_attn [k, v] | 10,485,760 | -0.00055 | |
| 172 2 | 29 self_attn [k, v] | 20,971,520 | -0.00008 | |
| 173 1 | 29 self_attn o | 62,914,560 | -0.00040 | |
| 173 2 | 29 self_attn o | 125,829,120 | -0.00062 | |
| 174 1 | 29 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00091 | |
| 174 2 | 29 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00076 | |
| 175 1 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00069 | |
| 175 2 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00057 | |
| 176 1 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00037 | |
| 176 2 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 | |
| 177 1 | 30 self_attn q | 62,914,560 | -0.00054 | |
| 177 2 | 30 self_attn q | 125,829,120 | -0.00042 | |
| 178 1 | 30 self_attn [k, v] | 10,485,760 | +0.00031 | |
| 178 2 | 30 self_attn [k, v] | 20,971,520 | +0.00004 | |
| 179 1 | 30 self_attn o | 62,914,560 | +0.00012 | |
| 179 2 | 30 self_attn o | 125,829,120 | +0.00010 | |
| 180 1 | 30 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00117 | |
| 180 2 | 30 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00137 | |
| 181 1 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00064 | |
| 181 2 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00072 | |
| 182 1 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 | |
| 182 2 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00029 | |
| 183 1 | 31 self_attn q | 62,914,560 | +0.00057 | |
| 183 2 | 31 self_attn q | 125,829,120 | +0.00081 | |
| 184 1 | 31 self_attn [k, v] | 10,485,760 | +0.00164 | |
| 184 2 | 31 self_attn [k, v] | 20,971,520 | +0.00088 | |
| 185 1 | 31 self_attn o | 62,914,560 | -0.00012 | |
| 185 2 | 31 self_attn o | 125,829,120 | -0.00022 | |
| 186 1 | 31 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00199 | |
| 186 2 | 31 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00063 | |
| 187 1 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00064 | |
| 187 2 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00092 | |
| 188 1 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007 | |
| 188 2 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00040 | |
| 189 1 | 32 self_attn q | 62,914,560 | +0.00057 | |
| 189 2 | 32 self_attn q | 125,829,120 | +0.00062 | |
| 190 1 | 32 self_attn [k, v] | 10,485,760 | -0.00001 | |
| 190 2 | 32 self_attn [k, v] | 20,971,520 | +0.00041 | |
| 191 1 | 32 self_attn o | 62,914,560 | -0.00070 | |
| 191 2 | 32 self_attn o | 125,829,120 | -0.00018 | |
| 192 1 | 32 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00119 | |
| 192 2 | 32 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00164 | |
| 193 1 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 | |
| 193 2 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00038 | |
| 194 1 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00030 | |
| 194 2 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00044 | |
| 195 1 | 33 self_attn q | 62,914,560 | -0.00042 | |
| 195 2 | 33 self_attn q | 125,829,120 | -0.00040 | |
| 196 1 | 33 self_attn [k, v] | 10,485,760 | -0.00040 | |
| 196 2 | 33 self_attn [k, v] | 20,971,520 | -0.00042 | |
| 197 1 | 33 self_attn o | 62,914,560 | +0.00082 | |
| 197 2 | 33 self_attn o | 125,829,120 | +0.00038 | |
| 198 1 | 33 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00144 | |
| 198 2 | 33 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00200 | |
| 199 1 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00027 | |
| 199 2 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00013 | |
| 200 1 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 | |
| 200 2 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00025 | |
| 201 1 | 34 self_attn q | 62,914,560 | +0.00033 | |
| 201 2 | 34 self_attn q | 125,829,120 | +0.00000 | |
| 202 1 | 34 self_attn [k, v] | 10,485,760 | +0.00022 | |
| 202 2 | 34 self_attn [k, v] | 20,971,520 | -0.00011 | |
| 203 1 | 34 self_attn o | 62,914,560 | -0.00080 | |
| 203 2 | 34 self_attn o | 125,829,120 | -0.00064 | |
| 204 1 | 34 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00241 | |
| 204 2 | 34 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00285 | |
| 205 1 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00084 | |
| 205 2 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00105 | |
| 206 1 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00040 | |
| 206 2 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00041 | |
| 207 1 | 35 self_attn q | 62,914,560 | -0.00040 | |
| 207 2 | 35 self_attn q | 125,829,120 | -0.00061 | |
| 208 1 | 35 self_attn [k, v] | 10,485,760 | -0.00090 | |
| 208 2 | 35 self_attn [k, v] | 20,971,520 | -0.00092 | |
| 209 1 | 35 self_attn o | 62,914,560 | -0.00054 | |
| 209 2 | 35 self_attn o | 125,829,120 | -0.00082 | |
| 210 1 | 35 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00015 | |
| 210 2 | 35 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016 | |
| 211 1 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00088 | |
| 211 2 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00108 | |
| 212 1 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00053 | |
| 212 2 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00071 | |
| 213 1 | 36 self_attn q | 62,914,560 | +0.00103 | |
| 213 2 | 36 self_attn q | 125,829,120 | +0.00090 | |
| 214 1 | 36 self_attn [k, v] | 10,485,760 | -0.00008 | |
| 214 2 | 36 self_attn [k, v] | 20,971,520 | -0.00176 | |
| 215 1 | 36 self_attn o | 62,914,560 | -0.00210 | |
| 215 2 | 36 self_attn o | 125,829,120 | -0.00219 | |
| 216 1 | 36 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00170 | |
| 216 2 | 36 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00136 | |
| 217 1 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00023 | |
| 217 2 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00053 | |
| 218 1 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 | |
| 218 2 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00036 | |
| 219 1 | 37 self_attn q | 62,914,560 | -0.00078 | |
| 219 2 | 37 self_attn q | 125,829,120 | -0.00057 | |
| 220 1 | 37 self_attn [k, v] | 10,485,760 | +0.00030 | |
| 220 2 | 37 self_attn [k, v] | 20,971,520 | -0.00025 | |
| 221 1 | 37 self_attn o | 62,914,560 | -0.00185 | |
| 221 2 | 37 self_attn o | 125,829,120 | -0.00134 | |
| 222 1 | 37 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00082 | |
| 222 2 | 37 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00095 | |
| 223 1 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00015 | |
| 223 2 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 | |
| 224 1 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 | |
| 224 2 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 | |
| 225 1 | 38 self_attn q | 62,914,560 | -0.00100 | |
| 225 2 | 38 self_attn q | 125,829,120 | -0.00120 | |
| 226 1 | 38 self_attn [k, v] | 10,485,760 | -0.00007 | |
| 226 2 | 38 self_attn [k, v] | 20,971,520 | -0.00013 | |
| 227 1 | 38 self_attn o | 62,914,560 | -0.00007 | |
| 227 2 | 38 self_attn o | 125,829,120 | -0.00017 | |
| 228 1 | 38 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 | |
| 228 2 | 38 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00028 | |
| 229 1 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00040 | |
| 229 2 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00061 | |
| 230 1 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 | |
| 230 2 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00034 | |
| 231 1 | 39 self_attn q | 62,914,560 | -0.00046 | |
| 231 2 | 39 self_attn q | 125,829,120 | -0.00052 | |
| 232 1 | 39 self_attn [k, v] | 10,485,760 | +0.00050 | |
| 232 2 | 39 self_attn [k, v] | 20,971,520 | -0.00047 | |
| 233 1 | 39 self_attn o | 62,914,560 | -0.00100 | |
| 233 2 | 39 self_attn o | 125,829,120 | -0.00038 | |
| 234 1 | 39 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00087 | |
| 234 2 | 39 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00101 | |
| 235 1 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00045 | |
| 235 2 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00057 | |
| 236 1 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013 | |
| 236 2 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00028 | |
| 237 1 | 40 self_attn q | 62,914,560 | -0.00051 | |
| 237 2 | 40 self_attn q | 125,829,120 | -0.00034 | |
| 238 1 | 40 self_attn [k, v] | 10,485,760 | -0.00089 | |
| 238 2 | 40 self_attn [k, v] | 20,971,520 | -0.00134 | |
| 239 1 | 40 self_attn o | 62,914,560 | +0.00010 | |
| 239 2 | 40 self_attn o | 125,829,120 | -0.00029 | |
| 240 1 | 40 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00051 | |
| 240 2 | 40 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00090 | |
| 241 1 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00055 | |
| 241 2 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00065 | |
| 242 1 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00036 | |
| 242 2 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00058 | |
| 243 1 | 41 self_attn q | 62,914,560 | +0.00062 | |
| 243 2 | 41 self_attn q | 125,829,120 | +0.00063 | |
| 244 1 | 41 self_attn [k, v] | 10,485,760 | -0.00110 | |
| 244 2 | 41 self_attn [k, v] | 20,971,520 | -0.00079 | |
| 245 1 | 41 self_attn o | 62,914,560 | +0.00059 | |
| 245 2 | 41 self_attn o | 125,829,120 | +0.00076 | |
| 246 1 | 41 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00056 | |
| 246 2 | 41 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00014 | |
| 247 1 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00074 | |
| 247 2 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00091 | |
| 248 1 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00021 | |
| 248 2 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 | |
| 249 1 | 42 self_attn q | 62,914,560 | -0.00035 | |
| 249 2 | 42 self_attn q | 125,829,120 | -0.00034 | |
| 250 1 | 42 self_attn [k, v] | 10,485,760 | +0.00023 | |
| 250 2 | 42 self_attn [k, v] | 20,971,520 | -0.00048 | |
| 251 1 | 42 self_attn o | 62,914,560 | -0.00097 | |
| 251 2 | 42 self_attn o | 125,829,120 | -0.00076 | |
| 252 1 | 42 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00072 | |
| 252 2 | 42 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00079 | |
| 253 1 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00038 | |
| 253 2 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00035 | |
| 254 1 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 254 2 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00028 | |
| 255 1 | 43 self_attn q | 62,914,560 | -0.00023 | |
| 255 2 | 43 self_attn q | 125,829,120 | -0.00029 | |
| 256 1 | 43 self_attn [k, v] | 10,485,760 | -0.00082 | |
| 256 2 | 43 self_attn [k, v] | 20,971,520 | -0.00066 | |
| 257 1 | 43 self_attn o | 62,914,560 | -0.00034 | |
| 257 2 | 43 self_attn o | 125,829,120 | -0.00058 | |
| 258 1 | 43 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005 | |
| 258 2 | 43 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00000 | |
| 259 1 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00054 | |
| 259 2 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00059 | |
| 260 1 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00045 | |
| 260 2 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00039 | |
| 261 1 | 44 self_attn q | 62,914,560 | -0.00056 | |
| 261 2 | 44 self_attn q | 125,829,120 | -0.00037 | |
| 262 1 | 44 self_attn [k, v] | 10,485,760 | +0.00008 | |
| 262 2 | 44 self_attn [k, v] | 20,971,520 | +0.00049 | |
| 263 1 | 44 self_attn o | 62,914,560 | -0.00041 | |
| 263 2 | 44 self_attn o | 125,829,120 | -0.00079 | |
| 264 1 | 44 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00005 | |
| 264 2 | 44 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016 | |
| 265 1 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00030 | |
| 265 2 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00033 | |
| 266 1 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00033 | |
| 266 2 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00030 | |
| 267 1 | 45 self_attn q | 62,914,560 | -0.00047 | |
| 267 2 | 45 self_attn q | 125,829,120 | -0.00035 | |
| 268 1 | 45 self_attn [k, v] | 10,485,760 | -0.00117 | |
| 268 2 | 45 self_attn [k, v] | 20,971,520 | -0.00121 | |
| 269 1 | 45 self_attn o | 62,914,560 | -0.00021 | |
| 269 2 | 45 self_attn o | 125,829,120 | -0.00019 | |
| 270 1 | 45 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00032 | |
| 270 2 | 45 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 | |
| 271 1 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00049 | |
| 271 2 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00046 | |
| 272 1 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 272 2 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00023 | |
| 273 1 | 46 self_attn q | 62,914,560 | -0.00018 | |
| 273 2 | 46 self_attn q | 125,829,120 | -0.00060 | |
| 274 1 | 46 self_attn [k, v] | 10,485,760 | -0.00030 | |
| 274 2 | 46 self_attn [k, v] | 20,971,520 | -0.00036 | |
| 275 1 | 46 self_attn o | 62,914,560 | -0.00001 | |
| 275 2 | 46 self_attn o | 125,829,120 | -0.00011 | |
| 276 1 | 46 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001 | |
| 276 2 | 46 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00012 | |
| 277 1 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 | |
| 277 2 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00013 | |
| 278 1 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00030 | |
| 278 2 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00033 | |
| 279 1 | 47 self_attn q | 62,914,560 | -0.00030 | |
| 279 2 | 47 self_attn q | 125,829,120 | -0.00023 | |
| 280 1 | 47 self_attn [k, v] | 10,485,760 | -0.00078 | |
| 280 2 | 47 self_attn [k, v] | 20,971,520 | -0.00070 | |
| 281 1 | 47 self_attn o | 62,914,560 | -0.00022 | |
| 281 2 | 47 self_attn o | 125,829,120 | -0.00030 | |
| 282 1 | 47 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00021 | |
| 282 2 | 47 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00015 | |
| 283 1 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 | |
| 283 2 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 | |
| 284 1 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 284 2 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00019 | |
| 285 1 | 48 self_attn q | 62,914,560 | +0.00035 | |
| 285 2 | 48 self_attn q | 125,829,120 | +0.00008 | |
| 286 1 | 48 self_attn [k, v] | 10,485,760 | -0.00085 | |
| 286 2 | 48 self_attn [k, v] | 20,971,520 | -0.00128 | |
| 287 1 | 48 self_attn o | 62,914,560 | -0.00057 | |
| 287 2 | 48 self_attn o | 125,829,120 | -0.00058 | |
| 288 1 | 48 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00055 | |
| 288 2 | 48 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00034 | |
| 289 1 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00039 | |
| 289 2 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00039 | |
| 290 1 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00025 | |
| 290 2 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00027 | |
| 291 1 | 49 self_attn q | 62,914,560 | -0.00038 | |
| 291 2 | 49 self_attn q | 125,829,120 | -0.00033 | |
| 292 1 | 49 self_attn [k, v] | 10,485,760 | +0.00027 | |
| 292 2 | 49 self_attn [k, v] | 20,971,520 | +0.00019 | |
| 293 1 | 49 self_attn o | 62,914,560 | -0.00006 | |
| 293 2 | 49 self_attn o | 125,829,120 | +0.00005 | |
| 294 1 | 49 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003 | |
| 294 2 | 49 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022 | |
| 295 1 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00031 | |
| 295 2 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00033 | |
| 296 1 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00020 | |
| 296 2 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 | |
| 297 1 | 50 self_attn q | 62,914,560 | -0.00012 | |
| 297 2 | 50 self_attn q | 125,829,120 | -0.00005 | |
| 298 1 | 50 self_attn [k, v] | 10,485,760 | +0.00018 | |
| 298 2 | 50 self_attn [k, v] | 20,971,520 | -0.00010 | |
| 299 1 | 50 self_attn o | 62,914,560 | -0.00006 | |
| 299 2 | 50 self_attn o | 125,829,120 | -0.00020 | |
| 300 1 | 50 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 | |
| 300 2 | 50 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011 | |
| 301 1 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 | |
| 301 2 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 | |
| 302 1 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 302 2 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00016 | |
| 303 1 | 51 self_attn q | 62,914,560 | -0.00014 | |
| 303 2 | 51 self_attn q | 125,829,120 | -0.00012 | |
| 304 1 | 51 self_attn [k, v] | 10,485,760 | -0.00033 | |
| 304 2 | 51 self_attn [k, v] | 20,971,520 | -0.00033 | |
| 305 1 | 51 self_attn o | 62,914,560 | -0.00015 | |
| 305 2 | 51 self_attn o | 125,829,120 | +0.00000 | |
| 306 1 | 51 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00016 | |
| 306 2 | 51 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00031 | |
| 307 1 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 | |
| 307 2 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00027 | |
| 308 1 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00026 | |
| 308 2 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 | |
| 309 1 | 52 self_attn q | 62,914,560 | +0.00001 | |
| 309 2 | 52 self_attn q | 125,829,120 | +0.00010 | |
| 310 1 | 52 self_attn [k, v] | 10,485,760 | -0.00075 | |
| 310 2 | 52 self_attn [k, v] | 20,971,520 | -0.00049 | |
| 311 1 | 52 self_attn o | 62,914,560 | -0.00009 | |
| 311 2 | 52 self_attn o | 125,829,120 | -0.00008 | |
| 312 1 | 52 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00026 | |
| 312 2 | 52 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00021 | |
| 313 1 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00019 | |
| 313 2 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00025 | |
| 314 1 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00024 | |
| 314 2 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00025 | |
| 315 1 | 53 self_attn q | 62,914,560 | -0.00007 | |
| 315 2 | 53 self_attn q | 125,829,120 | -0.00017 | |
| 316 1 | 53 self_attn [k, v] | 10,485,760 | -0.00069 | |
| 316 2 | 53 self_attn [k, v] | 20,971,520 | -0.00067 | |
| 317 1 | 53 self_attn o | 62,914,560 | +0.00003 | |
| 317 2 | 53 self_attn o | 125,829,120 | +0.00002 | |
| 318 1 | 53 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00047 | |
| 318 2 | 53 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 | |
| 319 1 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00013 | |
| 319 2 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00021 | |
| 320 1 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 320 2 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 | |
| 321 1 | 54 self_attn q | 62,914,560 | +0.00008 | |
| 321 2 | 54 self_attn q | 125,829,120 | +0.00002 | |
| 322 1 | 54 self_attn [k, v] | 10,485,760 | +0.00048 | |
| 322 2 | 54 self_attn [k, v] | 20,971,520 | +0.00017 | |
| 323 1 | 54 self_attn o | 62,914,560 | -0.00054 | |
| 323 2 | 54 self_attn o | 125,829,120 | -0.00040 | |
| 324 1 | 54 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011 | |
| 324 2 | 54 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 | |
| 325 1 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 | |
| 325 2 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015 | |
| 326 1 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00009 | |
| 326 2 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 | |
| 327 1 | 55 self_attn q | 62,914,560 | -0.00018 | |
| 327 2 | 55 self_attn q | 125,829,120 | -0.00009 | |
| 328 1 | 55 self_attn [k, v] | 10,485,760 | -0.00012 | |
| 328 2 | 55 self_attn [k, v] | 20,971,520 | +0.00007 | |
| 329 1 | 55 self_attn o | 62,914,560 | -0.00044 | |
| 329 2 | 55 self_attn o | 125,829,120 | -0.00046 | |
| 330 1 | 55 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 | |
| 330 2 | 55 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00028 | |
| 331 1 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 | |
| 331 2 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 | |
| 332 1 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 | |
| 332 2 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 | |
| 333 1 | 56 self_attn q | 62,914,560 | +0.00030 | |
| 333 2 | 56 self_attn q | 125,829,120 | +0.00025 | |
| 334 1 | 56 self_attn [k, v] | 10,485,760 | -0.00004 | |
| 334 2 | 56 self_attn [k, v] | 20,971,520 | +0.00010 | |
| 335 1 | 56 self_attn o | 62,914,560 | +0.00005 | |
| 335 2 | 56 self_attn o | 125,829,120 | +0.00013 | |
| 336 1 | 56 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 | |
| 336 2 | 56 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00013 | |
| 337 1 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00017 | |
| 337 2 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 | |
| 338 1 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 | |
| 338 2 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 | |
| 339 1 | 57 self_attn q | 62,914,560 | -0.00017 | |
| 339 2 | 57 self_attn q | 125,829,120 | -0.00020 | |
| 340 1 | 57 self_attn [k, v] | 10,485,760 | +0.00007 | |
| 340 2 | 57 self_attn [k, v] | 20,971,520 | +0.00004 | |
| 341 1 | 57 self_attn o | 62,914,560 | -0.00016 | |
| 341 2 | 57 self_attn o | 125,829,120 | -0.00005 | |
| 342 1 | 57 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00018 | |
| 342 2 | 57 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00032 | |
| 343 1 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018 | |
| 343 2 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015 | |
| 344 1 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 | |
| 344 2 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 | |
| 345 1 | 58 self_attn q | 62,914,560 | +0.00013 | |
| 345 2 | 58 self_attn q | 125,829,120 | +0.00023 | |
| 346 1 | 58 self_attn [k, v] | 10,485,760 | -0.00017 | |
| 346 2 | 58 self_attn [k, v] | 20,971,520 | -0.00011 | |
| 347 1 | 58 self_attn o | 62,914,560 | +0.00002 | |
| 347 2 | 58 self_attn o | 125,829,120 | +0.00002 | |
| 348 1 | 58 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00021 | |
| 348 2 | 58 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 | |
| 349 1 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 | |
| 349 2 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00023 | |
| 350 1 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00024 | |
| 350 2 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00020 | |
| 351 1 | 59 self_attn q | 62,914,560 | -0.00017 | |
| 351 2 | 59 self_attn q | 125,829,120 | -0.00017 | |
| 352 1 | 59 self_attn [k, v] | 10,485,760 | -0.00011 | |
| 352 2 | 59 self_attn [k, v] | 20,971,520 | -0.00027 | |
| 353 1 | 59 self_attn o | 62,914,560 | +0.00001 | |
| 353 2 | 59 self_attn o | 125,829,120 | +0.00009 | |
| 354 1 | 59 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00018 | |
| 354 2 | 59 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00032 | |
| 355 1 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 | |
| 355 2 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 | |
| 356 1 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00015 | |
| 356 2 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00011 | |
| 357 1 | 60 self_attn q | 62,914,560 | +0.00002 | |
| 357 2 | 60 self_attn q | 125,829,120 | -0.00008 | |
| 358 1 | 60 self_attn [k, v] | 10,485,760 | -0.00019 | |
| 358 2 | 60 self_attn [k, v] | 20,971,520 | -0.00029 | |
| 359 1 | 60 self_attn o | 62,914,560 | -0.00003 | |
| 359 2 | 60 self_attn o | 125,829,120 | -0.00002 | |
| 360 1 | 60 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00029 | |
| 360 2 | 60 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00039 | |
| 361 1 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018 | |
| 361 2 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00020 | |
| 362 1 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 | |
| 362 2 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014 | |
| 363 1 | 61 self_attn q | 62,914,560 | -0.00022 | |
| 363 2 | 61 self_attn q | 125,829,120 | -0.00023 | |
| 364 1 | 61 self_attn [k, v] | 10,485,760 | -0.00025 | |
| 364 2 | 61 self_attn [k, v] | 20,971,520 | -0.00026 | |
| 365 1 | 61 self_attn o | 62,914,560 | -0.00017 | |
| 365 2 | 61 self_attn o | 125,829,120 | -0.00021 | |
| 366 1 | 61 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012 | |
| 366 2 | 61 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010 | |
| 367 1 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 | |
| 367 2 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00009 | |
| 368 1 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 | |
| 368 2 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004 | |
| 369 1 | 62 self_attn q | 62,914,560 | -0.00008 | |
| 369 2 | 62 self_attn q | 125,829,120 | -0.00001 | |
| 370 1 | 62 self_attn [k, v] | 10,485,760 | -0.00008 | |
| 370 2 | 62 self_attn [k, v] | 20,971,520 | -0.00014 | |
| 371 1 | 62 self_attn o | 62,914,560 | -0.00012 | |
| 371 2 | 62 self_attn o | 125,829,120 | -0.00009 | |
| 372 1 | 62 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006 | |
| 372 2 | 62 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 | |
| 373 1 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00009 | |
| 373 2 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019 | |
| 374 1 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 | |
| 374 2 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00019 | |
| 375 1 | 63 self_attn q | 62,914,560 | +0.00009 | |
| 375 2 | 63 self_attn q | 125,829,120 | -0.00000 | |
| 376 1 | 63 self_attn [k, v] | 10,485,760 | -0.00013 | |
| 376 2 | 63 self_attn [k, v] | 20,971,520 | -0.00010 | |
| 377 1 | 63 self_attn o | 62,914,560 | -0.00003 | |
| 377 2 | 63 self_attn o | 125,829,120 | -0.00005 | |
| 378 1 | 63 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 | |
| 378 2 | 63 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003 | |
| 379 1 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004 | |
| 379 2 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 | |
| 380 1 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 | |
| 380 2 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 | |
| 381 1 | 64 self_attn q | 62,914,560 | -0.00011 | |
| 381 2 | 64 self_attn q | 125,829,120 | -0.00006 | |
| 382 1 | 64 self_attn [k, v] | 10,485,760 | -0.00028 | |
| 382 2 | 64 self_attn [k, v] | 20,971,520 | -0.00021 | |
| 383 1 | 64 self_attn o | 62,914,560 | -0.00011 | |
| 383 2 | 64 self_attn o | 125,829,120 | -0.00008 | |
| 384 1 | 64 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00008 | |
| 384 2 | 64 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 | |
| 385 1 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010 | |
| 385 2 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014 | |
| 386 1 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 | |
| 386 2 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 387 1 | 65 self_attn q | 62,914,560 | -0.00006 | |
| 387 2 | 65 self_attn q | 125,829,120 | +0.00001 | |
| 388 1 | 65 self_attn [k, v] | 10,485,760 | -0.00026 | |
| 388 2 | 65 self_attn [k, v] | 20,971,520 | -0.00039 | |
| 389 1 | 65 self_attn o | 62,914,560 | -0.00008 | |
| 389 2 | 65 self_attn o | 125,829,120 | -0.00009 | |
| 390 1 | 65 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00023 | |
| 390 2 | 65 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023 | |
| 391 1 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 | |
| 391 2 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 | |
| 392 1 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00011 | |
| 392 2 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00009 | |
| 393 1 | 66 self_attn q | 62,914,560 | -0.00060 | |
| 393 2 | 66 self_attn q | 125,829,120 | -0.00062 | |
| 394 1 | 66 self_attn [k, v] | 10,485,760 | +0.00028 | |
| 394 2 | 66 self_attn [k, v] | 20,971,520 | +0.00035 | |
| 395 1 | 66 self_attn o | 62,914,560 | -0.00016 | |
| 395 2 | 66 self_attn o | 125,829,120 | -0.00019 | |
| 396 1 | 66 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012 | |
| 396 2 | 66 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00019 | |
| 397 1 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00022 | |
| 397 2 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00020 | |
| 398 1 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00021 | |
| 398 2 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00024 | |
| 399 1 | 67 self_attn q | 62,914,560 | -0.00002 | |
| 399 2 | 67 self_attn q | 125,829,120 | -0.00006 | |
| 400 1 | 67 self_attn [k, v] | 10,485,760 | -0.00028 | |
| 400 2 | 67 self_attn [k, v] | 20,971,520 | -0.00019 | |
| 401 1 | 67 self_attn o | 62,914,560 | -0.00011 | |
| 401 2 | 67 self_attn o | 125,829,120 | -0.00009 | |
| 402 1 | 67 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 | |
| 402 2 | 67 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008 | |
| 403 1 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004 | |
| 403 2 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005 | |
| 404 1 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00026 | |
| 404 2 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00022 | |
| 405 1 | 68 self_attn q | 62,914,560 | -0.00009 | |
| 405 2 | 68 self_attn q | 125,829,120 | -0.00021 | |
| 406 1 | 68 self_attn [k, v] | 10,485,760 | -0.00031 | |
| 406 2 | 68 self_attn [k, v] | 20,971,520 | -0.00011 | |
| 407 1 | 68 self_attn o | 62,914,560 | -0.00022 | |
| 407 2 | 68 self_attn o | 125,829,120 | -0.00023 | |
| 408 1 | 68 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00007 | |
| 408 2 | 68 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002 | |
| 409 1 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006 | |
| 409 2 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00013 | |
| 410 1 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 | |
| 410 2 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00031 | |
| 411 1 | 69 self_attn q | 62,914,560 | -0.00007 | |
| 411 2 | 69 self_attn q | 125,829,120 | -0.00011 | |
| 412 1 | 69 self_attn [k, v] | 10,485,760 | -0.00018 | |
| 412 2 | 69 self_attn [k, v] | 20,971,520 | -0.00033 | |
| 413 1 | 69 self_attn o | 62,914,560 | -0.00012 | |
| 413 2 | 69 self_attn o | 125,829,120 | -0.00011 | |
| 414 1 | 69 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 | |
| 414 2 | 69 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008 | |
| 415 1 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 | |
| 415 2 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00030 | |
| 416 1 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002 | |
| 416 2 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 | |
| 417 1 | 70 self_attn q | 62,914,560 | +0.00002 | |
| 417 2 | 70 self_attn q | 125,829,120 | +0.00003 | |
| 418 1 | 70 self_attn [k, v] | 10,485,760 | -0.00014 | |
| 418 2 | 70 self_attn [k, v] | 20,971,520 | -0.00011 | |
| 419 1 | 70 self_attn o | 62,914,560 | -0.00001 | |
| 419 2 | 70 self_attn o | 125,829,120 | -0.00003 | |
| 420 1 | 70 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00035 | |
| 420 2 | 70 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00026 | |
| 421 1 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 | |
| 421 2 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 | |
| 422 1 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 | |
| 422 2 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00008 | |
| 423 1 | 71 self_attn q | 62,914,560 | -0.00000 | |
| 423 2 | 71 self_attn q | 125,829,120 | +0.00009 | |
| 424 1 | 71 self_attn [k, v] | 10,485,760 | -0.00033 | |
| 424 2 | 71 self_attn [k, v] | 20,971,520 | -0.00033 | |
| 425 1 | 71 self_attn o | 62,914,560 | -0.00023 | |
| 425 2 | 71 self_attn o | 125,829,120 | -0.00036 | |
| 426 1 | 71 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00057 | |
| 426 2 | 71 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00040 | |
| 427 1 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00038 | |
| 427 2 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00045 | |
| 428 1 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00017 | |
| 428 2 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014 | |
| 429 1 | 72 self_attn q | 62,914,560 | -0.00014 | |
| 429 2 | 72 self_attn q | 125,829,120 | -0.00011 | |
| 430 1 | 72 self_attn [k, v] | 10,485,760 | -0.00007 | |
| 430 2 | 72 self_attn [k, v] | 20,971,520 | -0.00003 | |
| 431 1 | 72 self_attn o | 62,914,560 | +0.00005 | |
| 431 2 | 72 self_attn o | 125,829,120 | -0.00009 | |
| 432 1 | 72 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00032 | |
| 432 2 | 72 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00051 | |
| 433 1 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00021 | |
| 433 2 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00013 | |
| 434 1 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 | |
| 434 2 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00022 | |
| 435 1 | 73 self_attn q | 62,914,560 | -0.00007 | |
| 435 2 | 73 self_attn q | 125,829,120 | -0.00014 | |
| 436 1 | 73 self_attn [k, v] | 10,485,760 | -0.00005 | |
| 436 2 | 73 self_attn [k, v] | 20,971,520 | +0.00005 | |
| 437 1 | 73 self_attn o | 62,914,560 | -0.00025 | |
| 437 2 | 73 self_attn o | 125,829,120 | -0.00022 | |
| 438 1 | 73 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 | |
| 438 2 | 73 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00029 | |
| 439 1 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00021 | |
| 439 2 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00017 | |
| 440 1 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00012 | |
| 440 2 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00015 | |
| 441 1 | 74 self_attn q | 62,914,560 | -0.00006 | |
| 441 2 | 74 self_attn q | 125,829,120 | -0.00002 | |
| 442 1 | 74 self_attn [k, v] | 10,485,760 | -0.00009 | |
| 442 2 | 74 self_attn [k, v] | 20,971,520 | -0.00012 | |
| 443 1 | 74 self_attn o | 62,914,560 | -0.00001 | |
| 443 2 | 74 self_attn o | 125,829,120 | +0.00004 | |
| 444 1 | 74 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 | |
| 444 2 | 74 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010 | |
| 445 1 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 | |
| 445 2 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00017 | |
| 446 1 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013 | |
| 446 2 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00016 | |
| 447 1 | 75 self_attn q | 62,914,560 | +0.00004 | |
| 447 2 | 75 self_attn q | 125,829,120 | -0.00002 | |
| 448 1 | 75 self_attn [k, v] | 10,485,760 | -0.00007 | |
| 448 2 | 75 self_attn [k, v] | 20,971,520 | -0.00005 | |
| 449 1 | 75 self_attn o | 62,914,560 | -0.00022 | |
| 449 2 | 75 self_attn o | 125,829,120 | -0.00012 | |
| 450 1 | 75 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 | |
| 450 2 | 75 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002 | |
| 451 1 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 | |
| 451 2 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00035 | |
| 452 1 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014 | |
| 452 2 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00015 | |
| 453 1 | 76 self_attn q | 62,914,560 | +0.00013 | |
| 453 2 | 76 self_attn q | 125,829,120 | -0.00011 | |
| 454 1 | 76 self_attn [k, v] | 10,485,760 | -0.00016 | |
| 454 2 | 76 self_attn [k, v] | 20,971,520 | -0.00029 | |
| 455 1 | 76 self_attn o | 62,914,560 | +0.00001 | |
| 455 2 | 76 self_attn o | 125,829,120 | -0.00010 | |
| 456 1 | 76 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 | |
| 456 2 | 76 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 | |
| 457 1 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00023 | |
| 457 2 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00024 | |
| 458 1 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 | |
| 458 2 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 459 1 | 77 self_attn q | 62,914,560 | -0.00028 | |
| 459 2 | 77 self_attn q | 125,829,120 | -0.00037 | |
| 460 1 | 77 self_attn [k, v] | 10,485,760 | -0.00024 | |
| 460 2 | 77 self_attn [k, v] | 20,971,520 | -0.00056 | |
| 461 1 | 77 self_attn o | 62,914,560 | -0.00006 | |
| 461 2 | 77 self_attn o | 125,829,120 | -0.00013 | |
| 462 1 | 77 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 | |
| 462 2 | 77 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00024 | |
| 463 1 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00030 | |
| 463 2 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00038 | |
| 464 1 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 | |
| 464 2 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 | |
| 465 1 | 78 self_attn q | 62,914,560 | -0.00010 | |
| 465 2 | 78 self_attn q | 125,829,120 | -0.00001 | |
| 466 1 | 78 self_attn [k, v] | 10,485,760 | -0.00040 | |
| 466 2 | 78 self_attn [k, v] | 20,971,520 | -0.00047 | |
| 467 1 | 78 self_attn o | 62,914,560 | -0.00016 | |
| 467 2 | 78 self_attn o | 125,829,120 | -0.00015 | |
| 468 1 | 78 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00010 | |
| 468 2 | 78 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00015 | |
| 469 1 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 | |
| 469 2 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 | |
| 470 1 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 | |
| 470 2 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 471 1 | 79 self_attn q | 62,914,560 | -0.00027 | |
| 471 2 | 79 self_attn q | 125,829,120 | -0.00028 | |
| 472 1 | 79 self_attn [k, v] | 10,485,760 | -0.00052 | |
| 472 2 | 79 self_attn [k, v] | 20,971,520 | -0.00096 | |
| 473 1 | 79 self_attn o | 62,914,560 | -0.00037 | |
| 473 2 | 79 self_attn o | 125,829,120 | -0.00035 | |
| 474 1 | 79 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00067 | |
| 474 2 | 79 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00070 | |
| 475 1 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 | |
| 475 2 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00040 | |
| 476 1 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014 | |
| 476 2 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 477 1 | 80 self_attn q | 62,914,560 | +0.00083 | |
| 477 2 | 80 self_attn q | 125,829,120 | +0.00137 | |
| 478 1 | 80 self_attn [k, v] | 10,485,760 | -0.00136 | |
| 478 2 | 80 self_attn [k, v] | 20,971,520 | -0.00107 | |
| 479 1 | 80 self_attn o | 62,914,560 | -0.00045 | |
| 479 2 | 80 self_attn o | 125,829,120 | -0.00038 | |
| 480 1 | 80 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 | |
| 480 2 | 80 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00046 | |
| 481 1 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010 | |
| 481 2 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 | |
| 482 1 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 | |
| 482 2 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00013 | |
| 483 1 | 81 self_attn q | 62,914,560 | -0.00027 | |
| 483 2 | 81 self_attn q | 125,829,120 | -0.00009 | |
| 484 1 | 81 self_attn [k, v] | 10,485,760 | -0.00097 | |
| 484 2 | 81 self_attn [k, v] | 20,971,520 | -0.00104 | |
| 485 1 | 81 self_attn o | 62,914,560 | -0.00048 | |
| 485 2 | 81 self_attn o | 125,829,120 | -0.00047 | |
| 486 1 | 81 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 | |
| 486 2 | 81 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00037 | |
| 487 1 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00015 | |
| 487 2 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00016 | |
| 488 1 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 | |
| 488 2 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 489 1 | 82 self_attn q | 62,914,560 | -0.00014 | |
| 489 2 | 82 self_attn q | 125,829,120 | -0.00028 | |
| 490 1 | 82 self_attn [k, v] | 10,485,760 | -0.00009 | |
| 490 2 | 82 self_attn [k, v] | 20,971,520 | -0.00017 | |
| 491 1 | 82 self_attn o | 62,914,560 | -0.00001 | |
| 491 2 | 82 self_attn o | 125,829,120 | -0.00023 | |
| 492 1 | 82 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00022 | |
| 492 2 | 82 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00029 | |
| 493 1 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00028 | |
| 493 2 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00029 | |
| 494 1 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 | |
| 494 2 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 | |
| 495 1 | 83 self_attn q | 62,914,560 | -0.00019 | |
| 495 2 | 83 self_attn q | 125,829,120 | -0.00016 | |
| 496 1 | 83 self_attn [k, v] | 10,485,760 | -0.00068 | |
| 496 2 | 83 self_attn [k, v] | 20,971,520 | -0.00093 | |
| 497 1 | 83 self_attn o | 62,914,560 | +0.00012 | |
| 497 2 | 83 self_attn o | 125,829,120 | -0.00005 | |
| 498 1 | 83 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00015 | |
| 498 2 | 83 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023 | |
| 499 1 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00020 | |
| 499 2 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019 | |
| 500 1 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 | |
| 500 2 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00012 | |
| 501 1 | 84 self_attn q | 62,914,560 | +0.00005 | |
| 501 2 | 84 self_attn q | 125,829,120 | -0.00014 | |
| 502 1 | 84 self_attn [k, v] | 10,485,760 | +0.00036 | |
| 502 2 | 84 self_attn [k, v] | 20,971,520 | -0.00004 | |
| 503 1 | 84 self_attn o | 62,914,560 | -0.00030 | |
| 503 2 | 84 self_attn o | 125,829,120 | -0.00008 | |
| 504 1 | 84 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00060 | |
| 504 2 | 84 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00033 | |
| 505 1 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00017 | |
| 505 2 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 | |
| 506 1 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 | |
| 506 2 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 | |
| 507 1 | 85 self_attn q | 62,914,560 | -0.00047 | |
| 507 2 | 85 self_attn q | 125,829,120 | -0.00052 | |
| 508 1 | 85 self_attn [k, v] | 10,485,760 | -0.00065 | |
| 508 2 | 85 self_attn [k, v] | 20,971,520 | -0.00086 | |
| 509 1 | 85 self_attn o | 62,914,560 | -0.00049 | |
| 509 2 | 85 self_attn o | 125,829,120 | -0.00066 | |
| 510 1 | 85 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00024 | |
| 510 2 | 85 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011 | |
| 511 1 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00007 | |
| 511 2 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00024 | |
| 512 1 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007 | |
| 512 2 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 | |
| 513 1 | 86 self_attn q | 62,914,560 | -0.00049 | |
| 513 2 | 86 self_attn q | 125,829,120 | -0.00040 | |
| 514 1 | 86 self_attn [k, v] | 10,485,760 | +0.00003 | |
| 514 2 | 86 self_attn [k, v] | 20,971,520 | -0.00036 | |
| 515 1 | 86 self_attn o | 62,914,560 | +0.00016 | |
| 515 2 | 86 self_attn o | 125,829,120 | -0.00005 | |
| 516 1 | 86 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00057 | |
| 516 2 | 86 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00053 | |
| 517 1 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 | |
| 517 2 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00022 | |
| 518 1 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 | |
| 518 2 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 | |
| 519 1 | 87 self_attn q | 62,914,560 | +0.00022 | |
| 519 2 | 87 self_attn q | 125,829,120 | +0.00020 | |
| 520 1 | 87 self_attn [k, v] | 10,485,760 | +0.00041 | |
| 520 2 | 87 self_attn [k, v] | 20,971,520 | +0.00044 | |
| 521 1 | 87 self_attn o | 62,914,560 | -0.00050 | |
| 521 2 | 87 self_attn o | 125,829,120 | -0.00042 | |
| 522 1 | 87 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 | |
| 522 2 | 87 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00033 | |
| 523 1 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 | |
| 523 2 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 | |
| 524 1 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010 | |
| 524 2 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 | |
| 525 1 | 88 self_attn q | 62,914,560 | -0.00005 | |
| 525 2 | 88 self_attn q | 125,829,120 | -0.00007 | |
| 526 1 | 88 self_attn [k, v] | 10,485,760 | -0.00073 | |
| 526 2 | 88 self_attn [k, v] | 20,971,520 | -0.00051 | |
| 527 1 | 88 self_attn o | 62,914,560 | -0.00012 | |
| 527 2 | 88 self_attn o | 125,829,120 | -0.00019 | |
| 528 1 | 88 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00008 | |
| 528 2 | 88 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00018 | |
| 529 1 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 | |
| 529 2 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014 | |
| 530 1 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00009 | |
| 530 2 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 | |
| 531 1 | 89 self_attn q | 62,914,560 | -0.00017 | |
| 531 2 | 89 self_attn q | 125,829,120 | -0.00023 | |
| 532 1 | 89 self_attn [k, v] | 10,485,760 | -0.00054 | |
| 532 2 | 89 self_attn [k, v] | 20,971,520 | -0.00070 | |
| 533 1 | 89 self_attn o | 62,914,560 | -0.00029 | |
| 533 2 | 89 self_attn o | 125,829,120 | -0.00023 | |
| 534 1 | 89 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00016 | |
| 534 2 | 89 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 | |
| 535 1 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 | |
| 535 2 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 | |
| 536 1 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 | |
| 536 2 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 | |
| 537 1 | 90 self_attn q | 62,914,560 | -0.00019 | |
| 537 2 | 90 self_attn q | 125,829,120 | -0.00024 | |
| 538 1 | 90 self_attn [k, v] | 10,485,760 | -0.00017 | |
| 538 2 | 90 self_attn [k, v] | 20,971,520 | -0.00028 | |
| 539 1 | 90 self_attn o | 62,914,560 | -0.00009 | |
| 539 2 | 90 self_attn o | 125,829,120 | -0.00009 | |
| 540 1 | 90 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00062 | |
| 540 2 | 90 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00064 | |
| 541 1 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 | |
| 541 2 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 | |
| 542 1 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010 | |
| 542 2 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 | |
| 543 1 | 91 self_attn q | 62,914,560 | -0.00020 | |
| 543 2 | 91 self_attn q | 125,829,120 | -0.00004 | |
| 544 1 | 91 self_attn [k, v] | 10,485,760 | -0.00072 | |
| 544 2 | 91 self_attn [k, v] | 20,971,520 | -0.00106 | |
| 545 1 | 91 self_attn o | 62,914,560 | -0.00016 | |
| 545 2 | 91 self_attn o | 125,829,120 | -0.00034 | |
| 546 1 | 91 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00076 | |
| 546 2 | 91 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 | |
| 547 1 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 | |
| 547 2 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 | |
| 548 1 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004 | |
| 548 2 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 | |