truncated :/ because no tmux 45 2 | 8 self_attn q | 125,829,120 | -0.00002 46 1 | 8 self_attn [k, v] | 10,485,760 | -0.00010 46 2 | 8 self_attn [k, v] | 20,971,520 | -0.00017 47 1 | 8 self_attn o | 62,914,560 | +0.00007 47 2 | 8 self_attn o | 125,829,120 | +0.00003 48 1 | 8 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012 48 2 | 8 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00009 49 1 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | +0.00002 49 2 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00004 50 1 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00007 50 2 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00005 51 1 | 9 self_attn q | 62,914,560 | -0.00004 51 2 | 9 self_attn q | 125,829,120 | -0.00000 52 1 | 9 self_attn [k, v] | 10,485,760 | +0.00007 52 2 | 9 self_attn [k, v] | 20,971,520 | +0.00002 53 1 | 9 self_attn o | 62,914,560 | -0.00005 53 2 | 9 self_attn o | 125,829,120 | -0.00012 54 1 | 9 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00008 54 2 | 9 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00007 55 1 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00004 55 2 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00008 56 1 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00003 56 2 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | +0.00001 57 1 | 10 self_attn q | 62,914,560 | -0.00004 57 2 | 10 self_attn q | 125,829,120 | +0.00000 58 1 | 10 self_attn [k, v] | 10,485,760 | +0.00011 58 2 | 10 self_attn [k, v] | 20,971,520 | -0.00013 59 1 | 10 self_attn o | 62,914,560 | -0.00020 59 2 | 10 self_attn o | 125,829,120 | -0.00016 60 1 | 10 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 60 2 | 10 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00001 61 1 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00005 61 2 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00009 62 1 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 62 2 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00007 63 1 | 11 self_attn q | 62,914,560 | -0.00008 63 2 | 11 self_attn q | 125,829,120 | -0.00003 64 1 | 11 self_attn [k, v] | 10,485,760 | +0.00008 64 2 | 11 self_attn [k, v] | 20,971,520 | +0.00008 65 1 | 11 self_attn o | 62,914,560 | -0.00031 65 2 | 11 self_attn o | 125,829,120 | -0.00032 66 1 | 11 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 66 2 | 11 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00000 67 1 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006 67 2 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004 68 1 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 68 2 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004 69 1 | 12 self_attn q | 62,914,560 | +0.00002 69 2 | 12 self_attn q | 125,829,120 | -0.00001 70 1 | 12 self_attn [k, v] | 10,485,760 | +0.00014 70 2 | 12 self_attn [k, v] | 20,971,520 | +0.00014 71 1 | 12 self_attn o | 62,914,560 | +0.00005 71 2 | 12 self_attn o | 125,829,120 | +0.00006 72 1 | 12 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00010 72 2 | 12 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011 73 1 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006 73 2 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 74 1 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 74 2 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 75 1 | 13 self_attn q | 62,914,560 | -0.00010 75 2 | 13 self_attn q | 125,829,120 | -0.00009 76 1 | 13 self_attn [k, v] | 10,485,760 | -0.00004 76 2 | 13 self_attn [k, v] | 20,971,520 | +0.00005 77 1 | 13 self_attn o | 62,914,560 | -0.00009 77 2 | 13 self_attn o | 125,829,120 | -0.00009 78 1 | 13 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00017 78 2 | 13 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00020 79 1 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007 79 2 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015 80 1 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013 80 2 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 81 1 | 14 self_attn q | 62,914,560 | +0.00002 81 2 | 14 self_attn q | 125,829,120 | -0.00003 82 1 | 14 self_attn [k, v] | 10,485,760 | +0.00011 82 2 | 14 self_attn [k, v] | 20,971,520 | +0.00010 83 1 | 14 self_attn o | 62,914,560 | -0.00024 83 2 | 14 self_attn o | 125,829,120 | -0.00023 84 1 | 14 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 84 2 | 14 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004 85 1 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010 85 2 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 86 1 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010 86 2 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014 87 1 | 15 self_attn q | 62,914,560 | -0.00000 87 2 | 15 self_attn q | 125,829,120 | -0.00002 88 1 | 15 self_attn [k, v] | 10,485,760 | -0.00009 88 2 | 15 self_attn [k, v] | 20,971,520 | -0.00007 89 1 | 15 self_attn o | 62,914,560 | -0.00006 89 2 | 15 self_attn o | 125,829,120 | -0.00012 90 1 | 15 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 90 2 | 15 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002 91 1 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 91 2 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 92 1 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007 92 2 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 93 1 | 16 self_attn q | 62,914,560 | +0.00004 93 2 | 16 self_attn q | 125,829,120 | +0.00001 94 1 | 16 self_attn [k, v] | 10,485,760 | -0.00002 94 2 | 16 self_attn [k, v] | 20,971,520 | -0.00004 95 1 | 16 self_attn o | 62,914,560 | -0.00015 95 2 | 16 self_attn o | 125,829,120 | -0.00012 96 1 | 16 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006 96 2 | 16 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 97 1 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018 97 2 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019 98 1 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 98 2 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 99 1 | 17 self_attn q | 62,914,560 | -0.00005 99 2 | 17 self_attn q | 125,829,120 | -0.00001 100 1 | 17 self_attn [k, v] | 10,485,760 | +0.00001 100 2 | 17 self_attn [k, v] | 20,971,520 | +0.00002 101 1 | 17 self_attn o | 62,914,560 | -0.00002 101 2 | 17 self_attn o | 125,829,120 | -0.00004 102 1 | 17 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006 102 2 | 17 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010 103 1 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00009 103 2 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 104 1 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 104 2 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004 105 1 | 18 self_attn q | 62,914,560 | +0.00011 105 2 | 18 self_attn q | 125,829,120 | +0.00003 106 1 | 18 self_attn [k, v] | 10,485,760 | +0.00004 106 2 | 18 self_attn [k, v] | 20,971,520 | +0.00004 107 1 | 18 self_attn o | 62,914,560 | +0.00003 107 2 | 18 self_attn o | 125,829,120 | -0.00007 108 1 | 18 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00024 108 2 | 18 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023 109 1 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 109 2 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004 110 1 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004 110 2 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00009 111 1 | 19 self_attn q | 62,914,560 | -0.00005 111 2 | 19 self_attn q | 125,829,120 | -0.00005 112 1 | 19 self_attn [k, v] | 10,485,760 | -0.00000 112 2 | 19 self_attn [k, v] | 20,971,520 | -0.00007 113 1 | 19 self_attn o | 62,914,560 | -0.00006 113 2 | 19 self_attn o | 125,829,120 | -0.00005 114 1 | 19 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00020 114 2 | 19 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00020 115 1 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007 115 2 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 116 1 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 116 2 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 117 1 | 20 self_attn q | 62,914,560 | +0.00002 117 2 | 20 self_attn q | 125,829,120 | +0.00002 118 1 | 20 self_attn [k, v] | 10,485,760 | -0.00009 118 2 | 20 self_attn [k, v] | 20,971,520 | -0.00003 119 1 | 20 self_attn o | 62,914,560 | -0.00026 119 2 | 20 self_attn o | 125,829,120 | -0.00020 120 1 | 20 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00023 120 2 | 20 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00027 121 1 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 121 2 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014 122 1 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 122 2 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 123 1 | 21 self_attn q | 62,914,560 | -0.00006 123 2 | 21 self_attn q | 125,829,120 | -0.00004 124 1 | 21 self_attn [k, v] | 10,485,760 | +0.00004 124 2 | 21 self_attn [k, v] | 20,971,520 | -0.00004 125 1 | 21 self_attn o | 62,914,560 | +0.00011 125 2 | 21 self_attn o | 125,829,120 | +0.00011 126 1 | 21 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00022 126 2 | 21 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00017 127 1 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010 127 2 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00009 128 1 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014 128 2 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 129 1 | 22 self_attn q | 62,914,560 | -0.00003 129 2 | 22 self_attn q | 125,829,120 | +0.00001 130 1 | 22 self_attn [k, v] | 10,485,760 | -0.00008 130 2 | 22 self_attn [k, v] | 20,971,520 | +0.00003 131 1 | 22 self_attn o | 62,914,560 | +0.00004 131 2 | 22 self_attn o | 125,829,120 | +0.00011 132 1 | 22 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003 132 2 | 22 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002 133 1 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007 133 2 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00012 134 1 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 134 2 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00007 135 1 | 23 self_attn q | 62,914,560 | -0.00004 135 2 | 23 self_attn q | 125,829,120 | -0.00001 136 1 | 23 self_attn [k, v] | 10,485,760 | -0.00010 136 2 | 23 self_attn [k, v] | 20,971,520 | -0.00004 137 1 | 23 self_attn o | 62,914,560 | +0.00008 137 2 | 23 self_attn o | 125,829,120 | +0.00008 138 1 | 23 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009 138 2 | 23 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00005 139 1 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 139 2 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00012 140 1 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004 140 2 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 141 1 | 24 self_attn q | 62,914,560 | -0.00010 141 2 | 24 self_attn q | 125,829,120 | -0.00010 142 1 | 24 self_attn [k, v] | 10,485,760 | -0.00006 142 2 | 24 self_attn [k, v] | 20,971,520 | -0.00013 143 1 | 24 self_attn o | 62,914,560 | -0.00002 143 2 | 24 self_attn o | 125,829,120 | -0.00015 144 1 | 24 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004 144 2 | 24 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00013 145 1 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000 145 2 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004 146 1 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 146 2 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 147 1 | 25 self_attn q | 62,914,560 | -0.00007 147 2 | 25 self_attn q | 125,829,120 | -0.00009 148 1 | 25 self_attn [k, v] | 10,485,760 | -0.00008 148 2 | 25 self_attn [k, v] | 20,971,520 | -0.00004 149 1 | 25 self_attn o | 62,914,560 | -0.00003 149 2 | 25 self_attn o | 125,829,120 | -0.00005 150 1 | 25 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00009 150 2 | 25 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00003 151 1 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 151 2 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 152 1 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 152 2 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 153 1 | 26 self_attn q | 62,914,560 | -0.00001 153 2 | 26 self_attn q | 125,829,120 | -0.00007 154 1 | 26 self_attn [k, v] | 10,485,760 | -0.00013 154 2 | 26 self_attn [k, v] | 20,971,520 | -0.00022 155 1 | 26 self_attn o | 62,914,560 | -0.00023 155 2 | 26 self_attn o | 125,829,120 | -0.00021 156 1 | 26 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 156 2 | 26 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 157 1 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00005 157 2 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00006 158 1 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 158 2 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004 159 1 | 27 self_attn q | 62,914,560 | +0.00004 159 2 | 27 self_attn q | 125,829,120 | -0.00002 160 1 | 27 self_attn [k, v] | 10,485,760 | -0.00003 160 2 | 27 self_attn [k, v] | 20,971,520 | -0.00001 161 1 | 27 self_attn o | 62,914,560 | -0.00000 161 2 | 27 self_attn o | 125,829,120 | -0.00013 162 1 | 27 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004 162 2 | 27 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00008 163 1 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 163 2 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002 164 1 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004 164 2 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 165 1 | 28 self_attn q | 62,914,560 | -0.00005 165 2 | 28 self_attn q | 125,829,120 | -0.00004 166 1 | 28 self_attn [k, v] | 10,485,760 | -0.00017 166 2 | 28 self_attn [k, v] | 20,971,520 | -0.00018 167 1 | 28 self_attn o | 62,914,560 | -0.00010 167 2 | 28 self_attn o | 125,829,120 | -0.00011 168 1 | 28 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00000 168 2 | 28 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00014 169 1 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000 169 2 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 170 1 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004 170 2 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004 171 1 | 29 self_attn q | 62,914,560 | -0.00003 171 2 | 29 self_attn q | 125,829,120 | -0.00001 172 1 | 29 self_attn [k, v] | 10,485,760 | +0.00009 172 2 | 29 self_attn [k, v] | 20,971,520 | +0.00001 173 1 | 29 self_attn o | 62,914,560 | -0.00006 173 2 | 29 self_attn o | 125,829,120 | -0.00003 174 1 | 29 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 174 2 | 29 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004 175 1 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000 175 2 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004 176 1 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 176 2 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 177 1 | 30 self_attn q | 62,914,560 | -0.00000 177 2 | 30 self_attn q | 125,829,120 | -0.00000 178 1 | 30 self_attn [k, v] | 10,485,760 | -0.00000 178 2 | 30 self_attn [k, v] | 20,971,520 | -0.00005 179 1 | 30 self_attn o | 62,914,560 | -0.00005 179 2 | 30 self_attn o | 125,829,120 | -0.00011 180 1 | 30 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012 180 2 | 30 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00007 181 1 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007 181 2 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 182 1 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 182 2 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 183 1 | 31 self_attn q | 62,914,560 | +0.00005 183 2 | 31 self_attn q | 125,829,120 | -0.00001 184 1 | 31 self_attn [k, v] | 10,485,760 | -0.00031 184 2 | 31 self_attn [k, v] | 20,971,520 | -0.00015 185 1 | 31 self_attn o | 62,914,560 | -0.00013 185 2 | 31 self_attn o | 125,829,120 | -0.00004 186 1 | 31 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00029 186 2 | 31 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00038 187 1 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00005 187 2 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00006 188 1 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 188 2 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00009 189 1 | 32 self_attn q | 62,914,560 | +0.00002 189 2 | 32 self_attn q | 125,829,120 | -0.00008 190 1 | 32 self_attn [k, v] | 10,485,760 | +0.00007 190 2 | 32 self_attn [k, v] | 20,971,520 | -0.00000 191 1 | 32 self_attn o | 62,914,560 | +0.00012 191 2 | 32 self_attn o | 125,829,120 | +0.00013 192 1 | 32 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002 192 2 | 32 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00013 193 1 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00005 193 2 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 194 1 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 194 2 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 195 1 | 33 self_attn q | 62,914,560 | +0.00002 195 2 | 33 self_attn q | 125,829,120 | +0.00002 196 1 | 33 self_attn [k, v] | 10,485,760 | -0.00004 196 2 | 33 self_attn [k, v] | 20,971,520 | -0.00004 197 1 | 33 self_attn o | 62,914,560 | -0.00021 197 2 | 33 self_attn o | 125,829,120 | -0.00015 198 1 | 33 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00029 198 2 | 33 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00019 199 1 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00004 199 2 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 200 1 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 200 2 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 201 1 | 34 self_attn q | 62,914,560 | -0.00010 201 2 | 34 self_attn q | 125,829,120 | -0.00008 202 1 | 34 self_attn [k, v] | 10,485,760 | -0.00013 202 2 | 34 self_attn [k, v] | 20,971,520 | -0.00009 203 1 | 34 self_attn o | 62,914,560 | +0.00009 203 2 | 34 self_attn o | 125,829,120 | +0.00008 204 1 | 34 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00022 204 2 | 34 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022 205 1 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007 205 2 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005 206 1 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004 206 2 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 207 1 | 35 self_attn q | 62,914,560 | -0.00007 207 2 | 35 self_attn q | 125,829,120 | -0.00004 208 1 | 35 self_attn [k, v] | 10,485,760 | +0.00018 208 2 | 35 self_attn [k, v] | 20,971,520 | +0.00018 209 1 | 35 self_attn o | 62,914,560 | -0.00018 209 2 | 35 self_attn o | 125,829,120 | -0.00017 210 1 | 35 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011 210 2 | 35 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004 211 1 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00005 211 2 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 212 1 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 212 2 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004 213 1 | 36 self_attn q | 62,914,560 | -0.00000 213 2 | 36 self_attn q | 125,829,120 | -0.00013 214 1 | 36 self_attn [k, v] | 10,485,760 | -0.00060 214 2 | 36 self_attn [k, v] | 20,971,520 | -0.00055 215 1 | 36 self_attn o | 62,914,560 | -0.00012 215 2 | 36 self_attn o | 125,829,120 | -0.00012 216 1 | 36 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00006 216 2 | 36 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00001 217 1 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00007 217 2 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00006 218 1 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 218 2 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00007 219 1 | 37 self_attn q | 62,914,560 | +0.00006 219 2 | 37 self_attn q | 125,829,120 | +0.00005 220 1 | 37 self_attn [k, v] | 10,485,760 | -0.00015 220 2 | 37 self_attn [k, v] | 20,971,520 | -0.00004 221 1 | 37 self_attn o | 62,914,560 | +0.00006 221 2 | 37 self_attn o | 125,829,120 | +0.00006 222 1 | 37 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 222 2 | 37 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008 223 1 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 223 2 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002 224 1 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 224 2 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000 225 1 | 38 self_attn q | 62,914,560 | -0.00011 225 2 | 38 self_attn q | 125,829,120 | -0.00011 226 1 | 38 self_attn [k, v] | 10,485,760 | +0.00001 226 2 | 38 self_attn [k, v] | 20,971,520 | +0.00001 227 1 | 38 self_attn o | 62,914,560 | -0.00009 227 2 | 38 self_attn o | 125,829,120 | -0.00015 228 1 | 38 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002 228 2 | 38 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004 229 1 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 229 2 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00011 230 1 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 230 2 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004 231 1 | 39 self_attn q | 62,914,560 | -0.00008 231 2 | 39 self_attn q | 125,829,120 | -0.00005 232 1 | 39 self_attn [k, v] | 10,485,760 | -0.00040 232 2 | 39 self_attn [k, v] | 20,971,520 | -0.00042 233 1 | 39 self_attn o | 62,914,560 | +0.00013 233 2 | 39 self_attn o | 125,829,120 | +0.00005 234 1 | 39 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00015 234 2 | 39 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00019 235 1 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003 235 2 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 236 1 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 236 2 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 237 1 | 40 self_attn q | 62,914,560 | -0.00001 237 2 | 40 self_attn q | 125,829,120 | -0.00002 238 1 | 40 self_attn [k, v] | 10,485,760 | -0.00018 238 2 | 40 self_attn [k, v] | 20,971,520 | -0.00011 239 1 | 40 self_attn o | 62,914,560 | -0.00014 239 2 | 40 self_attn o | 125,829,120 | -0.00021 240 1 | 40 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00007 240 2 | 40 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010 241 1 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 241 2 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 242 1 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007 242 2 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 243 1 | 41 self_attn q | 62,914,560 | -0.00001 243 2 | 41 self_attn q | 125,829,120 | +0.00001 244 1 | 41 self_attn [k, v] | 10,485,760 | -0.00008 244 2 | 41 self_attn [k, v] | 20,971,520 | -0.00011 245 1 | 41 self_attn o | 62,914,560 | -0.00002 245 2 | 41 self_attn o | 125,829,120 | -0.00004 246 1 | 41 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00009 246 2 | 41 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00003 247 1 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003 247 2 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004 248 1 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 248 2 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00005 249 1 | 42 self_attn q | 62,914,560 | -0.00000 249 2 | 42 self_attn q | 125,829,120 | -0.00001 250 1 | 42 self_attn [k, v] | 10,485,760 | -0.00025 250 2 | 42 self_attn [k, v] | 20,971,520 | -0.00026 251 1 | 42 self_attn o | 62,914,560 | +0.00005 251 2 | 42 self_attn o | 125,829,120 | +0.00008 252 1 | 42 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 252 2 | 42 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002 253 1 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 253 2 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004 254 1 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 254 2 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 255 1 | 43 self_attn q | 62,914,560 | -0.00008 255 2 | 43 self_attn q | 125,829,120 | -0.00007 256 1 | 43 self_attn [k, v] | 10,485,760 | +0.00000 256 2 | 43 self_attn [k, v] | 20,971,520 | +0.00001 257 1 | 43 self_attn o | 62,914,560 | -0.00010 257 2 | 43 self_attn o | 125,829,120 | -0.00013 258 1 | 43 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 258 2 | 43 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002 259 1 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 259 2 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002 260 1 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 260 2 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00001 261 1 | 44 self_attn q | 62,914,560 | +0.00000 261 2 | 44 self_attn q | 125,829,120 | -0.00003 262 1 | 44 self_attn [k, v] | 10,485,760 | +0.00009 262 2 | 44 self_attn [k, v] | 20,971,520 | +0.00014 263 1 | 44 self_attn o | 62,914,560 | -0.00008 263 2 | 44 self_attn o | 125,829,120 | -0.00008 264 1 | 44 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009 264 2 | 44 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 265 1 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 265 2 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001 266 1 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000 266 2 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 267 1 | 45 self_attn q | 62,914,560 | +0.00005 267 2 | 45 self_attn q | 125,829,120 | +0.00005 268 1 | 45 self_attn [k, v] | 10,485,760 | -0.00006 268 2 | 45 self_attn [k, v] | 20,971,520 | -0.00003 269 1 | 45 self_attn o | 62,914,560 | +0.00003 269 2 | 45 self_attn o | 125,829,120 | -0.00001 270 1 | 45 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001 270 2 | 45 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003 271 1 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002 271 2 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00001 272 1 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 272 2 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 273 1 | 46 self_attn q | 62,914,560 | -0.00012 273 2 | 46 self_attn q | 125,829,120 | -0.00007 274 1 | 46 self_attn [k, v] | 10,485,760 | +0.00001 274 2 | 46 self_attn [k, v] | 20,971,520 | -0.00001 275 1 | 46 self_attn o | 62,914,560 | +0.00001 275 2 | 46 self_attn o | 125,829,120 | -0.00000 276 1 | 46 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 276 2 | 46 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00005 277 1 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 277 2 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 278 1 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00003 278 2 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 279 1 | 47 self_attn q | 62,914,560 | -0.00001 279 2 | 47 self_attn q | 125,829,120 | +0.00002 280 1 | 47 self_attn [k, v] | 10,485,760 | +0.00006 280 2 | 47 self_attn [k, v] | 20,971,520 | +0.00002 281 1 | 47 self_attn o | 62,914,560 | -0.00005 281 2 | 47 self_attn o | 125,829,120 | -0.00002 282 1 | 47 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 282 2 | 47 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003 283 1 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002 283 2 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000 284 1 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 284 2 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 285 1 | 48 self_attn q | 62,914,560 | -0.00007 285 2 | 48 self_attn q | 125,829,120 | -0.00004 286 1 | 48 self_attn [k, v] | 10,485,760 | -0.00014 286 2 | 48 self_attn [k, v] | 20,971,520 | -0.00013 287 1 | 48 self_attn o | 62,914,560 | -0.00002 287 2 | 48 self_attn o | 125,829,120 | +0.00001 288 1 | 48 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00006 288 2 | 48 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 289 1 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001 289 2 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001 290 1 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 290 2 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 291 1 | 49 self_attn q | 62,914,560 | -0.00000 291 2 | 49 self_attn q | 125,829,120 | +0.00002 292 1 | 49 self_attn [k, v] | 10,485,760 | -0.00007 292 2 | 49 self_attn [k, v] | 20,971,520 | -0.00007 293 1 | 49 self_attn o | 62,914,560 | +0.00003 293 2 | 49 self_attn o | 125,829,120 | +0.00004 294 1 | 49 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 294 2 | 49 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001 295 1 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003 295 2 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001 296 1 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000 296 2 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 297 1 | 50 self_attn q | 62,914,560 | +0.00002 297 2 | 50 self_attn q | 125,829,120 | +0.00002 298 1 | 50 self_attn [k, v] | 10,485,760 | -0.00007 298 2 | 50 self_attn [k, v] | 20,971,520 | -0.00010 299 1 | 50 self_attn o | 62,914,560 | -0.00004 299 2 | 50 self_attn o | 125,829,120 | -0.00003 300 1 | 50 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 300 2 | 50 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00000 301 1 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00004 301 2 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004 302 1 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 302 2 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 303 1 | 51 self_attn q | 62,914,560 | -0.00000 303 2 | 51 self_attn q | 125,829,120 | +0.00001 304 1 | 51 self_attn [k, v] | 10,485,760 | -0.00002 304 2 | 51 self_attn [k, v] | 20,971,520 | -0.00006 305 1 | 51 self_attn o | 62,914,560 | +0.00006 305 2 | 51 self_attn o | 125,829,120 | +0.00005 306 1 | 51 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005 306 2 | 51 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004 307 1 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003 307 2 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004 308 1 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 308 2 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 309 1 | 52 self_attn q | 62,914,560 | +0.00004 309 2 | 52 self_attn q | 125,829,120 | -0.00000 310 1 | 52 self_attn [k, v] | 10,485,760 | +0.00010 310 2 | 52 self_attn [k, v] | 20,971,520 | +0.00003 311 1 | 52 self_attn o | 62,914,560 | -0.00002 311 2 | 52 self_attn o | 125,829,120 | -0.00004 312 1 | 52 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 312 2 | 52 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00000 313 1 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 313 2 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000 314 1 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 314 2 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000 315 1 | 53 self_attn q | 62,914,560 | -0.00003 315 2 | 53 self_attn q | 125,829,120 | -0.00001 316 1 | 53 self_attn [k, v] | 10,485,760 | +0.00002 316 2 | 53 self_attn [k, v] | 20,971,520 | +0.00000 317 1 | 53 self_attn o | 62,914,560 | -0.00005 317 2 | 53 self_attn o | 125,829,120 | -0.00005 318 1 | 53 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 318 2 | 53 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002 319 1 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000 319 2 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00001 320 1 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 320 2 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 321 1 | 54 self_attn q | 62,914,560 | -0.00007 321 2 | 54 self_attn q | 125,829,120 | -0.00007 322 1 | 54 self_attn [k, v] | 10,485,760 | -0.00008 322 2 | 54 self_attn [k, v] | 20,971,520 | -0.00008 323 1 | 54 self_attn o | 62,914,560 | +0.00004 323 2 | 54 self_attn o | 125,829,120 | +0.00001 324 1 | 54 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001 324 2 | 54 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00003 325 1 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 325 2 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000 326 1 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 326 2 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00001 327 1 | 55 self_attn q | 62,914,560 | +0.00004 327 2 | 55 self_attn q | 125,829,120 | +0.00003 328 1 | 55 self_attn [k, v] | 10,485,760 | +0.00005 328 2 | 55 self_attn [k, v] | 20,971,520 | +0.00006 329 1 | 55 self_attn o | 62,914,560 | -0.00001 329 2 | 55 self_attn o | 125,829,120 | -0.00000 330 1 | 55 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002 330 2 | 55 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001 331 1 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 331 2 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002 332 1 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 332 2 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000 333 1 | 56 self_attn q | 62,914,560 | +0.00001 333 2 | 56 self_attn q | 125,829,120 | -0.00001 334 1 | 56 self_attn [k, v] | 10,485,760 | +0.00005 334 2 | 56 self_attn [k, v] | 20,971,520 | +0.00006 335 1 | 56 self_attn o | 62,914,560 | +0.00001 335 2 | 56 self_attn o | 125,829,120 | +0.00001 336 1 | 56 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002 336 2 | 56 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00001 337 1 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002 337 2 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000 338 1 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000 338 2 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000 339 1 | 57 self_attn q | 62,914,560 | +0.00002 339 2 | 57 self_attn q | 125,829,120 | +0.00001 340 1 | 57 self_attn [k, v] | 10,485,760 | -0.00001 340 2 | 57 self_attn [k, v] | 20,971,520 | -0.00000 341 1 | 57 self_attn o | 62,914,560 | +0.00000 341 2 | 57 self_attn o | 125,829,120 | -0.00001 342 1 | 57 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 342 2 | 57 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00001 343 1 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001 343 2 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00000 344 1 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 344 2 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 345 1 | 58 self_attn q | 62,914,560 | +0.00002 345 2 | 58 self_attn q | 125,829,120 | -0.00001 346 1 | 58 self_attn [k, v] | 10,485,760 | +0.00000 346 2 | 58 self_attn [k, v] | 20,971,520 | -0.00001 347 1 | 58 self_attn o | 62,914,560 | -0.00001 347 2 | 58 self_attn o | 125,829,120 | -0.00001 348 1 | 58 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00005 348 2 | 58 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004 349 1 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001 349 2 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00002 350 1 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 350 2 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 351 1 | 59 self_attn q | 62,914,560 | +0.00000 351 2 | 59 self_attn q | 125,829,120 | +0.00003 352 1 | 59 self_attn [k, v] | 10,485,760 | -0.00004 352 2 | 59 self_attn [k, v] | 20,971,520 | -0.00003 353 1 | 59 self_attn o | 62,914,560 | -0.00002 353 2 | 59 self_attn o | 125,829,120 | -0.00002 354 1 | 59 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005 354 2 | 59 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004 355 1 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002 355 2 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00002 356 1 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 356 2 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 357 1 | 60 self_attn q | 62,914,560 | -0.00003 357 2 | 60 self_attn q | 125,829,120 | -0.00002 358 1 | 60 self_attn [k, v] | 10,485,760 | -0.00005 358 2 | 60 self_attn [k, v] | 20,971,520 | -0.00008 359 1 | 60 self_attn o | 62,914,560 | -0.00001 359 2 | 60 self_attn o | 125,829,120 | +0.00002 360 1 | 60 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 360 2 | 60 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004 361 1 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001 361 2 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000 362 1 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 362 2 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 363 1 | 61 self_attn q | 62,914,560 | +0.00002 363 2 | 61 self_attn q | 125,829,120 | +0.00002 364 1 | 61 self_attn [k, v] | 10,485,760 | -0.00000 364 2 | 61 self_attn [k, v] | 20,971,520 | -0.00000 365 1 | 61 self_attn o | 62,914,560 | -0.00001 365 2 | 61 self_attn o | 125,829,120 | -0.00002 366 1 | 61 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 366 2 | 61 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001 367 1 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 367 2 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 368 1 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002 368 2 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 369 1 | 62 self_attn q | 62,914,560 | +0.00001 369 2 | 62 self_attn q | 125,829,120 | +0.00001 370 1 | 62 self_attn [k, v] | 10,485,760 | +0.00002 370 2 | 62 self_attn [k, v] | 20,971,520 | +0.00002 371 1 | 62 self_attn o | 62,914,560 | +0.00001 371 2 | 62 self_attn o | 125,829,120 | +0.00001 372 1 | 62 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 372 2 | 62 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001 373 1 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 373 2 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001 374 1 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000 374 2 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 375 1 | 63 self_attn q | 62,914,560 | -0.00002 375 2 | 63 self_attn q | 125,829,120 | -0.00003 376 1 | 63 self_attn [k, v] | 10,485,760 | +0.00001 376 2 | 63 self_attn [k, v] | 20,971,520 | -0.00002 377 1 | 63 self_attn o | 62,914,560 | +0.00001 377 2 | 63 self_attn o | 125,829,120 | +0.00001 378 1 | 63 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002 378 2 | 63 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00000 379 1 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000 379 2 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000 380 1 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002 380 2 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00002 381 1 | 64 self_attn q | 62,914,560 | +0.00002 381 2 | 64 self_attn q | 125,829,120 | +0.00001 382 1 | 64 self_attn [k, v] | 10,485,760 | +0.00001 382 2 | 64 self_attn [k, v] | 20,971,520 | +0.00001 383 1 | 64 self_attn o | 62,914,560 | +0.00001 383 2 | 64 self_attn o | 125,829,120 | +0.00000 384 1 | 64 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00000 384 2 | 64 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001 385 1 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003 385 2 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002 386 1 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 386 2 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 387 1 | 65 self_attn q | 62,914,560 | +0.00002 387 2 | 65 self_attn q | 125,829,120 | +0.00002 388 1 | 65 self_attn [k, v] | 10,485,760 | -0.00005 388 2 | 65 self_attn [k, v] | 20,971,520 | -0.00002 389 1 | 65 self_attn o | 62,914,560 | -0.00001 389 2 | 65 self_attn o | 125,829,120 | -0.00001 390 1 | 65 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001 390 2 | 65 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001 391 1 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000 391 2 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001 392 1 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002 392 2 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00002 393 1 | 66 self_attn q | 62,914,560 | -0.00008 393 2 | 66 self_attn q | 125,829,120 | -0.00008 394 1 | 66 self_attn [k, v] | 10,485,760 | -0.00001 394 2 | 66 self_attn [k, v] | 20,971,520 | +0.00000 395 1 | 66 self_attn o | 62,914,560 | -0.00001 395 2 | 66 self_attn o | 125,829,120 | +0.00001 396 1 | 66 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005 396 2 | 66 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003 397 1 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000 397 2 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00001 398 1 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 398 2 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 399 1 | 67 self_attn q | 62,914,560 | -0.00002 399 2 | 67 self_attn q | 125,829,120 | -0.00003 400 1 | 67 self_attn [k, v] | 10,485,760 | +0.00003 400 2 | 67 self_attn [k, v] | 20,971,520 | +0.00002 401 1 | 67 self_attn o | 62,914,560 | +0.00002 401 2 | 67 self_attn o | 125,829,120 | +0.00002 402 1 | 67 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 402 2 | 67 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002 403 1 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 403 2 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00002 404 1 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000 404 2 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000 405 1 | 68 self_attn q | 62,914,560 | -0.00000 405 2 | 68 self_attn q | 125,829,120 | -0.00001 406 1 | 68 self_attn [k, v] | 10,485,760 | +0.00007 406 2 | 68 self_attn [k, v] | 20,971,520 | +0.00003 407 1 | 68 self_attn o | 62,914,560 | -0.00003 407 2 | 68 self_attn o | 125,829,120 | -0.00005 408 1 | 68 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 408 2 | 68 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002 409 1 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002 409 2 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00002 410 1 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 410 2 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 411 1 | 69 self_attn q | 62,914,560 | +0.00001 411 2 | 69 self_attn q | 125,829,120 | +0.00001 412 1 | 69 self_attn [k, v] | 10,485,760 | -0.00002 412 2 | 69 self_attn [k, v] | 20,971,520 | -0.00003 413 1 | 69 self_attn o | 62,914,560 | +0.00002 413 2 | 69 self_attn o | 125,829,120 | +0.00002 414 1 | 69 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004 414 2 | 69 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002 415 1 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 415 2 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002 416 1 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 416 2 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 417 1 | 70 self_attn q | 62,914,560 | +0.00001 417 2 | 70 self_attn q | 125,829,120 | -0.00000 418 1 | 70 self_attn [k, v] | 10,485,760 | +0.00002 418 2 | 70 self_attn [k, v] | 20,971,520 | +0.00001 419 1 | 70 self_attn o | 62,914,560 | -0.00000 419 2 | 70 self_attn o | 125,829,120 | -0.00001 420 1 | 70 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00000 420 2 | 70 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002 421 1 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001 421 2 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000 422 1 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 422 2 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 423 1 | 71 self_attn q | 62,914,560 | +0.00003 423 2 | 71 self_attn q | 125,829,120 | +0.00003 424 1 | 71 self_attn [k, v] | 10,485,760 | +0.00002 424 2 | 71 self_attn [k, v] | 20,971,520 | -0.00001 425 1 | 71 self_attn o | 62,914,560 | -0.00001 425 2 | 71 self_attn o | 125,829,120 | -0.00002 426 1 | 71 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 426 2 | 71 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004 427 1 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003 427 2 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002 428 1 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 428 2 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 429 1 | 72 self_attn q | 62,914,560 | +0.00001 429 2 | 72 self_attn q | 125,829,120 | +0.00001 430 1 | 72 self_attn [k, v] | 10,485,760 | -0.00000 430 2 | 72 self_attn [k, v] | 20,971,520 | -0.00002 431 1 | 72 self_attn o | 62,914,560 | -0.00002 431 2 | 72 self_attn o | 125,829,120 | -0.00002 432 1 | 72 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002 432 2 | 72 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00000 433 1 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00004 433 2 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005 434 1 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 434 2 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 435 1 | 73 self_attn q | 62,914,560 | +0.00001 435 2 | 73 self_attn q | 125,829,120 | +0.00001 436 1 | 73 self_attn [k, v] | 10,485,760 | +0.00003 436 2 | 73 self_attn [k, v] | 20,971,520 | +0.00003 437 1 | 73 self_attn o | 62,914,560 | +0.00001 437 2 | 73 self_attn o | 125,829,120 | +0.00002 438 1 | 73 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002 438 2 | 73 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003 439 1 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001 439 2 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00000 440 1 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 440 2 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 441 1 | 74 self_attn q | 62,914,560 | +0.00001 441 2 | 74 self_attn q | 125,829,120 | +0.00000 442 1 | 74 self_attn [k, v] | 10,485,760 | +0.00001 442 2 | 74 self_attn [k, v] | 20,971,520 | -0.00001 443 1 | 74 self_attn o | 62,914,560 | -0.00002 443 2 | 74 self_attn o | 125,829,120 | -0.00000 444 1 | 74 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003 444 2 | 74 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004 445 1 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000 445 2 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001 446 1 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 446 2 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 447 1 | 75 self_attn q | 62,914,560 | -0.00003 447 2 | 75 self_attn q | 125,829,120 | -0.00001 448 1 | 75 self_attn [k, v] | 10,485,760 | -0.00000 448 2 | 75 self_attn [k, v] | 20,971,520 | +0.00000 449 1 | 75 self_attn o | 62,914,560 | +0.00000 449 2 | 75 self_attn o | 125,829,120 | -0.00001 450 1 | 75 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005 450 2 | 75 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004 451 1 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003 451 2 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 452 1 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 452 2 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000 453 1 | 76 self_attn q | 62,914,560 | -0.00011 453 2 | 76 self_attn q | 125,829,120 | -0.00011 454 1 | 76 self_attn [k, v] | 10,485,760 | -0.00004 454 2 | 76 self_attn [k, v] | 20,971,520 | -0.00004 455 1 | 76 self_attn o | 62,914,560 | -0.00001 455 2 | 76 self_attn o | 125,829,120 | -0.00002 456 1 | 76 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005 456 2 | 76 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004 457 1 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 457 2 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002 458 1 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 458 2 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000 459 1 | 77 self_attn q | 62,914,560 | -0.00002 459 2 | 77 self_attn q | 125,829,120 | -0.00003 460 1 | 77 self_attn [k, v] | 10,485,760 | -0.00007 460 2 | 77 self_attn [k, v] | 20,971,520 | -0.00007 461 1 | 77 self_attn o | 62,914,560 | -0.00001 461 2 | 77 self_attn o | 125,829,120 | -0.00002 462 1 | 77 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002 462 2 | 77 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00003 463 1 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 463 2 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000 464 1 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 464 2 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 465 1 | 78 self_attn q | 62,914,560 | +0.00001 465 2 | 78 self_attn q | 125,829,120 | +0.00002 466 1 | 78 self_attn [k, v] | 10,485,760 | +0.00000 466 2 | 78 self_attn [k, v] | 20,971,520 | -0.00002 467 1 | 78 self_attn o | 62,914,560 | +0.00002 467 2 | 78 self_attn o | 125,829,120 | +0.00004 468 1 | 78 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001 468 2 | 78 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004 469 1 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001 469 2 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00000 470 1 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 470 2 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 471 1 | 79 self_attn q | 62,914,560 | +0.00002 471 2 | 79 self_attn q | 125,829,120 | +0.00001 472 1 | 79 self_attn [k, v] | 10,485,760 | -0.00016 472 2 | 79 self_attn [k, v] | 20,971,520 | -0.00019 473 1 | 79 self_attn o | 62,914,560 | -0.00002 473 2 | 79 self_attn o | 125,829,120 | +0.00000 474 1 | 79 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004 474 2 | 79 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002 475 1 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006 475 2 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005 476 1 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00003 476 2 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00002 477 1 | 80 self_attn q | 62,914,560 | +0.00019 477 2 | 80 self_attn q | 125,829,120 | +0.00008 478 1 | 80 self_attn [k, v] | 10,485,760 | +0.00005 478 2 | 80 self_attn [k, v] | 20,971,520 | +0.00000 479 1 | 80 self_attn o | 62,914,560 | -0.00000 479 2 | 80 self_attn o | 125,829,120 | +0.00000 480 1 | 80 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009 480 2 | 80 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 481 1 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00001 481 2 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00000 482 1 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 482 2 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 483 1 | 81 self_attn q | 62,914,560 | +0.00009 483 2 | 81 self_attn q | 125,829,120 | +0.00004 484 1 | 81 self_attn [k, v] | 10,485,760 | +0.00005 484 2 | 81 self_attn [k, v] | 20,971,520 | +0.00006 485 1 | 81 self_attn o | 62,914,560 | -0.00002 485 2 | 81 self_attn o | 125,829,120 | -0.00002 486 1 | 81 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00002 486 2 | 81 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 487 1 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 487 2 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002 488 1 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 488 2 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 489 1 | 82 self_attn q | 62,914,560 | -0.00002 489 2 | 82 self_attn q | 125,829,120 | -0.00002 490 1 | 82 self_attn [k, v] | 10,485,760 | -0.00009 490 2 | 82 self_attn [k, v] | 20,971,520 | -0.00008 491 1 | 82 self_attn o | 62,914,560 | -0.00007 491 2 | 82 self_attn o | 125,829,120 | -0.00007 492 1 | 82 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003 492 2 | 82 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002 493 1 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00002 493 2 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00001 494 1 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 494 2 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00001 495 1 | 83 self_attn q | 62,914,560 | +0.00001 495 2 | 83 self_attn q | 125,829,120 | -0.00000 496 1 | 83 self_attn [k, v] | 10,485,760 | -0.00013 496 2 | 83 self_attn [k, v] | 20,971,520 | -0.00014 497 1 | 83 self_attn o | 62,914,560 | -0.00005 497 2 | 83 self_attn o | 125,829,120 | -0.00003 498 1 | 83 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 498 2 | 83 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00004 499 1 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000 499 2 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00002 500 1 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 500 2 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 501 1 | 84 self_attn q | 62,914,560 | -0.00007 501 2 | 84 self_attn q | 125,829,120 | -0.00010 502 1 | 84 self_attn [k, v] | 10,485,760 | -0.00012 502 2 | 84 self_attn [k, v] | 20,971,520 | -0.00009 503 1 | 84 self_attn o | 62,914,560 | +0.00001 503 2 | 84 self_attn o | 125,829,120 | -0.00000 504 1 | 84 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00007 504 2 | 84 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00009 505 1 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 505 2 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 506 1 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 506 2 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 507 1 | 85 self_attn q | 62,914,560 | -0.00002 507 2 | 85 self_attn q | 125,829,120 | -0.00008 508 1 | 85 self_attn [k, v] | 10,485,760 | -0.00025 508 2 | 85 self_attn [k, v] | 20,971,520 | -0.00027 509 1 | 85 self_attn o | 62,914,560 | -0.00007 509 2 | 85 self_attn o | 125,829,120 | -0.00007 510 1 | 85 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003 510 2 | 85 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00004 511 1 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004 511 2 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00001 512 1 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 512 2 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 513 1 | 86 self_attn q | 62,914,560 | +0.00000 513 2 | 86 self_attn q | 125,829,120 | -0.00003 514 1 | 86 self_attn [k, v] | 10,485,760 | -0.00011 514 2 | 86 self_attn [k, v] | 20,971,520 | -0.00013 515 1 | 86 self_attn o | 62,914,560 | -0.00013 515 2 | 86 self_attn o | 125,829,120 | -0.00011 516 1 | 86 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001 516 2 | 86 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00001 517 1 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00003 517 2 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00004 518 1 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 518 2 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 519 1 | 87 self_attn q | 62,914,560 | +0.00001 519 2 | 87 self_attn q | 125,829,120 | +0.00000 520 1 | 87 self_attn [k, v] | 10,485,760 | -0.00013 520 2 | 87 self_attn [k, v] | 20,971,520 | -0.00035 521 1 | 87 self_attn o | 62,914,560 | +0.00000 521 2 | 87 self_attn o | 125,829,120 | +0.00000 522 1 | 87 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 522 2 | 87 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00005 523 1 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 523 2 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001 524 1 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 524 2 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 525 1 | 88 self_attn q | 62,914,560 | -0.00002 525 2 | 88 self_attn q | 125,829,120 | -0.00001 526 1 | 88 self_attn [k, v] | 10,485,760 | +0.00002 526 2 | 88 self_attn [k, v] | 20,971,520 | -0.00003 527 1 | 88 self_attn o | 62,914,560 | -0.00005 527 2 | 88 self_attn o | 125,829,120 | -0.00004 528 1 | 88 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 528 2 | 88 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003 529 1 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000 529 2 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001 530 1 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000 530 2 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000 531 1 | 89 self_attn q | 62,914,560 | -0.00004 531 2 | 89 self_attn q | 125,829,120 | -0.00004 532 1 | 89 self_attn [k, v] | 10,485,760 | -0.00010 532 2 | 89 self_attn [k, v] | 20,971,520 | -0.00008 533 1 | 89 self_attn o | 62,914,560 | -0.00002 533 2 | 89 self_attn o | 125,829,120 | -0.00002 534 1 | 89 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 534 2 | 89 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 535 1 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000 535 2 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000 536 1 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000 536 2 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00001 537 1 | 90 self_attn q | 62,914,560 | -0.00001 537 2 | 90 self_attn q | 125,829,120 | +0.00000 538 1 | 90 self_attn [k, v] | 10,485,760 | -0.00005 538 2 | 90 self_attn [k, v] | 20,971,520 | -0.00005 539 1 | 90 self_attn o | 62,914,560 | +0.00001 539 2 | 90 self_attn o | 125,829,120 | +0.00000 540 1 | 90 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00007 540 2 | 90 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00005 541 1 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00000 541 2 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00000 542 1 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 542 2 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00000 543 1 | 91 self_attn q | 62,914,560 | +0.00003 543 2 | 91 self_attn q | 125,829,120 | +0.00003 544 1 | 91 self_attn [k, v] | 10,485,760 | -0.00011 544 2 | 91 self_attn [k, v] | 20,971,520 | -0.00013 545 1 | 91 self_attn o | 62,914,560 | -0.00008 545 2 | 91 self_attn o | 125,829,120 | -0.00008 546 1 | 91 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 546 2 | 91 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016 547 1 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00000 547 2 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00001 548 1 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00000 548 2 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00000