-- Results idx | parts | d. bits | d. KL-div ---------|----------------------------------------------------------------------|----------------------|----------- - | | | 0.07912 0 1 | 0 self_attn q | 62,914,560 | -0.00014 0 2 | 0 self_attn q | 125,829,120 | -0.00009 0 3 | 0 self_attn q | 188,743,680 | +0.00003 0 4 | 0 self_attn q | 314,572,800 | -0.00010 1 1 | 0 self_attn [k, v] | 10,485,760 | -0.00005 1 2 | 0 self_attn [k, v] | 20,971,520 | -0.00006 1 3 | 0 self_attn [k, v] | 31,457,280 | +0.00004 1 4 | 0 self_attn [k, v] | 52,428,800 | -0.00007 2 1 | 0 self_attn o | 62,914,560 | -0.00040 2 2 | 0 self_attn o | 125,829,120 | -0.00022 2 3 | 0 self_attn o | 188,743,680 | -0.00015 2 4 | 0 self_attn o | 314,572,800 | -0.00023 3 1 | 0 mlp [gate, up] | 125,829,120 | -0.00044 3 2 | 0 mlp [gate, up] | 251,658,240 | -0.00052 3 3 | 0 mlp [gate, up] | 377,487,360 | -0.00053 3 4 | 0 mlp [gate, up] | 629,145,600 | -0.00054 4 1 | 0 mlp down | 62,914,560 | -0.00036 4 2 | 0 mlp down | 125,829,120 | -0.00018 4 3 | 0 mlp down | 188,743,680 | -0.00024 4 4 | 0 mlp down | 314,572,800 | -0.00028 5 1 | 1 self_attn q | 62,914,560 | -0.00008 5 2 | 1 self_attn q | 125,829,120 | +0.00000 5 3 | 1 self_attn q | 188,743,680 | +0.00008 5 4 | 1 self_attn q | 314,572,800 | -0.00003 6 1 | 1 self_attn [k, v] | 10,485,760 | -0.00012 6 2 | 1 self_attn [k, v] | 20,971,520 | -0.00034 6 3 | 1 self_attn [k, v] | 31,457,280 | -0.00022 6 4 | 1 self_attn [k, v] | 52,428,800 | -0.00012 7 1 | 1 self_attn o | 62,914,560 | +0.00001 7 2 | 1 self_attn o | 125,829,120 | +0.00007 7 3 | 1 self_attn o | 188,743,680 | -0.00012 7 4 | 1 self_attn o | 314,572,800 | +0.00000 8 1 | 1 mlp [gate, up] | 125,829,120 | -0.00024 8 2 | 1 mlp [gate, up] | 251,658,240 | -0.00037 8 3 | 1 mlp [gate, up] | 377,487,360 | -0.00039 8 4 | 1 mlp [gate, up] | 629,145,600 | -0.00040 9 1 | 1 mlp down | 62,914,560 | +0.00003 9 2 | 1 mlp down | 125,829,120 | -0.00005 9 3 | 1 mlp down | 188,743,680 | -0.00005 9 4 | 1 mlp down | 314,572,800 | +0.00004 10 1 | 2 self_attn q | 62,914,560 | +0.00001 10 2 | 2 self_attn q | 125,829,120 | -0.00007 10 3 | 2 self_attn q | 188,743,680 | -0.00009 10 4 | 2 self_attn q | 314,572,800 | -0.00010 11 1 | 2 self_attn [k, v] | 10,485,760 | -0.00062 11 2 | 2 self_attn [k, v] | 20,971,520 | -0.00047 11 3 | 2 self_attn [k, v] | 31,457,280 | -0.00038 11 4 | 2 self_attn [k, v] | 52,428,800 | -0.00046 12 1 | 2 self_attn o | 62,914,560 | -0.00006 12 2 | 2 self_attn o | 125,829,120 | +0.00002 12 3 | 2 self_attn o | 188,743,680 | +0.00014 12 4 | 2 self_attn o | 314,572,800 | -0.00012 13 1 | 2 mlp [gate, up] | 125,829,120 | -0.00008 13 2 | 2 mlp [gate, up] | 251,658,240 | -0.00058 13 3 | 2 mlp [gate, up] | 377,487,360 | -0.00064 13 4 | 2 mlp [gate, up] | 629,145,600 | -0.00061 14 1 | 2 mlp down | 62,914,560 | -0.00062 14 2 | 2 mlp down | 125,829,120 | -0.00037 14 3 | 2 mlp down | 188,743,680 | -0.00028 14 4 | 2 mlp down | 314,572,800 | -0.00035 15 1 | 3 self_attn q | 62,914,560 | -0.00025 15 2 | 3 self_attn q | 125,829,120 | -0.00012 15 3 | 3 self_attn q | 188,743,680 | -0.00027 15 4 | 3 self_attn q | 314,572,800 | -0.00017 16 1 | 3 self_attn [k, v] | 10,485,760 | -0.00060 16 2 | 3 self_attn [k, v] | 20,971,520 | -0.00071 16 3 | 3 self_attn [k, v] | 31,457,280 | -0.00056 16 4 | 3 self_attn [k, v] | 52,428,800 | -0.00047 17 1 | 3 self_attn o | 62,914,560 | -0.00006 17 2 | 3 self_attn o | 125,829,120 | -0.00007 17 3 | 3 self_attn o | 188,743,680 | +0.00000 17 4 | 3 self_attn o | 314,572,800 | +0.00003 18 1 | 3 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00031 18 2 | 3 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00011 18 3 | 3 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00016 18 4 | 3 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00015 19 1 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00042 19 2 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00027 19 3 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00030 19 4 | 3 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00032 20 1 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00019 20 2 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00020 20 3 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00028 20 4 | 3 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00020 21 1 | 4 self_attn q | 62,914,560 | +0.00006 21 2 | 4 self_attn q | 125,829,120 | +0.00013 21 3 | 4 self_attn q | 188,743,680 | -0.00003 21 4 | 4 self_attn q | 314,572,800 | +0.00012 22 1 | 4 self_attn [k, v] | 10,485,760 | +0.00022 22 2 | 4 self_attn [k, v] | 20,971,520 | +0.00011 22 3 | 4 self_attn [k, v] | 31,457,280 | +0.00009 22 4 | 4 self_attn [k, v] | 52,428,800 | +0.00014 23 1 | 4 self_attn o | 62,914,560 | -0.00001 23 2 | 4 self_attn o | 125,829,120 | +0.00003 23 3 | 4 self_attn o | 188,743,680 | +0.00015 23 4 | 4 self_attn o | 314,572,800 | +0.00009 24 1 | 4 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 24 2 | 4 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00036 24 3 | 4 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00022 24 4 | 4 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00040 25 1 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00018 25 2 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00031 25 3 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00043 25 4 | 4 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00032 26 1 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00026 26 2 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00026 26 3 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00040 26 4 | 4 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00030 27 1 | 5 self_attn q | 62,914,560 | -0.00009 27 2 | 5 self_attn q | 125,829,120 | -0.00008 27 3 | 5 self_attn q | 188,743,680 | -0.00020 27 4 | 5 self_attn q | 314,572,800 | -0.00015 28 1 | 5 self_attn [k, v] | 10,485,760 | +0.00007 28 2 | 5 self_attn [k, v] | 20,971,520 | +0.00015 28 3 | 5 self_attn [k, v] | 31,457,280 | +0.00006 28 4 | 5 self_attn [k, v] | 52,428,800 | +0.00007 29 1 | 5 self_attn o | 62,914,560 | -0.00026 29 2 | 5 self_attn o | 125,829,120 | -0.00013 29 3 | 5 self_attn o | 188,743,680 | -0.00028 29 4 | 5 self_attn o | 314,572,800 | -0.00020 30 1 | 5 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00048 30 2 | 5 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00034 30 3 | 5 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00034 30 4 | 5 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00049 31 1 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00020 31 2 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00015 31 3 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00013 31 4 | 5 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00024 32 1 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00021 32 2 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00025 32 3 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00022 32 4 | 5 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00030 33 1 | 6 self_attn q | 62,914,560 | +0.00010 33 2 | 6 self_attn q | 125,829,120 | +0.00016 33 3 | 6 self_attn q | 188,743,680 | +0.00016 33 4 | 6 self_attn q | 314,572,800 | +0.00015 34 1 | 6 self_attn [k, v] | 10,485,760 | -0.00018 34 2 | 6 self_attn [k, v] | 20,971,520 | -0.00023 34 3 | 6 self_attn [k, v] | 31,457,280 | -0.00020 34 4 | 6 self_attn [k, v] | 52,428,800 | -0.00012 35 1 | 6 self_attn o | 62,914,560 | -0.00030 35 2 | 6 self_attn o | 125,829,120 | -0.00031 35 3 | 6 self_attn o | 188,743,680 | -0.00030 35 4 | 6 self_attn o | 314,572,800 | -0.00038 36 1 | 6 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00009 36 2 | 6 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022 36 3 | 6 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00037 36 4 | 6 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00031 37 1 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00027 37 2 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00024 37 3 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00036 37 4 | 6 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00023 38 1 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00013 38 2 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00043 38 3 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00035 38 4 | 6 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00035 39 1 | 7 self_attn q | 62,914,560 | -0.00029 39 2 | 7 self_attn q | 125,829,120 | -0.00034 39 3 | 7 self_attn q | 188,743,680 | -0.00032 39 4 | 7 self_attn q | 314,572,800 | -0.00030 40 1 | 7 self_attn [k, v] | 10,485,760 | -0.00028 40 2 | 7 self_attn [k, v] | 20,971,520 | -0.00004 40 3 | 7 self_attn [k, v] | 31,457,280 | -0.00013 40 4 | 7 self_attn [k, v] | 52,428,800 | -0.00014 41 1 | 7 self_attn o | 62,914,560 | -0.00011 41 2 | 7 self_attn o | 125,829,120 | -0.00023 41 3 | 7 self_attn o | 188,743,680 | -0.00024 41 4 | 7 self_attn o | 314,572,800 | -0.00031 42 1 | 7 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00010 42 2 | 7 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00045 42 3 | 7 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00028 42 4 | 7 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00032 43 1 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00028 43 2 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00033 43 3 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00046 43 4 | 7 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00049 44 1 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00040 44 2 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00039 44 3 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00035 44 4 | 7 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00036 45 1 | 8 self_attn q | 62,914,560 | -0.00017 45 2 | 8 self_attn q | 125,829,120 | -0.00026 45 3 | 8 self_attn q | 188,743,680 | -0.00027 45 4 | 8 self_attn q | 314,572,800 | -0.00019 46 1 | 8 self_attn [k, v] | 10,485,760 | +0.00086 46 2 | 8 self_attn [k, v] | 20,971,520 | +0.00031 46 3 | 8 self_attn [k, v] | 31,457,280 | +0.00027 46 4 | 8 self_attn [k, v] | 52,428,800 | +0.00030 47 1 | 8 self_attn o | 62,914,560 | -0.00128 47 2 | 8 self_attn o | 125,829,120 | -0.00105 47 3 | 8 self_attn o | 188,743,680 | -0.00109 47 4 | 8 self_attn o | 314,572,800 | -0.00101 48 1 | 8 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00019 48 2 | 8 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00013 48 3 | 8 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00005 48 4 | 8 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00013 49 1 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00037 49 2 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00023 49 3 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00036 49 4 | 8 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00028 50 1 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00026 50 2 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00030 50 3 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00031 50 4 | 8 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00032 51 1 | 9 self_attn q | 62,914,560 | -0.00022 51 2 | 9 self_attn q | 125,829,120 | -0.00017 51 3 | 9 self_attn q | 188,743,680 | -0.00029 51 4 | 9 self_attn q | 314,572,800 | -0.00033 52 1 | 9 self_attn [k, v] | 10,485,760 | -0.00070 52 2 | 9 self_attn [k, v] | 20,971,520 | -0.00023 52 3 | 9 self_attn [k, v] | 31,457,280 | -0.00047 52 4 | 9 self_attn [k, v] | 52,428,800 | -0.00052 53 1 | 9 self_attn o | 62,914,560 | -0.00030 53 2 | 9 self_attn o | 125,829,120 | -0.00026 53 3 | 9 self_attn o | 188,743,680 | -0.00032 53 4 | 9 self_attn o | 314,572,800 | -0.00034 54 1 | 9 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 54 2 | 9 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00007 54 3 | 9 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00011 54 4 | 9 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00012 55 1 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 2,516,582,400 | -0.00047 55 2 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 5,033,164,800 | -0.00037 55 3 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 7,549,747,200 | -0.00044 55 4 | 9 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate,... | 12,582,912,000 | -0.00039 56 1 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 1,258,291,200 | -0.00025 56 2 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 2,516,582,400 | -0.00032 56 3 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 3,774,873,600 | -0.00047 56 4 | 9 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6 ... | 6,291,456,000 | -0.00036 57 1 | 10 self_attn q | 62,914,560 | -0.00030 57 2 | 10 self_attn q | 125,829,120 | -0.00026 57 3 | 10 self_attn q | 188,743,680 | -0.00017 57 4 | 10 self_attn q | 314,572,800 | -0.00021 58 1 | 10 self_attn [k, v] | 10,485,760 | -0.00006 58 2 | 10 self_attn [k, v] | 20,971,520 | +0.00030 58 3 | 10 self_attn [k, v] | 31,457,280 | +0.00009 58 4 | 10 self_attn [k, v] | 52,428,800 | +0.00008 59 1 | 10 self_attn o | 62,914,560 | +0.00033 59 2 | 10 self_attn o | 125,829,120 | +0.00003 59 3 | 10 self_attn o | 188,743,680 | +0.00007 59 4 | 10 self_attn o | 314,572,800 | +0.00018 60 1 | 10 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006 60 2 | 10 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00009 60 3 | 10 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00005 60 4 | 10 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00007 61 1 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00020 61 2 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00021 61 3 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00020 61 4 | 10 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00029 62 1 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 62 2 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00037 62 3 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00024 62 4 | 10 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00024 63 1 | 11 self_attn q | 62,914,560 | +0.00010 63 2 | 11 self_attn q | 125,829,120 | -0.00010 63 3 | 11 self_attn q | 188,743,680 | +0.00005 63 4 | 11 self_attn q | 314,572,800 | +0.00003 64 1 | 11 self_attn [k, v] | 10,485,760 | +0.00122 64 2 | 11 self_attn [k, v] | 20,971,520 | +0.00123 64 3 | 11 self_attn [k, v] | 31,457,280 | +0.00114 64 4 | 11 self_attn [k, v] | 52,428,800 | +0.00095 65 1 | 11 self_attn o | 62,914,560 | -0.00109 65 2 | 11 self_attn o | 125,829,120 | -0.00174 65 3 | 11 self_attn o | 188,743,680 | -0.00166 65 4 | 11 self_attn o | 314,572,800 | -0.00153 66 1 | 11 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00046 66 2 | 11 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00044 66 3 | 11 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00031 66 4 | 11 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00026 67 1 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00042 67 2 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00054 67 3 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00069 67 4 | 11 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00069 68 1 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00057 68 2 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00058 68 3 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00064 68 4 | 11 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00057 69 1 | 12 self_attn q | 62,914,560 | +0.00001 69 2 | 12 self_attn q | 125,829,120 | +0.00001 69 3 | 12 self_attn q | 188,743,680 | +0.00005 69 4 | 12 self_attn q | 314,572,800 | +0.00002 70 1 | 12 self_attn [k, v] | 10,485,760 | -0.00015 70 2 | 12 self_attn [k, v] | 20,971,520 | +0.00026 70 3 | 12 self_attn [k, v] | 31,457,280 | +0.00029 70 4 | 12 self_attn [k, v] | 52,428,800 | +0.00014 71 1 | 12 self_attn o | 62,914,560 | -0.00105 71 2 | 12 self_attn o | 125,829,120 | -0.00077 71 3 | 12 self_attn o | 188,743,680 | -0.00082 71 4 | 12 self_attn o | 314,572,800 | -0.00077 72 1 | 12 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00042 72 2 | 12 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00010 72 3 | 12 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00007 72 4 | 12 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00006 73 1 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 73 2 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00084 73 3 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00083 73 4 | 12 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00079 74 1 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00072 74 2 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00077 74 3 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00086 74 4 | 12 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00088 75 1 | 13 self_attn q | 62,914,560 | -0.00023 75 2 | 13 self_attn q | 125,829,120 | -0.00033 75 3 | 13 self_attn q | 188,743,680 | -0.00046 75 4 | 13 self_attn q | 314,572,800 | -0.00058 76 1 | 13 self_attn [k, v] | 10,485,760 | -0.00156 76 2 | 13 self_attn [k, v] | 20,971,520 | -0.00126 76 3 | 13 self_attn [k, v] | 31,457,280 | -0.00116 76 4 | 13 self_attn [k, v] | 52,428,800 | -0.00111 77 1 | 13 self_attn o | 62,914,560 | -0.00087 77 2 | 13 self_attn o | 125,829,120 | -0.00080 77 3 | 13 self_attn o | 188,743,680 | -0.00085 77 4 | 13 self_attn o | 314,572,800 | -0.00084 78 1 | 13 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00054 78 2 | 13 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00032 78 3 | 13 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00029 78 4 | 13 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00022 79 1 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00031 79 2 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00051 79 3 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00064 79 4 | 13 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00062 80 1 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 80 2 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00064 80 3 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00066 80 4 | 13 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00049 81 1 | 14 self_attn q | 62,914,560 | -0.00018 81 2 | 14 self_attn q | 125,829,120 | -0.00004 81 3 | 14 self_attn q | 188,743,680 | -0.00011 81 4 | 14 self_attn q | 314,572,800 | -0.00026 82 1 | 14 self_attn [k, v] | 10,485,760 | -0.00025 82 2 | 14 self_attn [k, v] | 20,971,520 | +0.00013 82 3 | 14 self_attn [k, v] | 31,457,280 | +0.00003 82 4 | 14 self_attn [k, v] | 52,428,800 | +0.00016 83 1 | 14 self_attn o | 62,914,560 | -0.00037 83 2 | 14 self_attn o | 125,829,120 | -0.00087 83 3 | 14 self_attn o | 188,743,680 | -0.00077 83 4 | 14 self_attn o | 314,572,800 | -0.00084 84 1 | 14 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00004 84 2 | 14 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00006 84 3 | 14 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00007 84 4 | 14 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00019 85 1 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 85 2 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00096 85 3 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00089 85 4 | 14 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00090 86 1 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00038 86 2 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00067 86 3 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00060 86 4 | 14 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00075 87 1 | 15 self_attn q | 62,914,560 | +0.00008 87 2 | 15 self_attn q | 125,829,120 | +0.00005 87 3 | 15 self_attn q | 188,743,680 | +0.00004 87 4 | 15 self_attn q | 314,572,800 | +0.00005 88 1 | 15 self_attn [k, v] | 10,485,760 | +0.00054 88 2 | 15 self_attn [k, v] | 20,971,520 | +0.00031 88 3 | 15 self_attn [k, v] | 31,457,280 | +0.00037 88 4 | 15 self_attn [k, v] | 52,428,800 | +0.00052 89 1 | 15 self_attn o | 62,914,560 | -0.00013 89 2 | 15 self_attn o | 125,829,120 | -0.00033 89 3 | 15 self_attn o | 188,743,680 | -0.00029 89 4 | 15 self_attn o | 314,572,800 | -0.00034 90 1 | 15 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00017 90 2 | 15 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00006 90 3 | 15 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00010 90 4 | 15 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00013 91 1 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00103 91 2 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00092 91 3 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00110 91 4 | 15 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00112 92 1 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00069 92 2 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00066 92 3 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00072 92 4 | 15 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00074 93 1 | 16 self_attn q | 62,914,560 | -0.00042 93 2 | 16 self_attn q | 125,829,120 | -0.00002 93 3 | 16 self_attn q | 188,743,680 | -0.00014 93 4 | 16 self_attn q | 314,572,800 | -0.00007 94 1 | 16 self_attn [k, v] | 10,485,760 | +0.00047 94 2 | 16 self_attn [k, v] | 20,971,520 | +0.00032 94 3 | 16 self_attn [k, v] | 31,457,280 | +0.00040 94 4 | 16 self_attn [k, v] | 52,428,800 | +0.00027 95 1 | 16 self_attn o | 62,914,560 | -0.00011 95 2 | 16 self_attn o | 125,829,120 | -0.00021 95 3 | 16 self_attn o | 188,743,680 | -0.00017 95 4 | 16 self_attn o | 314,572,800 | -0.00013 96 1 | 16 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 96 2 | 16 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00015 96 3 | 16 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00007 96 4 | 16 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00008 97 1 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00091 97 2 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00137 97 3 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00142 97 4 | 16 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00138 98 1 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00065 98 2 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00103 98 3 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00087 98 4 | 16 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00086 99 1 | 17 self_attn q | 62,914,560 | +0.00015 99 2 | 17 self_attn q | 125,829,120 | +0.00013 99 3 | 17 self_attn q | 188,743,680 | +0.00017 99 4 | 17 self_attn q | 314,572,800 | +0.00007 100 1 | 17 self_attn [k, v] | 10,485,760 | -0.00075 100 2 | 17 self_attn [k, v] | 20,971,520 | -0.00078 100 3 | 17 self_attn [k, v] | 31,457,280 | -0.00063 100 4 | 17 self_attn [k, v] | 52,428,800 | -0.00062 101 1 | 17 self_attn o | 62,914,560 | -0.00036 101 2 | 17 self_attn o | 125,829,120 | -0.00032 101 3 | 17 self_attn o | 188,743,680 | -0.00036 101 4 | 17 self_attn o | 314,572,800 | -0.00032 102 1 | 17 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00048 102 2 | 17 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00038 102 3 | 17 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00039 102 4 | 17 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00037 103 1 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00100 103 2 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00095 103 3 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00107 103 4 | 17 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00115 104 1 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00070 104 2 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00094 104 3 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00094 104 4 | 17 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00101 105 1 | 18 self_attn q | 62,914,560 | -0.00003 105 2 | 18 self_attn q | 125,829,120 | +0.00023 105 3 | 18 self_attn q | 188,743,680 | +0.00014 105 4 | 18 self_attn q | 314,572,800 | +0.00022 106 1 | 18 self_attn [k, v] | 10,485,760 | -0.00102 106 2 | 18 self_attn [k, v] | 20,971,520 | -0.00091 106 3 | 18 self_attn [k, v] | 31,457,280 | -0.00100 106 4 | 18 self_attn [k, v] | 52,428,800 | -0.00109 107 1 | 18 self_attn o | 62,914,560 | -0.00018 107 2 | 18 self_attn o | 125,829,120 | -0.00020 107 3 | 18 self_attn o | 188,743,680 | -0.00069 107 4 | 18 self_attn o | 314,572,800 | -0.00056 108 1 | 18 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00039 108 2 | 18 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00037 108 3 | 18 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00003 108 4 | 18 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00003 109 1 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00090 109 2 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00097 109 3 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00105 109 4 | 18 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00103 110 1 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 110 2 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00040 110 3 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00053 110 4 | 18 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00049 111 1 | 19 self_attn q | 62,914,560 | +0.00021 111 2 | 19 self_attn q | 125,829,120 | -0.00004 111 3 | 19 self_attn q | 188,743,680 | +0.00002 111 4 | 19 self_attn q | 314,572,800 | -0.00004 112 1 | 19 self_attn [k, v] | 10,485,760 | +0.00059 112 2 | 19 self_attn [k, v] | 20,971,520 | +0.00079 112 3 | 19 self_attn [k, v] | 31,457,280 | +0.00056 112 4 | 19 self_attn [k, v] | 52,428,800 | +0.00043 113 1 | 19 self_attn o | 62,914,560 | -0.00079 113 2 | 19 self_attn o | 125,829,120 | -0.00089 113 3 | 19 self_attn o | 188,743,680 | -0.00094 113 4 | 19 self_attn o | 314,572,800 | -0.00093 114 1 | 19 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00162 114 2 | 19 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00098 114 3 | 19 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00070 114 4 | 19 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00081 115 1 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00096 115 2 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00105 115 3 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00108 115 4 | 19 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00096 116 1 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00065 116 2 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00072 116 3 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00079 116 4 | 19 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00083 117 1 | 20 self_attn q | 62,914,560 | -0.00000 117 2 | 20 self_attn q | 125,829,120 | +0.00016 117 3 | 20 self_attn q | 188,743,680 | +0.00019 117 4 | 20 self_attn q | 314,572,800 | +0.00018 118 1 | 20 self_attn [k, v] | 10,485,760 | +0.00023 118 2 | 20 self_attn [k, v] | 20,971,520 | -0.00038 118 3 | 20 self_attn [k, v] | 31,457,280 | +0.00006 118 4 | 20 self_attn [k, v] | 52,428,800 | +0.00003 119 1 | 20 self_attn o | 62,914,560 | +0.00068 119 2 | 20 self_attn o | 125,829,120 | -0.00002 119 3 | 20 self_attn o | 188,743,680 | +0.00020 119 4 | 20 self_attn o | 314,572,800 | +0.00019 120 1 | 20 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00098 120 2 | 20 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00153 120 3 | 20 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00156 120 4 | 20 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00159 121 1 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00070 121 2 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00086 121 3 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00091 121 4 | 20 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00095 122 1 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00085 122 2 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00084 122 3 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00083 122 4 | 20 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00087 123 1 | 21 self_attn q | 62,914,560 | -0.00027 123 2 | 21 self_attn q | 125,829,120 | -0.00025 123 3 | 21 self_attn q | 188,743,680 | -0.00021 123 4 | 21 self_attn q | 314,572,800 | -0.00012 124 1 | 21 self_attn [k, v] | 10,485,760 | +0.00026 124 2 | 21 self_attn [k, v] | 20,971,520 | +0.00027 124 3 | 21 self_attn [k, v] | 31,457,280 | +0.00008 124 4 | 21 self_attn [k, v] | 52,428,800 | +0.00019 125 1 | 21 self_attn o | 62,914,560 | -0.00022 125 2 | 21 self_attn o | 125,829,120 | +0.00026 125 3 | 21 self_attn o | 188,743,680 | +0.00025 125 4 | 21 self_attn o | 314,572,800 | +0.00029 126 1 | 21 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011 126 2 | 21 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00072 126 3 | 21 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00048 126 4 | 21 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00040 127 1 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00056 127 2 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00089 127 3 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00086 127 4 | 21 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00079 128 1 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00017 128 2 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00042 128 3 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00040 128 4 | 21 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00036 129 1 | 22 self_attn q | 62,914,560 | -0.00027 129 2 | 22 self_attn q | 125,829,120 | -0.00029 129 3 | 22 self_attn q | 188,743,680 | -0.00038 129 4 | 22 self_attn q | 314,572,800 | -0.00037 130 1 | 22 self_attn [k, v] | 10,485,760 | +0.00042 130 2 | 22 self_attn [k, v] | 20,971,520 | +0.00001 130 3 | 22 self_attn [k, v] | 31,457,280 | +0.00037 130 4 | 22 self_attn [k, v] | 52,428,800 | +0.00031 131 1 | 22 self_attn o | 62,914,560 | -0.00048 131 2 | 22 self_attn o | 125,829,120 | -0.00034 131 3 | 22 self_attn o | 188,743,680 | -0.00027 131 4 | 22 self_attn o | 314,572,800 | -0.00026 132 1 | 22 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00094 132 2 | 22 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00085 132 3 | 22 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00064 132 4 | 22 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00058 133 1 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00045 133 2 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00058 133 3 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00056 133 4 | 22 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00064 134 1 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00023 134 2 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00033 134 3 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00033 134 4 | 22 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00036 135 1 | 23 self_attn q | 62,914,560 | -0.00067 135 2 | 23 self_attn q | 125,829,120 | -0.00063 135 3 | 23 self_attn q | 188,743,680 | -0.00054 135 4 | 23 self_attn q | 314,572,800 | -0.00063 136 1 | 23 self_attn [k, v] | 10,485,760 | -0.00013 136 2 | 23 self_attn [k, v] | 20,971,520 | -0.00052 136 3 | 23 self_attn [k, v] | 31,457,280 | -0.00018 136 4 | 23 self_attn [k, v] | 52,428,800 | -0.00035 137 1 | 23 self_attn o | 62,914,560 | -0.00050 137 2 | 23 self_attn o | 125,829,120 | +0.00007 137 3 | 23 self_attn o | 188,743,680 | -0.00016 137 4 | 23 self_attn o | 314,572,800 | -0.00019 138 1 | 23 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00094 138 2 | 23 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00065 138 3 | 23 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00069 138 4 | 23 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00083 139 1 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00082 139 2 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00101 139 3 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00101 139 4 | 23 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00094 140 1 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00034 140 2 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00046 140 3 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00044 140 4 | 23 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00051 141 1 | 24 self_attn q | 62,914,560 | -0.00007 141 2 | 24 self_attn q | 125,829,120 | -0.00033 141 3 | 24 self_attn q | 188,743,680 | -0.00039 141 4 | 24 self_attn q | 314,572,800 | -0.00034 142 1 | 24 self_attn [k, v] | 10,485,760 | -0.00082 142 2 | 24 self_attn [k, v] | 20,971,520 | -0.00100 142 3 | 24 self_attn [k, v] | 31,457,280 | -0.00122 142 4 | 24 self_attn [k, v] | 52,428,800 | -0.00114 143 1 | 24 self_attn o | 62,914,560 | -0.00023 143 2 | 24 self_attn o | 125,829,120 | -0.00007 143 3 | 24 self_attn o | 188,743,680 | -0.00037 143 4 | 24 self_attn o | 314,572,800 | -0.00027 144 1 | 24 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00058 144 2 | 24 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00050 144 3 | 24 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00073 144 4 | 24 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00060 145 1 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00105 145 2 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00106 145 3 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00105 145 4 | 24 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00102 146 1 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00036 146 2 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 146 3 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00046 146 4 | 24 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00057 147 1 | 25 self_attn q | 62,914,560 | +0.00065 147 2 | 25 self_attn q | 125,829,120 | +0.00064 147 3 | 25 self_attn q | 188,743,680 | +0.00071 147 4 | 25 self_attn q | 314,572,800 | +0.00071 148 1 | 25 self_attn [k, v] | 10,485,760 | +0.00041 148 2 | 25 self_attn [k, v] | 20,971,520 | +0.00035 148 3 | 25 self_attn [k, v] | 31,457,280 | +0.00044 148 4 | 25 self_attn [k, v] | 52,428,800 | +0.00036 149 1 | 25 self_attn o | 62,914,560 | -0.00089 149 2 | 25 self_attn o | 125,829,120 | -0.00086 149 3 | 25 self_attn o | 188,743,680 | -0.00074 149 4 | 25 self_attn o | 314,572,800 | -0.00084 150 1 | 25 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00115 150 2 | 25 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00084 150 3 | 25 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00093 150 4 | 25 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00084 151 1 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00041 151 2 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00066 151 3 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00069 151 4 | 25 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00073 152 1 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00043 152 2 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00060 152 3 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00054 152 4 | 25 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00050 153 1 | 26 self_attn q | 62,914,560 | -0.00035 153 2 | 26 self_attn q | 125,829,120 | -0.00033 153 3 | 26 self_attn q | 188,743,680 | -0.00045 153 4 | 26 self_attn q | 314,572,800 | -0.00041 154 1 | 26 self_attn [k, v] | 10,485,760 | +0.00018 154 2 | 26 self_attn [k, v] | 20,971,520 | -0.00017 154 3 | 26 self_attn [k, v] | 31,457,280 | -0.00029 154 4 | 26 self_attn [k, v] | 52,428,800 | -0.00026 155 1 | 26 self_attn o | 62,914,560 | -0.00035 155 2 | 26 self_attn o | 125,829,120 | -0.00106 155 3 | 26 self_attn o | 188,743,680 | -0.00104 155 4 | 26 self_attn o | 314,572,800 | -0.00103 156 1 | 26 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00090 156 2 | 26 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00073 156 3 | 26 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00065 156 4 | 26 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00068 157 1 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00071 157 2 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00085 157 3 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00077 157 4 | 26 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00087 158 1 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00035 158 2 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00044 158 3 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00043 158 4 | 26 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00043 159 1 | 27 self_attn q | 62,914,560 | -0.00028 159 2 | 27 self_attn q | 125,829,120 | -0.00024 159 3 | 27 self_attn q | 188,743,680 | -0.00022 159 4 | 27 self_attn q | 314,572,800 | -0.00035 160 1 | 27 self_attn [k, v] | 10,485,760 | -0.00073 160 2 | 27 self_attn [k, v] | 20,971,520 | -0.00092 160 3 | 27 self_attn [k, v] | 31,457,280 | -0.00095 160 4 | 27 self_attn [k, v] | 52,428,800 | -0.00090 161 1 | 27 self_attn o | 62,914,560 | -0.00115 161 2 | 27 self_attn o | 125,829,120 | -0.00110 161 3 | 27 self_attn o | 188,743,680 | -0.00145 161 4 | 27 self_attn o | 314,572,800 | -0.00134 162 1 | 27 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00105 162 2 | 27 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00132 162 3 | 27 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00127 162 4 | 27 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00128 163 1 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00055 163 2 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00056 163 3 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00055 163 4 | 27 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00054 164 1 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00037 164 2 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00052 164 3 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00056 164 4 | 27 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00051 165 1 | 28 self_attn q | 62,914,560 | -0.00011 165 2 | 28 self_attn q | 125,829,120 | -0.00024 165 3 | 28 self_attn q | 188,743,680 | -0.00023 165 4 | 28 self_attn q | 314,572,800 | -0.00028 166 1 | 28 self_attn [k, v] | 10,485,760 | +0.00080 166 2 | 28 self_attn [k, v] | 20,971,520 | +0.00048 166 3 | 28 self_attn [k, v] | 31,457,280 | +0.00031 166 4 | 28 self_attn [k, v] | 52,428,800 | +0.00038 167 1 | 28 self_attn o | 62,914,560 | -0.00065 167 2 | 28 self_attn o | 125,829,120 | -0.00095 167 3 | 28 self_attn o | 188,743,680 | -0.00106 167 4 | 28 self_attn o | 314,572,800 | -0.00104 168 1 | 28 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00038 168 2 | 28 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00026 168 3 | 28 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00003 168 4 | 28 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00022 169 1 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00087 169 2 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00083 169 3 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00094 169 4 | 28 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00095 170 1 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00033 170 2 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 170 3 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00042 170 4 | 28 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00047 171 1 | 29 self_attn q | 62,914,560 | -0.00006 171 2 | 29 self_attn q | 125,829,120 | +0.00003 171 3 | 29 self_attn q | 188,743,680 | +0.00005 171 4 | 29 self_attn q | 314,572,800 | +0.00019 172 1 | 29 self_attn [k, v] | 10,485,760 | -0.00055 172 2 | 29 self_attn [k, v] | 20,971,520 | -0.00008 172 3 | 29 self_attn [k, v] | 31,457,280 | -0.00048 172 4 | 29 self_attn [k, v] | 52,428,800 | -0.00042 173 1 | 29 self_attn o | 62,914,560 | -0.00040 173 2 | 29 self_attn o | 125,829,120 | -0.00062 173 3 | 29 self_attn o | 188,743,680 | -0.00056 173 4 | 29 self_attn o | 314,572,800 | -0.00057 174 1 | 29 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00091 174 2 | 29 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00076 174 3 | 29 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00066 174 4 | 29 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00072 175 1 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00069 175 2 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00057 175 3 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00071 175 4 | 29 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00076 176 1 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00037 176 2 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00047 176 3 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00043 176 4 | 29 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00052 177 1 | 30 self_attn q | 62,914,560 | -0.00054 177 2 | 30 self_attn q | 125,829,120 | -0.00042 177 3 | 30 self_attn q | 188,743,680 | -0.00054 177 4 | 30 self_attn q | 314,572,800 | -0.00059 178 1 | 30 self_attn [k, v] | 10,485,760 | +0.00031 178 2 | 30 self_attn [k, v] | 20,971,520 | +0.00004 178 3 | 30 self_attn [k, v] | 31,457,280 | -0.00011 178 4 | 30 self_attn [k, v] | 52,428,800 | +0.00000 179 1 | 30 self_attn o | 62,914,560 | +0.00012 179 2 | 30 self_attn o | 125,829,120 | +0.00010 179 3 | 30 self_attn o | 188,743,680 | +0.00016 179 4 | 30 self_attn o | 314,572,800 | +0.00015 180 1 | 30 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00117 180 2 | 30 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00137 180 3 | 30 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00110 180 4 | 30 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00120 181 1 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00064 181 2 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00072 181 3 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00077 181 4 | 30 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00079 182 1 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 182 2 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00029 182 3 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00035 182 4 | 30 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00034 183 1 | 31 self_attn q | 62,914,560 | +0.00057 183 2 | 31 self_attn q | 125,829,120 | +0.00081 183 3 | 31 self_attn q | 188,743,680 | +0.00078 183 4 | 31 self_attn q | 314,572,800 | +0.00087 184 1 | 31 self_attn [k, v] | 10,485,760 | +0.00164 184 2 | 31 self_attn [k, v] | 20,971,520 | +0.00088 184 3 | 31 self_attn [k, v] | 31,457,280 | +0.00136 184 4 | 31 self_attn [k, v] | 52,428,800 | +0.00115 185 1 | 31 self_attn o | 62,914,560 | -0.00012 185 2 | 31 self_attn o | 125,829,120 | -0.00022 185 3 | 31 self_attn o | 188,743,680 | +0.00026 185 4 | 31 self_attn o | 314,572,800 | +0.00032 186 1 | 31 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00199 186 2 | 31 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00063 186 3 | 31 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00042 186 4 | 31 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00051 187 1 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00064 187 2 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00092 187 3 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00091 187 4 | 31 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00098 188 1 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007 188 2 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00040 188 3 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00040 188 4 | 31 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00043 189 1 | 32 self_attn q | 62,914,560 | +0.00057 189 2 | 32 self_attn q | 125,829,120 | +0.00062 189 3 | 32 self_attn q | 188,743,680 | +0.00048 189 4 | 32 self_attn q | 314,572,800 | +0.00056 190 1 | 32 self_attn [k, v] | 10,485,760 | -0.00001 190 2 | 32 self_attn [k, v] | 20,971,520 | +0.00041 190 3 | 32 self_attn [k, v] | 31,457,280 | +0.00018 190 4 | 32 self_attn [k, v] | 52,428,800 | -0.00014 191 1 | 32 self_attn o | 62,914,560 | -0.00070 191 2 | 32 self_attn o | 125,829,120 | -0.00018 191 3 | 32 self_attn o | 188,743,680 | -0.00002 191 4 | 32 self_attn o | 314,572,800 | -0.00003 192 1 | 32 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00119 192 2 | 32 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00164 192 3 | 32 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00145 192 4 | 32 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00152 193 1 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 193 2 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00038 193 3 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00036 193 4 | 32 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00043 194 1 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00030 194 2 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00044 194 3 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00049 194 4 | 32 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00052 195 1 | 33 self_attn q | 62,914,560 | -0.00042 195 2 | 33 self_attn q | 125,829,120 | -0.00040 195 3 | 33 self_attn q | 188,743,680 | -0.00017 195 4 | 33 self_attn q | 314,572,800 | -0.00033 196 1 | 33 self_attn [k, v] | 10,485,760 | -0.00040 196 2 | 33 self_attn [k, v] | 20,971,520 | -0.00042 196 3 | 33 self_attn [k, v] | 31,457,280 | -0.00037 196 4 | 33 self_attn [k, v] | 52,428,800 | -0.00047 197 1 | 33 self_attn o | 62,914,560 | +0.00082 197 2 | 33 self_attn o | 125,829,120 | +0.00038 197 3 | 33 self_attn o | 188,743,680 | +0.00074 197 4 | 33 self_attn o | 314,572,800 | +0.00070 198 1 | 33 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00144 198 2 | 33 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00200 198 3 | 33 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00163 198 4 | 33 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00139 199 1 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00027 199 2 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00013 199 3 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00003 199 4 | 33 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | +0.00009 200 1 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 200 2 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00025 200 3 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00032 200 4 | 33 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00030 201 1 | 34 self_attn q | 62,914,560 | +0.00033 201 2 | 34 self_attn q | 125,829,120 | +0.00000 201 3 | 34 self_attn q | 188,743,680 | -0.00011 201 4 | 34 self_attn q | 314,572,800 | -0.00005 202 1 | 34 self_attn [k, v] | 10,485,760 | +0.00022 202 2 | 34 self_attn [k, v] | 20,971,520 | -0.00011 202 3 | 34 self_attn [k, v] | 31,457,280 | -0.00006 202 4 | 34 self_attn [k, v] | 52,428,800 | -0.00014 203 1 | 34 self_attn o | 62,914,560 | -0.00080 203 2 | 34 self_attn o | 125,829,120 | -0.00064 203 3 | 34 self_attn o | 188,743,680 | -0.00044 203 4 | 34 self_attn o | 314,572,800 | -0.00059 204 1 | 34 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00241 204 2 | 34 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00285 204 3 | 34 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00300 204 4 | 34 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00300 205 1 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00084 205 2 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00105 205 3 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00090 205 4 | 34 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00099 206 1 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00040 206 2 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00041 206 3 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00053 206 4 | 34 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00053 207 1 | 35 self_attn q | 62,914,560 | -0.00040 207 2 | 35 self_attn q | 125,829,120 | -0.00061 207 3 | 35 self_attn q | 188,743,680 | -0.00049 207 4 | 35 self_attn q | 314,572,800 | -0.00049 208 1 | 35 self_attn [k, v] | 10,485,760 | -0.00090 208 2 | 35 self_attn [k, v] | 20,971,520 | -0.00092 208 3 | 35 self_attn [k, v] | 31,457,280 | -0.00075 208 4 | 35 self_attn [k, v] | 52,428,800 | -0.00078 209 1 | 35 self_attn o | 62,914,560 | -0.00054 209 2 | 35 self_attn o | 125,829,120 | -0.00082 209 3 | 35 self_attn o | 188,743,680 | -0.00056 209 4 | 35 self_attn o | 314,572,800 | -0.00076 210 1 | 35 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00015 210 2 | 35 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016 210 3 | 35 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00018 210 4 | 35 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00006 211 1 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00088 211 2 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00108 211 3 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00118 211 4 | 35 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00120 212 1 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00053 212 2 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00071 212 3 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00077 212 4 | 35 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00080 213 1 | 36 self_attn q | 62,914,560 | +0.00103 213 2 | 36 self_attn q | 125,829,120 | +0.00090 213 3 | 36 self_attn q | 188,743,680 | +0.00064 213 4 | 36 self_attn q | 314,572,800 | +0.00064 214 1 | 36 self_attn [k, v] | 10,485,760 | -0.00008 214 2 | 36 self_attn [k, v] | 20,971,520 | -0.00176 214 3 | 36 self_attn [k, v] | 31,457,280 | -0.00158 214 4 | 36 self_attn [k, v] | 52,428,800 | -0.00160 215 1 | 36 self_attn o | 62,914,560 | -0.00210 215 2 | 36 self_attn o | 125,829,120 | -0.00219 215 3 | 36 self_attn o | 188,743,680 | -0.00214 215 4 | 36 self_attn o | 314,572,800 | -0.00225 216 1 | 36 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00170 216 2 | 36 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00136 216 3 | 36 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00144 216 4 | 36 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00137 217 1 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00023 217 2 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00053 217 3 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00060 217 4 | 36 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00061 218 1 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 218 2 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00036 218 3 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00051 218 4 | 36 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00051 219 1 | 37 self_attn q | 62,914,560 | -0.00078 219 2 | 37 self_attn q | 125,829,120 | -0.00057 219 3 | 37 self_attn q | 188,743,680 | -0.00060 219 4 | 37 self_attn q | 314,572,800 | -0.00062 220 1 | 37 self_attn [k, v] | 10,485,760 | +0.00030 220 2 | 37 self_attn [k, v] | 20,971,520 | -0.00025 220 3 | 37 self_attn [k, v] | 31,457,280 | +0.00002 220 4 | 37 self_attn [k, v] | 52,428,800 | -0.00029 221 1 | 37 self_attn o | 62,914,560 | -0.00185 221 2 | 37 self_attn o | 125,829,120 | -0.00134 221 3 | 37 self_attn o | 188,743,680 | -0.00124 221 4 | 37 self_attn o | 314,572,800 | -0.00116 222 1 | 37 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00082 222 2 | 37 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00095 222 3 | 37 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00077 222 4 | 37 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00073 223 1 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00015 223 2 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 223 3 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00026 223 4 | 37 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00025 224 1 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 224 2 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 224 3 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00020 224 4 | 37 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00022 225 1 | 38 self_attn q | 62,914,560 | -0.00100 225 2 | 38 self_attn q | 125,829,120 | -0.00120 225 3 | 38 self_attn q | 188,743,680 | -0.00101 225 4 | 38 self_attn q | 314,572,800 | -0.00101 226 1 | 38 self_attn [k, v] | 10,485,760 | -0.00007 226 2 | 38 self_attn [k, v] | 20,971,520 | -0.00013 226 3 | 38 self_attn [k, v] | 31,457,280 | -0.00005 226 4 | 38 self_attn [k, v] | 52,428,800 | -0.00021 227 1 | 38 self_attn o | 62,914,560 | -0.00007 227 2 | 38 self_attn o | 125,829,120 | -0.00017 227 3 | 38 self_attn o | 188,743,680 | -0.00017 227 4 | 38 self_attn o | 314,572,800 | -0.00019 228 1 | 38 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 228 2 | 38 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00028 228 3 | 38 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00043 228 4 | 38 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00045 229 1 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00040 229 2 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00061 229 3 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00063 229 4 | 38 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00060 230 1 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00028 230 2 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00034 230 3 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00039 230 4 | 38 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00043 231 1 | 39 self_attn q | 62,914,560 | -0.00046 231 2 | 39 self_attn q | 125,829,120 | -0.00052 231 3 | 39 self_attn q | 188,743,680 | -0.00052 231 4 | 39 self_attn q | 314,572,800 | -0.00051 232 1 | 39 self_attn [k, v] | 10,485,760 | +0.00050 232 2 | 39 self_attn [k, v] | 20,971,520 | -0.00047 232 3 | 39 self_attn [k, v] | 31,457,280 | -0.00067 232 4 | 39 self_attn [k, v] | 52,428,800 | -0.00052 233 1 | 39 self_attn o | 62,914,560 | -0.00100 233 2 | 39 self_attn o | 125,829,120 | -0.00038 233 3 | 39 self_attn o | 188,743,680 | -0.00060 233 4 | 39 self_attn o | 314,572,800 | -0.00053 234 1 | 39 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00087 234 2 | 39 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00101 234 3 | 39 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00109 234 4 | 39 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00110 235 1 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00045 235 2 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00057 235 3 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00064 235 4 | 39 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00067 236 1 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013 236 2 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00028 236 3 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00031 236 4 | 39 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00026 237 1 | 40 self_attn q | 62,914,560 | -0.00051 237 2 | 40 self_attn q | 125,829,120 | -0.00034 237 3 | 40 self_attn q | 188,743,680 | -0.00038 237 4 | 40 self_attn q | 314,572,800 | -0.00032 238 1 | 40 self_attn [k, v] | 10,485,760 | -0.00089 238 2 | 40 self_attn [k, v] | 20,971,520 | -0.00134 238 3 | 40 self_attn [k, v] | 31,457,280 | -0.00095 238 4 | 40 self_attn [k, v] | 52,428,800 | -0.00108 239 1 | 40 self_attn o | 62,914,560 | +0.00010 239 2 | 40 self_attn o | 125,829,120 | -0.00029 239 3 | 40 self_attn o | 188,743,680 | -0.00035 239 4 | 40 self_attn o | 314,572,800 | -0.00038 240 1 | 40 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00051 240 2 | 40 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00090 240 3 | 40 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00088 240 4 | 40 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00087 241 1 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00055 241 2 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00065 241 3 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00059 241 4 | 40 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00058 242 1 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00036 242 2 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00058 242 3 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00055 242 4 | 40 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00057 243 1 | 41 self_attn q | 62,914,560 | +0.00062 243 2 | 41 self_attn q | 125,829,120 | +0.00063 243 3 | 41 self_attn q | 188,743,680 | +0.00064 243 4 | 41 self_attn q | 314,572,800 | +0.00067 244 1 | 41 self_attn [k, v] | 10,485,760 | -0.00110 244 2 | 41 self_attn [k, v] | 20,971,520 | -0.00079 244 3 | 41 self_attn [k, v] | 31,457,280 | -0.00100 244 4 | 41 self_attn [k, v] | 52,428,800 | -0.00100 245 1 | 41 self_attn o | 62,914,560 | +0.00059 245 2 | 41 self_attn o | 125,829,120 | +0.00076 245 3 | 41 self_attn o | 188,743,680 | +0.00074 245 4 | 41 self_attn o | 314,572,800 | +0.00065 246 1 | 41 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00056 246 2 | 41 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00014 246 3 | 41 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00039 246 4 | 41 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00031 247 1 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00074 247 2 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00091 247 3 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00080 247 4 | 41 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00075 248 1 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00021 248 2 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 248 3 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00012 248 4 | 41 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00016 249 1 | 42 self_attn q | 62,914,560 | -0.00035 249 2 | 42 self_attn q | 125,829,120 | -0.00034 249 3 | 42 self_attn q | 188,743,680 | -0.00037 249 4 | 42 self_attn q | 314,572,800 | -0.00037 250 1 | 42 self_attn [k, v] | 10,485,760 | +0.00023 250 2 | 42 self_attn [k, v] | 20,971,520 | -0.00048 250 3 | 42 self_attn [k, v] | 31,457,280 | -0.00055 250 4 | 42 self_attn [k, v] | 52,428,800 | -0.00053 251 1 | 42 self_attn o | 62,914,560 | -0.00097 251 2 | 42 self_attn o | 125,829,120 | -0.00076 251 3 | 42 self_attn o | 188,743,680 | -0.00077 251 4 | 42 self_attn o | 314,572,800 | -0.00077 252 1 | 42 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00072 252 2 | 42 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00079 252 3 | 42 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00064 252 4 | 42 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00065 253 1 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00038 253 2 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00035 253 3 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00037 253 4 | 42 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00043 254 1 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 254 2 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00028 254 3 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00032 254 4 | 42 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00029 255 1 | 43 self_attn q | 62,914,560 | -0.00023 255 2 | 43 self_attn q | 125,829,120 | -0.00029 255 3 | 43 self_attn q | 188,743,680 | -0.00034 255 4 | 43 self_attn q | 314,572,800 | -0.00038 256 1 | 43 self_attn [k, v] | 10,485,760 | -0.00082 256 2 | 43 self_attn [k, v] | 20,971,520 | -0.00066 256 3 | 43 self_attn [k, v] | 31,457,280 | -0.00075 256 4 | 43 self_attn [k, v] | 52,428,800 | -0.00080 257 1 | 43 self_attn o | 62,914,560 | -0.00034 257 2 | 43 self_attn o | 125,829,120 | -0.00058 257 3 | 43 self_attn o | 188,743,680 | -0.00067 257 4 | 43 self_attn o | 314,572,800 | -0.00056 258 1 | 43 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00005 258 2 | 43 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00000 258 3 | 43 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00010 258 4 | 43 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00002 259 1 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00054 259 2 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00059 259 3 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00057 259 4 | 43 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00059 260 1 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00045 260 2 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00039 260 3 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00037 260 4 | 43 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00040 261 1 | 44 self_attn q | 62,914,560 | -0.00056 261 2 | 44 self_attn q | 125,829,120 | -0.00037 261 3 | 44 self_attn q | 188,743,680 | -0.00042 261 4 | 44 self_attn q | 314,572,800 | -0.00042 262 1 | 44 self_attn [k, v] | 10,485,760 | +0.00008 262 2 | 44 self_attn [k, v] | 20,971,520 | +0.00049 262 3 | 44 self_attn [k, v] | 31,457,280 | +0.00050 262 4 | 44 self_attn [k, v] | 52,428,800 | +0.00044 263 1 | 44 self_attn o | 62,914,560 | -0.00041 263 2 | 44 self_attn o | 125,829,120 | -0.00079 263 3 | 44 self_attn o | 188,743,680 | -0.00078 263 4 | 44 self_attn o | 314,572,800 | -0.00083 264 1 | 44 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00005 264 2 | 44 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00016 264 3 | 44 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00003 264 4 | 44 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00003 265 1 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00030 265 2 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00033 265 3 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00031 265 4 | 44 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00036 266 1 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00033 266 2 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00030 266 3 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00031 266 4 | 44 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00033 267 1 | 45 self_attn q | 62,914,560 | -0.00047 267 2 | 45 self_attn q | 125,829,120 | -0.00035 267 3 | 45 self_attn q | 188,743,680 | -0.00039 267 4 | 45 self_attn q | 314,572,800 | -0.00040 268 1 | 45 self_attn [k, v] | 10,485,760 | -0.00117 268 2 | 45 self_attn [k, v] | 20,971,520 | -0.00121 268 3 | 45 self_attn [k, v] | 31,457,280 | -0.00119 268 4 | 45 self_attn [k, v] | 52,428,800 | -0.00116 269 1 | 45 self_attn o | 62,914,560 | -0.00021 269 2 | 45 self_attn o | 125,829,120 | -0.00019 269 3 | 45 self_attn o | 188,743,680 | -0.00026 269 4 | 45 self_attn o | 314,572,800 | -0.00024 270 1 | 45 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00032 270 2 | 45 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 270 3 | 45 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00034 270 4 | 45 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00032 271 1 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00049 271 2 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00046 271 3 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00042 271 4 | 45 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00048 272 1 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 272 2 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00023 272 3 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00026 272 4 | 45 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00022 273 1 | 46 self_attn q | 62,914,560 | -0.00018 273 2 | 46 self_attn q | 125,829,120 | -0.00060 273 3 | 46 self_attn q | 188,743,680 | -0.00050 273 4 | 46 self_attn q | 314,572,800 | -0.00050 274 1 | 46 self_attn [k, v] | 10,485,760 | -0.00030 274 2 | 46 self_attn [k, v] | 20,971,520 | -0.00036 274 3 | 46 self_attn [k, v] | 31,457,280 | -0.00038 274 4 | 46 self_attn [k, v] | 52,428,800 | -0.00037 275 1 | 46 self_attn o | 62,914,560 | -0.00001 275 2 | 46 self_attn o | 125,829,120 | -0.00011 275 3 | 46 self_attn o | 188,743,680 | -0.00004 275 4 | 46 self_attn o | 314,572,800 | -0.00003 276 1 | 46 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00001 276 2 | 46 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00012 276 3 | 46 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00023 276 4 | 46 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00022 277 1 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 277 2 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00013 277 3 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00020 277 4 | 46 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00019 278 1 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00030 278 2 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00033 278 3 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00036 278 4 | 46 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00035 279 1 | 47 self_attn q | 62,914,560 | -0.00030 279 2 | 47 self_attn q | 125,829,120 | -0.00023 279 3 | 47 self_attn q | 188,743,680 | -0.00020 279 4 | 47 self_attn q | 314,572,800 | -0.00018 280 1 | 47 self_attn [k, v] | 10,485,760 | -0.00078 280 2 | 47 self_attn [k, v] | 20,971,520 | -0.00070 280 3 | 47 self_attn [k, v] | 31,457,280 | -0.00069 280 4 | 47 self_attn [k, v] | 52,428,800 | -0.00069 281 1 | 47 self_attn o | 62,914,560 | -0.00022 281 2 | 47 self_attn o | 125,829,120 | -0.00030 281 3 | 47 self_attn o | 188,743,680 | -0.00028 281 4 | 47 self_attn o | 314,572,800 | -0.00023 282 1 | 47 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00021 282 2 | 47 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00015 282 3 | 47 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00024 282 4 | 47 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00020 283 1 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 283 2 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 283 3 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00023 283 4 | 47 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00024 284 1 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 284 2 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00019 284 3 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00027 284 4 | 47 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00025 285 1 | 48 self_attn q | 62,914,560 | +0.00035 285 2 | 48 self_attn q | 125,829,120 | +0.00008 285 3 | 48 self_attn q | 188,743,680 | +0.00019 285 4 | 48 self_attn q | 314,572,800 | +0.00018 286 1 | 48 self_attn [k, v] | 10,485,760 | -0.00085 286 2 | 48 self_attn [k, v] | 20,971,520 | -0.00128 286 3 | 48 self_attn [k, v] | 31,457,280 | -0.00118 286 4 | 48 self_attn [k, v] | 52,428,800 | -0.00117 287 1 | 48 self_attn o | 62,914,560 | -0.00057 287 2 | 48 self_attn o | 125,829,120 | -0.00058 287 3 | 48 self_attn o | 188,743,680 | -0.00051 287 4 | 48 self_attn o | 314,572,800 | -0.00048 288 1 | 48 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00055 288 2 | 48 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00034 288 3 | 48 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00033 288 4 | 48 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00039 289 1 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00039 289 2 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00039 289 3 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00040 289 4 | 48 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00040 290 1 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00025 290 2 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00027 290 3 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00024 290 4 | 48 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00025 291 1 | 49 self_attn q | 62,914,560 | -0.00038 291 2 | 49 self_attn q | 125,829,120 | -0.00033 291 3 | 49 self_attn q | 188,743,680 | -0.00021 291 4 | 49 self_attn q | 314,572,800 | -0.00032 292 1 | 49 self_attn [k, v] | 10,485,760 | +0.00027 292 2 | 49 self_attn [k, v] | 20,971,520 | +0.00019 292 3 | 49 self_attn [k, v] | 31,457,280 | +0.00017 292 4 | 49 self_attn [k, v] | 52,428,800 | +0.00020 293 1 | 49 self_attn o | 62,914,560 | -0.00006 293 2 | 49 self_attn o | 125,829,120 | +0.00005 293 3 | 49 self_attn o | 188,743,680 | +0.00004 293 4 | 49 self_attn o | 314,572,800 | +0.00001 294 1 | 49 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00003 294 2 | 49 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00022 294 3 | 49 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00003 294 4 | 49 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00011 295 1 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00031 295 2 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00033 295 3 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00030 295 4 | 49 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00036 296 1 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00020 296 2 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 296 3 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00022 296 4 | 49 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00019 297 1 | 50 self_attn q | 62,914,560 | -0.00012 297 2 | 50 self_attn q | 125,829,120 | -0.00005 297 3 | 50 self_attn q | 188,743,680 | +0.00001 297 4 | 50 self_attn q | 314,572,800 | -0.00003 298 1 | 50 self_attn [k, v] | 10,485,760 | +0.00018 298 2 | 50 self_attn [k, v] | 20,971,520 | -0.00010 298 3 | 50 self_attn [k, v] | 31,457,280 | -0.00011 298 4 | 50 self_attn [k, v] | 52,428,800 | -0.00015 299 1 | 50 self_attn o | 62,914,560 | -0.00006 299 2 | 50 self_attn o | 125,829,120 | -0.00020 299 3 | 50 self_attn o | 188,743,680 | -0.00019 299 4 | 50 self_attn o | 314,572,800 | -0.00020 300 1 | 50 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 300 2 | 50 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011 300 3 | 50 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00003 300 4 | 50 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00002 301 1 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 301 2 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00024 301 3 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00029 301 4 | 50 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00031 302 1 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 302 2 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00016 302 3 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00017 302 4 | 50 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00020 303 1 | 51 self_attn q | 62,914,560 | -0.00014 303 2 | 51 self_attn q | 125,829,120 | -0.00012 303 3 | 51 self_attn q | 188,743,680 | -0.00013 303 4 | 51 self_attn q | 314,572,800 | -0.00010 304 1 | 51 self_attn [k, v] | 10,485,760 | -0.00033 304 2 | 51 self_attn [k, v] | 20,971,520 | -0.00033 304 3 | 51 self_attn [k, v] | 31,457,280 | -0.00047 304 4 | 51 self_attn [k, v] | 52,428,800 | -0.00034 305 1 | 51 self_attn o | 62,914,560 | -0.00015 305 2 | 51 self_attn o | 125,829,120 | +0.00000 305 3 | 51 self_attn o | 188,743,680 | +0.00004 305 4 | 51 self_attn o | 314,572,800 | -0.00001 306 1 | 51 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00016 306 2 | 51 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00031 306 3 | 51 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00024 306 4 | 51 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00025 307 1 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 307 2 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00027 307 3 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00029 307 4 | 51 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00029 308 1 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00026 308 2 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00021 308 3 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00022 308 4 | 51 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00021 309 1 | 52 self_attn q | 62,914,560 | +0.00001 309 2 | 52 self_attn q | 125,829,120 | +0.00010 309 3 | 52 self_attn q | 188,743,680 | +0.00009 309 4 | 52 self_attn q | 314,572,800 | +0.00007 310 1 | 52 self_attn [k, v] | 10,485,760 | -0.00075 310 2 | 52 self_attn [k, v] | 20,971,520 | -0.00049 310 3 | 52 self_attn [k, v] | 31,457,280 | -0.00063 310 4 | 52 self_attn [k, v] | 52,428,800 | -0.00057 311 1 | 52 self_attn o | 62,914,560 | -0.00009 311 2 | 52 self_attn o | 125,829,120 | -0.00008 311 3 | 52 self_attn o | 188,743,680 | -0.00025 311 4 | 52 self_attn o | 314,572,800 | -0.00023 312 1 | 52 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00026 312 2 | 52 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00021 312 3 | 52 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00019 312 4 | 52 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00022 313 1 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00019 313 2 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00025 313 3 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00019 313 4 | 52 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00023 314 1 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00024 314 2 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00025 314 3 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00023 314 4 | 52 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00027 315 1 | 53 self_attn q | 62,914,560 | -0.00007 315 2 | 53 self_attn q | 125,829,120 | -0.00017 315 3 | 53 self_attn q | 188,743,680 | -0.00016 315 4 | 53 self_attn q | 314,572,800 | -0.00018 316 1 | 53 self_attn [k, v] | 10,485,760 | -0.00069 316 2 | 53 self_attn [k, v] | 20,971,520 | -0.00067 316 3 | 53 self_attn [k, v] | 31,457,280 | -0.00058 316 4 | 53 self_attn [k, v] | 52,428,800 | -0.00061 317 1 | 53 self_attn o | 62,914,560 | +0.00003 317 2 | 53 self_attn o | 125,829,120 | +0.00002 317 3 | 53 self_attn o | 188,743,680 | -0.00003 317 4 | 53 self_attn o | 314,572,800 | -0.00002 318 1 | 53 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00047 318 2 | 53 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 318 3 | 53 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00036 318 4 | 53 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00031 319 1 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00013 319 2 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00021 319 3 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00023 319 4 | 53 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00024 320 1 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 320 2 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 320 3 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00021 320 4 | 53 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00019 321 1 | 54 self_attn q | 62,914,560 | +0.00008 321 2 | 54 self_attn q | 125,829,120 | +0.00002 321 3 | 54 self_attn q | 188,743,680 | -0.00009 321 4 | 54 self_attn q | 314,572,800 | -0.00010 322 1 | 54 self_attn [k, v] | 10,485,760 | +0.00048 322 2 | 54 self_attn [k, v] | 20,971,520 | +0.00017 322 3 | 54 self_attn [k, v] | 31,457,280 | +0.00017 322 4 | 54 self_attn [k, v] | 52,428,800 | +0.00020 323 1 | 54 self_attn o | 62,914,560 | -0.00054 323 2 | 54 self_attn o | 125,829,120 | -0.00040 323 3 | 54 self_attn o | 188,743,680 | -0.00045 323 4 | 54 self_attn o | 314,572,800 | -0.00045 324 1 | 54 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00011 324 2 | 54 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 324 3 | 54 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00001 324 4 | 54 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00004 325 1 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 325 2 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015 325 3 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00018 325 4 | 54 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00014 326 1 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00009 326 2 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 326 3 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00007 326 4 | 54 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00008 327 1 | 55 self_attn q | 62,914,560 | -0.00018 327 2 | 55 self_attn q | 125,829,120 | -0.00009 327 3 | 55 self_attn q | 188,743,680 | -0.00007 327 4 | 55 self_attn q | 314,572,800 | -0.00010 328 1 | 55 self_attn [k, v] | 10,485,760 | -0.00012 328 2 | 55 self_attn [k, v] | 20,971,520 | +0.00007 328 3 | 55 self_attn [k, v] | 31,457,280 | +0.00005 328 4 | 55 self_attn [k, v] | 52,428,800 | +0.00007 329 1 | 55 self_attn o | 62,914,560 | -0.00044 329 2 | 55 self_attn o | 125,829,120 | -0.00046 329 3 | 55 self_attn o | 188,743,680 | -0.00042 329 4 | 55 self_attn o | 314,572,800 | -0.00042 330 1 | 55 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 330 2 | 55 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00028 330 3 | 55 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00024 330 4 | 55 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00026 331 1 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 331 2 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 331 3 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00014 331 4 | 55 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00014 332 1 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 332 2 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 332 3 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00010 332 4 | 55 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00010 333 1 | 56 self_attn q | 62,914,560 | +0.00030 333 2 | 56 self_attn q | 125,829,120 | +0.00025 333 3 | 56 self_attn q | 188,743,680 | +0.00024 333 4 | 56 self_attn q | 314,572,800 | +0.00024 334 1 | 56 self_attn [k, v] | 10,485,760 | -0.00004 334 2 | 56 self_attn [k, v] | 20,971,520 | +0.00010 334 3 | 56 self_attn [k, v] | 31,457,280 | +0.00012 334 4 | 56 self_attn [k, v] | 52,428,800 | +0.00010 335 1 | 56 self_attn o | 62,914,560 | +0.00005 335 2 | 56 self_attn o | 125,829,120 | +0.00013 335 3 | 56 self_attn o | 188,743,680 | +0.00012 335 4 | 56 self_attn o | 314,572,800 | +0.00009 336 1 | 56 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 336 2 | 56 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00013 336 3 | 56 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00014 336 4 | 56 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00013 337 1 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00017 337 2 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 337 3 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00022 337 4 | 56 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00019 338 1 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 338 2 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 338 3 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00017 338 4 | 56 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00014 339 1 | 57 self_attn q | 62,914,560 | -0.00017 339 2 | 57 self_attn q | 125,829,120 | -0.00020 339 3 | 57 self_attn q | 188,743,680 | -0.00019 339 4 | 57 self_attn q | 314,572,800 | -0.00018 340 1 | 57 self_attn [k, v] | 10,485,760 | +0.00007 340 2 | 57 self_attn [k, v] | 20,971,520 | +0.00004 340 3 | 57 self_attn [k, v] | 31,457,280 | +0.00002 340 4 | 57 self_attn [k, v] | 52,428,800 | +0.00006 341 1 | 57 self_attn o | 62,914,560 | -0.00016 341 2 | 57 self_attn o | 125,829,120 | -0.00005 341 3 | 57 self_attn o | 188,743,680 | -0.00009 341 4 | 57 self_attn o | 314,572,800 | -0.00011 342 1 | 57 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00018 342 2 | 57 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00032 342 3 | 57 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00028 342 4 | 57 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00028 343 1 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018 343 2 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00015 343 3 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00014 343 4 | 57 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00017 344 1 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00001 344 2 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00002 344 3 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00003 344 4 | 57 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00005 345 1 | 58 self_attn q | 62,914,560 | +0.00013 345 2 | 58 self_attn q | 125,829,120 | +0.00023 345 3 | 58 self_attn q | 188,743,680 | +0.00008 345 4 | 58 self_attn q | 314,572,800 | +0.00016 346 1 | 58 self_attn [k, v] | 10,485,760 | -0.00017 346 2 | 58 self_attn [k, v] | 20,971,520 | -0.00011 346 3 | 58 self_attn [k, v] | 31,457,280 | -0.00019 346 4 | 58 self_attn [k, v] | 52,428,800 | -0.00013 347 1 | 58 self_attn o | 62,914,560 | +0.00002 347 2 | 58 self_attn o | 125,829,120 | +0.00002 347 3 | 58 self_attn o | 188,743,680 | -0.00007 347 4 | 58 self_attn o | 314,572,800 | -0.00010 348 1 | 58 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00021 348 2 | 58 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 348 3 | 58 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00001 348 4 | 58 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00001 349 1 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 349 2 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00023 349 3 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00024 349 4 | 58 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00025 350 1 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00024 350 2 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00020 350 3 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00025 350 4 | 58 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00023 351 1 | 59 self_attn q | 62,914,560 | -0.00017 351 2 | 59 self_attn q | 125,829,120 | -0.00017 351 3 | 59 self_attn q | 188,743,680 | -0.00020 351 4 | 59 self_attn q | 314,572,800 | -0.00022 352 1 | 59 self_attn [k, v] | 10,485,760 | -0.00011 352 2 | 59 self_attn [k, v] | 20,971,520 | -0.00027 352 3 | 59 self_attn [k, v] | 31,457,280 | -0.00023 352 4 | 59 self_attn [k, v] | 52,428,800 | -0.00023 353 1 | 59 self_attn o | 62,914,560 | +0.00001 353 2 | 59 self_attn o | 125,829,120 | +0.00009 353 3 | 59 self_attn o | 188,743,680 | +0.00004 353 4 | 59 self_attn o | 314,572,800 | +0.00005 354 1 | 59 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00018 354 2 | 59 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00032 354 3 | 59 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00031 354 4 | 59 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00030 355 1 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 355 2 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 355 3 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00013 355 4 | 59 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00013 356 1 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00015 356 2 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | +0.00011 356 3 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | +0.00012 356 4 | 59 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | +0.00014 357 1 | 60 self_attn q | 62,914,560 | +0.00002 357 2 | 60 self_attn q | 125,829,120 | -0.00008 357 3 | 60 self_attn q | 188,743,680 | -0.00009 357 4 | 60 self_attn q | 314,572,800 | -0.00008 358 1 | 60 self_attn [k, v] | 10,485,760 | -0.00019 358 2 | 60 self_attn [k, v] | 20,971,520 | -0.00029 358 3 | 60 self_attn [k, v] | 31,457,280 | -0.00041 358 4 | 60 self_attn [k, v] | 52,428,800 | -0.00035 359 1 | 60 self_attn o | 62,914,560 | -0.00003 359 2 | 60 self_attn o | 125,829,120 | -0.00002 359 3 | 60 self_attn o | 188,743,680 | -0.00002 359 4 | 60 self_attn o | 314,572,800 | -0.00004 360 1 | 60 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00029 360 2 | 60 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00039 360 3 | 60 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00035 360 4 | 60 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00035 361 1 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00018 361 2 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00020 361 3 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00018 361 4 | 60 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00019 362 1 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 362 2 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014 362 3 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00011 362 4 | 60 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00014 363 1 | 61 self_attn q | 62,914,560 | -0.00022 363 2 | 61 self_attn q | 125,829,120 | -0.00023 363 3 | 61 self_attn q | 188,743,680 | -0.00020 363 4 | 61 self_attn q | 314,572,800 | -0.00019 364 1 | 61 self_attn [k, v] | 10,485,760 | -0.00025 364 2 | 61 self_attn [k, v] | 20,971,520 | -0.00026 364 3 | 61 self_attn [k, v] | 31,457,280 | -0.00022 364 4 | 61 self_attn [k, v] | 52,428,800 | -0.00022 365 1 | 61 self_attn o | 62,914,560 | -0.00017 365 2 | 61 self_attn o | 125,829,120 | -0.00021 365 3 | 61 self_attn o | 188,743,680 | -0.00020 365 4 | 61 self_attn o | 314,572,800 | -0.00023 366 1 | 61 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012 366 2 | 61 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010 366 3 | 61 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00016 366 4 | 61 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00012 367 1 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00002 367 2 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00009 367 3 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00010 367 4 | 61 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00009 368 1 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 368 2 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00004 368 3 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00005 368 4 | 61 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00007 369 1 | 62 self_attn q | 62,914,560 | -0.00008 369 2 | 62 self_attn q | 125,829,120 | -0.00001 369 3 | 62 self_attn q | 188,743,680 | -0.00003 369 4 | 62 self_attn q | 314,572,800 | +0.00001 370 1 | 62 self_attn [k, v] | 10,485,760 | -0.00008 370 2 | 62 self_attn [k, v] | 20,971,520 | -0.00014 370 3 | 62 self_attn [k, v] | 31,457,280 | -0.00006 370 4 | 62 self_attn [k, v] | 52,428,800 | -0.00013 371 1 | 62 self_attn o | 62,914,560 | -0.00012 371 2 | 62 self_attn o | 125,829,120 | -0.00009 371 3 | 62 self_attn o | 188,743,680 | -0.00007 371 4 | 62 self_attn o | 314,572,800 | -0.00007 372 1 | 62 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00006 372 2 | 62 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 372 3 | 62 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00002 372 4 | 62 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00007 373 1 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00009 373 2 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019 373 3 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00016 373 4 | 62 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00018 374 1 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 374 2 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00019 374 3 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00015 374 4 | 62 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00014 375 1 | 63 self_attn q | 62,914,560 | +0.00009 375 2 | 63 self_attn q | 125,829,120 | -0.00000 375 3 | 63 self_attn q | 188,743,680 | +0.00000 375 4 | 63 self_attn q | 314,572,800 | -0.00001 376 1 | 63 self_attn [k, v] | 10,485,760 | -0.00013 376 2 | 63 self_attn [k, v] | 20,971,520 | -0.00010 376 3 | 63 self_attn [k, v] | 31,457,280 | -0.00013 376 4 | 63 self_attn [k, v] | 52,428,800 | -0.00014 377 1 | 63 self_attn o | 62,914,560 | -0.00003 377 2 | 63 self_attn o | 125,829,120 | -0.00005 377 3 | 63 self_attn o | 188,743,680 | -0.00006 377 4 | 63 self_attn o | 314,572,800 | -0.00002 378 1 | 63 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00003 378 2 | 63 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00003 378 3 | 63 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00000 378 4 | 63 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00001 379 1 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004 379 2 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00003 379 3 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00003 379 4 | 63 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00003 380 1 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 380 2 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 380 3 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00007 380 4 | 63 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00007 381 1 | 64 self_attn q | 62,914,560 | -0.00011 381 2 | 64 self_attn q | 125,829,120 | -0.00006 381 3 | 64 self_attn q | 188,743,680 | -0.00009 381 4 | 64 self_attn q | 314,572,800 | -0.00008 382 1 | 64 self_attn [k, v] | 10,485,760 | -0.00028 382 2 | 64 self_attn [k, v] | 20,971,520 | -0.00021 382 3 | 64 self_attn [k, v] | 31,457,280 | -0.00020 382 4 | 64 self_attn [k, v] | 52,428,800 | -0.00019 383 1 | 64 self_attn o | 62,914,560 | -0.00011 383 2 | 64 self_attn o | 125,829,120 | -0.00008 383 3 | 64 self_attn o | 188,743,680 | -0.00009 383 4 | 64 self_attn o | 314,572,800 | -0.00008 384 1 | 64 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00008 384 2 | 64 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00005 384 3 | 64 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00001 384 4 | 64 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00002 385 1 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010 385 2 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014 385 3 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00015 385 4 | 64 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00015 386 1 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00001 386 2 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 386 3 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00008 386 4 | 64 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00006 387 1 | 65 self_attn q | 62,914,560 | -0.00006 387 2 | 65 self_attn q | 125,829,120 | +0.00001 387 3 | 65 self_attn q | 188,743,680 | -0.00006 387 4 | 65 self_attn q | 314,572,800 | -0.00001 388 1 | 65 self_attn [k, v] | 10,485,760 | -0.00026 388 2 | 65 self_attn [k, v] | 20,971,520 | -0.00039 388 3 | 65 self_attn [k, v] | 31,457,280 | -0.00030 388 4 | 65 self_attn [k, v] | 52,428,800 | -0.00032 389 1 | 65 self_attn o | 62,914,560 | -0.00008 389 2 | 65 self_attn o | 125,829,120 | -0.00009 389 3 | 65 self_attn o | 188,743,680 | -0.00010 389 4 | 65 self_attn o | 314,572,800 | -0.00008 390 1 | 65 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00023 390 2 | 65 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023 390 3 | 65 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00013 390 4 | 65 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00016 391 1 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00001 391 2 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 391 3 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00007 391 4 | 65 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00007 392 1 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00011 392 2 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00009 392 3 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00012 392 4 | 65 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00012 393 1 | 66 self_attn q | 62,914,560 | -0.00060 393 2 | 66 self_attn q | 125,829,120 | -0.00062 393 3 | 66 self_attn q | 188,743,680 | -0.00061 393 4 | 66 self_attn q | 314,572,800 | -0.00063 394 1 | 66 self_attn [k, v] | 10,485,760 | +0.00028 394 2 | 66 self_attn [k, v] | 20,971,520 | +0.00035 394 3 | 66 self_attn [k, v] | 31,457,280 | +0.00030 394 4 | 66 self_attn [k, v] | 52,428,800 | +0.00033 395 1 | 66 self_attn o | 62,914,560 | -0.00016 395 2 | 66 self_attn o | 125,829,120 | -0.00019 395 3 | 66 self_attn o | 188,743,680 | -0.00024 395 4 | 66 self_attn o | 314,572,800 | -0.00026 396 1 | 66 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00012 396 2 | 66 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00019 396 3 | 66 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00017 396 4 | 66 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00014 397 1 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00022 397 2 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00020 397 3 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00021 397 4 | 66 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00020 398 1 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00021 398 2 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00024 398 3 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00024 398 4 | 66 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00026 399 1 | 67 self_attn q | 62,914,560 | -0.00002 399 2 | 67 self_attn q | 125,829,120 | -0.00006 399 3 | 67 self_attn q | 188,743,680 | -0.00009 399 4 | 67 self_attn q | 314,572,800 | -0.00004 400 1 | 67 self_attn [k, v] | 10,485,760 | -0.00028 400 2 | 67 self_attn [k, v] | 20,971,520 | -0.00019 400 3 | 67 self_attn [k, v] | 31,457,280 | -0.00021 400 4 | 67 self_attn [k, v] | 52,428,800 | -0.00020 401 1 | 67 self_attn o | 62,914,560 | -0.00011 401 2 | 67 self_attn o | 125,829,120 | -0.00009 401 3 | 67 self_attn o | 188,743,680 | -0.00018 401 4 | 67 self_attn o | 314,572,800 | -0.00015 402 1 | 67 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00001 402 2 | 67 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008 402 3 | 67 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00005 402 4 | 67 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00002 403 1 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00004 403 2 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00005 403 3 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | +0.00008 403 4 | 67 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | +0.00007 404 1 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00026 404 2 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00022 404 3 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00020 404 4 | 67 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00019 405 1 | 68 self_attn q | 62,914,560 | -0.00009 405 2 | 68 self_attn q | 125,829,120 | -0.00021 405 3 | 68 self_attn q | 188,743,680 | -0.00018 405 4 | 68 self_attn q | 314,572,800 | -0.00020 406 1 | 68 self_attn [k, v] | 10,485,760 | -0.00031 406 2 | 68 self_attn [k, v] | 20,971,520 | -0.00011 406 3 | 68 self_attn [k, v] | 31,457,280 | -0.00012 406 4 | 68 self_attn [k, v] | 52,428,800 | -0.00010 407 1 | 68 self_attn o | 62,914,560 | -0.00022 407 2 | 68 self_attn o | 125,829,120 | -0.00023 407 3 | 68 self_attn o | 188,743,680 | -0.00030 407 4 | 68 self_attn o | 314,572,800 | -0.00027 408 1 | 68 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00007 408 2 | 68 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00002 408 3 | 68 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00003 408 4 | 68 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00000 409 1 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00006 409 2 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00013 409 3 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00007 409 4 | 68 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00008 410 1 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 410 2 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00031 410 3 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00028 410 4 | 68 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00032 411 1 | 69 self_attn q | 62,914,560 | -0.00007 411 2 | 69 self_attn q | 125,829,120 | -0.00011 411 3 | 69 self_attn q | 188,743,680 | -0.00010 411 4 | 69 self_attn q | 314,572,800 | -0.00014 412 1 | 69 self_attn [k, v] | 10,485,760 | -0.00018 412 2 | 69 self_attn [k, v] | 20,971,520 | -0.00033 412 3 | 69 self_attn [k, v] | 31,457,280 | -0.00029 412 4 | 69 self_attn [k, v] | 52,428,800 | -0.00030 413 1 | 69 self_attn o | 62,914,560 | -0.00012 413 2 | 69 self_attn o | 125,829,120 | -0.00011 413 3 | 69 self_attn o | 188,743,680 | -0.00011 413 4 | 69 self_attn o | 314,572,800 | -0.00013 414 1 | 69 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00004 414 2 | 69 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00008 414 3 | 69 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00007 414 4 | 69 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00003 415 1 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 415 2 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00030 415 3 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00031 415 4 | 69 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00031 416 1 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | +0.00002 416 2 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 416 3 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00002 416 4 | 69 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00004 417 1 | 70 self_attn q | 62,914,560 | +0.00002 417 2 | 70 self_attn q | 125,829,120 | +0.00003 417 3 | 70 self_attn q | 188,743,680 | +0.00003 417 4 | 70 self_attn q | 314,572,800 | -0.00003 418 1 | 70 self_attn [k, v] | 10,485,760 | -0.00014 418 2 | 70 self_attn [k, v] | 20,971,520 | -0.00011 418 3 | 70 self_attn [k, v] | 31,457,280 | -0.00014 418 4 | 70 self_attn [k, v] | 52,428,800 | -0.00016 419 1 | 70 self_attn o | 62,914,560 | -0.00001 419 2 | 70 self_attn o | 125,829,120 | -0.00003 419 3 | 70 self_attn o | 188,743,680 | -0.00005 419 4 | 70 self_attn o | 314,572,800 | -0.00007 420 1 | 70 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00035 420 2 | 70 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00026 420 3 | 70 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00025 420 4 | 70 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00029 421 1 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 421 2 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 421 3 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00003 421 4 | 70 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00005 422 1 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00003 422 2 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00008 422 3 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00008 422 4 | 70 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00008 423 1 | 71 self_attn q | 62,914,560 | -0.00000 423 2 | 71 self_attn q | 125,829,120 | +0.00009 423 3 | 71 self_attn q | 188,743,680 | +0.00011 423 4 | 71 self_attn q | 314,572,800 | +0.00007 424 1 | 71 self_attn [k, v] | 10,485,760 | -0.00033 424 2 | 71 self_attn [k, v] | 20,971,520 | -0.00033 424 3 | 71 self_attn [k, v] | 31,457,280 | -0.00039 424 4 | 71 self_attn [k, v] | 52,428,800 | -0.00036 425 1 | 71 self_attn o | 62,914,560 | -0.00023 425 2 | 71 self_attn o | 125,829,120 | -0.00036 425 3 | 71 self_attn o | 188,743,680 | -0.00031 425 4 | 71 self_attn o | 314,572,800 | -0.00036 426 1 | 71 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00057 426 2 | 71 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00040 426 3 | 71 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00046 426 4 | 71 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00044 427 1 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00038 427 2 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00045 427 3 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00045 427 4 | 71 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00047 428 1 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00017 428 2 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00014 428 3 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00013 428 4 | 71 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00013 429 1 | 72 self_attn q | 62,914,560 | -0.00014 429 2 | 72 self_attn q | 125,829,120 | -0.00011 429 3 | 72 self_attn q | 188,743,680 | -0.00010 429 4 | 72 self_attn q | 314,572,800 | -0.00011 430 1 | 72 self_attn [k, v] | 10,485,760 | -0.00007 430 2 | 72 self_attn [k, v] | 20,971,520 | -0.00003 430 3 | 72 self_attn [k, v] | 31,457,280 | -0.00002 430 4 | 72 self_attn [k, v] | 52,428,800 | -0.00001 431 1 | 72 self_attn o | 62,914,560 | +0.00005 431 2 | 72 self_attn o | 125,829,120 | -0.00009 431 3 | 72 self_attn o | 188,743,680 | -0.00007 431 4 | 72 self_attn o | 314,572,800 | -0.00008 432 1 | 72 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00032 432 2 | 72 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00051 432 3 | 72 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00041 432 4 | 72 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00043 433 1 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00021 433 2 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00013 433 3 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | +0.00013 433 4 | 72 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | +0.00010 434 1 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00018 434 2 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00022 434 3 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00025 434 4 | 72 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00026 435 1 | 73 self_attn q | 62,914,560 | -0.00007 435 2 | 73 self_attn q | 125,829,120 | -0.00014 435 3 | 73 self_attn q | 188,743,680 | -0.00010 435 4 | 73 self_attn q | 314,572,800 | -0.00011 436 1 | 73 self_attn [k, v] | 10,485,760 | -0.00005 436 2 | 73 self_attn [k, v] | 20,971,520 | +0.00005 436 3 | 73 self_attn [k, v] | 31,457,280 | +0.00003 436 4 | 73 self_attn [k, v] | 52,428,800 | +0.00002 437 1 | 73 self_attn o | 62,914,560 | -0.00025 437 2 | 73 self_attn o | 125,829,120 | -0.00022 437 3 | 73 self_attn o | 188,743,680 | -0.00013 437 4 | 73 self_attn o | 314,572,800 | -0.00018 438 1 | 73 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 438 2 | 73 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00029 438 3 | 73 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00030 438 4 | 73 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00034 439 1 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00021 439 2 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00017 439 3 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00018 439 4 | 73 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00018 440 1 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00012 440 2 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00015 440 3 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00013 440 4 | 73 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00015 441 1 | 74 self_attn q | 62,914,560 | -0.00006 441 2 | 74 self_attn q | 125,829,120 | -0.00002 441 3 | 74 self_attn q | 188,743,680 | -0.00002 441 4 | 74 self_attn q | 314,572,800 | -0.00002 442 1 | 74 self_attn [k, v] | 10,485,760 | -0.00009 442 2 | 74 self_attn [k, v] | 20,971,520 | -0.00012 442 3 | 74 self_attn [k, v] | 31,457,280 | -0.00008 442 4 | 74 self_attn [k, v] | 52,428,800 | -0.00011 443 1 | 74 self_attn o | 62,914,560 | -0.00001 443 2 | 74 self_attn o | 125,829,120 | +0.00004 443 3 | 74 self_attn o | 188,743,680 | +0.00007 443 4 | 74 self_attn o | 314,572,800 | +0.00006 444 1 | 74 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00019 444 2 | 74 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00010 444 3 | 74 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00012 444 4 | 74 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00014 445 1 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00014 445 2 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00017 445 3 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00018 445 4 | 74 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00017 446 1 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00013 446 2 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00016 446 3 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00016 446 4 | 74 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00016 447 1 | 75 self_attn q | 62,914,560 | +0.00004 447 2 | 75 self_attn q | 125,829,120 | -0.00002 447 3 | 75 self_attn q | 188,743,680 | +0.00002 447 4 | 75 self_attn q | 314,572,800 | +0.00001 448 1 | 75 self_attn [k, v] | 10,485,760 | -0.00007 448 2 | 75 self_attn [k, v] | 20,971,520 | -0.00005 448 3 | 75 self_attn [k, v] | 31,457,280 | -0.00001 448 4 | 75 self_attn [k, v] | 52,428,800 | +0.00000 449 1 | 75 self_attn o | 62,914,560 | -0.00022 449 2 | 75 self_attn o | 125,829,120 | -0.00012 449 3 | 75 self_attn o | 188,743,680 | -0.00016 449 4 | 75 self_attn o | 314,572,800 | -0.00015 450 1 | 75 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 450 2 | 75 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00002 450 3 | 75 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00007 450 4 | 75 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00009 451 1 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00025 451 2 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00035 451 3 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00037 451 4 | 75 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00033 452 1 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014 452 2 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00015 452 3 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00010 452 4 | 75 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00013 453 1 | 76 self_attn q | 62,914,560 | +0.00013 453 2 | 76 self_attn q | 125,829,120 | -0.00011 453 3 | 76 self_attn q | 188,743,680 | -0.00011 453 4 | 76 self_attn q | 314,572,800 | -0.00015 454 1 | 76 self_attn [k, v] | 10,485,760 | -0.00016 454 2 | 76 self_attn [k, v] | 20,971,520 | -0.00029 454 3 | 76 self_attn [k, v] | 31,457,280 | -0.00032 454 4 | 76 self_attn [k, v] | 52,428,800 | -0.00026 455 1 | 76 self_attn o | 62,914,560 | +0.00001 455 2 | 76 self_attn o | 125,829,120 | -0.00010 455 3 | 76 self_attn o | 188,743,680 | -0.00014 455 4 | 76 self_attn o | 314,572,800 | -0.00014 456 1 | 76 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00002 456 2 | 76 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 456 3 | 76 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00005 456 4 | 76 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00010 457 1 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00023 457 2 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00024 457 3 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | +0.00027 457 4 | 76 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | +0.00022 458 1 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 458 2 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 458 3 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00010 458 4 | 76 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00012 459 1 | 77 self_attn q | 62,914,560 | -0.00028 459 2 | 77 self_attn q | 125,829,120 | -0.00037 459 3 | 77 self_attn q | 188,743,680 | -0.00040 459 4 | 77 self_attn q | 314,572,800 | -0.00040 460 1 | 77 self_attn [k, v] | 10,485,760 | -0.00024 460 2 | 77 self_attn [k, v] | 20,971,520 | -0.00056 460 3 | 77 self_attn [k, v] | 31,457,280 | -0.00062 460 4 | 77 self_attn [k, v] | 52,428,800 | -0.00057 461 1 | 77 self_attn o | 62,914,560 | -0.00006 461 2 | 77 self_attn o | 125,829,120 | -0.00013 461 3 | 77 self_attn o | 188,743,680 | -0.00015 461 4 | 77 self_attn o | 314,572,800 | -0.00009 462 1 | 77 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 462 2 | 77 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00024 462 3 | 77 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00021 462 4 | 77 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00018 463 1 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00030 463 2 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00038 463 3 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00034 463 4 | 77 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00034 464 1 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00002 464 2 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 464 3 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00005 464 4 | 77 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00005 465 1 | 78 self_attn q | 62,914,560 | -0.00010 465 2 | 78 self_attn q | 125,829,120 | -0.00001 465 3 | 78 self_attn q | 188,743,680 | -0.00003 465 4 | 78 self_attn q | 314,572,800 | -0.00002 466 1 | 78 self_attn [k, v] | 10,485,760 | -0.00040 466 2 | 78 self_attn [k, v] | 20,971,520 | -0.00047 466 3 | 78 self_attn [k, v] | 31,457,280 | -0.00052 466 4 | 78 self_attn [k, v] | 52,428,800 | -0.00049 467 1 | 78 self_attn o | 62,914,560 | -0.00016 467 2 | 78 self_attn o | 125,829,120 | -0.00015 467 3 | 78 self_attn o | 188,743,680 | -0.00011 467 4 | 78 self_attn o | 314,572,800 | -0.00013 468 1 | 78 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00010 468 2 | 78 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00015 468 3 | 78 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00006 468 4 | 78 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00007 469 1 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 469 2 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 469 3 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00002 469 4 | 78 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00008 470 1 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00008 470 2 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 470 3 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00013 470 4 | 78 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00011 471 1 | 79 self_attn q | 62,914,560 | -0.00027 471 2 | 79 self_attn q | 125,829,120 | -0.00028 471 3 | 79 self_attn q | 188,743,680 | -0.00032 471 4 | 79 self_attn q | 314,572,800 | -0.00032 472 1 | 79 self_attn [k, v] | 10,485,760 | -0.00052 472 2 | 79 self_attn [k, v] | 20,971,520 | -0.00096 472 3 | 79 self_attn [k, v] | 31,457,280 | -0.00106 472 4 | 79 self_attn [k, v] | 52,428,800 | -0.00103 473 1 | 79 self_attn o | 62,914,560 | -0.00037 473 2 | 79 self_attn o | 125,829,120 | -0.00035 473 3 | 79 self_attn o | 188,743,680 | -0.00030 473 4 | 79 self_attn o | 314,572,800 | -0.00034 474 1 | 79 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00067 474 2 | 79 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00070 474 3 | 79 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00069 474 4 | 79 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00075 475 1 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00024 475 2 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00040 475 3 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00033 475 4 | 79 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00036 476 1 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00014 476 2 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 476 3 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00010 476 4 | 79 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00009 477 1 | 80 self_attn q | 62,914,560 | +0.00083 477 2 | 80 self_attn q | 125,829,120 | +0.00137 477 3 | 80 self_attn q | 188,743,680 | +0.00106 477 4 | 80 self_attn q | 314,572,800 | +0.00083 478 1 | 80 self_attn [k, v] | 10,485,760 | -0.00136 478 2 | 80 self_attn [k, v] | 20,971,520 | -0.00107 478 3 | 80 self_attn [k, v] | 31,457,280 | -0.00109 478 4 | 80 self_attn [k, v] | 52,428,800 | -0.00118 479 1 | 80 self_attn o | 62,914,560 | -0.00045 479 2 | 80 self_attn o | 125,829,120 | -0.00038 479 3 | 80 self_attn o | 188,743,680 | -0.00033 479 4 | 80 self_attn o | 314,572,800 | -0.00038 480 1 | 80 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 480 2 | 80 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00046 480 3 | 80 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00048 480 4 | 80 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00044 481 1 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00010 481 2 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 481 3 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00009 481 4 | 80 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00009 482 1 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00015 482 2 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00013 482 3 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00012 482 4 | 80 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00016 483 1 | 81 self_attn q | 62,914,560 | -0.00027 483 2 | 81 self_attn q | 125,829,120 | -0.00009 483 3 | 81 self_attn q | 188,743,680 | -0.00009 483 4 | 81 self_attn q | 314,572,800 | -0.00016 484 1 | 81 self_attn [k, v] | 10,485,760 | -0.00097 484 2 | 81 self_attn [k, v] | 20,971,520 | -0.00104 484 3 | 81 self_attn [k, v] | 31,457,280 | -0.00096 484 4 | 81 self_attn [k, v] | 52,428,800 | -0.00096 485 1 | 81 self_attn o | 62,914,560 | -0.00048 485 2 | 81 self_attn o | 125,829,120 | -0.00047 485 3 | 81 self_attn o | 188,743,680 | -0.00044 485 4 | 81 self_attn o | 314,572,800 | -0.00047 486 1 | 81 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00020 486 2 | 81 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00037 486 3 | 81 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00040 486 4 | 81 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00041 487 1 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00015 487 2 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00016 487 3 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00016 487 4 | 81 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00014 488 1 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 488 2 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 488 3 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00010 488 4 | 81 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00012 489 1 | 82 self_attn q | 62,914,560 | -0.00014 489 2 | 82 self_attn q | 125,829,120 | -0.00028 489 3 | 82 self_attn q | 188,743,680 | -0.00020 489 4 | 82 self_attn q | 314,572,800 | -0.00021 490 1 | 82 self_attn [k, v] | 10,485,760 | -0.00009 490 2 | 82 self_attn [k, v] | 20,971,520 | -0.00017 490 3 | 82 self_attn [k, v] | 31,457,280 | -0.00016 490 4 | 82 self_attn [k, v] | 52,428,800 | -0.00019 491 1 | 82 self_attn o | 62,914,560 | -0.00001 491 2 | 82 self_attn o | 125,829,120 | -0.00023 491 3 | 82 self_attn o | 188,743,680 | -0.00020 491 4 | 82 self_attn o | 314,572,800 | -0.00021 492 1 | 82 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00022 492 2 | 82 mlp shared_experts [gate, up, down] | 47,185,920 | +0.00029 492 3 | 82 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00021 492 4 | 82 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00013 493 1 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00028 493 2 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00029 493 3 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00029 493 4 | 82 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00028 494 1 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00016 494 2 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00018 494 3 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00017 494 4 | 82 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00017 495 1 | 83 self_attn q | 62,914,560 | -0.00019 495 2 | 83 self_attn q | 125,829,120 | -0.00016 495 3 | 83 self_attn q | 188,743,680 | -0.00026 495 4 | 83 self_attn q | 314,572,800 | -0.00025 496 1 | 83 self_attn [k, v] | 10,485,760 | -0.00068 496 2 | 83 self_attn [k, v] | 20,971,520 | -0.00093 496 3 | 83 self_attn [k, v] | 31,457,280 | -0.00083 496 4 | 83 self_attn [k, v] | 52,428,800 | -0.00079 497 1 | 83 self_attn o | 62,914,560 | +0.00012 497 2 | 83 self_attn o | 125,829,120 | -0.00005 497 3 | 83 self_attn o | 188,743,680 | +0.00005 497 4 | 83 self_attn o | 314,572,800 | +0.00001 498 1 | 83 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00015 498 2 | 83 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00023 498 3 | 83 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00010 498 4 | 83 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00019 499 1 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00020 499 2 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00019 499 3 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00024 499 4 | 83 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00023 500 1 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00006 500 2 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00012 500 3 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00014 500 4 | 83 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00013 501 1 | 84 self_attn q | 62,914,560 | +0.00005 501 2 | 84 self_attn q | 125,829,120 | -0.00014 501 3 | 84 self_attn q | 188,743,680 | -0.00026 501 4 | 84 self_attn q | 314,572,800 | -0.00014 502 1 | 84 self_attn [k, v] | 10,485,760 | +0.00036 502 2 | 84 self_attn [k, v] | 20,971,520 | -0.00004 502 3 | 84 self_attn [k, v] | 31,457,280 | +0.00015 502 4 | 84 self_attn [k, v] | 52,428,800 | +0.00021 503 1 | 84 self_attn o | 62,914,560 | -0.00030 503 2 | 84 self_attn o | 125,829,120 | -0.00008 503 3 | 84 self_attn o | 188,743,680 | -0.00011 503 4 | 84 self_attn o | 314,572,800 | -0.00005 504 1 | 84 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00060 504 2 | 84 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00033 504 3 | 84 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00029 504 4 | 84 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00037 505 1 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00017 505 2 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00018 505 3 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00014 505 4 | 84 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00013 506 1 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00000 506 2 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 506 3 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00003 506 4 | 84 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00002 507 1 | 85 self_attn q | 62,914,560 | -0.00047 507 2 | 85 self_attn q | 125,829,120 | -0.00052 507 3 | 85 self_attn q | 188,743,680 | -0.00072 507 4 | 85 self_attn q | 314,572,800 | -0.00084 508 1 | 85 self_attn [k, v] | 10,485,760 | -0.00065 508 2 | 85 self_attn [k, v] | 20,971,520 | -0.00086 508 3 | 85 self_attn [k, v] | 31,457,280 | -0.00102 508 4 | 85 self_attn [k, v] | 52,428,800 | -0.00100 509 1 | 85 self_attn o | 62,914,560 | -0.00049 509 2 | 85 self_attn o | 125,829,120 | -0.00066 509 3 | 85 self_attn o | 188,743,680 | -0.00069 509 4 | 85 self_attn o | 314,572,800 | -0.00069 510 1 | 85 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00024 510 2 | 85 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00011 510 3 | 85 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00012 510 4 | 85 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00013 511 1 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | +0.00007 511 2 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | +0.00024 511 3 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | +0.00013 511 4 | 85 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | +0.00010 512 1 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00007 512 2 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 512 3 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00006 512 4 | 85 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00005 513 1 | 86 self_attn q | 62,914,560 | -0.00049 513 2 | 86 self_attn q | 125,829,120 | -0.00040 513 3 | 86 self_attn q | 188,743,680 | -0.00048 513 4 | 86 self_attn q | 314,572,800 | -0.00047 514 1 | 86 self_attn [k, v] | 10,485,760 | +0.00003 514 2 | 86 self_attn [k, v] | 20,971,520 | -0.00036 514 3 | 86 self_attn [k, v] | 31,457,280 | -0.00034 514 4 | 86 self_attn [k, v] | 52,428,800 | -0.00041 515 1 | 86 self_attn o | 62,914,560 | +0.00016 515 2 | 86 self_attn o | 125,829,120 | -0.00005 515 3 | 86 self_attn o | 188,743,680 | -0.00002 515 4 | 86 self_attn o | 314,572,800 | -0.00003 516 1 | 86 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00057 516 2 | 86 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00053 516 3 | 86 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00058 516 4 | 86 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00056 517 1 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 517 2 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00022 517 3 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00020 517 4 | 86 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00019 518 1 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 518 2 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00006 518 3 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00007 518 4 | 86 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00009 519 1 | 87 self_attn q | 62,914,560 | +0.00022 519 2 | 87 self_attn q | 125,829,120 | +0.00020 519 3 | 87 self_attn q | 188,743,680 | +0.00021 519 4 | 87 self_attn q | 314,572,800 | +0.00016 520 1 | 87 self_attn [k, v] | 10,485,760 | +0.00041 520 2 | 87 self_attn [k, v] | 20,971,520 | +0.00044 520 3 | 87 self_attn [k, v] | 31,457,280 | -0.00026 520 4 | 87 self_attn [k, v] | 52,428,800 | -0.00008 521 1 | 87 self_attn o | 62,914,560 | -0.00050 521 2 | 87 self_attn o | 125,829,120 | -0.00042 521 3 | 87 self_attn o | 188,743,680 | -0.00031 521 4 | 87 self_attn o | 314,572,800 | -0.00036 522 1 | 87 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00026 522 2 | 87 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00033 522 3 | 87 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00036 522 4 | 87 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00035 523 1 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 523 2 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00010 523 3 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00010 523 4 | 87 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00007 524 1 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010 524 2 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 524 3 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00011 524 4 | 87 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00012 525 1 | 88 self_attn q | 62,914,560 | -0.00005 525 2 | 88 self_attn q | 125,829,120 | -0.00007 525 3 | 88 self_attn q | 188,743,680 | -0.00005 525 4 | 88 self_attn q | 314,572,800 | -0.00008 526 1 | 88 self_attn [k, v] | 10,485,760 | -0.00073 526 2 | 88 self_attn [k, v] | 20,971,520 | -0.00051 526 3 | 88 self_attn [k, v] | 31,457,280 | -0.00051 526 4 | 88 self_attn [k, v] | 52,428,800 | -0.00057 527 1 | 88 self_attn o | 62,914,560 | -0.00012 527 2 | 88 self_attn o | 125,829,120 | -0.00019 527 3 | 88 self_attn o | 188,743,680 | -0.00019 527 4 | 88 self_attn o | 314,572,800 | -0.00018 528 1 | 88 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00008 528 2 | 88 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00018 528 3 | 88 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00023 528 4 | 88 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00023 529 1 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00011 529 2 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00014 529 3 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00012 529 4 | 88 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00013 530 1 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00009 530 2 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00011 530 3 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00009 530 4 | 88 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00011 531 1 | 89 self_attn q | 62,914,560 | -0.00017 531 2 | 89 self_attn q | 125,829,120 | -0.00023 531 3 | 89 self_attn q | 188,743,680 | -0.00028 531 4 | 89 self_attn q | 314,572,800 | -0.00026 532 1 | 89 self_attn [k, v] | 10,485,760 | -0.00054 532 2 | 89 self_attn [k, v] | 20,971,520 | -0.00070 532 3 | 89 self_attn [k, v] | 31,457,280 | -0.00049 532 4 | 89 self_attn [k, v] | 52,428,800 | -0.00057 533 1 | 89 self_attn o | 62,914,560 | -0.00029 533 2 | 89 self_attn o | 125,829,120 | -0.00023 533 3 | 89 self_attn o | 188,743,680 | -0.00028 533 4 | 89 self_attn o | 314,572,800 | -0.00027 534 1 | 89 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00016 534 2 | 89 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00030 534 3 | 89 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00034 534 4 | 89 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00029 535 1 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 535 2 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 535 3 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00009 535 4 | 89 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00009 536 1 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00005 536 2 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00005 536 3 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00006 536 4 | 89 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00006 537 1 | 90 self_attn q | 62,914,560 | -0.00019 537 2 | 90 self_attn q | 125,829,120 | -0.00024 537 3 | 90 self_attn q | 188,743,680 | -0.00023 537 4 | 90 self_attn q | 314,572,800 | -0.00022 538 1 | 90 self_attn [k, v] | 10,485,760 | -0.00017 538 2 | 90 self_attn [k, v] | 20,971,520 | -0.00028 538 3 | 90 self_attn [k, v] | 31,457,280 | -0.00025 538 4 | 90 self_attn [k, v] | 52,428,800 | -0.00022 539 1 | 90 self_attn o | 62,914,560 | -0.00009 539 2 | 90 self_attn o | 125,829,120 | -0.00009 539 3 | 90 self_attn o | 188,743,680 | -0.00011 539 4 | 90 self_attn o | 314,572,800 | -0.00014 540 1 | 90 mlp shared_experts [gate, up, down] | 23,592,960 | -0.00062 540 2 | 90 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00064 540 3 | 90 mlp shared_experts [gate, up, down] | 70,778,880 | -0.00055 540 4 | 90 mlp shared_experts [gate, up, down] | 117,964,800 | -0.00060 541 1 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00012 541 2 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00007 541 3 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00010 541 4 | 90 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00010 542 1 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00010 542 2 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00010 542 3 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00009 542 4 | 90 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00009 543 1 | 91 self_attn q | 62,914,560 | -0.00020 543 2 | 91 self_attn q | 125,829,120 | -0.00004 543 3 | 91 self_attn q | 188,743,680 | -0.00008 543 4 | 91 self_attn q | 314,572,800 | -0.00004 544 1 | 91 self_attn [k, v] | 10,485,760 | -0.00072 544 2 | 91 self_attn [k, v] | 20,971,520 | -0.00106 544 3 | 91 self_attn [k, v] | 31,457,280 | -0.00110 544 4 | 91 self_attn [k, v] | 52,428,800 | -0.00118 545 1 | 91 self_attn o | 62,914,560 | -0.00016 545 2 | 91 self_attn o | 125,829,120 | -0.00034 545 3 | 91 self_attn o | 188,743,680 | -0.00031 545 4 | 91 self_attn o | 314,572,800 | -0.00034 546 1 | 91 mlp shared_experts [gate, up, down] | 23,592,960 | +0.00076 546 2 | 91 mlp shared_experts [gate, up, down] | 47,185,920 | -0.00006 546 3 | 91 mlp shared_experts [gate, up, down] | 70,778,880 | +0.00013 546 4 | 91 mlp shared_experts [gate, up, down] | 117,964,800 | +0.00027 547 1 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 2,516,582,400 | -0.00008 547 2 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 5,033,164,800 | -0.00008 547 3 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 7,549,747,200 | -0.00009 547 4 | 91 mlp experts [0 [gate, up], 1 [gate, up], 2 [gate, up], 3 [gate... | 12,582,912,000 | -0.00010 548 1 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 1,258,291,200 | -0.00004 548 2 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 2,516,582,400 | -0.00003 548 3 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 3,774,873,600 | -0.00005 548 4 | 91 mlp experts [0 down, 1 down, 2 down, 3 down, 4 down, 5 down, 6... | 6,291,456,000 | -0.00005 -- Writing /home/beta/AI/local_models_exl/glm-4.5-measurements-3vs4vs5vs6vs8.json