Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See https://huggingface.co/docs/huggingface_hub/hf_transfer for more details.
bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 0.00/22.2G [00:00<?, ?B/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 0.00/22.2G [00:00<?, ?B/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 0.00/22.2G [00:00<?, ?B/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 0.00/22.2G [00:00<?, ?B/s][A[A[A


Upload 41 LFS files:   0%|          | 0/41 [00:00<?, ?it/s][A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 0.00/22.2G [00:00<?, ?B/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 16.4k/22.2G [00:00<82:28:53, 74.6kB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 16.4k/22.2G [00:00<83:43:52, 73.5kB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 16.4k/22.2G [00:00<84:13:11, 73.1kB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 16.4k/22.2G [00:00<85:43:59, 71.8kB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 16.4k/22.2G [00:00<105:54:16, 58.1kB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 147k/22.2G [00:00<12:35:55, 488kB/s]  [Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 197k/22.2G [00:00<9:35:40, 641kB/s]   


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 147k/22.2G [00:00<13:08:00, 469kB/s]  [A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 131k/22.2G [00:00<14:54:07, 413kB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 197k/22.2G [00:00<10:34:17, 582kB/s]   [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 557k/22.2G [00:00<4:09:00, 1.48MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 541k/22.2G [00:00<4:29:51, 1.37MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 623k/22.2G [00:00<3:51:07, 1.60MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 655k/22.2G [00:00<3:40:50, 1.67MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 541k/22.2G [00:00<4:40:02, 1.32MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 2.10M/22.2G [00:00<1:08:07, 5.42MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 1.56M/22.2G [00:00<1:43:08, 3.58MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 2.72M/22.2G [00:00<57:24, 6.43MB/s] 


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 2.29M/22.2G [00:00<1:11:03, 5.20MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 4.47M/22.2G [00:00<40:43, 9.07MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 2.15M/22.2G [00:00<1:33:36, 3.94MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 5.87M/22.2G [00:00<31:33, 11.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 4.85M/22.2G [00:00<39:26, 9.36MB/s]  [A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 7.45M/22.2G [00:00<26:23, 14.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 7.65M/22.2G [00:00<30:37, 12.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 9.63M/22.2G [00:00<23:12, 15.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 2.97M/22.2G [00:00<1:37:04, 3.80MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 4.90M/22.2G [00:01<58:07, 6.35MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 6.52M/22.2G [00:01<48:51, 7.56MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 12.9M/22.2G [00:01<20:07, 18.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 5.65M/22.2G [00:01<59:26, 6.21MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 14.8M/22.2G [00:01<22:20, 16.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 7.93M/22.2G [00:01<51:39, 7.15MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 8.86M/22.2G [00:01<35:48, 10.3MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 9.08M/22.2G [00:01<58:43, 6.28MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 18.0M/22.2G [00:01<23:06, 16.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 10.7M/22.2G [00:01<47:47, 7.72MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 9.47M/22.2G [00:01<52:18, 7.06MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 6.70M/22.2G [00:01<1:13:08, 5.05MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 12.9M/22.2G [00:01<33:24, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 12.0M/22.2G [00:01<46:54, 7.87MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 16.1M/22.2G [00:01<25:52, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 15.1M/22.2G [00:01<31:45, 11.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 8.57M/22.2G [00:01<1:09:15, 5.33MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 19.0M/22.2G [00:01<22:09, 16.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 19.4M/22.2G [00:01<21:15, 17.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 11.2M/22.2G [00:01<56:00, 6.59MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 22.2M/22.2G [00:02<18:33, 19.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 11.6M/22.2G [00:02<56:19, 6.55MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 12.7M/22.2G [00:02<57:47, 6.39MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 14.4M/22.2G [00:02<47:50, 7.71MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 13.5M/22.2G [00:02<57:08, 6.46MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 18.3M/22.2G [00:02<33:45, 10.9MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 22.2M/22.2G [00:02<41:11, 8.96MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 22.2M/22.2G [00:02<55:24, 6.66MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 21.8M/22.2G [00:02<25:06, 14.7MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 15.0M/22.2G [00:02<51:03, 7.23MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 26.3M/22.2G [00:02<30:25, 12.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 26.3M/22.2G [00:02<39:39, 9.30MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 28.4M/22.2G [00:02<29:06, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 29.4M/22.2G [00:02<33:08, 11.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 16.6M/22.2G [00:02<57:01, 6.47MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 24.5M/22.2G [00:02<52:41, 7.00MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 31.6M/22.2G [00:03<25:02, 14.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 32.6M/22.2G [00:03<27:54, 13.2MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 35.7M/22.2G [00:03<24:31, 15.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 23.8M/22.2G [00:03<42:45, 8.63MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 18.1M/22.2G [00:03<56:33, 6.52MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 26.3M/22.2G [00:03<54:12, 6.80MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 40.1M/22.2G [00:03<20:24, 18.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 26.3M/22.2G [00:03<38:30, 9.58MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 29.8M/22.2G [00:03<29:50, 12.4MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 42.4M/22.2G [00:03<22:33, 16.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 19.7M/22.2G [00:03<1:01:46, 5.97MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 30.6M/22.2G [00:03<41:07, 8.97MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 33.5M/22.2G [00:03<45:15, 8.15MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 21.4M/22.2G [00:03<52:23, 7.04MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 34.4M/22.2G [00:03<33:11, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 34.9M/22.2G [00:03<44:03, 8.37MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 37.7M/22.2G [00:03<30:12, 12.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 36.2M/22.2G [00:03<45:40, 8.07MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 32.3M/22.2G [00:04<43:49, 8.41MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 40.9M/22.2G [00:04<27:06, 13.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 39.7M/22.2G [00:04<33:31, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 42.6M/22.2G [00:04<26:46, 13.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 42.7M/22.2G [00:04<28:06, 13.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 44.3M/22.2G [00:04<48:53, 7.54MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 48.5M/22.2G [00:04<38:37, 9.54MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 22.2M/22.2G [00:04<2:08:13, 2.88MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 50.5M/22.2G [00:04<39:16, 9.38MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 44.4M/22.2G [00:04<47:27, 7.76MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 48.4M/22.2G [00:04<33:00, 11.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 52.9M/22.2G [00:05<37:45, 9.75MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 44.3M/22.2G [00:05<57:17, 6.43MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 33.8M/22.2G [00:05<1:22:46, 4.45MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 26.3M/22.2G [00:05<1:15:10, 4.91MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 52.1M/22.2G [00:05<26:27, 13.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 48.5M/22.2G [00:05<41:16, 8.93MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 55.2M/22.2G [00:05<23:50, 15.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 35.7M/22.2G [00:05<1:12:45, 5.07MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 27.7M/22.2G [00:05<1:13:43, 5.00MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 49.9M/22.2G [00:05<40:30, 9.09MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 38.4M/22.2G [00:05<52:51, 6.97MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 55.4M/22.2G [00:05<43:14, 8.52MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 58.4M/22.2G [00:05<21:46, 16.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 29.2M/22.2G [00:05<1:03:18, 5.83MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 51.7M/22.2G [00:05<36:49, 10.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 42.2M/22.2G [00:05<37:28, 9.84MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 43.9M/22.2G [00:05<37:36, 9.80MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 54.7M/22.2G [00:05<34:07, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 61.6M/22.2G [00:05<28:00, 13.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 57.3M/22.2G [00:05<51:38, 7.13MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 30.8M/22.2G [00:05<1:08:15, 5.40MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 65.2M/22.2G [00:05<23:42, 15.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 60.2M/22.2G [00:06<45:25, 8.11MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 32.2M/22.2G [00:06<1:10:04, 5.26MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 56.2M/22.2G [00:06<50:22, 7.31MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 45.4M/22.2G [00:06<55:40, 6.62MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 63.2M/22.2G [00:06<39:08, 9.41MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 33.6M/22.2G [00:06<1:05:13, 5.65MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 58.0M/22.2G [00:06<46:01, 8.00MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 64.6M/22.2G [00:06<37:41, 9.77MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 67.1M/22.2G [00:06<38:30, 9.56MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 48.4M/22.2G [00:06<43:48, 8.41MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 59.5M/22.2G [00:06<41:03, 8.97MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 70.6M/22.2G [00:06<30:42, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 60.9M/22.2G [00:06<38:24, 9.59MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 35.1M/22.2G [00:06<1:10:17, 5.25MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 62.3M/22.2G [00:06<36:13, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 72.3M/22.2G [00:06<30:47, 12.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 52.0M/22.2G [00:06<38:43, 9.51MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 63.8M/22.2G [00:06<34:59, 10.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 36.6M/22.2G [00:06<1:04:16, 5.73MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 55.0M/22.2G [00:06<32:43, 11.3MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 75.6M/22.2G [00:06<28:30, 12.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 65.5M/22.2G [00:06<32:04, 11.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 66.5M/22.2G [00:07<58:50, 6.26MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 57.0M/22.2G [00:07<32:51, 11.2MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 78.9M/22.2G [00:07<24:56, 14.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 38.1M/22.2G [00:07<1:06:53, 5.51MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 70.6M/22.2G [00:07<40:52, 9.01MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 80.6M/22.2G [00:07<26:01, 14.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 60.2M/22.2G [00:07<33:32, 11.0MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 83.8M/22.2G [00:07<23:04, 15.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 73.7M/22.2G [00:07<35:36, 10.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 39.7M/22.2G [00:07<1:06:05, 5.58MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 62.5M/22.2G [00:07<31:00, 11.9MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 85.5M/22.2G [00:07<24:35, 15.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 75.3M/22.2G [00:07<34:41, 10.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 64.0M/22.2G [00:07<31:57, 11.5MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 78.2M/22.2G [00:07<32:01, 11.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 41.1M/22.2G [00:07<1:14:34, 4.94MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 65.6M/22.2G [00:07<32:45, 11.2MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 66.7M/22.2G [00:07<1:28:50, 4.14MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 81.2M/22.2G [00:08<32:48, 11.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 70.6M/22.2G [00:08<53:28, 6.88MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 88.6M/22.2G [00:08<38:24, 9.57MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 83.6M/22.2G [00:08<29:37, 12.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 42.6M/22.2G [00:08<1:21:58, 4.50MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 72.1M/22.2G [00:08<48:57, 7.52MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 92.8M/22.2G [00:08<28:29, 12.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 85.0M/22.2G [00:08<30:53, 11.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 74.0M/22.2G [00:08<43:03, 8.55MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 66.8M/22.2G [00:08<58:20, 6.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 94.5M/22.2G [00:08<28:57, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 86.5M/22.2G [00:08<31:59, 11.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 75.9M/22.2G [00:08<38:30, 9.56MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 97.7M/22.2G [00:08<24:55, 14.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 87.9M/22.2G [00:08<32:57, 11.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 44.1M/22.2G [00:08<1:28:07, 4.18MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 77.9M/22.2G [00:08<34:45, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 70.6M/22.2G [00:08<44:15, 8.32MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 79.8M/22.2G [00:08<32:34, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 72.9M/22.2G [00:08<41:48, 8.80MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 81.6M/22.2G [00:08<31:33, 11.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 99.5M/22.2G [00:09<41:00, 8.96MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 83.6M/22.2G [00:09<30:01, 12.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 74.5M/22.2G [00:09<43:42, 8.42MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 85.3M/22.2G [00:09<30:00, 12.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 89.0M/22.2G [00:09<1:08:01, 5.41MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 75.9M/22.2G [00:09<43:02, 8.55MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 101M/22.2G [00:09<45:26, 8.09MB/s] [A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 88.5M/22.2G [00:09<27:44, 13.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 103M/22.2G [00:09<44:29, 8.26MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 77.6M/22.2G [00:09<44:45, 8.22MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 92.7M/22.2G [00:09<46:34, 7.90MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 104M/22.2G [00:09<42:49, 8.58MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 44.5M/22.2G [00:09<2:50:48, 2.16MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 79.5M/22.2G [00:09<44:31, 8.26MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 94.5M/22.2G [00:09<46:20, 7.93MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 107M/22.2G [00:09<32:24, 11.3MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 108M/22.2G [00:09<31:30, 11.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 81.1M/22.2G [00:09<41:49, 8.80MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 96.0M/22.2G [00:09<47:49, 7.69MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 89.9M/22.2G [00:09<52:10, 7.05MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   0%|          | 110M/22.2G [00:09<32:13, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 82.5M/22.2G [00:10<45:30, 8.09MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 92.7M/22.2G [00:10<42:58, 8.55MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 48.4M/22.2G [00:10<1:45:00, 3.51MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 97.4M/22.2G [00:10<58:55, 6.24MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 84.4M/22.2G [00:10<45:13, 8.13MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 95.0M/22.2G [00:10<36:52, 9.97MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 96.9M/22.2G [00:10<34:23, 10.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 86.3M/22.2G [00:10<44:17, 8.30MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 98.9M/22.2G [00:10<32:07, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 99.6M/22.2G [00:10<58:01, 6.33MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 111M/22.2G [00:10<1:10:07, 5.24MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 50.0M/22.2G [00:10<1:45:11, 3.50MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 88.2M/22.2G [00:10<43:52, 8.38MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 115M/22.2G [00:10<45:19, 8.11MB/s]  [A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 101M/22.2G [00:10<1:01:52, 5.94MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 118M/22.2G [00:11<37:23, 9.82MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 102M/22.2G [00:11<1:00:23, 6.09MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 51.3M/22.2G [00:11<1:51:27, 3.31MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 100M/22.2G [00:11<55:55, 6.57MB/s] 


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 120M/22.2G [00:11<36:02, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 102M/22.2G [00:11<49:15, 7.46MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 89.1M/22.2G [00:11<1:18:32, 4.68MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 104M/22.2G [00:11<1:03:24, 5.80MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 104M/22.2G [00:11<45:07, 8.15MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 52.9M/22.2G [00:11<1:38:30, 3.74MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 123M/22.2G [00:11<32:24, 11.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 92.7M/22.2G [00:11<50:20, 7.30MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 105M/22.2G [00:11<40:03, 9.17MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 105M/22.2G [00:11<59:39, 6.16MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 54.5M/22.2G [00:11<1:24:25, 4.36MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 126M/22.2G [00:11<30:36, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 108M/22.2G [00:11<33:16, 11.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 94.3M/22.2G [00:11<50:57, 7.22MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 107M/22.2G [00:11<59:23, 6.19MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   0%|          | 110M/22.2G [00:11<31:45, 11.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 129M/22.2G [00:11<29:14, 12.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 96.2M/22.2G [00:11<48:32, 7.57MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 56.0M/22.2G [00:11<1:23:59, 4.39MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 108M/22.2G [00:12<1:00:45, 6.05MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 132M/22.2G [00:12<28:18, 13.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 98.3M/22.2G [00:12<45:54, 8.01MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 57.4M/22.2G [00:12<1:18:22, 4.70MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   0%|          | 110M/22.2G [00:12<57:33, 6.38MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 99.8M/22.2G [00:12<43:26, 8.46MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 111M/22.2G [00:12<1:02:54, 5.84MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 58.8M/22.2G [00:12<1:15:14, 4.89MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 134M/22.2G [00:12<43:27, 8.45MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 102M/22.2G [00:12<44:04, 8.34MB/s] [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 115M/22.2G [00:12<42:03, 8.73MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 137M/22.2G [00:12<33:08, 11.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 103M/22.2G [00:12<40:35, 9.05MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 60.3M/22.2G [00:12<1:12:31, 5.08MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 117M/22.2G [00:12<36:22, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 138M/22.2G [00:12<33:55, 10.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 119M/22.2G [00:12<33:57, 10.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 105M/22.2G [00:12<42:52, 8.57MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 142M/22.2G [00:13<31:11, 11.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 121M/22.2G [00:13<31:51, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 107M/22.2G [00:13<40:58, 8.97MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 111M/22.2G [00:13<1:58:00, 3.11MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 61.7M/22.2G [00:13<1:23:54, 4.39MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 108M/22.2G [00:13<38:09, 9.63MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 145M/22.2G [00:13<29:11, 12.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   0%|          | 110M/22.2G [00:13<42:55, 8.56MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 148M/22.2G [00:13<27:52, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 63.2M/22.2G [00:13<1:25:50, 4.29MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 123M/22.2G [00:13<55:54, 6.57MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 115M/22.2G [00:13<1:16:57, 4.77MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 151M/22.2G [00:13<27:16, 13.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 124M/22.2G [00:13<47:52, 7.67MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 126M/22.2G [00:13<44:47, 8.20MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 154M/22.2G [00:13<26:30, 13.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 116M/22.2G [00:13<1:16:50, 4.78MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 64.6M/22.2G [00:14<1:35:11, 3.87MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 128M/22.2G [00:14<37:44, 9.73MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 111M/22.2G [00:14<1:19:09, 4.64MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 129M/22.2G [00:14<34:56, 10.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 115M/22.2G [00:14<43:53, 8.37MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 131M/22.2G [00:14<32:45, 11.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 66.4M/22.2G [00:14<1:23:11, 4.43MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 118M/22.2G [00:14<1:26:36, 4.24MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 116M/22.2G [00:14<46:48, 7.85MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 156M/22.2G [00:14<43:07, 8.50MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 118M/22.2G [00:14<41:02, 8.95MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 159M/22.2G [00:14<33:33, 10.9MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 161M/22.2G [00:14<34:15, 10.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 120M/22.2G [00:14<44:54, 8.18MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 119M/22.2G [00:14<1:35:29, 3.85MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 133M/22.2G [00:14<59:36, 6.16MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 66.9M/22.2G [00:14<2:09:26, 2.84MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 164M/22.2G [00:14<30:48, 11.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 121M/22.2G [00:14<43:48, 8.38MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 137M/22.2G [00:15<39:51, 9.21MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 167M/22.2G [00:15<29:25, 12.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 123M/22.2G [00:15<42:39, 8.61MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 70.4M/22.2G [00:15<1:16:30, 4.81MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 139M/22.2G [00:15<35:02, 10.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 120M/22.2G [00:15<1:42:33, 3.58MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 124M/22.2G [00:15<41:55, 8.76MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 170M/22.2G [00:15<26:02, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 141M/22.2G [00:15<32:59, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 172M/22.2G [00:15<26:59, 13.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 126M/22.2G [00:15<38:02, 9.65MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 143M/22.2G [00:15<30:39, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 122M/22.2G [00:15<1:33:03, 3.95MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 145M/22.2G [00:15<30:13, 12.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 175M/22.2G [00:15<26:38, 13.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 128M/22.2G [00:15<42:13, 8.70MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 71.8M/22.2G [00:15<1:31:14, 4.03MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 147M/22.2G [00:15<29:07, 12.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 123M/22.2G [00:15<1:27:11, 4.21MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 129M/22.2G [00:15<39:32, 9.28MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 131M/22.2G [00:16<38:28, 9.54MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 151M/22.2G [00:16<26:53, 13.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 125M/22.2G [00:16<1:21:48, 4.49MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 152M/22.2G [00:16<27:24, 13.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 73.4M/22.2G [00:16<1:34:53, 3.88MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 177M/22.2G [00:16<40:58, 8.94MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 133M/22.2G [00:16<42:29, 8.64MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 155M/22.2G [00:16<26:13, 14.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 181M/22.2G [00:16<29:53, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 126M/22.2G [00:16<1:17:55, 4.71MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 183M/22.2G [00:16<30:25, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 128M/22.2G [00:16<1:16:06, 4.82MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 186M/22.2G [00:16<28:53, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 156M/22.2G [00:16<47:57, 7.64MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 133M/22.2G [00:16<1:20:48, 4.54MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 74.9M/22.2G [00:16<1:59:17, 3.08MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 129M/22.2G [00:16<1:14:00, 4.96MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 189M/22.2G [00:16<27:29, 13.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 137M/22.2G [00:17<50:42, 7.24MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 159M/22.2G [00:17<41:49, 8.76MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 193M/22.2G [00:17<26:40, 13.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 139M/22.2G [00:17<47:01, 7.80MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 130M/22.2G [00:17<1:12:43, 5.05MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 162M/22.2G [00:17<33:49, 10.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 140M/22.2G [00:17<45:33, 8.05MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 164M/22.2G [00:17<33:52, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 196M/22.2G [00:17<26:02, 14.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 76.5M/22.2G [00:17<1:56:45, 3.15MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 132M/22.2G [00:17<1:13:18, 5.01MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 141M/22.2G [00:17<44:28, 8.25MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 165M/22.2G [00:17<34:31, 10.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 199M/22.2G [00:17<25:37, 14.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 143M/22.2G [00:17<43:01, 8.53MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 144M/22.2G [00:17<42:40, 8.60MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 78.0M/22.2G [00:17<1:51:12, 3.31MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 167M/22.2G [00:17<45:42, 8.02MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 146M/22.2G [00:18<46:50, 7.83MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 168M/22.2G [00:18<46:47, 7.83MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 201M/22.2G [00:18<39:57, 9.16MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 169M/22.2G [00:18<44:15, 8.28MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 204M/22.2G [00:18<33:30, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 147M/22.2G [00:18<48:49, 7.51MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 79.5M/22.2G [00:18<1:49:51, 3.35MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 171M/22.2G [00:18<41:25, 8.84MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 205M/22.2G [00:18<34:01, 10.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 149M/22.2G [00:18<51:18, 7.15MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 173M/22.2G [00:18<39:18, 9.32MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 208M/22.2G [00:18<30:43, 11.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 174M/22.2G [00:18<37:42, 9.72MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 133M/22.2G [00:18<2:28:59, 2.46MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 150M/22.2G [00:18<52:30, 6.99MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 176M/22.2G [00:18<36:07, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 80.9M/22.2G [00:18<1:58:04, 3.12MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 211M/22.2G [00:18<28:58, 12.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 152M/22.2G [00:18<50:23, 7.28MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 177M/22.2G [00:18<35:27, 10.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 215M/22.2G [00:19<27:20, 13.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 137M/22.2G [00:19<1:25:18, 4.30MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 153M/22.2G [00:19<52:53, 6.93MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 82.6M/22.2G [00:19<1:51:04, 3.31MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 218M/22.2G [00:19<26:29, 13.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 155M/22.2G [00:19<48:25, 7.57MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 178M/22.2G [00:19<1:03:18, 5.79MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 221M/22.2G [00:19<23:45, 15.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 139M/22.2G [00:19<1:25:26, 4.29MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 84.2M/22.2G [00:19<1:39:50, 3.68MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 181M/22.2G [00:19<45:05, 8.12MB/s]  bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 183M/22.2G [00:19<39:40, 9.23MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 140M/22.2G [00:19<1:25:33, 4.29MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 223M/22.2G [00:19<39:11, 9.33MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 185M/22.2G [00:19<37:47, 9.69MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 187M/22.2G [00:20<35:59, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 141M/22.2G [00:20<1:21:32, 4.50MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 226M/22.2G [00:20<32:49, 11.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 156M/22.2G [00:20<1:44:56, 3.49MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 188M/22.2G [00:20<34:43, 10.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 227M/22.2G [00:20<33:34, 10.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 190M/22.2G [00:20<33:45, 10.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 85.6M/22.2G [00:20<2:11:28, 2.80MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 143M/22.2G [00:20<1:23:20, 4.40MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 159M/22.2G [00:20<1:03:56, 5.73MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 230M/22.2G [00:20<30:06, 12.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 192M/22.2G [00:20<33:25, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 233M/22.2G [00:20<26:19, 13.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 193M/22.2G [00:20<32:24, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 87.2M/22.2G [00:20<1:49:18, 3.36MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 144M/22.2G [00:20<1:20:25, 4.56MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 161M/22.2G [00:20<1:02:26, 5.87MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 235M/22.2G [00:20<27:44, 13.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 195M/22.2G [00:20<32:06, 11.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 238M/22.2G [00:20<24:47, 14.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 197M/22.2G [00:20<31:59, 11.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 163M/22.2G [00:20<55:11, 6.64MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 240M/22.2G [00:21<26:31, 13.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 146M/22.2G [00:21<1:20:57, 4.53MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 198M/22.2G [00:21<31:45, 11.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 242M/22.2G [00:21<24:11, 15.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 165M/22.2G [00:21<58:52, 6.22MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 147M/22.2G [00:21<1:18:10, 4.69MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 166M/22.2G [00:21<58:20, 6.28MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 88.6M/22.2G [00:21<2:25:27, 2.53MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 149M/22.2G [00:21<1:15:44, 4.84MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 244M/22.2G [00:21<45:28, 8.03MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 168M/22.2G [00:21<55:07, 6.65MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 199M/22.2G [00:21<1:11:56, 5.09MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 248M/22.2G [00:21<34:26, 10.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 169M/22.2G [00:21<55:56, 6.55MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 92.8M/22.2G [00:21<1:23:15, 4.42MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 150M/22.2G [00:21<1:20:04, 4.58MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 203M/22.2G [00:21<44:28, 8.23MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 250M/22.2G [00:22<31:38, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 205M/22.2G [00:22<39:39, 9.23MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 171M/22.2G [00:22<56:42, 6.46MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 252M/22.2G [00:22<31:53, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 94.3M/22.2G [00:22<1:20:06, 4.59MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 207M/22.2G [00:22<37:58, 9.63MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 152M/22.2G [00:22<1:20:01, 4.58MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 253M/22.2G [00:22<32:36, 11.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 172M/22.2G [00:22<56:29, 6.49MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 209M/22.2G [00:22<36:11, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 95.8M/22.2G [00:22<1:15:55, 4.84MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 255M/22.2G [00:22<32:49, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 153M/22.2G [00:22<1:17:26, 4.74MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 174M/22.2G [00:22<56:14, 6.51MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 257M/22.2G [00:22<32:58, 11.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 97.4M/22.2G [00:22<1:08:51, 5.34MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 210M/22.2G [00:22<49:34, 7.38MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 258M/22.2G [00:22<33:05, 11.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 175M/22.2G [00:22<56:30, 6.48MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 260M/22.2G [00:22<33:13, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 212M/22.2G [00:23<50:23, 7.26MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 155M/22.2G [00:23<1:27:42, 4.18MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 98.8M/22.2G [00:23<1:14:56, 4.90MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 177M/22.2G [00:23<55:45, 6.57MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 261M/22.2G [00:23<32:57, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 214M/22.2G [00:23<44:01, 8.31MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 215M/22.2G [00:23<39:55, 9.16MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 263M/22.2G [00:23<33:07, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 216M/22.2G [00:23<38:30, 9.49MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 100M/22.2G [00:23<1:16:17, 4.82MB/s] [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 265M/22.2G [00:23<33:12, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 218M/22.2G [00:23<37:03, 9.87MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 219M/22.2G [00:23<34:23, 10.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 102M/22.2G [00:23<1:13:14, 5.02MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 221M/22.2G [00:23<32:47, 11.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 177M/22.2G [00:23<2:01:22, 3.02MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 266M/22.2G [00:24<1:18:38, 4.64MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 103M/22.2G [00:24<1:29:35, 4.10MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 181M/22.2G [00:24<1:07:27, 5.43MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 155M/22.2G [00:24<3:15:47, 1.87MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 222M/22.2G [00:24<1:12:22, 5.05MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 105M/22.2G [00:24<1:18:19, 4.69MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 270M/22.2G [00:24<48:40, 7.49MB/s]  [A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 183M/22.2G [00:24<1:02:53, 5.82MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 226M/22.2G [00:24<45:33, 8.02MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 272M/22.2G [00:24<44:08, 8.26MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 185M/22.2G [00:24<1:00:32, 6.05MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 228M/22.2G [00:24<39:37, 9.22MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 106M/22.2G [00:24<1:19:07, 4.64MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 273M/22.2G [00:24<41:08, 8.87MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 229M/22.2G [00:24<37:25, 9.76MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 159M/22.2G [00:24<1:49:17, 3.35MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 275M/22.2G [00:24<38:40, 9.43MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 186M/22.2G [00:24<58:52, 6.22MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 231M/22.2G [00:24<35:29, 10.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|          | 277M/22.2G [00:24<35:58, 10.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 108M/22.2G [00:25<1:23:54, 4.38MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 233M/22.2G [00:25<34:17, 10.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 188M/22.2G [00:25<1:02:43, 5.84MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 280M/22.2G [00:25<33:00, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 234M/22.2G [00:25<33:09, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 161M/22.2G [00:25<1:45:48, 3.46MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   0%|          | 109M/22.2G [00:25<1:15:24, 4.87MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 282M/22.2G [00:25<33:04, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 236M/22.2G [00:25<32:12, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 189M/22.2G [00:25<1:01:28, 5.96MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 283M/22.2G [00:25<32:26, 11.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 238M/22.2G [00:25<31:36, 11.6MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 240M/22.2G [00:25<31:06, 11.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 191M/22.2G [00:25<1:03:33, 5.76MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 286M/22.2G [00:25<30:23, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 162M/22.2G [00:25<1:50:52, 3.31MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 241M/22.2G [00:25<30:53, 11.8MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 243M/22.2G [00:25<30:36, 11.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 192M/22.2G [00:26<1:04:06, 5.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 164M/22.2G [00:26<1:50:49, 3.31MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 194M/22.2G [00:26<1:07:39, 5.41MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 288M/22.2G [00:26<53:32, 6.81MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 111M/22.2G [00:26<2:21:42, 2.59MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 244M/22.2G [00:26<1:03:48, 5.72MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 292M/22.2G [00:26<38:08, 9.55MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 196M/22.2G [00:26<1:06:09, 5.53MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 165M/22.2G [00:26<1:49:02, 3.36MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 294M/22.2G [00:26<36:03, 10.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 248M/22.2G [00:26<44:10, 8.27MB/s]  bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 250M/22.2G [00:26<39:01, 9.35MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 197M/22.2G [00:26<1:07:28, 5.42MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 297M/22.2G [00:26<32:55, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 251M/22.2G [00:27<37:07, 9.83MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 115M/22.2G [00:27<1:33:04, 3.95MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 299M/22.2G [00:27<32:56, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 167M/22.2G [00:27<1:47:37, 3.41MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 199M/22.2G [00:27<1:05:01, 5.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 253M/22.2G [00:27<35:09, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 302M/22.2G [00:27<30:47, 11.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 255M/22.2G [00:27<33:41, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 303M/22.2G [00:27<31:47, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 257M/22.2G [00:27<32:41, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 168M/22.2G [00:27<1:46:25, 3.44MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 116M/22.2G [00:27<1:39:22, 3.70MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 258M/22.2G [00:27<31:59, 11.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 306M/22.2G [00:27<29:32, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 260M/22.2G [00:27<31:22, 11.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 308M/22.2G [00:27<30:31, 11.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 262M/22.2G [00:27<30:57, 11.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 170M/22.2G [00:27<1:45:30, 3.47MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 310M/22.2G [00:27<30:37, 11.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 199M/22.2G [00:27<1:58:53, 3.08MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 118M/22.2G [00:27<1:39:53, 3.68MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 264M/22.2G [00:28<30:43, 11.9MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 265M/22.2G [00:28<30:37, 11.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 171M/22.2G [00:28<1:41:27, 3.61MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 119M/22.2G [00:28<1:33:25, 3.93MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 204M/22.2G [00:28<1:09:55, 5.23MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 311M/22.2G [00:28<52:21, 6.95MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 121M/22.2G [00:28<1:19:16, 4.63MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 205M/22.2G [00:28<1:07:32, 5.42MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 314M/22.2G [00:28<38:51, 9.37MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 122M/22.2G [00:28<1:11:51, 5.11MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 173M/22.2G [00:28<1:42:15, 3.58MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 316M/22.2G [00:28<36:04, 10.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 267M/22.2G [00:28<1:04:34, 5.65MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 207M/22.2G [00:28<1:06:23, 5.51MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 318M/22.2G [00:28<33:32, 10.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 270M/22.2G [00:29<44:21, 8.22MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 320M/22.2G [00:29<32:37, 11.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 174M/22.2G [00:29<1:42:21, 3.58MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 272M/22.2G [00:29<38:52, 9.38MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 322M/22.2G [00:29<32:40, 11.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 124M/22.2G [00:29<1:31:22, 4.02MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 209M/22.2G [00:29<1:04:46, 5.65MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 274M/22.2G [00:29<36:14, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 324M/22.2G [00:29<31:12, 11.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|          | 276M/22.2G [00:29<34:20, 10.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 176M/22.2G [00:29<1:42:23, 3.58MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 211M/22.2G [00:29<1:04:10, 5.70MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 125M/22.2G [00:29<1:25:54, 4.27MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 277M/22.2G [00:29<32:47, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 326M/22.2G [00:29<29:41, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 127M/22.2G [00:29<1:10:10, 5.23MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 279M/22.2G [00:29<31:39, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 212M/22.2G [00:29<1:03:04, 5.80MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 329M/22.2G [00:29<29:10, 12.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 281M/22.2G [00:29<30:56, 11.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 177M/22.2G [00:29<1:38:48, 3.71MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   1%|▏         | 331M/22.2G [00:29<30:01, 12.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 283M/22.2G [00:30<30:20, 12.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 128M/22.2G [00:30<1:14:47, 4.91MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 214M/22.2G [00:30<1:02:28, 5.85MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 284M/22.2G [00:30<30:30, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 130M/22.2G [00:30<1:11:40, 5.12MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 286M/22.2G [00:30<30:39, 11.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 216M/22.2G [00:30<1:02:33, 5.85MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 332M/22.2G [00:30<56:40, 6.42MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 131M/22.2G [00:30<1:14:45, 4.91MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 218M/22.2G [00:30<58:23, 6.26MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 336M/22.2G [00:30<39:01, 9.32MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 178M/22.2G [00:30<3:01:34, 2.02MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 133M/22.2G [00:30<1:09:45, 5.26MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 338M/22.2G [00:30<36:40, 9.92MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 288M/22.2G [00:30<59:39, 6.11MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 219M/22.2G [00:30<1:02:50, 5.82MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 340M/22.2G [00:31<34:40, 10.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 292M/22.2G [00:31<39:30, 9.22MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 221M/22.2G [00:31<59:01, 6.19MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 342M/22.2G [00:31<33:38, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 181M/22.2G [00:31<1:37:50, 3.74MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 294M/22.2G [00:31<35:19, 10.3MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 296M/22.2G [00:31<33:11, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 345M/22.2G [00:31<30:18, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 298M/22.2G [00:31<31:33, 11.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 347M/22.2G [00:31<30:59, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 134M/22.2G [00:31<2:05:35, 2.92MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 183M/22.2G [00:31<1:40:09, 3.66MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 300M/22.2G [00:31<30:31, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 350M/22.2G [00:31<29:33, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 137M/22.2G [00:31<1:08:28, 5.36MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 302M/22.2G [00:31<29:02, 12.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 351M/22.2G [00:31<30:06, 12.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 304M/22.2G [00:32<28:41, 12.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 184M/22.2G [00:32<1:36:50, 3.78MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 139M/22.2G [00:32<1:05:01, 5.64MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 306M/22.2G [00:32<27:42, 13.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 222M/22.2G [00:32<2:04:25, 2.94MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 354M/22.2G [00:32<28:50, 12.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 140M/22.2G [00:32<56:44, 6.47MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 308M/22.2G [00:32<28:03, 13.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 142M/22.2G [00:32<49:16, 7.45MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 186M/22.2G [00:32<1:32:59, 3.94MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 226M/22.2G [00:32<1:10:37, 5.18MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 143M/22.2G [00:32<54:56, 6.68MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 356M/22.2G [00:32<50:35, 7.18MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 187M/22.2G [00:32<1:32:12, 3.97MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 227M/22.2G [00:32<1:11:07, 5.14MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 145M/22.2G [00:32<57:03, 6.43MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 310M/22.2G [00:32<51:27, 7.08MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 359M/22.2G [00:32<41:57, 8.66MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 146M/22.2G [00:32<47:29, 7.72MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 229M/22.2G [00:33<1:08:21, 5.35MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 360M/22.2G [00:33<38:56, 9.33MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 189M/22.2G [00:33<1:31:54, 3.98MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 314M/22.2G [00:33<36:36, 9.95MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 148M/22.2G [00:33<49:55, 7.35MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 317M/22.2G [00:33<32:21, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 364M/22.2G [00:33<33:53, 10.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 231M/22.2G [00:33<1:06:02, 5.53MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 149M/22.2G [00:33<44:08, 8.31MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 319M/22.2G [00:33<30:20, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 190M/22.2G [00:33<1:30:22, 4.05MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 365M/22.2G [00:33<33:40, 10.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 152M/22.2G [00:33<35:09, 10.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 321M/22.2G [00:33<29:29, 12.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 233M/22.2G [00:33<1:04:29, 5.67MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 368M/22.2G [00:33<30:43, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 192M/22.2G [00:33<1:29:07, 4.11MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 324M/22.2G [00:33<26:29, 13.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 370M/22.2G [00:33<31:14, 11.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 234M/22.2G [00:33<59:11, 6.17MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 326M/22.2G [00:33<26:21, 13.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 373M/22.2G [00:34<29:12, 12.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 193M/22.2G [00:34<1:28:18, 4.14MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   1%|▏         | 330M/22.2G [00:34<24:43, 14.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 236M/22.2G [00:34<1:03:34, 5.75MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 375M/22.2G [00:34<29:57, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 155M/22.2G [00:34<1:02:03, 5.91MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 238M/22.2G [00:34<58:02, 6.29MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 159M/22.2G [00:34<45:06, 8.13MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 195M/22.2G [00:34<1:37:09, 3.77MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 239M/22.2G [00:34<58:02, 6.29MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 377M/22.2G [00:34<51:23, 7.06MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 161M/22.2G [00:34<41:32, 8.82MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 332M/22.2G [00:34<45:14, 8.04MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 240M/22.2G [00:34<58:11, 6.28MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 336M/22.2G [00:34<32:23, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 381M/22.2G [00:34<37:18, 9.73MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 196M/22.2G [00:35<1:38:44, 3.71MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 242M/22.2G [00:35<58:00, 6.30MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 338M/22.2G [00:35<32:04, 11.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 384M/22.2G [00:35<33:16, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 164M/22.2G [00:35<50:19, 7.28MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 243M/22.2G [00:35<58:00, 6.30MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 342M/22.2G [00:35<28:29, 12.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 386M/22.2G [00:35<32:41, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 197M/22.2G [00:35<1:38:21, 3.72MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 166M/22.2G [00:35<43:41, 8.39MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 345M/22.2G [00:35<25:40, 14.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 167M/22.2G [00:35<42:19, 8.66MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 389M/22.2G [00:35<30:32, 11.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 349M/22.2G [00:35<22:13, 16.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 199M/22.2G [00:35<1:35:11, 3.84MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 391M/22.2G [00:35<30:19, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 169M/22.2G [00:35<43:23, 8.45MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 351M/22.2G [00:35<23:11, 15.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 394M/22.2G [00:35<28:40, 12.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 354M/22.2G [00:36<22:33, 16.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 244M/22.2G [00:36<1:59:43, 3.05MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 397M/22.2G [00:36<27:48, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 172M/22.2G [00:36<52:40, 6.96MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 248M/22.2G [00:36<1:05:58, 5.53MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 356M/22.2G [00:36<35:55, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 174M/22.2G [00:36<54:06, 6.77MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 359M/22.2G [00:36<31:49, 11.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 249M/22.2G [00:36<1:01:35, 5.93MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 175M/22.2G [00:36<46:14, 7.92MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 399M/22.2G [00:36<44:28, 8.15MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 360M/22.2G [00:36<31:01, 11.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 251M/22.2G [00:36<59:22, 6.15MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 364M/22.2G [00:36<24:27, 14.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 199M/22.2G [00:36<3:09:56, 1.93MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 403M/22.2G [00:36<34:20, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 368M/22.2G [00:37<21:10, 17.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 253M/22.2G [00:37<57:46, 6.32MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 406M/22.2G [00:37<30:52, 11.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 370M/22.2G [00:37<20:13, 18.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 254M/22.2G [00:37<55:53, 6.53MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 373M/22.2G [00:37<19:28, 18.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 204M/22.2G [00:37<1:38:49, 3.70MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 410M/22.2G [00:37<29:06, 12.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 376M/22.2G [00:37<18:49, 19.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 256M/22.2G [00:37<53:52, 6.77MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 413M/22.2G [00:37<27:54, 13.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 205M/22.2G [00:37<1:35:09, 3.84MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 177M/22.2G [00:37<1:30:10, 4.06MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 258M/22.2G [00:37<52:31, 6.95MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 416M/22.2G [00:37<26:49, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 259M/22.2G [00:37<50:49, 7.18MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 378M/22.2G [00:37<35:30, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 207M/22.2G [00:38<1:33:00, 3.93MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 419M/22.2G [00:38<26:14, 13.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 381M/22.2G [00:38<31:57, 11.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 181M/22.2G [00:38<1:06:23, 5.52MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 261M/22.2G [00:38<49:05, 7.43MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 384M/22.2G [00:38<25:28, 14.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 184M/22.2G [00:38<54:18, 6.74MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 208M/22.2G [00:38<1:31:10, 4.01MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 263M/22.2G [00:38<47:49, 7.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 388M/22.2G [00:38<22:18, 16.3MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 391M/22.2G [00:38<20:14, 17.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 264M/22.2G [00:38<45:30, 8.02MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 187M/22.2G [00:38<44:20, 8.26MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 421M/22.2G [00:38<43:23, 8.35MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 394M/22.2G [00:38<18:53, 19.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 190M/22.2G [00:38<35:13, 10.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 210M/22.2G [00:38<1:28:49, 4.12MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 425M/22.2G [00:38<31:49, 11.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 398M/22.2G [00:38<17:44, 20.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 427M/22.2G [00:38<35:14, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 211M/22.2G [00:39<1:28:22, 4.14MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 430M/22.2G [00:39<29:07, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 193M/22.2G [00:39<42:48, 8.55MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 431M/22.2G [00:39<30:02, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 196M/22.2G [00:39<34:09, 10.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 266M/22.2G [00:39<1:30:06, 4.05MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 433M/22.2G [00:39<30:34, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 212M/22.2G [00:39<1:34:23, 3.87MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 400M/22.2G [00:39<41:02, 8.83MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 435M/22.2G [00:39<30:22, 11.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 199M/22.2G [00:39<36:41, 9.97MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 436M/22.2G [00:39<31:04, 11.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 270M/22.2G [00:39<57:26, 6.35MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 403M/22.2G [00:39<35:53, 10.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 214M/22.2G [00:39<1:37:45, 3.74MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 408M/22.2G [00:39<26:08, 13.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 439M/22.2G [00:39<29:09, 12.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 272M/22.2G [00:39<55:39, 6.55MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 411M/22.2G [00:40<22:19, 16.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 441M/22.2G [00:40<29:57, 12.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 273M/22.2G [00:40<55:12, 6.61MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 201M/22.2G [00:40<53:54, 6.79MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 415M/22.2G [00:40<19:45, 18.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 215M/22.2G [00:40<1:38:13, 3.72MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 419M/22.2G [00:40<17:46, 20.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 203M/22.2G [00:40<44:58, 8.14MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 275M/22.2G [00:40<54:35, 6.68MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|          | 276M/22.2G [00:40<53:36, 6.80MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 217M/22.2G [00:40<1:33:31, 3.91MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 207M/22.2G [00:40<43:54, 8.33MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 443M/22.2G [00:40<56:03, 6.45MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 278M/22.2G [00:40<52:17, 6.97MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 421M/22.2G [00:40<31:07, 11.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 210M/22.2G [00:40<37:18, 9.80MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 447M/22.2G [00:41<40:00, 9.04MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 425M/22.2G [00:41<25:22, 14.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 280M/22.2G [00:41<50:39, 7.20MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 218M/22.2G [00:41<1:41:04, 3.62MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 430M/22.2G [00:41<20:15, 17.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 450M/22.2G [00:41<34:46, 10.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 213M/22.2G [00:41<38:10, 9.58MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 281M/22.2G [00:41<49:36, 7.35MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 434M/22.2G [00:41<18:11, 19.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 452M/22.2G [00:41<34:04, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 438M/22.2G [00:41<15:55, 22.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 283M/22.2G [00:41<48:42, 7.48MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 455M/22.2G [00:41<31:19, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 216M/22.2G [00:41<39:19, 9.30MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 220M/22.2G [00:41<1:41:27, 3.60MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 441M/22.2G [00:41<18:40, 19.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 285M/22.2G [00:41<47:37, 7.65MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 459M/22.2G [00:41<28:51, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 219M/22.2G [00:41<35:35, 10.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 287M/22.2G [00:41<47:03, 7.75MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 462M/22.2G [00:42<27:37, 13.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 220M/22.2G [00:42<39:26, 9.27MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 222M/22.2G [00:42<1:45:23, 3.47MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 465M/22.2G [00:42<26:54, 13.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 288M/22.2G [00:42<1:27:35, 4.16MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 222M/22.2G [00:42<1:05:16, 5.60MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 443M/22.2G [00:42<52:54, 6.84MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 466M/22.2G [00:42<48:19, 7.48MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 292M/22.2G [00:42<52:52, 6.89MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 469M/22.2G [00:43<40:29, 8.93MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 447M/22.2G [00:43<43:11, 8.38MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 294M/22.2G [00:43<53:31, 6.81MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 222M/22.2G [00:43<3:20:19, 1.82MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 452M/22.2G [00:43<32:17, 11.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 226M/22.2G [00:43<57:09, 6.40MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 473M/22.2G [00:43<34:24, 10.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 455M/22.2G [00:43<27:56, 12.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 296M/22.2G [00:43<50:14, 7.25MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 228M/22.2G [00:43<49:10, 7.43MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 476M/22.2G [00:43<31:31, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 229M/22.2G [00:43<45:00, 8.12MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 457M/22.2G [00:43<29:36, 12.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 297M/22.2G [00:43<48:02, 7.58MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 479M/22.2G [00:43<27:05, 13.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 226M/22.2G [00:43<1:51:20, 3.28MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 231M/22.2G [00:43<45:50, 7.97MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 461M/22.2G [00:43<24:40, 14.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 481M/22.2G [00:43<28:09, 12.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 299M/22.2G [00:43<45:23, 8.02MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 464M/22.2G [00:43<23:48, 15.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 232M/22.2G [00:43<47:10, 7.74MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 301M/22.2G [00:44<44:46, 8.13MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 484M/22.2G [00:44<26:41, 13.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 227M/22.2G [00:44<1:49:33, 3.34MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 302M/22.2G [00:44<43:19, 8.41MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 234M/22.2G [00:44<51:05, 7.15MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 487M/22.2G [00:44<25:54, 13.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 236M/22.2G [00:44<44:08, 8.27MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 304M/22.2G [00:44<43:42, 8.33MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 229M/22.2G [00:44<1:48:56, 3.35MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 466M/22.2G [00:44<39:56, 9.05MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 306M/22.2G [00:44<41:37, 8.75MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 237M/22.2G [00:44<49:27, 7.39MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 489M/22.2G [00:44<38:39, 9.34MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 469M/22.2G [00:44<31:33, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 307M/22.2G [00:44<42:39, 8.53MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 239M/22.2G [00:44<46:54, 7.79MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 471M/22.2G [00:44<31:10, 11.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 492M/22.2G [00:44<32:53, 11.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 309M/22.2G [00:44<41:19, 8.81MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 230M/22.2G [00:44<1:46:57, 3.42MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 493M/22.2G [00:44<33:23, 10.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 242M/22.2G [00:44<37:26, 9.76MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 475M/22.2G [00:45<27:16, 13.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 496M/22.2G [00:45<30:04, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 243M/22.2G [00:45<42:50, 8.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 479M/22.2G [00:45<25:03, 14.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 499M/22.2G [00:45<25:39, 14.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 231M/22.2G [00:45<1:47:26, 3.40MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 482M/22.2G [00:45<23:29, 15.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 501M/22.2G [00:45<27:04, 13.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 310M/22.2G [00:45<1:17:08, 4.72MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 504M/22.2G [00:45<23:41, 15.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 486M/22.2G [00:45<22:21, 16.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 244M/22.2G [00:45<1:10:45, 5.16MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 233M/22.2G [00:45<1:46:14, 3.44MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 506M/22.2G [00:45<25:18, 14.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 314M/22.2G [00:45<46:46, 7.78MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 248M/22.2G [00:45<42:21, 8.62MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 509M/22.2G [00:45<23:20, 15.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 249M/22.2G [00:46<39:45, 9.18MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 316M/22.2G [00:46<48:01, 7.58MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 317M/22.2G [00:46<45:38, 7.97MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 234M/22.2G [00:46<1:45:28, 3.46MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 488M/22.2G [00:46<37:40, 9.58MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 252M/22.2G [00:46<39:16, 9.29MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 492M/22.2G [00:46<29:48, 12.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 319M/22.2G [00:46<44:46, 8.13MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 254M/22.2G [00:46<37:18, 9.78MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 493M/22.2G [00:46<29:30, 12.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 510M/22.2G [00:46<45:55, 7.85MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 321M/22.2G [00:46<43:22, 8.39MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 255M/22.2G [00:46<35:58, 10.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 236M/22.2G [00:46<1:43:27, 3.53MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 514M/22.2G [00:46<35:01, 10.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 497M/22.2G [00:46<25:43, 14.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 322M/22.2G [00:46<42:10, 8.63MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 258M/22.2G [00:46<31:36, 11.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 515M/22.2G [00:46<34:50, 10.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 501M/22.2G [00:46<22:36, 16.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 324M/22.2G [00:46<43:32, 8.36MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 503M/22.2G [00:46<23:07, 15.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 237M/22.2G [00:47<1:42:54, 3.55MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 259M/22.2G [00:47<38:33, 9.46MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 518M/22.2G [00:47<30:57, 11.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 325M/22.2G [00:47<42:20, 8.59MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 506M/22.2G [00:47<19:58, 18.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 261M/22.2G [00:47<36:09, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 522M/22.2G [00:47<26:19, 13.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 327M/22.2G [00:47<40:57, 8.88MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 523M/22.2G [00:47<27:13, 13.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 509M/22.2G [00:47<21:30, 16.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 239M/22.2G [00:47<1:39:21, 3.68MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 264M/22.2G [00:47<32:03, 11.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 329M/22.2G [00:47<41:58, 8.67MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 526M/22.2G [00:47<23:56, 15.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 330M/22.2G [00:47<41:08, 8.84MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 266M/22.2G [00:47<37:47, 9.65MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 528M/22.2G [00:47<25:12, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 511M/22.2G [00:47<32:02, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   1%|▏         | 331M/22.2G [00:47<40:41, 8.94MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 240M/22.2G [00:47<1:40:22, 3.64MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 531M/22.2G [00:47<22:50, 15.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 514M/22.2G [00:47<28:43, 12.6MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 517M/22.2G [00:48<27:02, 13.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 242M/22.2G [00:48<1:37:23, 3.75MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 267M/22.2G [00:48<1:03:12, 5.77MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 521M/22.2G [00:48<21:40, 16.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 533M/22.2G [00:48<41:14, 8.74MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 523M/22.2G [00:48<22:34, 16.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 270M/22.2G [00:48<50:19, 7.25MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 536M/22.2G [00:48<32:42, 11.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 332M/22.2G [00:48<1:27:28, 4.16MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 243M/22.2G [00:48<1:35:10, 3.84MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 526M/22.2G [00:48<20:51, 17.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 271M/22.2G [00:48<46:04, 7.92MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 537M/22.2G [00:48<32:54, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 336M/22.2G [00:48<49:19, 7.37MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 529M/22.2G [00:48<21:19, 16.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 273M/22.2G [00:48<46:48, 7.79MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 541M/22.2G [00:48<29:06, 12.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 338M/22.2G [00:48<50:04, 7.26MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 544M/22.2G [00:48<25:10, 14.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 274M/22.2G [00:48<50:33, 7.21MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 546M/22.2G [00:49<26:22, 13.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 340M/22.2G [00:49<47:14, 7.70MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|          | 276M/22.2G [00:49<50:48, 7.18MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 532M/22.2G [00:49<38:56, 9.26MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 549M/22.2G [00:49<25:23, 14.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 342M/22.2G [00:49<45:19, 8.02MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 279M/22.2G [00:49<40:02, 9.11MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 536M/22.2G [00:49<28:58, 12.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 552M/22.2G [00:49<22:46, 15.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 344M/22.2G [00:49<42:30, 8.55MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 244M/22.2G [00:49<3:02:14, 2.00MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   2%|▏         | 554M/22.2G [00:49<24:23, 14.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 280M/22.2G [00:49<43:41, 8.34MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 345M/22.2G [00:49<43:03, 8.44MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 539M/22.2G [00:49<29:28, 12.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 282M/22.2G [00:49<41:41, 8.74MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 347M/22.2G [00:49<41:44, 8.71MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 544M/22.2G [00:49<24:27, 14.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 248M/22.2G [00:49<1:35:06, 3.84MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 284M/22.2G [00:50<42:53, 8.50MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 555M/22.2G [00:50<42:05, 8.55MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 349M/22.2G [00:50<42:06, 8.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 547M/22.2G [00:50<24:42, 14.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 285M/22.2G [00:50<41:13, 8.84MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 558M/22.2G [00:50<34:18, 10.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 350M/22.2G [00:50<40:32, 8.96MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 249M/22.2G [00:50<1:33:00, 3.93MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 560M/22.2G [00:50<34:13, 10.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 550M/22.2G [00:50<24:48, 14.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 352M/22.2G [00:50<41:04, 8.85MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 288M/22.2G [00:50<39:35, 9.21MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 563M/22.2G [00:50<29:45, 12.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   2%|▏         | 553M/22.2G [00:50<24:43, 14.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 354M/22.2G [00:50<40:01, 9.08MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 251M/22.2G [00:50<1:31:20, 4.00MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 566M/22.2G [00:50<25:17, 14.2MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 568M/22.2G [00:50<26:41, 13.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 289M/22.2G [00:51<1:07:01, 5.44MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 252M/22.2G [00:51<1:29:49, 4.06MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 571M/22.2G [00:51<23:44, 15.2MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 572M/22.2G [00:51<25:30, 14.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 292M/22.2G [00:51<47:12, 7.72MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 355M/22.2G [00:51<1:23:56, 4.33MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 294M/22.2G [00:51<42:12, 8.63MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 576M/22.2G [00:51<22:06, 16.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 254M/22.2G [00:51<1:28:32, 4.12MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 359M/22.2G [00:51<48:52, 7.43MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 297M/22.2G [00:51<35:04, 10.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 555M/22.2G [00:51<56:58, 6.32MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 255M/22.2G [00:51<1:24:08, 4.34MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 360M/22.2G [00:51<45:24, 8.00MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 558M/22.2G [00:51<45:48, 7.86MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 298M/22.2G [00:51<41:14, 8.83MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 577M/22.2G [00:51<39:18, 9.15MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 362M/22.2G [00:51<43:56, 8.27MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 560M/22.2G [00:51<42:29, 8.47MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 300M/22.2G [00:51<38:13, 9.53MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 257M/22.2G [00:51<1:19:01, 4.62MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 580M/22.2G [00:51<32:54, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 363M/22.2G [00:51<42:06, 8.63MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 301M/22.2G [00:52<35:50, 10.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 563M/22.2G [00:52<35:48, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 582M/22.2G [00:52<33:07, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 365M/22.2G [00:52<40:42, 8.92MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 303M/22.2G [00:52<34:32, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 258M/22.2G [00:52<1:16:01, 4.80MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 366M/22.2G [00:52<40:13, 9.03MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 566M/22.2G [00:52<31:44, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 305M/22.2G [00:52<36:03, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 585M/22.2G [00:52<28:46, 12.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 367M/22.2G [00:52<39:13, 9.26MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 568M/22.2G [00:52<30:20, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 588M/22.2G [00:52<24:35, 14.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 260M/22.2G [00:52<1:14:56, 4.87MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 306M/22.2G [00:52<37:24, 9.74MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 369M/22.2G [00:52<38:53, 9.34MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 590M/22.2G [00:52<25:58, 13.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 572M/22.2G [00:52<27:19, 13.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 309M/22.2G [00:52<32:25, 11.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 370M/22.2G [00:52<38:19, 9.48MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 261M/22.2G [00:52<1:12:09, 5.06MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 593M/22.2G [00:52<25:32, 14.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 372M/22.2G [00:52<37:52, 9.59MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 575M/22.2G [00:52<25:47, 13.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 263M/22.2G [00:52<1:06:18, 5.50MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 596M/22.2G [00:53<23:00, 15.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 373M/22.2G [00:53<37:52, 9.59MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 598M/22.2G [00:53<24:22, 14.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 375M/22.2G [00:53<37:30, 9.68MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 264M/22.2G [00:53<1:06:04, 5.52MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 376M/22.2G [00:53<36:44, 9.88MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 577M/22.2G [00:53<43:05, 8.35MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 310M/22.2G [00:53<1:10:22, 5.17MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 265M/22.2G [00:53<1:02:54, 5.80MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 580M/22.2G [00:53<33:02, 10.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 600M/22.2G [00:53<41:01, 8.76MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 314M/22.2G [00:53<46:37, 7.81MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 582M/22.2G [00:53<33:04, 10.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 602M/22.2G [00:53<33:50, 10.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 316M/22.2G [00:53<43:07, 8.44MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 377M/22.2G [00:53<1:13:35, 4.93MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 604M/22.2G [00:53<33:28, 10.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 585M/22.2G [00:53<28:56, 12.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 607M/22.2G [00:54<26:51, 13.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 317M/22.2G [00:54<46:48, 7.78MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 381M/22.2G [00:54<47:45, 7.60MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 589M/22.2G [00:54<26:36, 13.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 609M/22.2G [00:54<28:11, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 266M/22.2G [00:54<1:59:37, 3.05MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 319M/22.2G [00:54<45:57, 7.92MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 383M/22.2G [00:54<42:51, 8.47MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 612M/22.2G [00:54<23:49, 15.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 592M/22.2G [00:54<25:07, 14.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 384M/22.2G [00:54<40:25, 8.98MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 270M/22.2G [00:54<1:05:39, 5.56MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 614M/22.2G [00:54<25:25, 14.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 320M/22.2G [00:54<52:22, 6.95MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 386M/22.2G [00:54<38:34, 9.41MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 596M/22.2G [00:54<24:14, 14.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 617M/22.2G [00:54<22:23, 16.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 322M/22.2G [00:54<46:33, 7.82MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 272M/22.2G [00:54<1:00:58, 5.98MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 387M/22.2G [00:54<37:14, 9.74MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 618M/22.2G [00:54<24:07, 14.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 323M/22.2G [00:54<42:21, 8.59MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 273M/22.2G [00:54<54:41, 6.67MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 389M/22.2G [00:54<35:53, 10.1MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 325M/22.2G [00:54<39:11, 9.29MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 391M/22.2G [00:55<34:54, 10.4MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 274M/22.2G [00:55<54:03, 6.75MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 598M/22.2G [00:55<35:49, 10.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 326M/22.2G [00:55<43:26, 8.38MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 392M/22.2G [00:55<34:02, 10.7MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|          | 276M/22.2G [00:55<49:47, 7.32MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 602M/22.2G [00:55<27:26, 13.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 328M/22.2G [00:55<39:31, 9.21MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 394M/22.2G [00:55<33:08, 10.9MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 277M/22.2G [00:55<46:10, 7.90MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 604M/22.2G [00:55<27:47, 12.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 620M/22.2G [00:55<48:20, 7.42MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 330M/22.2G [00:55<37:07, 9.80MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 396M/22.2G [00:55<34:37, 10.5MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 279M/22.2G [00:55<46:53, 7.78MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 624M/22.2G [00:55<32:28, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 607M/22.2G [00:55<26:08, 13.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 398M/22.2G [00:55<31:44, 11.4MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   1%|▏         | 331M/22.2G [00:55<42:29, 8.56MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 280M/22.2G [00:55<43:51, 8.31MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 626M/22.2G [00:55<32:24, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 282M/22.2G [00:55<41:18, 8.82MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 629M/22.2G [00:55<26:31, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 611M/22.2G [00:55<26:59, 13.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 283M/22.2G [00:55<39:11, 9.30MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 631M/22.2G [00:55<27:20, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 614M/22.2G [00:56<24:41, 14.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 285M/22.2G [00:56<38:10, 9.55MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 634M/22.2G [00:56<23:26, 15.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 286M/22.2G [00:56<37:02, 9.84MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 617M/22.2G [00:56<25:22, 14.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 636M/22.2G [00:56<24:47, 14.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 399M/22.2G [00:56<1:12:30, 5.00MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 332M/22.2G [00:56<1:23:02, 4.38MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 288M/22.2G [00:56<35:42, 10.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 618M/22.2G [00:56<26:33, 13.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 639M/22.2G [00:56<22:12, 16.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 620M/22.2G [00:56<28:18, 12.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 403M/22.2G [00:56<44:48, 8.09MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 336M/22.2G [00:56<50:27, 7.21MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 642M/22.2G [00:56<23:12, 15.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 405M/22.2G [00:56<39:10, 9.25MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 338M/22.2G [00:56<45:56, 7.92MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 407M/22.2G [00:56<36:49, 9.84MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 289M/22.2G [00:56<1:07:22, 5.41MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 409M/22.2G [00:56<33:50, 10.7MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 621M/22.2G [00:57<50:06, 7.16MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 340M/22.2G [00:57<50:52, 7.15MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 292M/22.2G [00:57<43:44, 8.33MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 644M/22.2G [00:57<38:54, 9.22MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 411M/22.2G [00:57<31:53, 11.4MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 294M/22.2G [00:57<40:55, 8.90MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 624M/22.2G [00:57<37:48, 9.49MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 647M/22.2G [00:57<32:40, 11.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 413M/22.2G [00:57<30:31, 11.9MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 341M/22.2G [00:57<56:59, 6.38MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 627M/22.2G [00:57<34:11, 10.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 648M/22.2G [00:57<32:04, 11.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 297M/22.2G [00:57<34:41, 10.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 343M/22.2G [00:57<50:53, 7.14MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 416M/22.2G [00:57<28:51, 12.6MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 628M/22.2G [00:57<32:26, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 652M/22.2G [00:57<25:24, 14.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 344M/22.2G [00:57<45:39, 7.96MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 300M/22.2G [00:57<31:42, 11.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 630M/22.2G [00:57<31:28, 11.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 654M/22.2G [00:57<26:35, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 419M/22.2G [00:57<31:04, 11.7MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 346M/22.2G [00:57<45:41, 7.96MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 632M/22.2G [00:57<34:18, 10.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 302M/22.2G [00:57<32:50, 11.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 657M/22.2G [00:57<26:11, 13.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 633M/22.2G [00:58<34:40, 10.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 660M/22.2G [00:58<22:48, 15.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 305M/22.2G [00:58<31:12, 11.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 635M/22.2G [00:58<35:18, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 663M/22.2G [00:58<23:58, 14.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 636M/22.2G [00:58<35:26, 10.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 307M/22.2G [00:58<33:14, 11.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 421M/22.2G [00:58<51:46, 7.00MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 665M/22.2G [00:58<25:36, 14.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 638M/22.2G [00:58<35:27, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 347M/22.2G [00:58<1:21:11, 4.48MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 309M/22.2G [00:58<37:02, 9.83MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 639M/22.2G [00:58<35:42, 10.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 425M/22.2G [00:58<38:37, 9.38MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 310M/22.2G [00:58<36:36, 9.95MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 641M/22.2G [00:58<35:33, 10.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 427M/22.2G [00:58<36:20, 9.97MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 642M/22.2G [00:58<35:31, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 349M/22.2G [00:58<1:25:45, 4.24MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 666M/22.2G [00:58<43:51, 8.17MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 429M/22.2G [00:58<35:31, 10.2MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 669M/22.2G [00:59<35:02, 10.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 430M/22.2G [00:59<35:56, 10.1MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 670M/22.2G [00:59<35:24, 10.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 350M/22.2G [00:59<1:22:56, 4.38MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 432M/22.2G [00:59<34:45, 10.4MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 433M/22.2G [00:59<32:40, 11.1MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 673M/22.2G [00:59<31:22, 11.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 311M/22.2G [00:59<1:19:12, 4.60MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 435M/22.2G [00:59<32:24, 11.2MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 352M/22.2G [00:59<1:21:46, 4.44MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 643M/22.2G [00:59<1:24:48, 4.23MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 436M/22.2G [00:59<33:01, 11.0MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 676M/22.2G [00:59<29:08, 12.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 314M/22.2G [00:59<58:01, 6.27MB/s]  [Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 647M/22.2G [00:59<53:05, 6.75MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 439M/22.2G [00:59<30:59, 11.7MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 316M/22.2G [00:59<53:28, 6.81MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 680M/22.2G [00:59<27:34, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 353M/22.2G [00:59<1:22:04, 4.43MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 648M/22.2G [00:59<46:44, 7.67MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 441M/22.2G [01:00<30:25, 11.9MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 317M/22.2G [01:00<52:12, 6.97MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 683M/22.2G [01:00<26:06, 13.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 650M/22.2G [01:00<44:04, 8.13MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 443M/22.2G [01:00<30:06, 12.0MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 319M/22.2G [01:00<47:23, 7.68MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 651M/22.2G [01:00<41:38, 8.61MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 686M/22.2G [01:00<25:26, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 653M/22.2G [01:00<39:44, 9.02MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 320M/22.2G [01:00<48:44, 7.47MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 654M/22.2G [01:00<38:19, 9.35MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 322M/22.2G [01:00<45:10, 8.05MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 444M/22.2G [01:00<51:52, 6.98MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 656M/22.2G [01:00<37:21, 9.59MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 323M/22.2G [01:00<46:24, 7.84MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 354M/22.2G [01:00<2:08:33, 2.83MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 447M/22.2G [01:00<40:11, 9.00MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 688M/22.2G [01:00<42:39, 8.39MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 657M/22.2G [01:00<36:12, 9.90MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 325M/22.2G [01:00<43:47, 8.31MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 449M/22.2G [01:00<35:15, 10.3MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 659M/22.2G [01:00<35:44, 10.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 691M/22.2G [01:00<34:11, 10.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 326M/22.2G [01:01<45:36, 7.98MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 660M/22.2G [01:01<34:58, 10.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 451M/22.2G [01:01<34:07, 10.6MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 692M/22.2G [01:01<34:08, 10.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 359M/22.2G [01:01<1:16:24, 4.75MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 662M/22.2G [01:01<34:54, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 328M/22.2G [01:01<47:30, 7.66MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 453M/22.2G [01:01<36:24, 9.93MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 696M/22.2G [01:01<29:47, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 663M/22.2G [01:01<34:28, 10.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 456M/22.2G [01:01<31:05, 11.6MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 699M/22.2G [01:01<25:19, 14.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 360M/22.2G [01:01<1:14:43, 4.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 329M/22.2G [01:01<51:39, 7.04MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 701M/22.2G [01:01<26:31, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 457M/22.2G [01:01<32:21, 11.2MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 704M/22.2G [01:01<23:18, 15.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 459M/22.2G [01:01<33:44, 10.7MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 331M/22.2G [01:01<52:06, 6.98MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 362M/22.2G [01:01<1:14:50, 4.85MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 706M/22.2G [01:01<24:57, 14.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 460M/22.2G [01:01<34:27, 10.5MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   1%|▏         | 332M/22.2G [01:02<51:26, 7.07MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 709M/22.2G [01:02<21:57, 16.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 461M/22.2G [01:02<35:28, 10.2MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 665M/22.2G [01:02<1:15:50, 4.72MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 363M/22.2G [01:02<1:15:52, 4.79MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 463M/22.2G [01:02<35:49, 10.1MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 669M/22.2G [01:02<45:09, 7.93MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 365M/22.2G [01:02<1:08:04, 5.33MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 464M/22.2G [01:02<35:34, 10.2MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 670M/22.2G [01:02<46:42, 7.67MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 367M/22.2G [01:02<1:01:59, 5.86MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 368M/22.2G [01:02<55:05, 6.59MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 672M/22.2G [01:02<48:19, 7.41MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 333M/22.2G [01:02<1:54:35, 3.17MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 369M/22.2G [01:02<54:21, 6.68MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 673M/22.2G [01:02<49:14, 7.27MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 465M/22.2G [01:03<1:18:30, 4.60MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 371M/22.2G [01:03<52:09, 6.96MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 336M/22.2G [01:03<1:08:56, 5.28MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 675M/22.2G [01:03<50:00, 7.16MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 372M/22.2G [01:03<47:22, 7.66MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 469M/22.2G [01:03<49:08, 7.36MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 338M/22.2G [01:03<1:03:38, 5.71MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 676M/22.2G [01:03<49:59, 7.16MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 374M/22.2G [01:03<47:52, 7.58MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 339M/22.2G [01:03<1:02:08, 5.85MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 471M/22.2G [01:03<51:57, 6.95MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 678M/22.2G [01:03<49:55, 7.17MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 375M/22.2G [01:03<47:33, 7.63MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 474M/22.2G [01:03<41:42, 8.66MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 341M/22.2G [01:03<59:33, 6.10MB/s]  [Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 679M/22.2G [01:03<49:47, 7.19MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 475M/22.2G [01:03<44:43, 8.08MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 342M/22.2G [01:04<58:07, 6.25MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 681M/22.2G [01:04<49:14, 7.27MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 477M/22.2G [01:04<45:50, 7.88MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 344M/22.2G [01:04<56:23, 6.45MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 682M/22.2G [01:04<48:46, 7.34MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 711M/22.2G [01:04<1:58:23, 3.02MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 479M/22.2G [01:04<46:05, 7.84MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 345M/22.2G [01:04<56:07, 6.48MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 684M/22.2G [01:04<48:59, 7.30MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 377M/22.2G [01:04<1:40:43, 3.60MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 347M/22.2G [01:04<54:10, 6.71MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 480M/22.2G [01:04<45:42, 7.90MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 686M/22.2G [01:04<48:39, 7.35MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 381M/22.2G [01:04<58:27, 6.21MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 713M/22.2G [01:04<1:43:20, 3.46MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 348M/22.2G [01:04<53:43, 6.76MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 482M/22.2G [01:04<45:47, 7.89MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 715M/22.2G [01:04<1:26:05, 4.15MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 382M/22.2G [01:04<54:47, 6.62MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 350M/22.2G [01:05<52:42, 6.90MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 718M/22.2G [01:05<58:47, 6.08MB/s]  [A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 484M/22.2G [01:05<45:46, 7.89MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 384M/22.2G [01:05<53:48, 6.74MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 720M/22.2G [01:05<52:56, 6.75MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 351M/22.2G [01:05<51:38, 7.04MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 385M/22.2G [01:05<48:33, 7.47MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 723M/22.2G [01:05<39:40, 9.00MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 486M/22.2G [01:05<49:35, 7.28MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 353M/22.2G [01:05<51:44, 7.02MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 687M/22.2G [01:05<1:37:47, 3.66MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 487M/22.2G [01:05<46:35, 7.75MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 726M/22.2G [01:05<33:43, 10.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 354M/22.2G [01:05<50:18, 7.22MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 387M/22.2G [01:05<1:02:25, 5.81MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 729M/22.2G [01:05<28:42, 12.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 691M/22.2G [01:05<57:59, 6.17MB/s]  bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 693M/22.2G [01:06<53:46, 6.65MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 388M/22.2G [01:06<1:08:20, 5.31MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 694M/22.2G [01:06<51:47, 6.90MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 488M/22.2G [01:06<1:35:45, 3.77MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 731M/22.2G [01:06<47:30, 7.52MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 390M/22.2G [01:06<1:10:18, 5.16MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 355M/22.2G [01:06<1:43:43, 3.50MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 696M/22.2G [01:06<49:42, 7.19MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 391M/22.2G [01:06<1:01:19, 5.92MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 492M/22.2G [01:06<1:03:25, 5.69MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 735M/22.2G [01:06<38:43, 9.22MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 698M/22.2G [01:06<48:40, 7.35MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 359M/22.2G [01:06<1:04:47, 5.61MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 739M/22.2G [01:06<31:19, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 493M/22.2G [01:06<1:00:08, 6.00MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 393M/22.2G [01:06<1:06:09, 5.48MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 700M/22.2G [01:06<47:22, 7.55MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 360M/22.2G [01:06<1:00:51, 5.97MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 740M/22.2G [01:06<31:07, 11.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 495M/22.2G [01:07<58:47, 6.14MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 701M/22.2G [01:07<47:22, 7.55MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 742M/22.2G [01:07<31:51, 11.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 362M/22.2G [01:07<57:42, 6.29MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 394M/22.2G [01:07<1:09:16, 5.24MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 703M/22.2G [01:07<44:11, 8.09MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 496M/22.2G [01:07<58:33, 6.16MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 745M/22.2G [01:07<30:11, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 363M/22.2G [01:07<55:00, 6.60MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 704M/22.2G [01:07<46:18, 7.72MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 746M/22.2G [01:07<31:29, 11.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 498M/22.2G [01:07<56:52, 6.35MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 396M/22.2G [01:07<1:13:19, 4.95MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 365M/22.2G [01:07<53:51, 6.74MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 706M/22.2G [01:07<45:43, 7.82MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 398M/22.2G [01:07<1:00:25, 6.00MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 749M/22.2G [01:07<29:21, 12.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 499M/22.2G [01:07<56:14, 6.42MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 366M/22.2G [01:07<52:32, 6.91MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 751M/22.2G [01:07<30:03, 11.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 708M/22.2G [01:07<45:42, 7.82MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 501M/22.2G [01:07<55:10, 6.54MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 368M/22.2G [01:08<51:42, 7.02MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 753M/22.2G [01:08<31:10, 11.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 369M/22.2G [01:08<52:07, 6.97MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 502M/22.2G [01:08<55:23, 6.52MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 371M/22.2G [01:08<47:25, 7.65MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 504M/22.2G [01:08<54:32, 6.62MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 399M/22.2G [01:08<1:41:38, 3.57MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 754M/22.2G [01:08<52:29, 6.80MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 372M/22.2G [01:08<49:02, 7.40MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 709M/22.2G [01:08<1:21:56, 4.36MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 505M/22.2G [01:08<54:01, 6.68MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 757M/22.2G [01:08<38:16, 9.32MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 403M/22.2G [01:08<58:43, 6.17MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 374M/22.2G [01:08<50:20, 7.21MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 404M/22.2G [01:08<52:48, 6.86MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 507M/22.2G [01:08<52:56, 6.81MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 713M/22.2G [01:08<51:48, 6.90MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 761M/22.2G [01:08<33:23, 10.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 375M/22.2G [01:08<50:35, 7.17MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 406M/22.2G [01:09<51:25, 7.05MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 762M/22.2G [01:09<31:56, 11.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 508M/22.2G [01:09<52:48, 6.83MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 715M/22.2G [01:09<49:28, 7.22MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 377M/22.2G [01:09<50:03, 7.25MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 407M/22.2G [01:09<47:32, 7.63MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 764M/22.2G [01:09<32:01, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 717M/22.2G [01:09<48:06, 7.43MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 409M/22.2G [01:09<47:20, 7.66MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 767M/22.2G [01:09<29:02, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 410M/22.2G [01:09<43:36, 8.31MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 718M/22.2G [01:09<47:11, 7.57MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 769M/22.2G [01:09<29:30, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 412M/22.2G [01:09<40:20, 8.98MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 720M/22.2G [01:09<46:22, 7.70MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 377M/22.2G [01:09<1:30:59, 3.99MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 772M/22.2G [01:09<27:47, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 413M/22.2G [01:09<42:30, 8.52MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 510M/22.2G [01:09<1:34:27, 3.82MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 774M/22.2G [01:09<27:07, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 722M/22.2G [01:09<46:14, 7.73MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 415M/22.2G [01:09<40:25, 8.96MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 381M/22.2G [01:10<57:42, 6.29MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   3%|▎         | 775M/22.2G [01:10<27:19, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 416M/22.2G [01:10<38:01, 9.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 723M/22.2G [01:10<46:07, 7.74MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 514M/22.2G [01:10<1:00:44, 5.94MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 382M/22.2G [01:10<55:44, 6.51MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 418M/22.2G [01:10<40:15, 9.00MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 725M/22.2G [01:10<46:26, 7.69MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 419M/22.2G [01:10<38:09, 9.49MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 516M/22.2G [01:10<56:29, 6.39MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 384M/22.2G [01:10<54:53, 6.61MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 727M/22.2G [01:10<43:08, 8.28MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 777M/22.2G [01:10<52:30, 6.79MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 421M/22.2G [01:10<36:17, 9.98MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 385M/22.2G [01:10<52:45, 6.88MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 517M/22.2G [01:10<55:01, 6.55MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 728M/22.2G [01:10<45:18, 7.88MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 780M/22.2G [01:10<41:50, 8.52MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 387M/22.2G [01:10<49:26, 7.34MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 519M/22.2G [01:10<54:23, 6.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 730M/22.2G [01:10<45:58, 7.77MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 783M/22.2G [01:11<35:00, 10.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 388M/22.2G [01:11<51:58, 6.98MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 731M/22.2G [01:11<43:29, 8.21MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 520M/22.2G [01:11<53:09, 6.78MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 785M/22.2G [01:11<33:27, 10.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 422M/22.2G [01:11<1:16:09, 4.76MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 390M/22.2G [01:11<51:59, 6.98MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 787M/22.2G [01:11<28:35, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 425M/22.2G [01:11<48:26, 7.48MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 522M/22.2G [01:11<57:09, 6.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 789M/22.2G [01:11<29:44, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 391M/22.2G [01:11<51:00, 7.11MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 427M/22.2G [01:11<47:01, 7.70MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 393M/22.2G [01:11<49:29, 7.33MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 524M/22.2G [01:11<55:18, 6.52MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 792M/22.2G [01:11<28:23, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 430M/22.2G [01:11<39:03, 9.27MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 732M/22.2G [01:11<1:29:48, 3.98MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 394M/22.2G [01:11<48:47, 7.43MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 795M/22.2G [01:11<27:24, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 431M/22.2G [01:11<37:41, 9.61MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 525M/22.2G [01:11<1:01:51, 5.83MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 396M/22.2G [01:12<47:49, 7.58MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 432M/22.2G [01:12<36:36, 9.89MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 735M/22.2G [01:12<59:08, 6.04MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 434M/22.2G [01:12<35:11, 10.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 397M/22.2G [01:12<46:29, 7.80MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 527M/22.2G [01:12<1:03:23, 5.69MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 737M/22.2G [01:12<54:35, 6.54MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 436M/22.2G [01:12<34:03, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 398M/22.2G [01:12<47:36, 7.62MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 437M/22.2G [01:12<33:03, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 798M/22.2G [01:12<44:41, 7.97MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 739M/22.2G [01:12<51:07, 6.98MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 439M/22.2G [01:12<32:43, 11.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 528M/22.2G [01:12<1:03:07, 5.71MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 440M/22.2G [01:12<32:37, 11.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 741M/22.2G [01:12<48:33, 7.35MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 802M/22.2G [01:12<34:24, 10.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 530M/22.2G [01:12<1:01:13, 5.89MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 442M/22.2G [01:12<31:46, 11.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 742M/22.2G [01:12<47:25, 7.53MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 805M/22.2G [01:12<30:54, 11.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 531M/22.2G [01:13<1:00:04, 6.00MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 399M/22.2G [01:13<1:36:19, 3.76MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 807M/22.2G [01:13<30:42, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 744M/22.2G [01:13<43:21, 8.23MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 746M/22.2G [01:13<44:30, 8.02MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 810M/22.2G [01:13<28:38, 12.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 403M/22.2G [01:13<57:56, 6.26MB/s]  [Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 747M/22.2G [01:13<41:29, 8.60MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 404M/22.2G [01:13<51:59, 6.97MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 813M/22.2G [01:13<27:10, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 749M/22.2G [01:13<44:12, 8.07MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 406M/22.2G [01:13<51:38, 7.02MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 443M/22.2G [01:13<1:24:34, 4.28MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 816M/22.2G [01:13<26:40, 13.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 750M/22.2G [01:13<40:43, 8.76MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 407M/22.2G [01:13<46:54, 7.73MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 532M/22.2G [01:13<1:58:54, 3.03MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 447M/22.2G [01:13<49:37, 7.29MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 819M/22.2G [01:14<26:19, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 752M/22.2G [01:14<43:03, 8.28MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 409M/22.2G [01:14<48:11, 7.52MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 536M/22.2G [01:14<1:07:32, 5.33MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 410M/22.2G [01:14<44:47, 8.09MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 449M/22.2G [01:14<52:10, 6.93MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 537M/22.2G [01:14<1:03:17, 5.69MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 412M/22.2G [01:14<44:31, 8.14MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 452M/22.2G [01:14<41:31, 8.71MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 414M/22.2G [01:14<41:39, 8.70MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 539M/22.2G [01:14<1:00:32, 5.95MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 455M/22.2G [01:14<35:19, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 415M/22.2G [01:14<40:55, 8.86MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 821M/22.2G [01:14<51:59, 6.84MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 753M/22.2G [01:14<1:22:07, 4.34MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 458M/22.2G [01:14<31:00, 11.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 540M/22.2G [01:14<58:57, 6.11MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 416M/22.2G [01:14<40:24, 8.97MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 824M/22.2G [01:14<43:27, 8.18MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 461M/22.2G [01:14<27:09, 13.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 757M/22.2G [01:14<48:29, 7.35MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 418M/22.2G [01:15<41:09, 8.80MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 542M/22.2G [01:15<57:04, 6.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 825M/22.2G [01:15<41:19, 8.60MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 419M/22.2G [01:15<39:15, 9.23MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 462M/22.2G [01:15<31:51, 11.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 759M/22.2G [01:15<48:39, 7.33MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 544M/22.2G [01:15<55:47, 6.46MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 421M/22.2G [01:15<37:49, 9.58MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▎         | 829M/22.2G [01:15<35:37, 9.98MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 761M/22.2G [01:15<43:38, 8.17MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 762M/22.2G [01:15<40:54, 8.72MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 545M/22.2G [01:15<54:11, 6.65MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 832M/22.2G [01:15<32:12, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 764M/22.2G [01:15<39:46, 8.96MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 547M/22.2G [01:15<53:34, 6.72MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 835M/22.2G [01:15<29:50, 11.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 765M/22.2G [01:15<38:45, 9.20MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 422M/22.2G [01:15<1:15:07, 4.82MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 767M/22.2G [01:15<37:53, 9.41MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 548M/22.2G [01:15<52:33, 6.85MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 838M/22.2G [01:15<28:27, 12.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 768M/22.2G [01:16<37:18, 9.56MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 425M/22.2G [01:16<48:37, 7.45MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 465M/22.2G [01:16<1:02:03, 5.83MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 550M/22.2G [01:16<51:48, 6.95MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 841M/22.2G [01:16<27:34, 12.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 769M/22.2G [01:16<37:00, 9.63MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 427M/22.2G [01:16<43:49, 8.26MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 469M/22.2G [01:16<42:17, 8.55MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 771M/22.2G [01:16<35:53, 9.93MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 428M/22.2G [01:16<41:31, 8.72MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 551M/22.2G [01:16<52:30, 6.86MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 772M/22.2G [01:16<35:22, 10.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 430M/22.2G [01:16<39:06, 9.26MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 472M/22.2G [01:16<38:12, 9.46MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   2%|▏         | 553M/22.2G [01:16<51:22, 7.01MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   3%|▎         | 774M/22.2G [01:16<34:41, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 431M/22.2G [01:16<39:03, 9.27MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 842M/22.2G [01:16<45:36, 7.79MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 475M/22.2G [01:16<33:40, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 775M/22.2G [01:16<34:16, 10.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 433M/22.2G [01:16<37:31, 9.65MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 477M/22.2G [01:16<33:19, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 434M/22.2G [01:16<36:31, 9.91MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 846M/22.2G [01:16<36:00, 9.86MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 848M/22.2G [01:17<33:20, 10.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 437M/22.2G [01:17<32:57, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 481M/22.2G [01:17<32:21, 11.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 852M/22.2G [01:17<29:47, 11.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 439M/22.2G [01:17<34:32, 10.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 776M/22.2G [01:17<1:09:26, 5.13MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 483M/22.2G [01:17<33:26, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 440M/22.2G [01:17<33:38, 10.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 554M/22.2G [01:17<1:52:41, 3.19MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 484M/22.2G [01:17<33:36, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 855M/22.2G [01:17<28:15, 12.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 780M/22.2G [01:17<47:34, 7.49MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 442M/22.2G [01:17<32:05, 11.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 486M/22.2G [01:17<31:07, 11.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 781M/22.2G [01:17<41:39, 8.55MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 858M/22.2G [01:17<27:23, 13.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 783M/22.2G [01:17<38:35, 9.23MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 558M/22.2G [01:17<1:07:49, 5.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 861M/22.2G [01:17<26:38, 13.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 785M/22.2G [01:17<36:16, 9.82MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 443M/22.2G [01:18<1:00:02, 6.03MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 787M/22.2G [01:18<38:30, 9.25MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 864M/22.2G [01:18<26:06, 13.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 560M/22.2G [01:18<1:06:37, 5.40MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 487M/22.2G [01:18<54:58, 6.57MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 447M/22.2G [01:18<38:28, 9.40MB/s]  [Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 789M/22.2G [01:18<33:10, 10.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 492M/22.2G [01:18<34:39, 10.4MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 493M/22.2G [01:18<33:36, 10.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 561M/22.2G [01:18<1:05:56, 5.46MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 450M/22.2G [01:18<33:31, 10.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 790M/22.2G [01:18<37:35, 9.47MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 865M/22.2G [01:18<37:53, 9.36MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 496M/22.2G [01:18<30:43, 11.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 792M/22.2G [01:18<36:24, 9.78MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 453M/22.2G [01:18<30:33, 11.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 563M/22.2G [01:18<1:05:29, 5.49MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 868M/22.2G [01:18<35:03, 10.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 499M/22.2G [01:18<26:01, 13.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 793M/22.2G [01:18<35:06, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 501M/22.2G [01:18<26:07, 13.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 870M/22.2G [01:18<31:56, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 457M/22.2G [01:18<28:25, 12.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 795M/22.2G [01:19<35:41, 9.98MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 504M/22.2G [01:19<23:22, 15.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 565M/22.2G [01:19<1:02:58, 5.71MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 796M/22.2G [01:19<35:56, 9.90MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 460M/22.2G [01:19<27:14, 13.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 874M/22.2G [01:19<29:09, 12.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 505M/22.2G [01:19<23:43, 15.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 463M/22.2G [01:19<24:37, 14.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 508M/22.2G [01:19<21:59, 16.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 567M/22.2G [01:19<1:01:18, 5.87MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 877M/22.2G [01:19<27:47, 12.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 464M/22.2G [01:19<25:25, 14.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 568M/22.2G [01:19<55:51, 6.44MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 880M/22.2G [01:19<32:07, 11.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 570M/22.2G [01:19<1:00:47, 5.92MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 883M/22.2G [01:19<26:37, 13.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 466M/22.2G [01:20<47:25, 7.62MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 510M/22.2G [01:20<49:08, 7.34MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 885M/22.2G [01:20<27:59, 12.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 571M/22.2G [01:20<1:02:56, 5.72MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 469M/22.2G [01:20<33:09, 10.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 514M/22.2G [01:20<37:52, 9.52MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 471M/22.2G [01:20<32:42, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 515M/22.2G [01:20<36:01, 10.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 472M/22.2G [01:20<33:45, 10.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 573M/22.2G [01:20<1:06:27, 5.41MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 798M/22.2G [01:20<2:02:33, 2.90MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 476M/22.2G [01:20<26:41, 13.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 518M/22.2G [01:20<32:27, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 477M/22.2G [01:20<27:41, 13.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 520M/22.2G [01:20<32:00, 11.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 886M/22.2G [01:20<54:43, 6.48MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 574M/22.2G [01:20<1:08:23, 5.26MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 802M/22.2G [01:20<1:09:32, 5.12MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 479M/22.2G [01:20<29:52, 12.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 521M/22.2G [01:20<36:23, 9.91MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 890M/22.2G [01:20<39:19, 9.01MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 803M/22.2G [01:20<1:02:51, 5.66MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 480M/22.2G [01:21<31:54, 11.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 576M/22.2G [01:21<1:09:54, 5.15MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 523M/22.2G [01:21<33:43, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 892M/22.2G [01:21<37:12, 9.52MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 524M/22.2G [01:21<31:43, 11.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 805M/22.2G [01:21<58:31, 6.08MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 482M/22.2G [01:21<35:50, 10.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 894M/22.2G [01:21<35:51, 9.88MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 526M/22.2G [01:21<31:11, 11.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 483M/22.2G [01:21<36:40, 9.85MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 527M/22.2G [01:21<29:36, 12.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 896M/22.2G [01:21<34:28, 10.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 807M/22.2G [01:21<55:42, 6.39MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 529M/22.2G [01:21<28:48, 12.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 484M/22.2G [01:21<38:41, 9.34MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 897M/22.2G [01:21<33:05, 10.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 809M/22.2G [01:21<52:20, 6.80MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 899M/22.2G [01:21<31:57, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 486M/22.2G [01:21<37:14, 9.70MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 531M/22.2G [01:21<30:37, 11.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 810M/22.2G [01:21<50:22, 7.06MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 901M/22.2G [01:21<32:03, 11.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 576M/22.2G [01:21<2:19:34, 2.58MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 902M/22.2G [01:21<32:04, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 812M/22.2G [01:22<48:34, 7.32MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 813M/22.2G [01:22<45:26, 7.83MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 905M/22.2G [01:22<29:13, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 533M/22.2G [01:22<54:21, 6.63MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 487M/22.2G [01:22<1:13:55, 4.88MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 580M/22.2G [01:22<1:19:28, 4.52MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 907M/22.2G [01:22<29:34, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 815M/22.2G [01:22<46:56, 7.58MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 536M/22.2G [01:22<40:54, 8.81MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 816M/22.2G [01:22<43:34, 8.16MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 491M/22.2G [01:22<45:59, 7.85MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 539M/22.2G [01:22<34:21, 10.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 582M/22.2G [01:22<1:16:22, 4.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 493M/22.2G [01:22<42:43, 8.45MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 818M/22.2G [01:22<45:41, 7.78MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 541M/22.2G [01:22<32:50, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 495M/22.2G [01:22<41:01, 8.80MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 908M/22.2G [01:22<55:53, 6.34MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 819M/22.2G [01:22<42:09, 8.44MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 584M/22.2G [01:22<1:12:16, 4.97MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 543M/22.2G [01:22<30:21, 11.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 496M/22.2G [01:23<39:38, 9.11MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 545M/22.2G [01:23<29:38, 12.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 912M/22.2G [01:23<37:53, 9.35MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 497M/22.2G [01:23<38:17, 9.43MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 585M/22.2G [01:23<1:10:44, 5.08MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 548M/22.2G [01:23<25:59, 13.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 914M/22.2G [01:23<35:16, 10.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 499M/22.2G [01:23<37:40, 9.58MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 549M/22.2G [01:23<26:50, 13.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 587M/22.2G [01:23<1:06:38, 5.39MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 500M/22.2G [01:23<41:28, 8.70MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 918M/22.2G [01:23<31:16, 11.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   2%|▏         | 552M/22.2G [01:23<26:02, 13.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 820M/22.2G [01:23<1:26:27, 4.11MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 919M/22.2G [01:23<31:09, 11.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 502M/22.2G [01:23<40:33, 8.90MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 588M/22.2G [01:23<1:10:39, 5.09MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 824M/22.2G [01:23<51:15, 6.94MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 923M/22.2G [01:23<28:30, 12.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 504M/22.2G [01:23<43:28, 8.30MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 825M/22.2G [01:24<51:47, 6.86MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 505M/22.2G [01:24<41:55, 8.61MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 926M/22.2G [01:24<27:41, 12.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 590M/22.2G [01:24<1:08:39, 5.23MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 554M/22.2G [01:24<52:20, 6.88MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 827M/22.2G [01:24<48:44, 7.29MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 927M/22.2G [01:24<27:45, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 506M/22.2G [01:24<44:53, 8.04MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 591M/22.2G [01:24<1:07:13, 5.35MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 508M/22.2G [01:24<42:15, 8.54MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 829M/22.2G [01:24<46:31, 7.64MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 558M/22.2G [01:24<39:54, 9.02MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▎         | 830M/22.2G [01:24<44:16, 8.03MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 509M/22.2G [01:24<44:22, 8.13MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 593M/22.2G [01:24<1:06:28, 5.41MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 560M/22.2G [01:24<39:24, 9.13MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 832M/22.2G [01:24<41:55, 8.48MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 562M/22.2G [01:24<38:03, 9.46MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 931M/22.2G [01:24<45:15, 7.82MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 833M/22.2G [01:24<43:52, 8.10MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 595M/22.2G [01:24<1:05:40, 5.47MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 564M/22.2G [01:25<37:26, 9.61MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 835M/22.2G [01:25<42:33, 8.35MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 935M/22.2G [01:25<34:51, 10.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 596M/22.2G [01:25<1:02:46, 5.72MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 836M/22.2G [01:25<40:27, 8.78MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 510M/22.2G [01:25<1:30:09, 4.00MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 937M/22.2G [01:25<32:47, 10.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 838M/22.2G [01:25<41:51, 8.49MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 598M/22.2G [01:25<1:05:19, 5.50MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 940M/22.2G [01:25<29:39, 11.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 514M/22.2G [01:25<57:12, 6.31MB/s]  [Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 839M/22.2G [01:25<41:04, 8.65MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 515M/22.2G [01:25<50:18, 7.17MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 943M/22.2G [01:25<27:57, 12.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 841M/22.2G [01:25<39:35, 8.97MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 517M/22.2G [01:25<50:38, 7.12MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 946M/22.2G [01:25<27:01, 13.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 518M/22.2G [01:26<45:39, 7.90MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 566M/22.2G [01:26<1:23:30, 4.31MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 950M/22.2G [01:26<26:22, 13.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 520M/22.2G [01:26<47:09, 7.65MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 521M/22.2G [01:26<43:24, 8.31MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 953M/22.2G [01:26<25:57, 13.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 598M/22.2G [01:26<2:13:27, 2.69MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 842M/22.2G [01:26<1:26:09, 4.12MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 523M/22.2G [01:26<44:29, 8.10MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 567M/22.2G [01:26<1:32:51, 3.87MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 602M/22.2G [01:26<1:11:04, 5.05MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 846M/22.2G [01:26<48:49, 7.27MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 524M/22.2G [01:26<41:56, 8.60MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 954M/22.2G [01:26<37:06, 9.52MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 526M/22.2G [01:26<40:38, 8.87MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 568M/22.2G [01:26<1:27:33, 4.11MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 957M/22.2G [01:26<31:34, 11.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 847M/22.2G [01:26<49:22, 7.19MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 604M/22.2G [01:27<1:12:58, 4.92MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 527M/22.2G [01:27<43:28, 8.29MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 958M/22.2G [01:27<32:29, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 570M/22.2G [01:27<1:17:58, 4.61MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 849M/22.2G [01:27<46:55, 7.57MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 529M/22.2G [01:27<40:50, 8.82MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 961M/22.2G [01:27<29:38, 11.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 605M/22.2G [01:27<1:10:28, 5.10MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 530M/22.2G [01:27<39:59, 9.01MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 851M/22.2G [01:27<44:18, 8.01MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 571M/22.2G [01:27<1:17:06, 4.67MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 853M/22.2G [01:27<40:17, 8.81MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 964M/22.2G [01:27<27:58, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 532M/22.2G [01:27<42:20, 8.51MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 573M/22.2G [01:27<1:05:27, 5.50MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 607M/22.2G [01:27<1:07:58, 5.28MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 855M/22.2G [01:27<43:03, 8.25MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 968M/22.2G [01:27<26:48, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 574M/22.2G [01:27<1:01:11, 5.88MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 856M/22.2G [01:27<38:39, 9.18MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 609M/22.2G [01:27<1:07:58, 5.28MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 971M/22.2G [01:27<26:01, 13.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 576M/22.2G [01:27<56:34, 6.36MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 858M/22.2G [01:28<42:25, 8.37MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 532M/22.2G [01:28<1:23:04, 4.34MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 974M/22.2G [01:28<25:39, 13.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 610M/22.2G [01:28<1:06:46, 5.38MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 860M/22.2G [01:28<41:30, 8.55MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 536M/22.2G [01:28<54:47, 6.58MB/s]  [Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 861M/22.2G [01:28<37:40, 9.42MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 612M/22.2G [01:28<1:04:40, 5.55MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 577M/22.2G [01:28<1:37:47, 3.68MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 863M/22.2G [01:28<41:37, 8.52MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 975M/22.2G [01:28<41:08, 8.58MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 537M/22.2G [01:28<56:43, 6.35MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 580M/22.2G [01:28<58:26, 6.15MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 979M/22.2G [01:28<31:41, 11.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 614M/22.2G [01:28<1:07:35, 5.31MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 539M/22.2G [01:28<55:34, 6.48MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 980M/22.2G [01:28<32:23, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 581M/22.2G [01:28<53:04, 6.77MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 583M/22.2G [01:29<51:14, 7.02MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 983M/22.2G [01:29<29:48, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 540M/22.2G [01:29<58:36, 6.15MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 615M/22.2G [01:29<1:12:13, 4.97MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 584M/22.2G [01:29<46:49, 7.68MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 987M/22.2G [01:29<27:55, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 542M/22.2G [01:29<56:09, 6.41MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 864M/22.2G [01:29<1:29:07, 3.98MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 586M/22.2G [01:29<47:40, 7.54MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 617M/22.2G [01:29<1:15:36, 4.75MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 543M/22.2G [01:29<54:48, 6.57MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 990M/22.2G [01:29<26:43, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 587M/22.2G [01:29<45:48, 7.85MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 868M/22.2G [01:29<51:18, 6.92MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 589M/22.2G [01:29<44:08, 8.14MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 993M/22.2G [01:29<25:58, 13.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 618M/22.2G [01:29<1:13:03, 4.91MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 545M/22.2G [01:29<58:38, 6.14MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 870M/22.2G [01:29<51:46, 6.85MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 590M/22.2G [01:30<44:25, 8.09MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   4%|▍         | 996M/22.2G [01:30<25:29, 13.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 546M/22.2G [01:30<57:06, 6.31MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 871M/22.2G [01:30<49:05, 7.23MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 592M/22.2G [01:30<41:16, 8.71MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 620M/22.2G [01:30<1:11:59, 4.99MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 548M/22.2G [01:30<55:08, 6.53MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 873M/22.2G [01:30<46:11, 7.68MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 593M/22.2G [01:30<42:29, 8.46MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 875M/22.2G [01:30<42:55, 8.26MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 595M/22.2G [01:30<40:18, 8.92MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 998M/22.2G [01:30<39:29, 8.93MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 549M/22.2G [01:30<54:37, 6.59MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [01:30<31:02, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 596M/22.2G [01:30<42:12, 8.51MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 876M/22.2G [01:30<43:38, 8.13MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 551M/22.2G [01:30<54:10, 6.65MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [01:30<31:49, 11.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 598M/22.2G [01:30<40:22, 8.90MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 878M/22.2G [01:30<42:07, 8.42MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 552M/22.2G [01:30<53:53, 6.68MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 879M/22.2G [01:30<39:33, 8.97MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 620M/22.2G [01:31<2:13:49, 2.68MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [01:31<29:24, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   2%|▏         | 553M/22.2G [01:31<53:03, 6.79MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 881M/22.2G [01:31<41:18, 8.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [01:31<27:48, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 882M/22.2G [01:31<40:17, 8.80MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 624M/22.2G [01:31<1:16:18, 4.70MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 599M/22.2G [01:31<1:22:24, 4.36MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [01:31<26:58, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 884M/22.2G [01:31<38:38, 9.17MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 626M/22.2G [01:31<1:13:10, 4.90MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 886M/22.2G [01:31<39:50, 8.90MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [01:31<26:15, 13.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 602M/22.2G [01:31<50:35, 7.10MB/s]  [A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 604M/22.2G [01:31<48:40, 7.38MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [01:31<25:45, 13.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 554M/22.2G [01:31<1:54:55, 3.13MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 628M/22.2G [01:31<1:10:48, 5.07MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 605M/22.2G [01:32<44:43, 8.03MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 558M/22.2G [01:32<1:03:33, 5.66MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 629M/22.2G [01:32<1:09:35, 5.16MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 887M/22.2G [01:32<1:20:49, 4.39MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [01:32<40:06, 8.78MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 607M/22.2G [01:32<53:52, 6.67MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 560M/22.2G [01:32<1:00:01, 6.00MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [01:32<30:18, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 890M/22.2G [01:32<48:11, 7.35MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 631M/22.2G [01:32<1:07:29, 5.32MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 561M/22.2G [01:32<57:54, 6.21MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [01:32<31:02, 11.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 609M/22.2G [01:32<56:01, 6.41MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 892M/22.2G [01:32<48:21, 7.33MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [01:32<30:16, 11.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 562M/22.2G [01:32<56:39, 6.35MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 633M/22.2G [01:32<1:06:29, 5.39MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 894M/22.2G [01:32<43:30, 8.14MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 610M/22.2G [01:32<55:59, 6.41MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 895M/22.2G [01:33<39:10, 9.04MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 564M/22.2G [01:33<55:04, 6.53MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 634M/22.2G [01:33<1:06:04, 5.43MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 897M/22.2G [01:33<38:23, 9.23MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 612M/22.2G [01:33<1:04:03, 5.60MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 565M/22.2G [01:33<54:58, 6.55MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [01:33<38:18, 9.19MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 898M/22.2G [01:33<37:49, 9.37MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 613M/22.2G [01:33<57:02, 6.29MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 900M/22.2G [01:33<36:50, 9.62MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 636M/22.2G [01:33<1:05:08, 5.51MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [01:33<42:23, 8.30MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 567M/22.2G [01:33<59:24, 6.06MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 901M/22.2G [01:33<36:24, 9.73MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 615M/22.2G [01:33<1:01:41, 5.82MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [01:33<43:17, 8.13MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 903M/22.2G [01:33<35:48, 9.89MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 638M/22.2G [01:33<1:04:23, 5.57MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 568M/22.2G [01:33<1:01:47, 5.82MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 904M/22.2G [01:33<35:26, 10.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [01:33<34:48, 10.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 616M/22.2G [01:33<1:01:47, 5.81MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [01:34<32:54, 10.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 905M/22.2G [01:34<35:03, 10.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 640M/22.2G [01:34<1:03:36, 5.64MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 570M/22.2G [01:34<1:04:01, 5.62MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 907M/22.2G [01:34<34:46, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [01:34<29:52, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 618M/22.2G [01:34<1:04:14, 5.59MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 641M/22.2G [01:34<1:02:58, 5.69MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 571M/22.2G [01:34<1:05:02, 5.53MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 619M/22.2G [01:34<1:08:39, 5.23MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 573M/22.2G [01:34<1:05:03, 5.53MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [01:34<52:37, 6.69MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 908M/22.2G [01:34<1:14:42, 4.74MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:34<36:32, 9.63MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 574M/22.2G [01:35<1:11:50, 5.01MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 912M/22.2G [01:35<44:04, 8.03MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:35<36:08, 9.73MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 643M/22.2G [01:35<1:53:04, 3.17MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 914M/22.2G [01:35<43:10, 8.20MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:35<31:16, 11.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 576M/22.2G [01:35<1:14:41, 4.82MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 620M/22.2G [01:35<1:55:42, 3.10MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:35<26:45, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 916M/22.2G [01:35<41:11, 8.59MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:35<27:54, 12.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 624M/22.2G [01:35<1:01:53, 5.80MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 918M/22.2G [01:35<41:35, 8.51MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 647M/22.2G [01:35<1:13:23, 4.88MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [01:35<24:33, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 919M/22.2G [01:35<39:53, 8.87MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 626M/22.2G [01:35<59:17, 6.05MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [01:35<26:00, 13.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 628M/22.2G [01:36<53:56, 6.65MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 648M/22.2G [01:36<1:10:22, 5.09MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [01:36<23:22, 15.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 921M/22.2G [01:36<47:30, 7.45MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 629M/22.2G [01:36<48:57, 7.33MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 922M/22.2G [01:36<44:13, 8.00MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 576M/22.2G [01:36<2:31:05, 2.38MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 650M/22.2G [01:36<1:15:08, 4.77MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 924M/22.2G [01:36<43:13, 8.19MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 631M/22.2G [01:36<54:57, 6.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 925M/22.2G [01:36<41:20, 8.56MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 580M/22.2G [01:36<1:19:42, 4.51MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [01:36<47:04, 7.47MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 632M/22.2G [01:36<56:27, 6.35MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 652M/22.2G [01:36<1:12:16, 4.96MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 927M/22.2G [01:36<40:08, 8.82MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [01:36<32:52, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 634M/22.2G [01:36<49:44, 7.21MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 928M/22.2G [01:36<38:08, 9.27MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [01:36<32:46, 10.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 582M/22.2G [01:36<1:21:55, 4.39MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 653M/22.2G [01:37<1:13:10, 4.90MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 635M/22.2G [01:37<49:15, 7.28MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 930M/22.2G [01:37<37:29, 9.43MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [01:37<28:40, 12.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 583M/22.2G [01:37<1:18:57, 4.55MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 654M/22.2G [01:37<1:13:57, 4.85MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 637M/22.2G [01:37<51:43, 6.93MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [01:37<24:30, 14.3MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [01:37<25:34, 13.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 638M/22.2G [01:37<51:12, 7.00MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 584M/22.2G [01:37<1:16:49, 4.68MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 656M/22.2G [01:37<1:13:40, 4.86MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [01:37<22:36, 15.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 640M/22.2G [01:37<45:57, 7.80MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 931M/22.2G [01:37<1:18:59, 4.48MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [01:37<24:34, 14.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 641M/22.2G [01:37<41:35, 8.62MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 586M/22.2G [01:37<1:14:24, 4.83MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 935M/22.2G [01:37<46:57, 7.53MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 657M/22.2G [01:37<1:12:04, 4.97MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [01:37<22:27, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 936M/22.2G [01:38<42:20, 8.35MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 659M/22.2G [01:38<1:08:31, 5.23MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 587M/22.2G [01:38<1:14:07, 4.85MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 938M/22.2G [01:38<40:47, 8.67MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 939M/22.2G [01:38<38:40, 9.14MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 643M/22.2G [01:38<1:16:35, 4.68MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 589M/22.2G [01:38<1:13:23, 4.90MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 660M/22.2G [01:38<1:09:25, 5.16MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [01:38<41:26, 8.47MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 941M/22.2G [01:38<37:28, 9.44MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [01:38<33:39, 10.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 942M/22.2G [01:38<36:07, 9.79MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 647M/22.2G [01:38<44:20, 8.09MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 590M/22.2G [01:38<1:12:39, 4.95MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 662M/22.2G [01:38<1:09:14, 5.17MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [01:38<33:11, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 944M/22.2G [01:38<35:34, 9.94MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 648M/22.2G [01:38<44:28, 8.06MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [01:38<26:16, 13.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 945M/22.2G [01:38<34:44, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 650M/22.2G [01:38<41:11, 8.70MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 663M/22.2G [01:38<1:06:21, 5.40MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 592M/22.2G [01:38<1:12:09, 4.98MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [01:39<27:00, 13.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 947M/22.2G [01:39<34:08, 10.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 651M/22.2G [01:39<38:50, 9.23MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 948M/22.2G [01:39<33:47, 10.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [01:39<22:18, 15.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 653M/22.2G [01:39<37:06, 9.66MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 593M/22.2G [01:39<1:14:05, 4.85MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 950M/22.2G [01:39<33:27, 10.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 654M/22.2G [01:39<35:38, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [01:39<23:09, 15.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 951M/22.2G [01:39<33:13, 10.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 656M/22.2G [01:39<35:03, 10.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▍         | 1.11G/22.2G [01:39<20:44, 16.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 595M/22.2G [01:39<1:22:13, 4.37MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 657M/22.2G [01:39<42:25, 8.45MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 665M/22.2G [01:39<1:56:44, 3.07MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 659M/22.2G [01:39<45:16, 7.91MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [01:40<35:12, 9.96MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 953M/22.2G [01:40<1:08:50, 5.13MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 596M/22.2G [01:40<1:24:27, 4.25MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 660M/22.2G [01:40<43:35, 8.22MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [01:40<29:54, 11.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 957M/22.2G [01:40<41:22, 8.54MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 669M/22.2G [01:40<1:10:25, 5.08MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 661M/22.2G [01:40<48:35, 7.37MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 598M/22.2G [01:40<1:23:32, 4.30MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 959M/22.2G [01:40<37:41, 9.37MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 663M/22.2G [01:40<42:43, 8.39MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [01:40<27:32, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 960M/22.2G [01:40<36:14, 9.75MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 671M/22.2G [01:40<1:08:48, 5.20MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 664M/22.2G [01:40<39:25, 9.08MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [01:40<27:47, 12.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 962M/22.2G [01:40<35:00, 10.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 672M/22.2G [01:40<1:05:38, 5.45MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 963M/22.2G [01:40<34:01, 10.4MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 965M/22.2G [01:41<33:19, 10.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [01:41<41:34, 8.43MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 674M/22.2G [01:41<1:06:02, 5.42MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 665M/22.2G [01:41<1:14:31, 4.81MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 967M/22.2G [01:41<32:27, 10.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [01:41<43:12, 8.11MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 968M/22.2G [01:41<32:07, 11.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 675M/22.2G [01:41<1:04:21, 5.56MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 669M/22.2G [01:41<50:50, 7.04MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 970M/22.2G [01:41<31:53, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [01:41<42:46, 8.20MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 598M/22.2G [01:41<2:43:30, 2.20MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 972M/22.2G [01:41<31:51, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [01:41<40:11, 8.72MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 677M/22.2G [01:41<1:02:37, 5.72MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 670M/22.2G [01:41<51:11, 7.00MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [01:41<28:38, 12.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 973M/22.2G [01:41<31:23, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [01:41<28:25, 12.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 678M/22.2G [01:41<1:03:37, 5.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 975M/22.2G [01:41<31:49, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 602M/22.2G [01:41<1:28:52, 4.04MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 673M/22.2G [01:41<45:35, 7.85MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [01:42<32:44, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 675M/22.2G [01:42<42:06, 8.50MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 680M/22.2G [01:42<1:03:10, 5.67MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 604M/22.2G [01:42<1:23:14, 4.31MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 677M/22.2G [01:42<45:18, 7.90MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 976M/22.2G [01:42<1:00:35, 5.83MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 681M/22.2G [01:42<1:01:52, 5.78MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 605M/22.2G [01:42<1:20:18, 4.47MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 979M/22.2G [01:42<43:36, 8.09MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [01:42<1:00:25, 5.80MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 682M/22.2G [01:42<1:00:44, 5.89MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 680M/22.2G [01:42<40:56, 8.74MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 981M/22.2G [01:42<38:54, 9.07MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 607M/22.2G [01:42<1:20:54, 4.44MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [01:42<43:36, 8.03MB/s]  [A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 982M/22.2G [01:42<36:33, 9.65MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 684M/22.2G [01:42<1:03:14, 5.66MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 682M/22.2G [01:42<42:03, 8.51MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [01:42<37:56, 9.23MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 984M/22.2G [01:43<34:25, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 608M/22.2G [01:43<1:18:27, 4.58MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 685M/22.2G [01:43<35:37, 10.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 986M/22.2G [01:43<33:36, 10.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [01:43<32:12, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 686M/22.2G [01:43<1:02:39, 5.71MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 988M/22.2G [01:43<32:19, 10.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 687M/22.2G [01:43<36:16, 9.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 610M/22.2G [01:43<1:16:17, 4.71MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [01:43<29:04, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 989M/22.2G [01:43<32:06, 11.0MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 991M/22.2G [01:43<31:08, 11.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [01:43<27:03, 12.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 611M/22.2G [01:43<1:15:14, 4.77MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 993M/22.2G [01:43<30:54, 11.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [01:43<25:48, 13.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 994M/22.2G [01:43<30:43, 11.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 688M/22.2G [01:43<1:06:36, 5.37MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   4%|▍         | 996M/22.2G [01:44<30:34, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 687M/22.2G [01:44<1:50:21, 3.24MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 613M/22.2G [01:44<1:20:02, 4.49MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 691M/22.2G [01:44<44:43, 8.00MB/s]  [A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 692M/22.2G [01:44<46:45, 7.65MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 691M/22.2G [01:44<1:03:50, 5.60MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 696M/22.2G [01:44<33:47, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [01:44<41:11, 8.50MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 614M/22.2G [01:44<1:25:44, 4.19MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 697M/22.2G [01:44<32:43, 10.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 997M/22.2G [01:44<1:03:19, 5.57MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [01:44<30:38, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 692M/22.2G [01:44<1:06:15, 5.40MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 700M/22.2G [01:44<26:46, 13.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [01:44<30:48, 11.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [01:44<39:56, 8.83MB/s] 
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 616M/22.2G [01:44<1:25:24, 4.20MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 703M/22.2G [01:44<23:16, 15.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 694M/22.2G [01:44<1:03:31, 5.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [01:44<37:59, 9.28MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [01:44<28:33, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 706M/22.2G [01:44<20:45, 17.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [01:45<37:05, 9.50MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 695M/22.2G [01:45<1:04:12, 5.57MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [01:45<35:09, 10.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 617M/22.2G [01:45<1:29:16, 4.02MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [01:45<34:04, 10.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 697M/22.2G [01:45<1:01:41, 5.80MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [01:45<40:20, 8.67MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [01:45<33:16, 10.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 709M/22.2G [01:45<37:29, 9.53MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 619M/22.2G [01:45<1:28:50, 4.04MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [01:45<32:50, 10.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 699M/22.2G [01:45<1:02:11, 5.75MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [01:45<43:23, 8.06MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 713M/22.2G [01:45<27:31, 13.0MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 716M/22.2G [01:45<23:50, 15.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [01:45<42:20, 8.26MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 620M/22.2G [01:45<1:24:39, 4.24MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [01:45<30:38, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 700M/22.2G [01:45<1:00:36, 5.90MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 719M/22.2G [01:45<21:33, 16.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [01:46<30:29, 11.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [01:46<34:14, 10.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 722M/22.2G [01:46<19:46, 18.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [01:46<30:42, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 702M/22.2G [01:46<1:00:58, 5.86MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [01:46<29:51, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 725M/22.2G [01:46<19:14, 18.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [01:46<30:36, 11.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [01:46<28:14, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 729M/22.2G [01:46<18:04, 19.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 704M/22.2G [01:46<59:37, 6.00MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 706M/22.2G [01:46<54:28, 6.56MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [01:46<47:28, 7.37MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 731M/22.2G [01:46<29:52, 12.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 620M/22.2G [01:46<2:48:08, 2.13MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [01:46<1:04:44, 5.44MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [01:46<34:53, 10.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 735M/22.2G [01:47<25:14, 14.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 707M/22.2G [01:47<1:03:39, 5.62MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [01:47<34:13, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [01:47<44:05, 7.99MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 739M/22.2G [01:47<23:11, 15.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 624M/22.2G [01:47<1:28:35, 4.05MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [01:47<39:34, 8.90MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [01:47<29:04, 12.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 709M/22.2G [01:47<1:01:44, 5.79MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [01:47<37:06, 9.49MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 742M/22.2G [01:47<23:13, 15.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [01:47<25:14, 13.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [01:47<35:15, 9.99MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 744M/22.2G [01:47<23:58, 14.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [01:47<25:48, 13.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 626M/22.2G [01:47<1:26:42, 4.14MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 745M/22.2G [01:47<24:32, 14.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [01:47<33:52, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [01:47<22:42, 15.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 747M/22.2G [01:47<25:43, 13.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [01:47<32:48, 10.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [01:47<24:09, 14.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 748M/22.2G [01:47<27:18, 13.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 628M/22.2G [01:48<1:26:28, 4.15MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [01:48<32:01, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [01:48<22:12, 15.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [01:48<31:10, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 751M/22.2G [01:48<27:24, 13.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [01:48<30:51, 11.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 753M/22.2G [01:48<28:18, 12.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 629M/22.2G [01:48<1:25:58, 4.17MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 709M/22.2G [01:48<2:05:53, 2.84MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [01:48<30:41, 11.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [01:48<38:51, 8.99MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [01:48<30:08, 11.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 630M/22.2G [01:48<1:25:29, 4.20MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [01:48<31:26, 11.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 713M/22.2G [01:48<1:19:01, 4.52MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [01:48<31:45, 11.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 754M/22.2G [01:48<57:15, 6.23MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 714M/22.2G [01:49<1:18:16, 4.56MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 632M/22.2G [01:49<1:25:11, 4.21MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [01:49<27:28, 12.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 757M/22.2G [01:49<41:02, 8.69MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [01:49<1:04:12, 5.48MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [01:49<25:16, 13.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 716M/22.2G [01:49<1:14:12, 4.82MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 759M/22.2G [01:49<43:41, 8.16MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 633M/22.2G [01:49<1:24:46, 4.23MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:49<40:38, 8.66MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [01:49<22:21, 15.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 760M/22.2G [01:49<40:32, 8.80MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:49<36:33, 9.62MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [01:49<23:42, 14.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 718M/22.2G [01:49<1:10:25, 5.07MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:49<34:37, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 635M/22.2G [01:49<1:24:26, 4.25MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   5%|▌         | 1.22G/22.2G [01:49<21:06, 16.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 762M/22.2G [01:49<47:29, 7.51MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 719M/22.2G [01:49<1:05:33, 5.45MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:49<32:46, 10.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 765M/22.2G [01:49<38:12, 9.33MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:49<31:43, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 636M/22.2G [01:50<1:25:03, 4.22MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 766M/22.2G [01:50<36:57, 9.65MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [01:50<30:42, 11.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 721M/22.2G [01:50<1:06:18, 5.39MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [01:50<38:43, 9.01MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [01:50<29:54, 11.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 768M/22.2G [01:50<43:52, 8.13MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 723M/22.2G [01:50<1:01:44, 5.79MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [01:50<30:08, 11.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 638M/22.2G [01:50<1:23:38, 4.29MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [01:50<29:35, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [01:50<30:25, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [01:50<28:50, 12.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 769M/22.2G [01:50<47:26, 7.51MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 724M/22.2G [01:50<1:02:51, 5.68MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 639M/22.2G [01:50<1:20:31, 4.45MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [01:50<28:15, 12.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [01:50<26:54, 13.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 726M/22.2G [01:50<1:01:02, 5.85MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 772M/22.2G [01:50<41:53, 8.51MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [01:51<25:07, 13.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 641M/22.2G [01:51<1:21:32, 4.40MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   3%|▎         | 774M/22.2G [01:51<39:52, 8.94MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 727M/22.2G [01:51<59:47, 5.97MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [01:51<22:11, 15.7MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [01:51<23:24, 14.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [01:51<55:01, 6.39MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 728M/22.2G [01:51<58:48, 6.07MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 642M/22.2G [01:51<1:18:18, 4.58MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [01:51<21:14, 16.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [01:51<37:12, 9.44MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 730M/22.2G [01:51<57:26, 6.22MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [01:51<33:06, 10.6MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [01:51<31:06, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 775M/22.2G [01:51<1:21:48, 4.36MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [01:51<30:29, 11.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [01:52<40:44, 8.56MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 780M/22.2G [01:52<50:18, 7.08MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [01:52<27:57, 12.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [01:52<31:21, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 643M/22.2G [01:52<2:27:42, 2.43MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 781M/22.2G [01:52<45:21, 7.85MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [01:52<31:22, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [01:52<26:54, 13.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 783M/22.2G [01:52<41:54, 8.50MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 731M/22.2G [01:52<1:48:31, 3.29MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [01:52<27:15, 12.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [01:52<27:42, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 647M/22.2G [01:52<1:19:38, 4.50MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 786M/22.2G [01:52<34:43, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [01:52<24:01, 14.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 735M/22.2G [01:52<1:02:41, 5.69MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [01:52<25:20, 13.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 787M/22.2G [01:52<33:39, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [01:52<24:59, 13.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 789M/22.2G [01:52<32:26, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 648M/22.2G [01:52<1:21:27, 4.40MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 737M/22.2G [01:52<59:39, 5.98MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [01:52<22:04, 15.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 792M/22.2G [01:53<29:38, 12.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [01:53<23:31, 14.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 738M/22.2G [01:53<58:28, 6.11MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 649M/22.2G [01:53<1:17:57, 4.60MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 793M/22.2G [01:53<29:22, 12.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 740M/22.2G [01:53<56:42, 6.29MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 796M/22.2G [01:53<27:35, 12.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 651M/22.2G [01:53<1:16:00, 4.72MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [01:53<54:51, 6.40MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 741M/22.2G [01:53<56:04, 6.36MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [01:53<36:21, 9.58MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [01:53<41:32, 8.45MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 652M/22.2G [01:53<1:13:35, 4.87MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [01:53<26:51, 13.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [01:53<39:03, 8.99MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 743M/22.2G [01:53<54:52, 6.50MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 798M/22.2G [01:53<49:28, 7.19MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [01:53<27:21, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [01:53<34:12, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 654M/22.2G [01:54<1:11:07, 5.04MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 744M/22.2G [01:54<54:47, 6.51MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [01:54<30:58, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 802M/22.2G [01:54<35:06, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [01:54<25:37, 13.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 746M/22.2G [01:54<54:34, 6.54MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 655M/22.2G [01:54<1:10:01, 5.12MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [01:54<22:39, 15.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [01:54<27:10, 12.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 803M/22.2G [01:54<38:33, 9.23MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [01:54<23:45, 14.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 747M/22.2G [01:54<53:47, 6.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [01:54<25:01, 14.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 657M/22.2G [01:54<1:09:05, 5.19MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 805M/22.2G [01:54<40:32, 8.78MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [01:54<21:00, 16.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 749M/22.2G [01:54<53:48, 6.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▍         | 1.11G/22.2G [01:54<23:34, 14.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [01:54<22:27, 15.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 807M/22.2G [01:54<39:01, 9.12MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 658M/22.2G [01:54<1:07:36, 5.30MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 750M/22.2G [01:54<53:51, 6.62MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 809M/22.2G [01:55<44:15, 8.04MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 660M/22.2G [01:55<1:01:39, 5.81MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 752M/22.2G [01:55<53:11, 6.71MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [01:55<39:09, 8.96MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [01:55<36:07, 9.63MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 661M/22.2G [01:55<1:03:30, 5.64MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 812M/22.2G [01:55<41:11, 8.64MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [01:55<29:12, 12.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [01:55<26:42, 13.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 663M/22.2G [01:55<1:00:01, 5.97MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [01:55<29:17, 12.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [01:55<27:10, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 813M/22.2G [01:55<45:02, 7.90MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [01:55<23:08, 15.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [01:55<25:39, 13.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 815M/22.2G [01:55<41:05, 8.66MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 664M/22.2G [01:55<1:01:27, 5.83MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [01:55<24:08, 14.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [01:55<23:12, 15.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 817M/22.2G [01:56<44:38, 7.97MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [01:56<21:44, 16.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [01:56<19:52, 17.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 753M/22.2G [01:56<1:46:54, 3.34MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 818M/22.2G [01:56<41:51, 8.50MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [01:56<23:06, 15.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [01:56<20:49, 16.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [01:56<20:57, 16.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 757M/22.2G [01:56<1:03:18, 5.63MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [01:56<22:35, 15.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 665M/22.2G [01:56<1:56:10, 3.08MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 759M/22.2G [01:56<1:00:09, 5.93MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [01:56<35:07, 9.98MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 760M/22.2G [01:56<58:41, 6.08MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 669M/22.2G [01:56<1:09:42, 5.14MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 820M/22.2G [01:56<1:16:45, 4.63MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [01:57<43:12, 8.04MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 762M/22.2G [01:57<57:32, 6.20MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 824M/22.2G [01:57<46:48, 7.59MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [01:57<31:21, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [01:57<30:04, 11.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 670M/22.2G [01:57<1:07:45, 5.29MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [01:57<25:00, 14.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [01:57<30:04, 11.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 763M/22.2G [01:57<56:34, 6.30MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 826M/22.2G [01:57<47:40, 7.46MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [01:57<24:50, 14.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [01:57<24:16, 14.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 672M/22.2G [01:57<1:06:47, 5.36MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [01:57<25:05, 14.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 765M/22.2G [01:57<55:52, 6.38MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [01:57<25:08, 13.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 673M/22.2G [01:57<1:02:46, 5.70MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 828M/22.2G [01:57<44:08, 8.05MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [01:57<24:32, 14.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [01:57<22:20, 15.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 766M/22.2G [01:57<55:21, 6.44MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▎         | 830M/22.2G [01:57<41:40, 8.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [01:57<23:38, 14.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 675M/22.2G [01:57<1:03:09, 5.67MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [01:57<22:38, 15.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 768M/22.2G [01:58<54:17, 6.57MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 832M/22.2G [01:58<43:59, 8.08MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [01:58<20:51, 16.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [01:58<23:13, 15.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 833M/22.2G [01:58<39:05, 9.09MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 676M/22.2G [01:58<59:13, 6.04MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 769M/22.2G [01:58<53:28, 6.67MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 835M/22.2G [01:58<43:02, 8.26MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 678M/22.2G [01:58<1:01:03, 5.86MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 771M/22.2G [01:58<53:37, 6.65MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 836M/22.2G [01:58<40:03, 8.87MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [01:58<36:53, 9.41MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 679M/22.2G [01:58<58:49, 6.08MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 773M/22.2G [01:58<52:26, 6.80MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 839M/22.2G [01:58<34:18, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [01:58<27:03, 12.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [01:58<47:08, 7.43MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 681M/22.2G [01:58<56:26, 6.34MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   3%|▎         | 774M/22.2G [01:58<52:08, 6.83MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [01:58<27:12, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 842M/22.2G [01:58<31:03, 11.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [01:58<34:27, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [01:59<22:23, 15.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 682M/22.2G [01:59<54:58, 6.51MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [01:59<33:47, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [01:59<23:12, 14.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 684M/22.2G [01:59<54:27, 6.57MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [01:59<29:14, 12.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [01:59<21:11, 16.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 843M/22.2G [01:59<55:29, 6.40MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 685M/22.2G [01:59<54:05, 6.62MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [01:59<19:35, 17.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [01:59<26:04, 13.4MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [01:59<22:49, 15.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 687M/22.2G [01:59<53:48, 6.65MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [01:59<20:54, 16.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 846M/22.2G [01:59<46:35, 7.62MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [01:59<23:43, 14.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 775M/22.2G [01:59<1:42:29, 3.48MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 848M/22.2G [01:59<42:30, 8.35MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [02:00<22:40, 15.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 851M/22.2G [02:00<35:24, 10.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [02:00<32:15, 10.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 780M/22.2G [02:00<1:01:00, 5.84MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:00<25:36, 13.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 854M/22.2G [02:00<31:39, 11.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 781M/22.2G [02:00<56:57, 6.25MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:00<25:37, 13.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 855M/22.2G [02:00<31:29, 11.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 687M/22.2G [02:00<1:50:55, 3.23MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:00<21:40, 16.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 783M/22.2G [02:00<54:03, 6.59MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 858M/22.2G [02:00<28:47, 12.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:00<20:10, 17.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 691M/22.2G [02:00<1:03:24, 5.64MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 785M/22.2G [02:00<51:51, 6.87MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 861M/22.2G [02:00<27:14, 13.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:00<19:01, 18.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 693M/22.2G [02:00<59:47, 5.98MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 862M/22.2G [02:00<27:46, 12.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:01<18:04, 19.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 787M/22.2G [02:01<50:45, 7.02MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 694M/22.2G [02:01<58:15, 6.14MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:01<17:39, 19.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 788M/22.2G [02:01<48:30, 7.34MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 695M/22.2G [02:01<56:12, 6.36MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 790M/22.2G [02:01<46:46, 7.61MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 864M/22.2G [02:01<47:48, 7.42MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [02:01<1:19:55, 4.38MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 697M/22.2G [02:01<54:42, 6.54MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 868M/22.2G [02:01<33:57, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 792M/22.2G [02:01<47:08, 7.55MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 698M/22.2G [02:01<58:15, 6.14MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 871M/22.2G [02:01<30:29, 11.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 794M/22.2G [02:01<45:21, 7.85MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [02:01<1:08:10, 5.13MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 874M/22.2G [02:02<28:27, 12.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [02:02<56:12, 6.22MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 796M/22.2G [02:02<43:45, 8.14MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 700M/22.2G [02:02<1:05:39, 5.45MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:02<54:59, 6.30MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 877M/22.2G [02:02<27:01, 13.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [02:02<41:30, 8.42MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [02:02<31:21, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 880M/22.2G [02:02<26:12, 13.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:02<53:15, 6.50MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [02:02<30:34, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 702M/22.2G [02:02<1:08:55, 5.19MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 883M/22.2G [02:02<25:20, 14.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:02<40:23, 8.57MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 798M/22.2G [02:02<1:08:17, 5.21MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [02:02<26:54, 13.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.38G/22.2G [02:02<30:23, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 886M/22.2G [02:02<24:37, 14.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [02:02<25:20, 13.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 802M/22.2G [02:02<44:22, 8.02MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 703M/22.2G [02:03<1:16:12, 4.69MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [02:03<27:51, 12.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 803M/22.2G [02:03<46:17, 7.69MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [02:03<38:55, 8.97MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [02:03<21:58, 15.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 705M/22.2G [02:03<1:19:25, 4.50MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 888M/22.2G [02:03<42:21, 8.37MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 805M/22.2G [02:03<47:31, 7.49MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [02:03<28:48, 12.1MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [02:03<28:29, 12.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 806M/22.2G [02:03<47:33, 7.48MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 890M/22.2G [02:03<40:06, 8.84MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 706M/22.2G [02:03<1:21:32, 4.38MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 893M/22.2G [02:03<33:52, 10.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [02:03<24:25, 14.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 808M/22.2G [02:03<47:30, 7.49MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 894M/22.2G [02:03<33:29, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [02:03<38:48, 8.92MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [02:03<20:38, 16.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 708M/22.2G [02:04<1:22:21, 4.34MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 896M/22.2G [02:04<34:04, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [02:04<30:49, 11.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 810M/22.2G [02:04<47:08, 7.55MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [02:04<21:32, 16.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [02:04<30:24, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 897M/22.2G [02:04<36:06, 9.81MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 811M/22.2G [02:04<46:42, 7.61MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   5%|▌         | 1.22G/22.2G [02:04<19:21, 18.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:04<24:57, 13.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 899M/22.2G [02:04<38:07, 9.29MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 813M/22.2G [02:04<41:41, 8.53MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 900M/22.2G [02:04<38:02, 9.31MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 815M/22.2G [02:04<45:16, 7.86MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [02:04<30:47, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 902M/22.2G [02:04<40:20, 8.78MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:04<40:14, 8.59MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [02:04<24:54, 14.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 817M/22.2G [02:04<44:19, 8.02MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 903M/22.2G [02:04<38:52, 9.11MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [02:05<24:56, 14.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 818M/22.2G [02:05<42:41, 8.33MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 905M/22.2G [02:05<41:25, 8.55MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:05<46:33, 7.43MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [02:05<21:41, 16.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 709M/22.2G [02:05<2:31:52, 2.35MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 906M/22.2G [02:05<42:23, 8.35MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:05<45:47, 7.55MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [02:05<20:58, 16.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 908M/22.2G [02:05<40:00, 8.85MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:05<34:38, 9.98MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [02:05<17:53, 19.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [02:05<29:05, 11.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 713M/22.2G [02:05<1:27:55, 4.06MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [02:05<17:43, 19.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 820M/22.2G [02:05<1:17:12, 4.61MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 715M/22.2G [02:06<1:26:15, 4.14MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 824M/22.2G [02:06<49:10, 7.23MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 909M/22.2G [02:06<1:32:17, 3.84MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 825M/22.2G [02:06<47:53, 7.42MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [02:06<46:34, 7.42MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [02:06<37:12, 9.37MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 716M/22.2G [02:06<1:25:40, 4.17MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 827M/22.2G [02:06<45:06, 7.88MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [02:06<28:15, 12.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [02:06<35:06, 9.84MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 828M/22.2G [02:06<45:19, 7.84MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 912M/22.2G [02:06<1:00:38, 5.84MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [02:06<24:13, 14.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:06<29:31, 11.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 718M/22.2G [02:06<1:24:21, 4.23MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▎         | 830M/22.2G [02:06<41:28, 8.57MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [02:06<22:21, 15.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:06<29:15, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 914M/22.2G [02:06<58:26, 6.06MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [02:06<21:01, 16.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 832M/22.2G [02:06<41:04, 8.65MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:07<27:50, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 916M/22.2G [02:07<55:02, 6.43MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:07<20:02, 17.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 719M/22.2G [02:07<1:23:52, 4.26MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 833M/22.2G [02:07<39:19, 9.04MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:07<27:30, 12.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 917M/22.2G [02:07<48:48, 7.25MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:07<19:11, 18.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 835M/22.2G [02:07<38:51, 9.14MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:07<27:46, 12.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:07<18:38, 18.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 836M/22.2G [02:07<38:21, 9.26MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 721M/22.2G [02:07<1:23:31, 4.28MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 919M/22.2G [02:07<56:35, 6.25MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [02:07<26:53, 12.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 838M/22.2G [02:07<37:58, 9.36MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 920M/22.2G [02:07<49:22, 7.17MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 839M/22.2G [02:07<37:23, 9.50MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 722M/22.2G [02:07<1:20:18, 4.45MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [02:07<25:53, 13.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 840M/22.2G [02:07<37:25, 9.49MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 922M/22.2G [02:07<48:30, 7.30MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 842M/22.2G [02:08<37:05, 9.58MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:08<40:31, 8.59MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 724M/22.2G [02:08<1:21:16, 4.39MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 923M/22.2G [02:08<50:38, 6.99MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [02:08<34:56, 9.96MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 925M/22.2G [02:08<45:53, 7.71MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.44G/22.2G [02:08<42:01, 8.21MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [02:08<28:02, 12.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 926M/22.2G [02:08<43:28, 8.14MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.44G/22.2G [02:08<33:06, 10.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 725M/22.2G [02:08<1:22:06, 4.35MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [02:08<24:45, 14.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [02:08<33:32, 10.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 927M/22.2G [02:08<44:39, 7.92MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 843M/22.2G [02:08<1:14:34, 4.76MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [02:08<22:25, 15.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [02:08<32:09, 10.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 727M/22.2G [02:08<1:18:27, 4.55MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 929M/22.2G [02:08<41:12, 8.58MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [02:08<20:51, 16.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 846M/22.2G [02:08<49:54, 7.12MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 930M/22.2G [02:08<39:44, 8.90MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [02:08<19:47, 17.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [02:08<27:49, 12.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 848M/22.2G [02:08<44:25, 7.99MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [02:09<19:03, 18.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 728M/22.2G [02:09<1:19:59, 4.46MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 849M/22.2G [02:09<42:12, 8.41MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [02:09<26:31, 13.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 851M/22.2G [02:09<40:30, 8.77MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 729M/22.2G [02:09<1:17:27, 4.61MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [02:09<25:28, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 852M/22.2G [02:09<39:18, 9.03MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 854M/22.2G [02:09<38:59, 9.11MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 931M/22.2G [02:09<1:27:24, 4.05MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [02:09<35:39, 9.75MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [02:09<24:47, 13.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 731M/22.2G [02:09<1:19:39, 4.48MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [02:09<31:13, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 935M/22.2G [02:09<53:49, 6.57MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 855M/22.2G [02:09<42:47, 8.30MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [02:09<25:38, 13.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 936M/22.2G [02:09<51:57, 6.81MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:10<23:11, 15.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 857M/22.2G [02:10<43:40, 8.13MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 938M/22.2G [02:10<46:35, 7.59MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [02:10<41:01, 8.41MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:10<21:14, 16.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 858M/22.2G [02:10<45:58, 7.72MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [02:10<29:27, 11.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:10<20:00, 17.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 939M/22.2G [02:10<51:03, 6.93MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [02:10<29:50, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:10<19:14, 18.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 860M/22.2G [02:10<46:04, 7.70MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 731M/22.2G [02:10<2:28:43, 2.40MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 941M/22.2G [02:10<47:06, 7.50MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [02:10<18:31, 18.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [02:10<27:40, 12.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 862M/22.2G [02:10<45:26, 7.81MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 942M/22.2G [02:10<50:04, 7.06MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [02:10<26:03, 13.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 735M/22.2G [02:10<1:19:39, 4.48MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 864M/22.2G [02:10<46:13, 7.68MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 944M/22.2G [02:10<51:20, 6.89MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [02:11<32:06, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [02:11<25:13, 13.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 945M/22.2G [02:11<45:49, 7.72MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [02:11<28:56, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 737M/22.2G [02:11<1:21:08, 4.40MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [02:11<24:30, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [02:11<23:58, 14.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [02:11<22:09, 15.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 947M/22.2G [02:11<54:37, 6.47MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [02:11<21:54, 15.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 738M/22.2G [02:11<1:18:13, 4.56MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 864M/22.2G [02:11<1:23:34, 4.25MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [02:11<20:25, 17.0MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [02:11<19:26, 17.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 739M/22.2G [02:11<1:16:07, 4.69MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 868M/22.2G [02:11<54:21, 6.53MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 949M/22.2G [02:11<53:02, 6.66MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [02:11<18:40, 18.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 870M/22.2G [02:11<50:11, 7.07MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [02:11<39:14, 8.78MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 951M/22.2G [02:11<46:47, 7.55MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 741M/22.2G [02:12<1:14:36, 4.78MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [02:12<22:48, 15.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [02:12<33:51, 10.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 953M/22.2G [02:12<46:13, 7.64MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 871M/22.2G [02:12<56:06, 6.32MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [02:12<33:35, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 742M/22.2G [02:12<1:13:14, 4.87MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [02:12<29:14, 11.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 873M/22.2G [02:12<57:31, 6.17MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [02:12<38:16, 9.07MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 744M/22.2G [02:12<1:11:28, 4.99MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [02:12<28:47, 12.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [02:12<27:01, 12.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 874M/22.2G [02:12<56:36, 6.26MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 745M/22.2G [02:12<1:06:10, 5.39MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [02:12<29:02, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [02:12<25:10, 13.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [02:12<28:58, 12.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 876M/22.2G [02:13<58:34, 6.06MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 747M/22.2G [02:13<1:07:03, 5.32MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 953M/22.2G [02:13<1:44:53, 3.37MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [02:13<24:21, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [02:13<25:50, 13.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 878M/22.2G [02:13<57:01, 6.22MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [02:13<24:59, 13.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 957M/22.2G [02:13<1:05:59, 5.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 748M/22.2G [02:13<1:11:11, 5.01MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 879M/22.2G [02:13<54:46, 6.47MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [02:13<24:16, 14.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 958M/22.2G [02:13<1:00:47, 5.81MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 960M/22.2G [02:13<53:12, 6.64MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [02:13<37:17, 9.23MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 881M/22.2G [02:13<55:11, 6.42MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [02:13<23:15, 14.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 750M/22.2G [02:13<1:15:45, 4.71MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 961M/22.2G [02:13<47:16, 7.47MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [02:13<28:22, 12.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 882M/22.2G [02:13<54:56, 6.45MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 963M/22.2G [02:14<47:34, 7.42MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [02:14<28:44, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 751M/22.2G [02:14<1:13:26, 4.86MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 964M/22.2G [02:14<43:15, 8.16MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 883M/22.2G [02:14<55:15, 6.42MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [02:14<26:54, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 966M/22.2G [02:14<39:37, 8.91MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [02:14<39:18, 8.82MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 752M/22.2G [02:14<1:16:59, 4.63MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 885M/22.2G [02:14<54:47, 6.47MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 967M/22.2G [02:14<41:39, 8.48MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [02:14<25:33, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:14<30:58, 11.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 969M/22.2G [02:14<37:40, 9.37MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:14<29:13, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [02:14<24:18, 14.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 970M/22.2G [02:14<36:36, 9.65MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:14<29:32, 11.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 972M/22.2G [02:14<35:44, 9.88MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [02:14<23:40, 14.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:15<30:06, 11.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 973M/22.2G [02:15<39:38, 8.91MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:15<30:35, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 886M/22.2G [02:15<1:40:22, 3.53MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:15<31:18, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 753M/22.2G [02:15<2:23:04, 2.49MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:15<31:08, 11.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 890M/22.2G [02:15<59:29, 5.96MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:15<31:28, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [02:15<38:46, 8.86MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:15<31:16, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 757M/22.2G [02:15<1:18:13, 4.56MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [02:15<29:27, 11.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 892M/22.2G [02:15<56:32, 6.27MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:15<31:10, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [02:15<29:33, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:16<31:05, 11.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 893M/22.2G [02:16<55:54, 6.34MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 975M/22.2G [02:16<1:33:50, 3.76MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [02:16<27:29, 12.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 759M/22.2G [02:16<1:22:28, 4.32MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:16<30:56, 11.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 895M/22.2G [02:16<54:55, 6.45MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 979M/22.2G [02:16<55:44, 6.33MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [02:16<25:48, 13.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 897M/22.2G [02:16<53:57, 6.57MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 760M/22.2G [02:16<1:23:46, 4.26MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 981M/22.2G [02:16<52:52, 6.67MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [02:16<24:34, 14.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:16<58:34, 5.91MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 898M/22.2G [02:16<53:16, 6.65MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 982M/22.2G [02:16<51:31, 6.85MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [02:16<23:51, 14.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 762M/22.2G [02:16<1:23:56, 4.25MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:16<42:25, 8.16MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 983M/22.2G [02:16<47:18, 7.46MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 900M/22.2G [02:16<53:25, 6.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:16<37:04, 9.34MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 985M/22.2G [02:17<47:38, 7.41MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:17<35:32, 9.74MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 986M/22.2G [02:17<44:06, 8.00MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 901M/22.2G [02:17<57:05, 6.20MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 763M/22.2G [02:17<1:27:58, 4.05MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:17<33:34, 10.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 988M/22.2G [02:17<44:55, 7.85MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 903M/22.2G [02:17<55:29, 6.38MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:17<32:29, 10.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [02:17<38:22, 8.95MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 989M/22.2G [02:17<41:22, 8.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [02:17<31:19, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [02:17<29:11, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 765M/22.2G [02:17<1:27:31, 4.07MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 904M/22.2G [02:17<54:15, 6.53MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [02:17<30:28, 11.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 991M/22.2G [02:17<42:50, 8.23MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [02:17<29:19, 11.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [02:17<29:58, 11.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 992M/22.2G [02:17<39:55, 8.83MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 906M/22.2G [02:17<53:24, 6.63MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 766M/22.2G [02:17<1:26:16, 4.13MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [02:17<27:17, 12.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [02:18<29:36, 11.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 994M/22.2G [02:18<41:35, 8.48MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [02:18<29:34, 11.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 907M/22.2G [02:18<58:52, 6.02MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [02:18<25:59, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 995M/22.2G [02:18<38:32, 9.15MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 768M/22.2G [02:18<1:24:00, 4.24MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [02:18<29:50, 11.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:18<23:10, 14.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   4%|▍         | 997M/22.2G [02:18<40:40, 8.67MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:18<24:15, 14.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 769M/22.2G [02:18<1:23:30, 4.27MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:18<22:13, 15.4MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:18<23:35, 14.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [02:18<1:00:17, 5.74MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 998M/22.2G [02:18<1:11:03, 4.96MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 770M/22.2G [02:18<1:23:17, 4.28MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [02:19<39:18, 8.80MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [02:19<45:26, 7.76MB/s] [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 908M/22.2G [02:19<1:57:23, 3.02MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [02:19<35:04, 9.86MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 772M/22.2G [02:19<1:20:53, 4.41MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [02:19<46:11, 7.63MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:19<46:55, 7.31MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [02:19<36:16, 9.54MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [02:19<36:09, 9.49MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 912M/22.2G [02:19<1:10:16, 5.04MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [02:19<48:34, 7.26MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 773M/22.2G [02:19<1:21:23, 4.38MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:19<34:39, 9.98MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [02:19<35:17, 9.72MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:19<45:42, 7.71MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:19<33:25, 10.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 914M/22.2G [02:19<1:07:54, 5.21MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   3%|▎         | 775M/22.2G [02:19<1:19:40, 4.47MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [02:19<30:10, 11.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:19<37:33, 9.21MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:19<52:36, 6.70MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:20<25:27, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 916M/22.2G [02:20<1:05:26, 5.41MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:20<34:58, 9.89MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:20<46:50, 7.52MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:20<26:20, 13.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:20<33:48, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:20<41:05, 8.58MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:20<22:32, 15.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 917M/22.2G [02:20<1:05:16, 5.42MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:20<34:11, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:20<23:57, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [02:20<34:22, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:20<51:43, 6.81MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 919M/22.2G [02:20<1:01:56, 5.71MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:20<21:24, 16.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [02:20<34:37, 9.98MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [02:20<34:36, 9.99MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:20<53:14, 6.62MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 920M/22.2G [02:20<1:02:30, 5.66MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:20<45:55, 7.67MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 775M/22.2G [02:20<2:42:56, 2.19MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 922M/22.2G [02:21<57:28, 6.16MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [02:21<45:21, 7.77MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [02:21<40:30, 8.46MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [02:21<41:37, 8.46MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [02:21<1:06:06, 5.23MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [02:21<33:11, 10.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 924M/22.2G [02:21<56:32, 6.26MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 780M/22.2G [02:21<1:28:45, 4.01MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [02:21<32:57, 10.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [02:21<46:37, 7.41MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 925M/22.2G [02:21<58:32, 6.04MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [02:21<25:52, 13.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 781M/22.2G [02:21<1:26:00, 4.14MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [02:21<26:50, 12.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [02:21<41:44, 8.28MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 926M/22.2G [02:21<57:53, 6.11MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [02:21<23:05, 14.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:21<42:49, 8.07MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 928M/22.2G [02:22<56:43, 6.24MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [02:22<24:32, 14.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 782M/22.2G [02:22<1:25:33, 4.16MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [02:22<21:30, 15.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:22<42:52, 8.06MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [02:22<1:35:05, 3.70MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 929M/22.2G [02:22<55:03, 6.42MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [02:22<23:06, 14.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 784M/22.2G [02:22<1:24:50, 4.20MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:22<43:12, 7.99MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [02:22<59:52, 5.88MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:22<42:33, 8.12MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 785M/22.2G [02:22<1:21:11, 4.39MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [02:22<54:23, 6.48MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:22<39:02, 8.84MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [02:22<42:07, 8.13MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:22<53:22, 6.60MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 787M/22.2G [02:22<1:18:31, 4.54MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [02:22<42:23, 8.15MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [02:23<28:42, 11.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:23<48:28, 7.26MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [02:23<28:56, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:23<43:53, 8.02MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 931M/22.2G [02:23<1:55:17, 3.07MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [02:23<41:26, 8.33MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:23<40:52, 8.61MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 788M/22.2G [02:23<1:19:20, 4.49MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [02:23<38:44, 8.91MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [02:23<25:39, 13.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 935M/22.2G [02:23<1:05:06, 5.43MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [02:23<22:19, 15.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [02:23<41:29, 8.32MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 790M/22.2G [02:23<1:16:09, 4.68MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:23<51:03, 6.89MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 936M/22.2G [02:23<1:00:46, 5.82MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [02:23<22:35, 15.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:23<44:55, 7.84MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 791M/22.2G [02:23<1:14:55, 4.75MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:23<40:12, 8.75MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [02:23<20:23, 16.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 938M/22.2G [02:23<59:02, 5.99MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:24<37:53, 9.29MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 939M/22.2G [02:24<57:02, 6.20MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 793M/22.2G [02:24<1:14:38, 4.77MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:24<41:54, 8.40MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.44G/22.2G [02:24<1:23:44, 4.12MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 941M/22.2G [02:24<55:32, 6.37MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:24<38:40, 9.10MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.44G/22.2G [02:24<47:24, 7.28MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 794M/22.2G [02:24<1:15:02, 4.74MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [02:24<36:13, 9.44MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:24<41:36, 8.46MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 942M/22.2G [02:24<54:17, 6.51MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [02:24<47:56, 7.20MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [02:24<27:34, 12.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 795M/22.2G [02:24<1:14:05, 4.80MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [02:24<42:17, 8.16MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 944M/22.2G [02:24<54:05, 6.54MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [02:24<27:29, 12.4MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [02:24<23:04, 14.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 797M/22.2G [02:25<1:11:54, 4.95MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [02:25<44:05, 7.83MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 945M/22.2G [02:25<53:18, 6.63MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:25<1:11:14, 4.94MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [02:25<20:48, 16.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [02:25<39:59, 8.63MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [02:25<21:34, 15.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 947M/22.2G [02:25<52:38, 6.71MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:25<45:38, 7.71MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [02:25<42:03, 8.20MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [02:25<21:17, 16.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:25<45:14, 7.78MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 948M/22.2G [02:25<52:51, 6.69MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [02:25<37:34, 9.18MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [02:25<19:39, 17.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:25<41:53, 8.40MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 950M/22.2G [02:25<53:42, 6.58MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [02:25<40:42, 8.48MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:25<39:24, 8.92MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:25<37:47, 9.31MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [02:25<37:08, 9.29MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 798M/22.2G [02:25<2:17:34, 2.59MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 951M/22.2G [02:25<52:54, 6.68MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:26<36:40, 9.59MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [02:26<35:56, 9.60MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.66G/22.2G [02:26<35:28, 9.63MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:26<38:29, 9.14MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [02:26<26:04, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [02:26<39:56, 8.64MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 802M/22.2G [02:26<1:14:23, 4.78MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:26<36:30, 9.63MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [02:26<22:40, 15.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [02:26<35:50, 9.62MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:26<35:38, 9.87MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [02:26<20:56, 16.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 803M/22.2G [02:26<1:11:54, 4.95MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:26<35:03, 10.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [02:26<19:41, 17.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:26<34:11, 10.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 953M/22.2G [02:26<1:38:26, 3.59MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [02:26<18:41, 18.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 805M/22.2G [02:26<1:09:56, 5.09MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:26<34:02, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [02:26<17:58, 19.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:27<33:54, 10.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 806M/22.2G [02:27<1:04:21, 5.53MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [02:27<1:14:46, 4.61MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 957M/22.2G [02:27<58:40, 6.02MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 958M/22.2G [02:27<56:01, 6.31MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 808M/22.2G [02:27<1:05:00, 5.47MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [02:27<52:24, 6.58MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 960M/22.2G [02:27<54:22, 6.50MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 809M/22.2G [02:27<1:02:07, 5.73MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [02:27<33:16, 10.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [02:27<50:17, 6.86MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:27<1:09:26, 5.06MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [02:27<25:29, 13.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 811M/22.2G [02:27<58:52, 6.04MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 962M/22.2G [02:27<53:36, 6.59MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:27<42:18, 8.31MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [02:27<50:12, 6.87MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [02:27<21:38, 15.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:27<39:25, 8.91MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 812M/22.2G [02:27<56:28, 6.30MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 963M/22.2G [02:27<53:07, 6.65MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [02:27<20:02, 17.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [02:27<49:38, 6.95MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:28<37:57, 9.26MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [02:28<18:26, 18.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 814M/22.2G [02:28<55:20, 6.43MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:28<36:43, 9.57MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 965M/22.2G [02:28<52:38, 6.71MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [02:28<48:34, 7.10MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [02:28<17:37, 19.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:28<35:47, 9.82MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 815M/22.2G [02:28<53:44, 6.62MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [02:28<47:47, 7.21MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [02:28<16:56, 20.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:28<35:03, 10.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 966M/22.2G [02:28<56:21, 6.27MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 816M/22.2G [02:28<51:51, 6.86MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:28<34:00, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [02:28<46:42, 7.38MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:28<33:40, 10.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 818M/22.2G [02:28<48:11, 7.38MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 968M/22.2G [02:28<58:18, 6.06MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:28<33:01, 10.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [02:28<47:34, 7.24MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [02:28<29:39, 11.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 819M/22.2G [02:28<48:11, 7.38MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:28<32:40, 10.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 969M/22.2G [02:29<57:13, 6.17MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [02:29<46:02, 7.48MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [02:29<25:28, 13.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:29<32:23, 10.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [02:29<40:48, 8.44MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [02:29<20:35, 16.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:29<32:12, 10.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 971M/22.2G [02:29<1:01:52, 5.71MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [02:29<18:48, 18.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:29<32:00, 11.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [02:29<43:41, 7.89MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [02:29<17:42, 19.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 972M/22.2G [02:29<1:00:21, 5.85MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [02:29<39:58, 8.62MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [02:29<16:46, 20.3MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [02:29<16:01, 21.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 820M/22.2G [02:29<1:58:24, 3.00MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 974M/22.2G [02:29<1:08:28, 5.16MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:29<1:12:19, 4.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 824M/22.2G [02:30<1:07:09, 5.29MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [02:30<25:03, 13.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [02:30<1:20:42, 4.27MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:30<46:46, 7.50MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [02:30<24:35, 13.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:30<42:25, 8.27MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 825M/22.2G [02:30<1:06:40, 5.33MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [02:30<47:58, 7.18MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [02:30<19:37, 17.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:30<39:31, 8.88MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [02:30<17:22, 19.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 827M/22.2G [02:30<1:02:41, 5.67MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:30<36:52, 9.52MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [02:30<48:22, 7.12MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [02:30<17:04, 19.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:30<35:17, 9.95MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 975M/22.2G [02:30<2:00:28, 2.93MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 828M/22.2G [02:30<58:45, 6.05MB/s]  [Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [02:30<45:53, 7.50MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [02:30<15:17, 22.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:30<31:14, 11.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [02:31<15:12, 22.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▎         | 830M/22.2G [02:31<56:44, 6.26MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [02:31<43:30, 7.91MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 979M/22.2G [02:31<1:07:35, 5.22MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:31<31:07, 11.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 831M/22.2G [02:31<51:50, 6.86MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [02:31<38:55, 8.85MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:31<30:26, 11.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 980M/22.2G [02:31<1:03:59, 5.52MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 833M/22.2G [02:31<51:12, 6.94MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [02:31<42:05, 8.18MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:31<28:42, 12.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [02:31<26:45, 12.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 982M/22.2G [02:31<1:01:30, 5.74MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [02:31<25:43, 13.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 834M/22.2G [02:31<50:52, 6.98MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [02:31<40:54, 8.42MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▍         | 1.11G/22.2G [02:31<27:26, 12.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [02:31<19:31, 17.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 983M/22.2G [02:31<1:00:09, 5.87MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [02:31<37:19, 9.22MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 836M/22.2G [02:31<49:18, 7.21MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [02:31<16:16, 20.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 985M/22.2G [02:31<58:09, 6.07MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [02:31<40:02, 8.60MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 837M/22.2G [02:32<48:28, 7.33MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [02:32<15:58, 21.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [02:32<37:28, 9.18MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 839M/22.2G [02:32<45:31, 7.80MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [02:32<49:22, 7.10MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [02:32<13:45, 24.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 986M/22.2G [02:32<56:11, 6.28MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [02:32<35:48, 9.61MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 840M/22.2G [02:32<46:49, 7.59MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [02:32<38:05, 9.21MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 988M/22.2G [02:32<55:41, 6.33MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [02:32<35:28, 9.89MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 842M/22.2G [02:32<44:17, 8.02MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 989M/22.2G [02:32<54:22, 6.49MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [02:32<30:45, 11.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 991M/22.2G [02:32<54:06, 6.52MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [02:32<28:20, 12.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [02:33<1:16:30, 4.50MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [02:33<26:45, 13.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 842M/22.2G [02:33<1:25:58, 4.13MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 992M/22.2G [02:33<53:14, 6.62MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [02:33<39:29, 8.60MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [02:33<45:00, 7.65MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [02:33<25:36, 13.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 994M/22.2G [02:33<52:41, 6.69MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 846M/22.2G [02:33<53:05, 6.69MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [02:33<23:05, 15.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [02:33<45:32, 7.55MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [02:33<39:39, 8.56MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 848M/22.2G [02:33<47:47, 7.43MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   4%|▍         | 995M/22.2G [02:33<52:13, 6.75MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [02:33<40:14, 8.55MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [02:33<30:04, 11.3MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [02:33<24:51, 13.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 849M/22.2G [02:33<48:06, 7.38MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [02:33<42:55, 8.01MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [02:33<38:03, 9.04MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [02:33<40:28, 8.66MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [02:33<27:04, 12.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 851M/22.2G [02:34<52:26, 6.77MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [02:34<28:51, 12.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [02:34<19:22, 17.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [02:34<41:00, 8.39MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [02:34<28:15, 12.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 852M/22.2G [02:34<52:03, 6.82MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [02:34<36:27, 9.43MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [02:34<25:38, 13.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 997M/22.2G [02:34<1:37:42, 3.61MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [02:34<25:35, 13.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 854M/22.2G [02:34<56:51, 6.24MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [02:34<39:55, 8.61MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [02:34<28:03, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [02:34<22:50, 15.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [02:34<22:47, 14.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 855M/22.2G [02:34<53:51, 6.59MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [02:34<38:57, 8.83MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [02:34<22:46, 15.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [02:34<1:02:38, 5.63MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [02:34<20:33, 16.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [02:34<35:29, 9.69MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [02:34<20:37, 17.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 857M/22.2G [02:34<55:55, 6.35MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [02:35<17:01, 19.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [02:35<35:38, 9.65MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [02:35<19:15, 18.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [02:35<58:59, 5.98MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [02:35<16:26, 20.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 858M/22.2G [02:35<56:15, 6.31MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [02:35<19:32, 17.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 860M/22.2G [02:35<53:53, 6.59MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [02:35<1:01:44, 5.71MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [02:35<36:06, 9.69MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 861M/22.2G [02:35<52:50, 6.72MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:35<1:02:39, 5.63MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 863M/22.2G [02:35<52:01, 6.82MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [02:35<1:20:44, 4.26MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [02:35<35:47, 9.78MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [02:35<27:36, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [02:36<33:38, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:36<1:03:25, 5.56MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 864M/22.2G [02:36<52:10, 6.80MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [02:36<22:39, 15.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [02:36<50:49, 6.76MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [02:36<22:54, 14.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [02:36<35:46, 9.78MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:36<1:02:31, 5.64MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [02:36<49:20, 6.97MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [02:36<20:55, 16.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [02:36<36:38, 9.55MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [02:36<19:32, 17.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [02:36<48:59, 7.01MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:36<1:01:19, 5.75MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [02:36<18:13, 18.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 865M/22.2G [02:36<1:40:29, 3.53MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [02:36<48:37, 7.07MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [02:36<44:49, 7.80MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [02:36<17:42, 19.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:36<59:20, 5.94MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [02:36<17:25, 19.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [02:36<47:50, 7.18MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [02:36<42:58, 8.14MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 868M/22.2G [02:37<1:02:47, 5.65MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [02:37<58:51, 5.99MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [02:37<47:19, 7.26MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [02:37<39:32, 8.85MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [02:37<37:40, 9.28MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 870M/22.2G [02:37<1:06:30, 5.33MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [02:37<57:26, 6.13MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [02:37<45:47, 7.50MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [02:37<32:26, 10.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [02:37<35:45, 9.78MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [02:37<55:49, 6.31MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [02:37<34:11, 10.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [02:37<45:13, 7.60MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 871M/22.2G [02:37<1:10:31, 5.03MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [02:37<32:10, 10.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [02:37<33:00, 10.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [02:37<44:24, 7.73MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [02:37<27:41, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 873M/22.2G [02:38<1:10:46, 5.01MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [02:38<44:22, 7.74MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [02:38<26:26, 12.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 874M/22.2G [02:38<1:04:40, 5.48MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [02:38<43:08, 7.96MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [02:38<25:57, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [02:38<1:05:17, 5.36MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 876M/22.2G [02:38<1:00:01, 5.91MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [02:38<1:42:03, 3.45MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [02:38<25:37, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [02:38<44:43, 7.82MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 877M/22.2G [02:38<57:10, 6.20MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [02:38<41:57, 8.33MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [02:38<25:10, 13.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [02:38<39:00, 8.96MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [02:38<1:01:35, 5.72MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 879M/22.2G [02:38<54:26, 6.51MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [02:38<33:03, 10.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [02:39<1:16:44, 4.47MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [02:39<58:10, 6.05MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 880M/22.2G [02:39<52:39, 6.73MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [02:39<32:55, 10.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [02:39<46:38, 7.36MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:39<56:31, 6.23MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [02:39<31:38, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 882M/22.2G [02:39<51:14, 6.92MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [02:39<47:12, 7.27MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [02:39<29:02, 12.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:39<55:11, 6.38MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [02:39<43:12, 7.83MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 883M/22.2G [02:39<51:25, 6.89MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [02:39<29:12, 12.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [02:39<31:46, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [02:39<45:01, 7.62MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:39<54:07, 6.50MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 885M/22.2G [02:39<51:05, 6.94MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [02:39<29:13, 12.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [02:39<31:35, 10.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [02:39<42:42, 8.04MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:39<53:17, 6.61MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [02:39<27:26, 12.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [02:39<29:01, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [02:40<38:58, 8.81MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:40<52:40, 6.68MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [02:40<41:26, 8.28MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [02:40<27:13, 12.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [02:40<52:15, 6.74MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:40<38:34, 8.90MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [02:40<25:41, 13.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:40<37:16, 9.20MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:40<51:52, 6.79MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [02:40<24:47, 13.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:40<40:58, 8.37MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 886M/22.2G [02:40<1:54:09, 3.11MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:40<51:39, 6.81MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:40<38:54, 8.82MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.88G/22.2G [02:40<24:03, 14.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:41<36:02, 9.52MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:41<50:51, 6.92MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 890M/22.2G [02:41<1:08:10, 5.20MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:41<36:27, 9.41MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [02:41<1:30:41, 3.85MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:41<50:54, 6.91MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.88G/22.2G [02:41<33:57, 9.95MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 892M/22.2G [02:41<1:03:44, 5.56MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [02:41<29:43, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [02:41<1:05:25, 5.34MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [02:41<30:09, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [02:41<57:59, 6.02MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 893M/22.2G [02:41<1:04:46, 5.47MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [02:41<51:02, 6.84MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [02:41<26:59, 12.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 895M/22.2G [02:41<1:01:23, 5.77MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [02:41<45:25, 7.69MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [02:41<1:21:32, 4.21MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [02:42<25:24, 13.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 896M/22.2G [02:42<57:47, 6.13MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [02:42<36:35, 9.54MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [02:42<48:58, 7.00MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [02:42<1:44:15, 3.38MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [02:42<34:50, 10.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [02:42<24:23, 13.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 898M/22.2G [02:42<59:33, 5.95MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [02:42<49:04, 6.99MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [02:42<30:46, 11.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:42<1:00:01, 5.86MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [02:42<21:31, 15.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [02:42<44:50, 7.65MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 899M/22.2G [02:42<57:27, 6.17MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [02:42<30:41, 11.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [02:42<22:47, 14.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:42<57:02, 6.17MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [02:42<43:20, 7.91MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 901M/22.2G [02:42<55:59, 6.33MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.22G/22.2G [02:42<28:10, 12.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [02:42<40:55, 8.38MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:42<55:01, 6.39MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   5%|▌         | 1.22G/22.2G [02:42<31:50, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 902M/22.2G [02:42<54:53, 6.45MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:42<37:09, 9.23MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:43<54:00, 6.51MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [02:43<41:54, 8.05MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:43<40:37, 8.44MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 903M/22.2G [02:43<59:23, 5.96MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [02:43<29:50, 11.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:43<53:11, 6.61MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:43<39:10, 8.75MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [02:43<30:04, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [02:43<55:08, 6.33MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 905M/22.2G [02:43<1:02:39, 5.65MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:43<51:59, 6.76MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:43<42:52, 7.99MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [02:43<27:15, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [02:43<41:32, 8.40MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [02:43<51:45, 6.79MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [02:43<23:36, 14.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 906M/22.2G [02:43<1:04:13, 5.51MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:43<44:46, 7.65MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [02:43<42:19, 8.24MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [02:43<24:30, 13.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [02:43<39:45, 8.77MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:43<42:14, 8.11MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:43<51:32, 6.82MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 908M/22.2G [02:44<1:05:39, 5.39MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [02:44<21:26, 15.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [02:44<37:46, 9.23MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [02:44<44:57, 7.62MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:44<52:01, 6.76MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [02:44<22:51, 14.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [02:44<36:16, 9.62MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [02:44<20:50, 16.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [02:44<34:56, 9.98MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:44<51:06, 6.88MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [02:44<37:55, 9.19MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [02:44<36:30, 9.55MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:44<56:09, 6.26MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [02:44<35:28, 9.83MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [02:44<36:43, 9.18MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [02:44<1:32:25, 3.71MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 908M/22.2G [02:44<2:14:00, 2.64MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [02:44<31:05, 10.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [02:44<37:13, 9.36MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:44<53:06, 6.62MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [02:45<34:38, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.94G/22.2G [02:45<27:05, 12.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [02:45<53:11, 6.44MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [02:45<33:18, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 912M/22.2G [02:45<1:12:19, 4.89MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▊         | 1.94G/22.2G [02:45<23:07, 14.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [02:45<32:19, 10.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [02:45<49:35, 6.91MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [02:45<24:11, 13.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 914M/22.2G [02:45<1:09:34, 5.09MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [02:45<47:36, 7.19MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [02:45<21:50, 15.4MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [02:45<23:17, 14.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [02:45<46:06, 7.43MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [02:45<1:47:41, 3.26MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 916M/22.2G [02:45<1:11:21, 4.96MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [02:45<21:06, 16.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [02:45<1:08:21, 5.10MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [02:46<45:31, 7.52MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [02:46<23:14, 14.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 917M/22.2G [02:46<1:09:22, 5.10MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [02:46<45:23, 7.68MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:46<1:09:57, 5.02MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [02:46<43:42, 7.83MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [02:46<40:52, 8.52MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [02:46<39:45, 8.61MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [02:46<38:30, 9.05MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 919M/22.2G [02:46<1:08:53, 5.14MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:46<1:06:57, 5.25MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [02:46<40:09, 8.39MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [02:46<37:10, 9.37MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [02:46<42:33, 8.05MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [02:46<31:48, 10.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [02:46<35:46, 9.74MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:46<1:00:32, 5.80MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 920M/22.2G [02:46<1:13:49, 4.79MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [02:46<31:27, 10.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [02:46<41:03, 8.34MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [02:46<34:34, 10.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:46<59:27, 5.91MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [02:46<33:40, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [02:47<27:44, 12.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [02:47<40:33, 8.44MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 921M/22.2G [02:47<1:14:00, 4.78MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:47<33:13, 10.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [02:47<23:16, 14.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:47<33:00, 10.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [02:47<39:45, 8.61MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [02:47<24:20, 13.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:47<32:20, 10.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [02:47<1:12:38, 4.84MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 923M/22.2G [02:47<1:16:37, 4.62MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [02:47<21:14, 15.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:47<32:08, 10.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [02:47<22:43, 14.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:47<31:31, 11.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:47<1:02:21, 5.63MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 924M/22.2G [02:47<1:18:18, 4.52MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [02:47<1:06:31, 5.15MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:47<31:24, 11.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [02:47<20:48, 16.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [02:47<44:02, 7.77MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:47<1:02:39, 5.61MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 926M/22.2G [02:48<1:16:11, 4.64MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [02:48<45:07, 7.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [02:48<35:23, 9.50MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:48<1:05:36, 5.35MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 927M/22.2G [02:48<1:14:57, 4.72MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [02:48<1:07:44, 5.14MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [02:48<30:15, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [02:48<42:39, 8.02MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [02:48<30:32, 11.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [02:48<44:43, 7.78MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:48<1:06:54, 5.25MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [02:48<41:22, 8.27MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 929M/22.2G [02:48<1:17:13, 4.58MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [02:48<24:16, 13.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [02:48<40:46, 8.54MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [02:48<37:26, 9.14MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [02:48<25:28, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [02:48<37:58, 9.16MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:48<1:07:14, 5.22MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 930M/22.2G [02:48<1:14:21, 4.76MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [02:48<35:54, 9.69MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [02:48<22:12, 15.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [02:48<39:54, 8.57MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [02:49<35:24, 9.83MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [02:49<23:46, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [02:49<37:54, 9.02MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [02:49<1:08:16, 5.14MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [02:49<34:38, 10.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [02:49<21:00, 16.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [02:49<35:33, 9.62MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [02:49<32:58, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [02:49<22:40, 14.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [02:49<39:27, 8.67MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:49<1:07:22, 5.21MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [02:49<32:07, 10.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [02:49<37:20, 9.16MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [02:49<31:48, 10.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [02:49<35:04, 9.75MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [02:49<31:38, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 931M/22.2G [02:49<2:27:31, 2.40MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [02:49<31:29, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [02:49<40:31, 8.29MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [02:49<39:12, 8.72MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [02:50<31:27, 11.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [02:50<27:54, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 935M/22.2G [02:50<1:17:24, 4.57MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [02:50<28:15, 11.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:50<2:10:10, 2.70MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [02:50<23:45, 14.1MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [02:50<24:48, 13.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 936M/22.2G [02:50<1:19:15, 4.46MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [02:50<21:24, 15.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [02:50<1:11:37, 4.86MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:50<1:19:22, 4.42MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [02:50<22:45, 14.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 938M/22.2G [02:50<1:16:51, 4.60MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [02:50<1:40:29, 3.40MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [02:50<41:46, 8.32MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [02:50<20:38, 16.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:50<1:14:02, 4.74MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [02:51<38:43, 8.98MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 939M/22.2G [02:51<1:17:04, 4.59MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [02:51<21:33, 15.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [02:51<36:39, 9.49MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [02:51<1:08:00, 5.03MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:51<1:13:13, 4.79MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [02:51<37:19, 9.32MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 940M/22.2G [02:51<1:14:29, 4.75MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [02:51<1:08:02, 5.16MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [02:51<1:04:52, 5.27MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:51<38:50, 8.95MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [02:51<32:47, 10.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:51<36:00, 9.66MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [02:51<27:59, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 942M/22.2G [02:51<1:14:22, 4.75MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:51<34:20, 10.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:51<1:10:29, 4.98MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [02:51<1:02:58, 5.43MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [02:51<28:11, 11.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:51<33:57, 10.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [02:51<23:09, 14.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:52<32:42, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 943M/22.2G [02:52<1:17:58, 4.53MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [02:52<1:01:09, 5.59MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:52<1:08:01, 5.16MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:52<31:51, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [02:52<23:17, 14.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [02:52<58:28, 5.84MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [02:52<31:38, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [02:52<20:41, 16.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 945M/22.2G [02:52<1:14:50, 4.72MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:52<1:08:50, 5.10MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [02:52<31:09, 11.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [02:52<54:56, 6.22MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [02:52<21:29, 15.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 946M/22.2G [02:52<1:12:36, 4.87MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:52<1:06:58, 5.24MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [02:52<19:37, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [02:52<58:28, 5.84MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 948M/22.2G [02:52<1:15:55, 4.66MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [02:53<52:51, 6.46MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:53<1:07:45, 5.18MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [02:53<1:09:32, 5.00MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [02:53<32:46, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [02:53<52:34, 6.50MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 949M/22.2G [02:53<1:12:25, 4.88MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [02:53<40:48, 8.51MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [02:53<29:14, 11.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [02:53<1:06:42, 5.26MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [02:53<37:39, 9.22MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [02:53<52:23, 6.52MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [02:53<28:43, 11.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 951M/22.2G [02:53<1:10:00, 5.05MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [02:53<36:14, 9.58MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [02:53<23:47, 14.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.11G/22.2G [02:53<1:05:27, 5.36MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [02:53<51:55, 6.58MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [02:53<34:38, 10.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [02:53<21:11, 15.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 952M/22.2G [02:53<1:09:08, 5.11MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [02:53<33:42, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [02:53<51:53, 6.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [02:53<19:16, 17.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▍         | 1.11G/22.2G [02:53<1:05:19, 5.37MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [02:53<32:36, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [02:54<18:51, 17.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [02:54<31:46, 10.9MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [02:54<31:31, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [02:54<18:18, 18.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [02:54<31:21, 11.1MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [02:54<31:18, 11.1MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [02:54<30:58, 11.2MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [02:54<30:45, 11.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 953M/22.2G [02:54<2:25:10, 2.43MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [02:54<2:08:30, 2.73MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [02:55<46:12, 7.25MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 957M/22.2G [02:55<1:21:24, 4.34MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [02:55<39:07, 8.56MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [02:55<1:15:38, 4.64MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [02:55<1:02:55, 5.52MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [02:55<29:31, 11.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [02:55<37:57, 9.14MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [02:55<25:37, 13.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [02:55<1:12:11, 4.86MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 959M/22.2G [02:55<1:22:14, 4.30MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [02:55<36:11, 9.59MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [02:55<22:52, 14.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [02:55<34:57, 9.93MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [02:55<20:49, 16.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [02:55<1:09:50, 5.02MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [02:55<33:32, 10.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 960M/22.2G [02:56<1:23:35, 4.23MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [02:56<19:16, 17.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [02:56<29:55, 11.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [02:56<1:08:32, 5.12MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [02:56<30:00, 11.6MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [02:56<30:21, 11.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 962M/22.2G [02:56<1:24:29, 4.18MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [02:56<29:46, 11.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [02:56<33:32, 9.97MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [02:56<1:08:06, 5.15MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [02:56<25:22, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [02:56<27:48, 12.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 963M/22.2G [02:56<1:24:30, 4.18MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [02:56<21:19, 15.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [02:56<1:05:18, 5.37MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [02:56<28:24, 12.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [02:56<19:43, 17.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 965M/22.2G [02:57<1:18:51, 4.48MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [02:57<18:18, 18.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [02:57<1:03:46, 5.50MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [02:57<17:23, 19.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 966M/22.2G [02:57<1:20:59, 4.36MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [02:57<51:01, 6.80MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [02:57<16:51, 19.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [02:57<1:02:48, 5.58MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:57<34:32, 10.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [02:57<58:35, 5.98MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:57<33:01, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 967M/22.2G [02:57<1:27:22, 4.04MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [02:57<26:12, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:57<29:33, 11.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [02:57<22:20, 15.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [02:57<1:00:41, 5.77MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [02:58<20:41, 16.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:58<29:25, 11.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [02:58<55:32, 6.31MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 969M/22.2G [02:58<1:30:57, 3.88MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [02:58<18:40, 17.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [02:58<27:40, 12.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [02:58<17:28, 19.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:58<26:29, 13.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [02:58<16:39, 20.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 970M/22.2G [02:58<1:28:55, 3.97MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [02:58<14:24, 23.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:58<25:28, 13.6MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:58<25:54, 13.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 972M/22.2G [02:58<1:28:29, 3.99MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [02:59<22:39, 14.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [02:59<1:45:43, 3.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [02:59<21:36, 15.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 973M/22.2G [02:59<1:27:39, 4.03MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [02:59<17:59, 18.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [02:59<1:02:30, 5.60MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [02:59<48:35, 7.13MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [02:59<17:02, 19.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:59<34:06, 10.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [02:59<58:49, 5.96MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [02:59<14:26, 23.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 975M/22.2G [02:59<1:32:23, 3.82MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:59<29:57, 11.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [02:59<14:52, 22.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.66G/22.2G [02:59<8:23:53, 678kB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [02:59<56:18, 6.22MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [02:59<27:31, 12.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:00<54:20, 6.45MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [03:00<26:06, 13.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [03:00<25:59, 12.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:00<52:13, 6.71MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [03:00<23:21, 14.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:00<4:01:32, 1.41MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [03:00<21:10, 15.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:00<48:37, 7.20MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [03:00<23:43, 14.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [03:00<18:51, 17.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:00<3:10:27, 1.79MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [03:00<22:10, 15.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 975M/22.2G [03:00<2:57:06, 1.99MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [03:00<15:23, 21.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:00<54:41, 6.40MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:00<2:37:02, 2.17MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [03:00<13:35, 24.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:00<53:13, 6.58MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [03:01<13:35, 24.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:01<2:09:48, 2.63MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 979M/22.2G [03:01<1:33:16, 3.78MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [03:01<42:28, 8.14MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:01<53:54, 6.49MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:01<1:48:30, 3.15MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [03:01<30:23, 11.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 980M/22.2G [03:01<1:30:54, 3.88MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:01<59:32, 5.88MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [03:01<32:32, 10.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:01<1:36:48, 3.53MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [03:01<27:42, 12.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [03:01<28:22, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [03:01<27:46, 12.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 982M/22.2G [03:01<1:29:55, 3.92MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:01<1:25:00, 4.01MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [03:01<27:17, 12.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:01<1:07:18, 5.20MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [03:01<23:33, 14.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [03:01<26:47, 12.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:02<1:20:41, 4.23MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [03:02<27:24, 12.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [03:02<18:14, 18.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 983M/22.2G [03:02<1:29:20, 3.95MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:02<1:05:44, 5.33MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [03:02<15:06, 22.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [03:02<26:03, 13.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:02<1:16:24, 4.47MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [03:02<26:47, 12.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 985M/22.2G [03:02<1:28:42, 3.98MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [03:02<25:34, 13.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:02<1:12:28, 4.71MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [03:02<22:23, 14.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 986M/22.2G [03:02<1:27:16, 4.04MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:02<1:05:11, 5.23MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [03:02<19:28, 17.1MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [03:03<16:02, 20.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:03<2:05:01, 2.80MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [03:03<45:42, 7.56MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:03<1:06:02, 5.17MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [03:03<14:00, 23.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 988M/22.2G [03:03<1:32:44, 3.80MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [03:03<36:16, 9.53MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [03:03<12:46, 26.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:03<1:13:07, 4.79MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [03:03<31:10, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 989M/22.2G [03:03<1:38:42, 3.57MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [03:03<28:40, 12.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:03<1:11:06, 4.92MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [03:03<26:33, 13.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [03:04<24:24, 13.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:04<1:11:00, 4.93MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:04<1:53:08, 3.02MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [03:04<25:28, 13.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [03:04<20:55, 15.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 990M/22.2G [03:04<1:40:18, 3.52MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [03:04<16:37, 20.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [03:04<24:35, 14.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [03:04<1:04:28, 5.29MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:04<1:09:22, 5.04MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [03:04<13:22, 24.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 992M/22.2G [03:04<1:42:47, 3.43MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [03:04<15:35, 21.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [03:04<1:07:45, 5.03MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:04<1:19:44, 4.39MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.44G/22.2G [03:05<41:08, 8.39MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [03:05<1:06:03, 5.16MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 993M/22.2G [03:05<1:55:50, 3.04MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.44G/22.2G [03:05<31:52, 10.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:05<1:25:50, 4.08MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [03:05<26:50, 12.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [03:05<1:06:47, 5.11MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [03:05<27:09, 12.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [03:05<22:34, 14.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [03:05<26:55, 12.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:05<1:22:56, 4.22MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [03:05<18:05, 18.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [03:05<1:07:22, 5.06MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [03:05<24:23, 14.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [03:05<17:16, 19.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [03:05<24:39, 14.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 995M/22.2G [03:05<2:01:41, 2.90MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [03:05<14:47, 22.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [03:05<22:38, 15.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:05<1:07:49, 5.03MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:05<1:25:35, 4.09MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [03:06<23:09, 14.9MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [03:06<21:42, 15.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:06<1:07:20, 5.06MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   4%|▍         | 996M/22.2G [03:06<2:00:05, 2.94MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [03:06<22:22, 15.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:06<1:25:09, 4.11MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [03:06<21:27, 15.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:06<1:06:11, 5.15MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:06<1:22:13, 4.25MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [03:06<20:33, 16.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [03:06<36:33, 9.43MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:06<1:06:05, 5.16MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [03:06<16:36, 19.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [03:06<29:36, 11.6MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [03:06<29:43, 11.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [03:06<16:41, 19.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:06<1:22:46, 4.23MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:07<1:04:23, 5.29MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [03:07<16:35, 20.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [03:07<26:28, 13.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [03:07<16:25, 20.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [03:07<22:35, 15.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:07<1:21:15, 4.30MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:07<1:03:58, 5.33MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [03:07<16:18, 20.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [03:07<23:10, 14.9MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [03:07<21:14, 16.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:07<1:03:47, 5.34MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [03:07<21:50, 15.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 997M/22.2G [03:07<3:37:33, 1.62MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:07<1:22:13, 4.25MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [03:07<24:47, 13.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [03:07<20:24, 16.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [03:07<24:00, 13.8MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [03:08<20:06, 16.5MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [03:08<18:51, 17.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [03:08<1:58:39, 2.97MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [03:08<17:35, 18.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [03:08<41:25, 8.32MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [03:08<30:10, 11.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [03:08<16:51, 19.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:08<2:26:14, 2.39MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:08<1:50:22, 3.09MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [03:08<29:19, 11.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [03:08<16:21, 20.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [03:08<1:56:58, 3.01MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [03:08<25:15, 13.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [03:08<1:20:39, 4.33MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [03:08<25:05, 13.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:08<1:07:17, 5.06MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [03:09<22:15, 15.5MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [03:09<22:30, 15.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [03:09<30:16, 10.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.00G/22.2G [03:09<1:56:40, 3.02MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [03:09<1:22:27, 4.24MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [03:09<20:58, 16.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [03:09<25:06, 13.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:09<1:08:32, 4.97MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [03:09<20:40, 16.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [03:09<21:14, 16.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [03:09<18:52, 17.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [03:09<1:52:28, 3.13MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [03:09<1:21:07, 4.31MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:09<1:11:03, 4.79MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [03:09<17:46, 18.6MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [03:09<17:01, 19.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [03:09<1:19:44, 4.38MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:10<1:12:15, 4.71MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  10%|█         | 2.33G/22.2G [03:10<16:26, 20.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [03:10<1:55:05, 3.06MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [03:10<50:19, 6.84MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [03:10<1:14:49, 4.67MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:10<1:10:02, 4.86MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [03:10<36:25, 9.45MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [03:10<28:43, 11.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [03:10<1:52:26, 3.13MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [03:10<34:44, 9.90MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:10<1:14:44, 4.68MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:10<1:12:36, 4.69MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [03:10<24:47, 13.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [03:10<30:58, 11.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [03:10<20:38, 16.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:10<1:08:45, 4.95MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:10<1:15:24, 4.63MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [03:10<30:49, 11.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [03:10<1:50:22, 3.19MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [03:10<18:48, 17.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [03:11<30:23, 11.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [03:11<17:44, 18.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:11<1:08:45, 4.95MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [03:11<27:41, 12.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [03:11<16:49, 19.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:11<1:20:11, 4.36MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [03:11<1:48:27, 3.25MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [03:11<28:08, 12.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [03:11<15:59, 20.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:11<1:07:57, 5.01MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [03:11<27:56, 12.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:11<1:14:29, 4.69MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [03:11<26:44, 12.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [03:11<1:42:28, 3.44MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:11<1:07:09, 5.07MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:11<1:19:20, 4.40MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [03:12<1:06:48, 5.10MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.01G/22.2G [03:12<1:42:50, 3.43MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [03:12<52:00, 6.61MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:12<1:19:59, 4.37MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [03:12<1:05:42, 5.18MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [03:12<43:21, 7.61MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [03:12<1:38:17, 3.58MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [03:12<35:26, 9.70MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [03:12<38:20, 8.61MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [03:12<31:08, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [03:12<31:42, 10.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [03:12<1:27:13, 4.00MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [03:12<30:55, 11.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [03:12<26:17, 12.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [03:12<1:42:06, 3.45MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [03:13<22:18, 14.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [03:13<34:37, 9.93MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [03:13<19:59, 16.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [03:13<26:54, 12.8MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [03:13<27:31, 12.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [03:13<18:30, 17.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [03:13<1:40:31, 3.50MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [03:13<1:57:07, 2.91MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [03:13<30:58, 11.1MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [03:13<31:05, 11.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [03:13<1:09:51, 4.87MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [03:13<29:45, 11.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [03:13<31:34, 10.9MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [03:13<31:04, 11.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [03:13<23:48, 13.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [03:13<1:07:01, 5.08MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [03:14<20:43, 15.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [03:14<2:43:43, 2.13MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [03:14<19:42, 16.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:14<1:04:48, 5.25MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [03:14<16:28, 20.0MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [03:14<15:59, 20.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [03:14<1:33:52, 3.72MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:14<1:02:59, 5.40MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [03:14<1:10:13, 4.89MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [03:14<15:57, 20.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [03:14<3:26:01, 1.71MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [03:14<42:11, 8.14MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:14<1:01:49, 5.50MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [03:14<1:30:46, 3.85MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [03:14<38:53, 8.83MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [03:15<23:35, 14.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [03:15<38:15, 8.97MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:15<1:01:26, 5.54MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [03:15<23:37, 13.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [03:15<36:10, 9.49MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [03:15<1:28:20, 3.95MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [03:15<19:20, 17.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [03:15<1:54:22, 3.08MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [03:15<34:47, 9.86MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:15<1:00:27, 5.63MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [03:15<17:43, 18.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [03:15<34:10, 10.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:15<1:21:59, 4.26MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [03:15<16:44, 19.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:15<57:39, 5.90MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [03:15<33:14, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [03:15<16:02, 20.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [03:15<32:56, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:15<1:21:50, 4.27MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [03:15<15:31, 21.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [03:15<32:18, 10.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:15<1:00:43, 5.60MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.02G/22.2G [03:15<2:01:23, 2.90MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [03:16<36:13, 9.47MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:16<59:59, 5.67MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [03:16<35:06, 9.77MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:16<1:25:44, 4.07MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [03:16<25:23, 13.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [03:16<1:55:48, 3.04MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:16<56:29, 6.02MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [03:16<38:22, 8.94MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [03:16<23:56, 13.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:16<1:18:33, 4.44MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [03:16<37:32, 9.14MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [03:16<19:25, 16.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:16<59:39, 5.70MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [03:16<1:47:21, 3.28MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [03:16<17:44, 18.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:16<1:17:26, 4.51MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [03:16<16:24, 20.0MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [03:17<16:06, 20.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [03:17<1:46:28, 3.31MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:17<1:16:36, 4.56MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [03:17<15:30, 21.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [03:17<1:19:30, 4.31MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [03:17<49:19, 6.95MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [03:17<1:41:13, 3.48MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:17<1:58:54, 2.86MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:17<1:22:24, 4.24MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [03:17<49:33, 6.92MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [03:17<26:36, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [03:17<44:38, 7.68MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [03:17<26:16, 12.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:17<1:06:36, 5.10MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [03:17<1:40:34, 3.50MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [03:17<20:50, 15.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [03:17<44:46, 7.66MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.22G/22.2G [03:18<1:29:08, 3.92MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [03:18<18:55, 17.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [03:18<41:19, 8.30MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:18<1:05:03, 5.23MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [03:18<17:59, 18.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [03:18<1:40:23, 3.51MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [03:18<41:59, 8.16MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [03:18<16:54, 19.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.22G/22.2G [03:18<1:28:05, 3.96MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [03:18<39:35, 8.66MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:18<1:02:36, 5.43MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [03:18<38:09, 8.98MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.03G/22.2G [03:18<1:38:19, 3.58MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [03:18<36:02, 9.51MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:18<1:01:21, 5.54MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   5%|▌         | 1.22G/22.2G [03:18<1:29:35, 3.89MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [03:18<38:35, 8.88MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:18<59:11, 5.74MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [03:19<28:37, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [03:19<37:34, 9.12MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [03:19<1:37:27, 3.61MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [03:19<23:16, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:19<1:00:43, 5.60MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [03:19<37:20, 9.18MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [03:19<19:41, 16.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [03:19<36:02, 9.51MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [03:19<1:34:38, 3.72MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:19<58:32, 5.81MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [03:19<18:09, 18.1MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [03:19<17:02, 19.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:19<56:48, 5.98MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [03:19<2:49:15, 2.06MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [03:19<16:14, 20.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:19<55:59, 6.07MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [03:19<1:41:25, 3.47MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [03:19<15:38, 21.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:20<58:30, 5.81MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [03:20<1:35:33, 3.65MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [03:20<1:32:51, 3.69MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [03:20<24:51, 13.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:20<56:33, 6.01MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [03:20<1:43:43, 3.39MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [03:20<24:22, 13.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [03:20<58:06, 5.90MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [03:20<1:30:54, 3.84MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:20<55:11, 6.16MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [03:20<19:41, 16.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [03:20<53:47, 6.37MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.49G/22.2G [03:20<17:58, 18.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:20<53:37, 6.34MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [03:20<48:38, 7.04MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [03:20<16:54, 19.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [03:20<1:28:26, 3.94MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [03:20<44:15, 7.74MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [03:21<16:02, 20.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [03:21<41:19, 8.29MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [03:21<1:20:15, 4.35MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [03:21<15:37, 21.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [03:21<41:13, 8.31MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [03:21<39:04, 8.77MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [03:21<1:24:31, 4.13MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [03:21<36:36, 9.35MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:21<1:50:00, 3.09MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [03:21<24:49, 13.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.04G/22.2G [03:21<3:15:57, 1.80MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [03:21<35:45, 9.58MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [03:21<23:27, 14.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:21<1:02:49, 5.41MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [03:21<1:18:47, 4.43MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [03:21<37:17, 9.18MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [03:21<20:06, 16.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [03:22<35:32, 9.63MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [03:22<18:23, 17.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:22<57:37, 5.89MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [03:22<1:44:49, 3.36MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [03:22<34:33, 9.90MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [03:22<17:09, 19.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:22<54:56, 6.18MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [03:22<34:23, 9.95MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [03:22<1:28:17, 3.95MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [03:22<14:41, 22.3MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [03:22<14:38, 22.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:22<52:47, 6.43MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [03:22<1:43:43, 3.39MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:22<50:38, 6.70MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [03:22<1:09:05, 4.95MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [03:22<1:28:10, 3.95MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [03:22<1:40:50, 3.49MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [03:22<24:28, 13.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:22<49:05, 6.91MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [03:23<43:51, 7.80MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [03:23<25:05, 13.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:23<46:57, 7.23MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [03:23<20:04, 16.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [03:23<44:27, 7.70MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [03:23<1:29:31, 3.89MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [03:23<1:41:52, 3.45MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:23<45:21, 7.48MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [03:23<18:05, 18.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [03:23<41:07, 8.32MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [03:23<17:57, 18.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:23<40:06, 8.46MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [03:23<37:18, 9.17MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [03:23<17:38, 18.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [03:23<1:30:14, 3.86MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:23<42:34, 7.97MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [03:23<39:33, 8.65MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [03:23<17:37, 18.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [03:23<1:40:34, 3.50MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [03:23<36:53, 9.27MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:23<38:25, 8.83MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [03:24<34:36, 9.89MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [03:24<1:30:31, 3.85MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:24<40:50, 8.31MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [03:24<1:37:28, 3.61MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [03:24<37:20, 9.16MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [03:24<29:05, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [03:24<36:05, 9.48MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [03:24<1:29:30, 3.89MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [03:24<22:52, 14.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [03:24<34:47, 9.83MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.05G/22.2G [03:24<1:38:44, 3.56MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [03:24<19:02, 17.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [03:24<33:25, 10.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:24<1:17:32, 4.38MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [03:24<17:50, 18.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [03:24<36:20, 9.41MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [03:24<16:38, 19.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [03:24<1:35:55, 3.67MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:24<48:35, 6.98MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [03:25<15:36, 20.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:25<43:08, 7.86MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:25<40:34, 8.36MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [03:25<15:38, 20.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [03:25<1:37:25, 3.61MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:25<38:04, 8.91MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:25<35:49, 9.47MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [03:25<2:32:55, 2.28MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [03:25<21:57, 14.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [03:25<1:15:43, 4.51MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:25<34:33, 9.81MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [03:25<22:02, 14.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [03:25<1:34:59, 3.70MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [03:25<43:51, 7.80MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:25<32:57, 10.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [03:25<18:08, 18.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [03:25<1:25:18, 4.09MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:25<31:46, 10.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [03:25<44:07, 7.75MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [03:25<17:00, 19.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:26<30:34, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [03:26<41:07, 8.31MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [03:26<1:35:37, 3.68MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [03:26<15:57, 20.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [03:26<37:27, 9.12MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:26<29:43, 11.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [03:26<1:24:51, 4.11MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [03:26<13:55, 23.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [03:26<36:14, 9.43MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [03:26<29:36, 11.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [03:26<1:34:11, 3.73MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [03:26<38:30, 8.87MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [03:26<1:26:20, 4.04MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [03:26<36:03, 9.48MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [03:26<35:07, 9.73MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [03:26<27:57, 11.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [03:26<1:35:43, 3.67MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [03:26<33:56, 10.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [03:26<55:50, 6.07MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [03:26<1:23:14, 4.19MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [03:27<33:08, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [03:27<24:30, 13.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [03:27<36:32, 9.27MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [03:27<33:14, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [03:27<21:01, 15.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [03:27<31:47, 10.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [03:27<33:51, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [03:27<19:33, 16.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [03:27<1:26:02, 4.05MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [03:27<29:49, 11.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [03:27<20:07, 16.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [03:27<37:05, 9.21MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [03:27<28:19, 12.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [03:27<19:50, 16.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [03:27<1:23:17, 4.18MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [03:27<25:37, 13.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [03:27<20:46, 15.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [03:27<24:59, 13.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [03:27<19:20, 16.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.66G/22.2G [03:28<1:08:50, 4.96MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [03:28<1:30:26, 3.85MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.06G/22.2G [03:28<3:05:05, 1.90MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [03:28<23:08, 14.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:28<42:37, 8.01MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [03:28<35:46, 9.10MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:28<1:27:26, 3.98MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:28<47:38, 7.17MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [03:28<1:38:10, 3.58MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [03:28<35:02, 9.66MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [03:28<28:48, 11.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:28<39:38, 8.61MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [03:28<26:04, 13.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [03:28<22:38, 14.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:28<1:29:11, 3.91MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:28<40:27, 8.44MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [03:28<26:19, 12.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [03:28<1:39:14, 3.54MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [03:28<23:09, 14.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:29<41:40, 8.19MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [03:29<20:49, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [03:29<23:21, 14.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:29<1:29:08, 3.91MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [03:29<39:45, 8.58MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [03:29<19:09, 17.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [03:29<19:24, 17.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [03:29<1:36:11, 3.65MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [03:29<20:15, 16.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [03:29<18:40, 18.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:29<41:04, 8.31MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [03:29<18:07, 18.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [03:29<19:46, 16.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [03:29<1:30:49, 3.87MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:29<1:26:30, 4.03MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:29<40:33, 8.42MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:29<40:32, 8.42MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:29<1:20:28, 4.33MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [03:29<1:29:37, 3.92MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:29<38:02, 8.97MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [03:30<36:58, 8.80MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:30<39:26, 8.65MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:30<1:20:53, 4.30MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [03:30<36:18, 9.31MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [03:30<26:59, 12.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.07G/22.2G [03:30<1:28:02, 3.99MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:30<37:38, 9.06MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [03:30<26:47, 12.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [03:30<27:38, 12.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [03:30<23:01, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [03:30<23:02, 14.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [03:30<1:24:35, 4.15MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [03:30<20:36, 16.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [03:30<20:03, 16.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [03:30<18:54, 17.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [03:30<18:32, 17.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [03:30<1:15:35, 4.51MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [03:30<1:21:20, 4.32MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [03:30<17:28, 19.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [03:31<20:11, 16.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [03:31<16:17, 20.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [03:31<47:32, 7.18MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:31<2:36:18, 2.23MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [03:31<18:46, 17.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [03:31<1:17:39, 4.52MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [03:31<46:54, 7.27MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [03:31<43:20, 7.87MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [03:31<1:14:17, 4.73MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.88G/22.2G [03:31<27:08, 12.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [03:31<1:26:15, 4.04MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [03:31<39:56, 8.54MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [03:31<24:34, 13.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [03:31<33:31, 9.69MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [03:31<37:29, 9.09MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [03:31<1:11:44, 4.90MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [03:31<19:30, 17.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [03:31<28:29, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [03:31<1:22:42, 4.21MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:31<39:21, 8.66MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [03:32<27:30, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [03:32<1:10:58, 4.95MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:32<36:31, 9.34MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [03:32<20:00, 16.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [03:32<23:24, 13.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [03:32<1:20:53, 4.30MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [03:32<17:18, 19.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:32<39:59, 8.53MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [03:32<21:25, 15.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.08G/22.2G [03:32<1:08:57, 5.09MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [03:32<16:51, 20.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:32<41:47, 8.16MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [03:32<19:55, 16.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [03:32<1:23:50, 4.15MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [03:32<18:37, 17.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:32<42:39, 7.99MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [03:32<17:16, 18.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:32<43:08, 7.90MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [03:32<1:16:31, 4.55MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [03:32<30:21, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [03:33<43:21, 7.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [03:33<1:59:27, 2.94MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [03:33<24:00, 14.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [03:33<1:15:41, 4.60MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:33<44:01, 7.74MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [03:33<21:00, 16.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [03:33<1:03:48, 5.50MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [03:33<19:44, 17.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:33<1:20:28, 4.32MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [03:33<18:41, 18.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [03:33<1:00:00, 5.85MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [03:33<17:49, 18.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [03:33<57:22, 6.12MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:33<1:20:13, 4.25MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [03:33<17:04, 19.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:33<1:17:39, 4.48MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.09G/22.2G [03:33<54:51, 6.40MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:34<50:57, 6.69MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:34<1:13:07, 4.76MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [03:34<58:33, 5.99MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:34<48:42, 7.00MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [03:34<30:48, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [03:34<49:39, 7.07MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:34<44:27, 7.66MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [03:34<23:39, 14.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:34<1:13:02, 4.76MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [03:34<45:09, 7.54MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▊         | 1.94G/22.2G [03:34<20:04, 16.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [03:34<53:58, 6.50MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [03:34<18:32, 18.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:34<1:13:10, 4.75MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:34<45:34, 7.48MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [03:34<52:58, 6.62MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [03:34<17:17, 19.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:34<41:38, 8.18MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [03:35<16:28, 20.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [03:35<53:29, 6.56MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:35<1:13:45, 4.72MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:35<42:42, 7.97MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [03:35<15:46, 21.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:35<43:05, 7.90MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [03:35<57:14, 6.13MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:35<1:13:46, 4.72MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:35<40:32, 8.40MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.10G/22.2G [03:35<52:23, 6.70MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [03:35<27:02, 12.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:35<41:56, 8.12MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.11G/22.2G [03:35<52:47, 6.64MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [03:35<24:32, 13.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [03:35<39:53, 8.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [03:35<20:03, 16.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▍         | 1.11G/22.2G [03:35<53:03, 6.61MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [03:36<41:16, 8.25MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [03:36<18:08, 18.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [03:36<38:53, 8.75MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [03:36<16:49, 20.0MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [03:36<15:49, 21.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [03:36<2:32:53, 2.28MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [03:36<15:08, 22.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [03:36<1:48:31, 3.23MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [03:36<1:25:58, 3.96MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [03:36<1:28:38, 3.92MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [03:36<23:28, 14.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [03:36<1:00:49, 5.77MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [03:37<23:56, 14.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [03:37<51:06, 6.66MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [03:37<19:21, 17.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [03:37<57:58, 6.05MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [03:37<1:29:30, 3.89MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [03:37<48:49, 6.97MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [03:37<17:18, 19.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.11G/22.2G [03:37<55:13, 6.35MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:37<43:44, 7.78MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [03:37<15:59, 21.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [03:37<1:30:04, 3.86MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [03:37<15:05, 22.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [03:37<53:22, 6.57MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:37<43:37, 7.80MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [03:37<3:09:18, 1.71MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [03:37<50:09, 6.99MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:37<44:10, 7.70MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [03:37<1:29:35, 3.88MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:37<39:48, 8.55MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [03:38<49:58, 7.02MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:38<41:22, 8.22MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [03:38<2:26:31, 2.21MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [03:38<48:47, 7.19MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [03:38<1:25:40, 4.06MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [03:38<2:02:17, 2.65MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:38<41:46, 8.14MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [03:38<34:17, 9.80MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [03:38<45:55, 7.63MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [03:38<1:21:38, 3.97MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [03:38<39:38, 8.58MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [03:38<28:28, 11.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.12G/22.2G [03:38<47:00, 7.46MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [03:38<59:29, 5.45MB/s]  [A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [03:38<1:27:15, 3.98MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:38<40:21, 8.43MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [03:38<22:01, 15.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [03:38<45:14, 7.17MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [03:38<46:09, 7.59MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:38<38:17, 8.88MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [03:38<19:21, 17.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [03:38<36:44, 8.82MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [03:38<1:22:12, 4.23MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [03:39<45:24, 7.72MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:39<40:58, 8.30MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [03:39<30:49, 10.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [03:39<43:57, 7.97MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [03:39<23:34, 14.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [03:39<1:21:14, 4.28MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [03:39<19:51, 16.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [03:39<41:01, 7.90MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [03:39<1:29:01, 3.90MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [03:39<34:36, 9.37MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:39<1:26:13, 3.94MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [03:39<33:04, 9.80MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [03:39<31:23, 10.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [03:40<26:59, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [03:40<24:58, 13.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [03:40<50:40, 6.71MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [03:40<1:43:47, 3.38MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [03:40<1:35:13, 3.65MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [03:40<23:30, 13.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [03:40<24:42, 13.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:40<46:29, 7.31MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [03:40<20:13, 16.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [03:40<20:34, 15.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.13G/22.2G [03:40<58:46, 5.96MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:40<45:38, 7.45MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [03:40<20:37, 16.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [03:40<18:48, 17.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [03:40<1:31:55, 3.78MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:40<55:00, 6.37MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [03:40<19:22, 17.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:40<44:53, 7.57MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [03:40<17:32, 18.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:40<50:13, 6.97MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [03:40<17:48, 18.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:40<41:55, 8.11MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [03:40<17:28, 19.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [03:40<1:33:13, 3.73MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:40<49:44, 7.04MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:40<42:14, 8.05MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:41<45:42, 7.66MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:41<39:36, 8.58MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [03:41<31:06, 10.4MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [03:41<27:54, 11.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [03:41<1:33:06, 3.73MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [03:41<40:42, 8.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:41<47:04, 7.44MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [03:41<23:30, 13.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:41<38:31, 8.82MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:41<44:16, 7.91MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [03:41<36:02, 9.30MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [03:41<21:23, 15.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:41<39:47, 8.54MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.14G/22.2G [03:41<45:58, 7.62MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [03:41<30:26, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [03:41<19:56, 16.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:41<38:40, 8.79MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:41<43:13, 8.10MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [03:41<24:37, 13.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [03:41<18:54, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [03:41<22:08, 15.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:41<40:49, 8.32MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:42<45:30, 7.69MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [03:42<18:12, 17.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [03:42<20:24, 16.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:42<38:53, 8.73MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [03:42<17:37, 18.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:42<45:50, 7.64MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [03:42<19:08, 17.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:42<42:37, 8.21MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [03:42<18:06, 18.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [03:42<2:50:26, 2.04MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:42<43:54, 7.97MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [03:42<32:39, 9.90MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [03:42<1:21:22, 4.17MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [03:42<28:58, 11.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [03:42<1:33:17, 3.72MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [03:42<32:15, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [03:42<24:00, 13.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:43<51:30, 6.59MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [03:43<24:27, 13.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [03:43<21:40, 14.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:43<45:39, 7.44MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.15G/22.2G [03:43<1:27:22, 4.01MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [03:43<21:02, 15.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [03:43<19:59, 16.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [03:43<1:32:01, 3.77MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:43<44:53, 7.56MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [03:43<19:28, 17.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [03:43<18:57, 17.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:43<54:25, 6.43MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:43<40:53, 8.30MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [03:43<19:07, 17.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [03:43<22:02, 14.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:43<38:45, 8.76MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [03:43<18:07, 18.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [03:43<1:31:17, 3.80MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:43<51:33, 6.79MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [03:43<20:37, 15.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [03:43<17:22, 19.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [03:43<40:14, 8.44MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:43<46:45, 7.48MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:43<37:41, 9.01MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:44<47:21, 7.39MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [03:44<1:30:19, 3.85MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:44<36:15, 9.36MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:44<46:45, 7.48MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:44<39:06, 8.68MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [03:44<30:40, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.16G/22.2G [03:44<43:20, 8.07MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [03:44<42:31, 7.60MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [03:44<1:29:43, 3.87MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:44<35:57, 9.44MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [03:44<23:30, 14.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [03:44<30:14, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:44<34:19, 9.89MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [03:44<20:09, 16.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:44<44:50, 7.80MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [03:44<28:47, 11.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [03:44<18:55, 17.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:44<37:02, 9.16MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [03:44<1:29:13, 3.89MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:44<45:42, 7.65MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [03:44<21:39, 14.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [03:44<17:54, 18.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:44<43:01, 8.13MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [03:45<20:04, 16.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [03:45<21:19, 15.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [03:45<19:02, 17.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:45<43:06, 8.11MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [03:45<1:29:00, 3.90MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [03:45<18:56, 17.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [03:45<18:12, 17.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:45<41:00, 8.53MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [03:45<17:35, 18.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:45<43:13, 8.09MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [03:45<1:29:40, 3.87MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [03:45<1:21:11, 4.18MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.10G/22.2G [03:45<31:11, 10.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:45<46:27, 7.30MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [03:45<29:15, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [03:45<23:57, 13.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [03:45<1:29:32, 3.88MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:45<42:44, 7.94MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [03:46<27:20, 11.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [03:46<24:02, 13.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:46<39:59, 8.48MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [03:46<24:33, 13.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [03:46<22:43, 14.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:46<38:00, 8.92MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.17G/22.2G [03:46<1:22:08, 4.26MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [03:46<1:27:55, 3.95MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [03:46<35:58, 9.43MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [03:46<20:40, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [03:46<20:56, 15.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:46<34:24, 9.86MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [03:46<21:44, 15.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [03:46<19:17, 16.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [03:46<50:08, 6.97MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:46<33:06, 10.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [03:46<20:46, 16.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [03:46<46:14, 7.56MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [03:46<18:20, 17.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [03:46<1:27:35, 3.96MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:46<32:42, 10.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [03:46<21:58, 15.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [03:46<17:32, 18.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [03:46<45:40, 7.65MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:46<31:38, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [03:46<19:52, 16.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [03:46<43:33, 8.02MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:46<30:40, 11.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [03:47<1:26:36, 4.01MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [03:47<31:30, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.18G/22.2G [03:47<43:29, 8.04MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [03:47<29:28, 11.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:47<40:53, 8.55MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [03:47<32:51, 9.81MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [03:47<1:25:58, 4.04MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [03:47<40:55, 8.16MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:47<41:22, 8.45MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [03:47<26:09, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [03:47<28:50, 11.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:47<40:03, 8.72MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [03:47<24:59, 12.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [03:47<28:28, 11.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [03:47<24:27, 13.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:47<38:19, 9.12MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [03:47<1:04:36, 5.25MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [03:47<21:49, 14.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:47<38:01, 9.19MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [03:47<24:57, 13.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [03:48<19:42, 16.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [03:48<39:31, 8.58MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [03:48<20:58, 15.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.19G/22.2G [03:48<39:45, 8.79MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [03:48<18:25, 17.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [03:48<35:15, 9.61MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [03:48<21:43, 15.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [03:48<38:03, 9.18MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [03:48<2:38:47, 2.19MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [03:48<33:26, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [03:48<17:35, 18.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [03:48<20:09, 16.5MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [03:48<21:20, 15.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [03:48<29:34, 11.5MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [03:48<29:11, 11.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [03:48<1:26:04, 4.03MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [03:48<30:39, 10.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [03:48<26:35, 12.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [03:48<1:12:05, 4.85MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [03:49<23:20, 13.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [03:49<1:27:28, 3.97MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [03:49<25:17, 13.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [03:49<42:42, 8.18MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [03:49<20:12, 15.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [03:49<39:56, 8.74MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [03:49<24:20, 13.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [03:49<18:44, 17.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [03:49<48:08, 6.93MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [03:49<1:22:08, 4.22MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.20G/22.2G [03:49<38:44, 9.01MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [03:49<17:54, 18.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:49<37:16, 9.37MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [03:49<35:26, 9.41MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [03:49<16:59, 18.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [03:49<1:19:48, 4.35MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:49<36:04, 9.68MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [03:49<32:31, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [03:49<16:24, 19.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:49<35:02, 9.96MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [03:49<1:12:42, 4.77MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [03:49<27:14, 12.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:49<34:08, 10.2MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:50<33:18, 10.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [03:50<24:09, 13.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [03:50<1:13:35, 4.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:50<33:56, 10.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [03:50<20:32, 16.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [03:50<31:38, 10.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.21G/22.2G [03:50<32:22, 10.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [03:50<21:21, 15.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [03:50<24:20, 13.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [03:50<1:11:22, 4.86MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.22G/22.2G [03:50<31:56, 10.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [03:50<20:49, 15.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   5%|▌         | 1.22G/22.2G [03:50<31:40, 11.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [03:50<1:05:17, 5.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [03:50<18:43, 17.2MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [03:50<17:31, 18.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [03:51<1:07:31, 5.14MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [03:51<36:15, 9.19MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [03:51<20:51, 15.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [03:51<29:06, 11.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [03:51<1:00:44, 5.71MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [03:51<18:13, 17.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [03:51<1:09:53, 4.99MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [03:51<27:09, 12.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [03:51<59:11, 5.86MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [03:51<41:37, 8.38MB/s]  [Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [03:51<27:40, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.22G/22.2G [03:51<38:28, 9.07MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [03:51<28:05, 11.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [03:51<57:17, 6.05MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [03:51<28:13, 11.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [03:51<33:44, 10.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [03:51<33:49, 9.50MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [03:51<55:59, 6.19MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [03:51<28:32, 11.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [03:52<25:20, 12.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [03:52<2:22:14, 2.38MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [03:52<30:15, 11.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [03:52<28:29, 11.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [03:52<25:11, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [03:52<28:26, 11.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.23G/22.2G [03:52<28:35, 12.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [03:52<21:39, 14.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [03:52<28:10, 11.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [03:52<22:22, 14.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [03:52<27:10, 12.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [03:52<28:41, 11.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [03:52<1:38:52, 3.51MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [03:52<1:42:39, 3.30MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [03:52<27:38, 12.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [03:52<1:26:02, 3.93MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [03:52<23:49, 13.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [03:52<25:58, 13.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [03:52<58:20, 5.94MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [03:52<20:31, 15.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [03:52<1:02:15, 5.44MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [03:53<47:30, 7.12MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [03:53<53:08, 6.52MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [03:53<21:40, 14.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [03:53<38:04, 8.89MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [03:53<42:55, 8.12MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [03:53<48:07, 7.20MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [03:53<59:51, 5.56MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [03:53<31:16, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.24G/22.2G [03:53<30:04, 11.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [03:53<48:26, 7.15MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [03:53<26:28, 12.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [03:53<29:52, 11.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [03:53<40:09, 8.28MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [03:53<43:26, 7.98MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [03:53<25:53, 13.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [03:53<35:02, 9.49MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [03:53<40:00, 8.02MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [03:53<40:16, 8.61MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [03:53<26:23, 13.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [03:53<32:46, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [03:53<28:45, 11.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [03:53<39:34, 8.76MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.25G/22.2G [03:53<23:03, 15.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [03:53<37:31, 9.01MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [03:53<31:05, 10.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [03:54<28:35, 11.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [03:54<38:16, 9.05MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:54<23:45, 14.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [03:54<29:31, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [03:54<32:32, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [03:54<28:56, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:54<21:44, 16.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [03:54<37:11, 9.32MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [03:54<26:49, 12.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:54<20:03, 17.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [03:54<36:05, 9.60MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [03:54<26:28, 12.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [03:54<39:49, 8.35MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [03:54<34:52, 9.93MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [03:54<32:37, 9.83MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [03:54<36:01, 9.23MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [03:54<25:47, 13.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [03:54<33:37, 10.3MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [03:54<26:44, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [03:54<37:03, 8.97MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [03:54<27:34, 11.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [03:54<25:23, 13.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [03:54<34:54, 9.52MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [03:54<28:31, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [03:55<24:10, 14.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.26G/22.2G [03:55<45:40, 7.62MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [03:55<28:47, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [03:55<36:51, 9.02MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [03:55<33:03, 10.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [03:55<23:36, 14.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [03:55<1:06:30, 5.21MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [03:55<29:43, 10.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|▉         | 2.22G/22.2G [03:55<35:31, 9.35MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [03:55<29:41, 11.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [03:55<29:41, 10.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [03:55<44:12, 7.83MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.27G/22.2G [03:55<25:50, 13.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [03:55<36:49, 9.40MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:55<22:44, 15.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.88G/22.2G [03:55<42:10, 8.01MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:55<20:49, 16.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [03:55<29:28, 11.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:55<19:23, 17.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [03:55<41:35, 8.32MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [03:56<59:36, 5.38MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [03:56<28:49, 11.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.28G/22.2G [03:56<18:23, 18.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [03:56<1:20:40, 4.12MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [03:56<30:57, 11.2MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [03:56<26:56, 12.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [03:56<37:50, 8.47MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [03:56<31:08, 11.1MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [03:56<55:13, 6.02MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [03:56<23:23, 14.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [03:56<35:15, 9.09MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [03:56<31:40, 10.9MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [03:56<23:44, 14.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [03:56<33:29, 9.57MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [03:56<32:09, 10.8MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [03:56<51:24, 6.46MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [03:56<21:47, 15.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [03:56<32:12, 9.95MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [03:56<32:30, 10.6MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [03:56<22:20, 15.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [03:56<50:03, 6.64MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [03:56<31:28, 10.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [03:56<32:29, 10.7MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [03:56<21:40, 15.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [03:56<31:31, 10.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [03:57<32:23, 10.7MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [03:57<48:30, 6.85MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [03:57<30:32, 10.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [03:57<47:23, 7.01MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [03:57<30:09, 10.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [03:57<37:14, 9.06MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [03:57<29:34, 10.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [03:57<46:27, 7.15MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [03:57<26:52, 12.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [03:57<29:31, 10.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [03:57<1:07:22, 5.14MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [03:57<45:47, 7.25MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [03:57<29:27, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [03:57<29:21, 11.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [03:57<47:49, 7.23MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [03:57<26:00, 13.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [03:57<44:56, 7.39MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [03:58<41:19, 8.37MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [03:58<25:02, 13.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [03:58<44:13, 7.51MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [03:58<40:10, 8.61MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [03:58<25:36, 13.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [03:58<54:10, 5.91MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [03:58<43:43, 7.59MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [03:58<37:05, 9.32MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [03:58<26:31, 12.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [03:58<34:37, 9.99MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [03:58<43:19, 7.66MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [03:58<35:23, 9.04MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [03:58<25:55, 13.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [03:58<32:45, 10.6MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [03:58<26:27, 12.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [03:58<32:28, 9.85MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [03:58<32:03, 10.8MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [03:58<31:10, 10.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [03:58<25:18, 13.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [03:58<29:52, 10.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [03:58<30:01, 11.5MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [03:59<29:41, 10.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [03:59<29:34, 11.7MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [03:59<28:44, 11.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [03:59<32:07, 10.8MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [03:59<1:17:33, 4.28MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [03:59<44:37, 7.55MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [03:59<28:50, 11.1MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [03:59<28:49, 11.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [03:59<33:55, 9.94MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [03:59<49:30, 6.70MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [03:59<32:08, 10.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [03:59<26:45, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [03:59<46:40, 7.11MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▊         | 1.94G/22.2G [03:59<28:39, 11.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [03:59<27:49, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:00<45:13, 7.34MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [04:00<26:47, 12.6MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [04:00<26:33, 12.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:00<43:49, 7.57MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [04:00<25:27, 13.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:00<43:25, 7.64MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [04:00<56:15, 5.68MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [04:00<24:36, 13.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:00<42:15, 7.85MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [04:00<37:02, 8.63MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:00<42:19, 7.84MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [04:00<34:12, 9.35MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:01<39:06, 8.48MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [04:01<33:01, 9.68MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [04:01<31:28, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:01<41:51, 7.93MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [04:01<40:08, 8.39MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [04:01<29:13, 10.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [04:01<3:40:27, 1.58MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [04:01<30:51, 10.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:01<41:33, 7.98MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [04:01<28:38, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [04:01<28:05, 12.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:01<41:50, 7.93MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [04:01<28:45, 11.1MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [04:01<28:31, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [04:01<26:31, 12.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [04:01<3:04:46, 1.88MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.29G/22.2G [04:02<2:06:58, 2.74MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [04:02<25:25, 13.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [04:02<26:34, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [04:02<1:30:14, 3.85MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [04:02<27:12, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [04:02<24:43, 13.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [04:02<1:06:36, 5.22MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:02<1:22:32, 4.02MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [04:02<24:01, 14.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.30G/22.2G [04:02<44:19, 7.84MB/s]  [A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [04:02<36:06, 9.62MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [04:02<23:27, 14.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:02<50:06, 6.62MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 2.99G/22.2G [04:02<49:58, 6.39MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:02<46:40, 7.10MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [04:02<33:44, 9.46MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [04:03<31:40, 10.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [04:03<38:12, 8.80MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:03<45:10, 7.34MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [04:03<31:04, 10.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [04:03<33:41, 9.98MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:03<43:22, 7.64MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [04:03<55:39, 6.24MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [04:03<30:03, 10.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [04:03<29:43, 11.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [04:03<29:51, 10.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:03<43:02, 7.70MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.31G/22.2G [04:03<45:35, 7.62MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:03<40:24, 8.20MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [04:03<27:30, 12.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [04:03<32:00, 10.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [04:03<27:42, 11.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [04:03<27:08, 12.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:03<42:14, 7.84MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [04:03<28:01, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:03<26:08, 12.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.32G/22.2G [04:04<21:33, 16.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:04<39:58, 8.29MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [04:04<27:49, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:04<24:47, 13.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [04:04<17:57, 19.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:04<41:36, 7.96MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [04:04<26:22, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:04<24:14, 13.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:04<39:30, 8.39MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [04:04<26:37, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:04<23:54, 14.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [04:04<5:32:06, 1.04MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:04<41:21, 8.01MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [04:04<29:22, 11.8MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.33G/22.2G [04:04<26:16, 13.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:04<41:35, 7.96MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [04:04<46:33, 6.85MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [04:04<24:47, 14.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:05<42:10, 7.97MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [04:05<3:07:45, 1.84MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [04:05<21:18, 16.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [04:05<36:24, 8.76MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [04:05<19:30, 17.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [04:05<2:35:16, 2.23MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:05<36:14, 9.27MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [04:05<33:36, 9.49MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.34G/22.2G [04:05<18:41, 18.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:05<2:09:40, 2.66MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:05<34:35, 9.71MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:05<1:17:51, 4.25MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [04:05<32:50, 9.71MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [04:05<16:22, 21.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:05<1:46:33, 3.24MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:05<30:33, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [04:05<16:05, 21.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [04:05<30:57, 10.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:05<47:12, 7.01MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:05<1:29:11, 3.87MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:05<30:20, 11.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [04:05<30:15, 10.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:05<30:18, 11.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:05<1:14:28, 4.64MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:05<47:50, 6.92MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.35G/22.2G [04:05<25:00, 13.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:05<30:30, 11.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:05<1:03:12, 5.46MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [04:05<27:21, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:06<30:27, 11.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:06<44:49, 7.39MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [04:06<24:20, 14.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [04:06<27:44, 11.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:06<54:28, 6.34MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:06<30:43, 10.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [04:06<21:28, 16.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [04:06<27:30, 11.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:06<48:50, 7.07MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:06<43:18, 7.65MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:06<30:33, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.36G/22.2G [04:06<17:21, 20.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [04:06<27:49, 11.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [04:06<44:12, 7.81MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:06<39:40, 8.34MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:06<30:02, 11.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [04:06<20:09, 17.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:06<29:54, 11.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [04:06<18:31, 18.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [04:06<51:33, 6.70MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:06<46:02, 7.19MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:06<30:02, 11.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [04:06<18:09, 19.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [04:06<45:59, 7.51MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:06<46:35, 7.10MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [04:07<47:56, 6.65MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:07<49:35, 6.67MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [04:07<34:17, 9.29MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.37G/22.2G [04:07<33:48, 10.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:07<1:10:09, 4.78MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:07<49:14, 6.72MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [04:07<32:33, 9.78MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [04:07<25:35, 13.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:07<41:49, 8.02MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [04:07<25:08, 13.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:07<47:45, 6.93MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [04:07<29:14, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.44G/22.2G [04:07<1:30:35, 3.81MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:07<38:23, 8.74MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▌         | 1.38G/22.2G [04:07<22:35, 15.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:07<45:33, 7.26MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:07<36:23, 9.22MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▎        | 3.05G/22.2G [04:07<29:12, 10.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [04:07<20:49, 16.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:08<34:19, 9.77MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.44G/22.2G [04:08<55:10, 6.26MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:08<45:41, 7.24MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [04:08<19:30, 17.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [04:08<26:57, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [04:08<32:58, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [04:08<18:47, 18.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [04:08<27:08, 11.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [04:08<51:11, 6.74MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [04:08<29:08, 11.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.39G/22.2G [04:08<18:11, 19.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [04:08<27:05, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [04:08<29:18, 11.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [04:08<49:32, 6.97MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [04:08<28:58, 11.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [04:08<25:06, 12.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [04:08<48:43, 7.08MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [04:08<28:37, 11.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [04:08<25:44, 12.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:08<1:31:35, 3.61MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [04:08<28:46, 11.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [04:08<35:11, 9.83MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [04:08<49:07, 7.02MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [04:09<27:39, 12.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [04:09<27:41, 12.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [04:09<55:02, 6.01MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [04:09<44:46, 7.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.40G/22.2G [04:09<23:55, 14.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [04:09<44:00, 7.23MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [04:09<50:48, 6.51MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [04:09<45:57, 7.51MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [04:09<21:46, 15.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [04:09<35:40, 8.92MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [04:09<20:17, 17.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [04:09<44:05, 7.82MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [04:09<48:06, 6.87MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [04:09<19:21, 17.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [04:09<41:43, 8.27MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [04:09<1:02:54, 5.33MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [04:09<30:06, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [04:09<46:04, 7.18MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.41G/22.2G [04:09<18:36, 18.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [04:09<39:53, 8.65MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [04:09<29:23, 10.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [04:09<39:24, 8.51MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [04:09<18:14, 19.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:09<45:09, 7.32MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [04:10<36:58, 9.07MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [04:10<43:28, 7.93MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [04:10<26:41, 11.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [04:10<34:56, 9.59MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [04:10<41:11, 8.37MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:10<43:25, 7.61MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [04:10<32:44, 10.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [04:10<24:52, 12.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [04:10<30:19, 11.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:10<42:38, 7.75MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.42G/22.2G [04:10<27:02, 12.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [04:10<29:22, 11.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [04:10<23:54, 13.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:10<42:04, 7.86MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:10<23:19, 14.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [04:10<29:40, 11.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [04:10<23:06, 13.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [04:10<29:23, 11.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [04:10<1:19:43, 4.33MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:10<22:19, 15.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:10<41:20, 8.00MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [04:10<29:08, 11.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:10<20:35, 16.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:10<38:07, 8.67MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [04:10<46:55, 7.35MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [04:11<28:37, 11.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.43G/22.2G [04:11<19:15, 17.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [04:11<34:12, 9.29MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  10%|█         | 2.33G/22.2G [04:11<41:11, 8.02MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [04:11<48:04, 7.17MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [04:11<18:19, 18.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [04:11<25:56, 12.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [04:11<28:52, 11.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   6%|▋         | 1.44G/22.2G [04:11<17:35, 19.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [04:11<29:15, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [04:11<45:33, 7.57MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [04:11<40:22, 8.54MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [04:11<26:29, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [04:11<49:39, 6.74MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.44G/22.2G [04:11<30:48, 11.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [04:11<22:52, 13.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [04:11<43:40, 7.89MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [04:11<1:22:38, 4.00MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [04:11<23:55, 13.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.44G/22.2G [04:11<28:52, 12.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [04:11<41:55, 8.22MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [04:11<34:33, 9.69MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [04:12<24:15, 14.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [04:12<20:53, 15.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [04:12<33:06, 10.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [04:12<46:59, 7.03MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:12<41:40, 8.27MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [04:12<21:36, 16.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [04:12<32:12, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [04:12<22:08, 14.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:12<40:20, 8.54MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [04:12<47:29, 6.96MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.45G/22.2G [04:12<19:43, 17.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [04:12<36:26, 9.19MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [04:12<18:27, 18.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [04:12<24:59, 12.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:12<40:45, 8.45MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [04:12<45:19, 7.29MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [04:12<17:40, 19.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [04:12<31:23, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:12<43:33, 7.58MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:12<43:58, 7.84MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [04:12<31:13, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:12<39:51, 8.29MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:13<46:41, 7.38MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [04:13<34:11, 9.79MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [04:13<41:35, 7.63MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:13<41:49, 7.90MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [04:13<33:40, 9.94MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.46G/22.2G [04:13<33:35, 10.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:13<48:18, 7.13MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [04:13<34:04, 9.32MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:13<37:16, 8.86MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [04:13<33:17, 10.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [04:13<25:35, 13.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [04:13<32:07, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [04:13<30:04, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:13<39:54, 8.28MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [04:13<22:42, 15.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [04:13<33:08, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [04:13<29:31, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.47G/22.2G [04:13<20:27, 16.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:13<40:03, 8.24MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:13<18:57, 18.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:13<37:31, 8.80MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [04:13<27:45, 11.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:13<18:11, 18.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:13<1:39:57, 3.45MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:14<39:47, 8.30MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [04:14<26:26, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:14<17:21, 19.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:14<36:49, 8.97MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [04:14<1:08:18, 4.90MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [04:14<59:02, 5.83MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [04:14<25:08, 12.6MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [04:14<24:00, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [04:14<40:37, 8.23MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [04:14<54:42, 6.30MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [04:14<24:18, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [04:14<37:43, 8.87MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.48G/22.2G [04:14<31:29, 10.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [04:14<52:22, 6.57MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [04:14<35:58, 9.29MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [04:14<24:17, 14.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [04:14<34:28, 9.70MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:14<1:16:18, 4.33MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [04:14<21:42, 15.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [04:14<50:44, 6.79MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [04:14<33:10, 10.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.49G/22.2G [04:15<20:10, 17.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [04:15<47:47, 6.64MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [04:15<32:18, 10.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:15<51:30, 6.41MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [04:15<48:51, 7.05MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [04:15<18:59, 18.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [04:15<31:42, 10.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [04:15<31:40, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [04:15<18:10, 18.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [04:15<31:09, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [04:15<31:37, 10.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [04:15<47:00, 7.32MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:15<51:15, 6.44MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [04:15<17:23, 19.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [04:15<30:27, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [04:15<27:51, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [04:15<45:42, 7.53MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [04:15<29:55, 11.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:15<52:01, 6.34MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [04:15<43:29, 7.92MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [04:15<29:36, 11.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [04:15<25:40, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [04:15<29:21, 11.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [04:15<45:38, 7.54MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [04:16<31:21, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:16<52:10, 6.32MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [04:16<23:51, 13.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [04:16<24:18, 14.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.50G/22.2G [04:16<43:51, 7.85MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:16<48:34, 6.79MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [04:16<22:58, 13.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [04:16<21:56, 15.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [04:16<43:38, 7.88MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [04:16<19:52, 17.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [04:16<22:09, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:16<52:18, 6.31MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.10G/22.2G [04:16<1:02:50, 5.32MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [04:16<18:43, 18.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:16<49:30, 6.66MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [04:16<17:58, 19.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [04:16<38:38, 8.65MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [04:16<17:13, 20.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [04:16<31:47, 9.96MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [04:16<36:07, 9.25MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:16<47:56, 6.88MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [04:17<27:35, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [04:17<34:34, 9.66MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [04:17<1:19:40, 4.32MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:17<47:58, 6.88MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [04:17<28:14, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [04:17<32:59, 10.1MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [04:17<31:47, 10.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [04:17<47:45, 7.21MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [04:17<30:30, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:17<48:31, 6.80MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [04:17<25:22, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [04:17<31:25, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [04:17<23:41, 14.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [04:17<22:08, 14.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [04:17<48:52, 7.04MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:17<48:07, 6.85MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [04:17<37:22, 8.94MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [04:17<21:00, 16.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [04:17<22:50, 13.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [04:17<33:46, 9.88MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:17<48:07, 6.85MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.51G/22.2G [04:17<46:44, 7.36MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [04:17<19:49, 17.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [04:17<19:59, 15.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [04:17<32:00, 10.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [04:17<18:49, 18.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [04:17<21:30, 14.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [04:18<44:56, 7.65MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [04:18<30:23, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [04:18<18:01, 19.1MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [04:18<18:07, 19.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [04:18<44:02, 7.81MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [04:18<38:44, 8.62MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [04:18<26:45, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [04:18<20:11, 17.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [04:18<30:43, 10.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [04:18<49:24, 6.96MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:18<1:42:50, 3.21MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [04:18<39:02, 8.11MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [04:18<49:59, 6.88MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:18<1:03:46, 5.17MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [04:18<37:58, 9.04MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [04:18<33:25, 9.47MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [04:19<50:37, 6.79MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [04:19<1:02:33, 5.34MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [04:19<31:34, 10.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [04:19<31:27, 10.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [04:19<31:12, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:19<1:02:27, 5.28MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.52G/22.2G [04:19<48:53, 7.03MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [04:19<30:51, 10.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [04:19<44:11, 7.55MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [04:19<29:47, 10.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [04:19<27:39, 12.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [04:19<44:12, 7.55MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [04:19<49:11, 6.99MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:19<1:02:44, 5.25MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [04:19<24:38, 13.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [04:19<26:49, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [04:19<44:30, 7.50MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:19<21:55, 15.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [04:19<54:34, 6.30MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [04:19<27:14, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:19<1:02:17, 5.29MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:19<22:24, 15.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [04:19<40:38, 8.21MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [04:19<27:20, 11.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:20<21:42, 15.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:20<1:01:56, 5.32MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [04:20<45:28, 7.34MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [04:20<25:23, 12.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:20<20:14, 17.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [04:20<26:09, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [04:20<45:48, 7.28MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:20<1:00:56, 5.41MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [04:20<45:38, 7.31MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:20<34:18, 10.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:20<59:43, 5.52MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [04:20<45:32, 7.32MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [04:20<29:51, 11.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [04:20<44:19, 7.13MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [04:20<29:25, 11.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [04:20<1:54:01, 3.01MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [04:20<45:17, 7.36MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:20<59:20, 5.55MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [04:21<24:20, 14.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [04:21<31:40, 9.98MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [04:21<45:21, 7.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [04:21<21:50, 15.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [04:21<30:09, 10.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:21<58:22, 5.64MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [04:21<1:09:47, 4.92MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [04:21<42:00, 7.94MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [04:21<19:58, 17.2MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [04:21<20:32, 16.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [04:21<27:25, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [04:21<43:25, 7.68MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:21<57:03, 5.77MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.53G/22.2G [04:21<1:09:02, 4.98MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [04:21<20:13, 16.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [04:21<30:00, 10.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [04:21<43:57, 7.58MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [04:21<18:53, 18.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [04:21<1:05:54, 5.21MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:21<56:50, 5.80MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [04:21<30:39, 10.3MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [04:22<27:19, 11.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [04:22<1:06:44, 5.15MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:22<55:59, 5.88MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [04:22<34:50, 9.83MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [04:22<27:07, 11.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [04:22<1:25:20, 3.91MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [04:22<1:02:31, 5.49MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [04:22<31:00, 11.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [04:22<25:01, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [04:22<27:42, 12.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [04:22<53:03, 6.28MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [04:22<1:04:43, 5.31MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [04:22<23:52, 14.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [04:22<49:59, 6.67MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [04:22<21:22, 16.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:22<1:40:47, 3.27MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [04:22<1:00:51, 5.65MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [04:22<48:35, 6.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [04:22<19:47, 17.3MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [04:23<18:26, 18.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [04:23<1:02:52, 5.46MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.21G/22.2G [04:23<50:03, 6.31MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [04:23<47:20, 7.04MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:23<1:03:07, 5.22MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [04:23<43:35, 7.65MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [04:23<36:28, 8.66MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.54G/22.2G [04:23<59:20, 5.79MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:23<1:00:10, 5.47MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [04:23<43:50, 7.60MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [04:23<30:57, 10.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [04:23<30:51, 11.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [04:23<1:02:46, 5.47MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [04:23<43:37, 7.64MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [04:23<29:59, 10.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [04:23<23:47, 14.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:23<58:40, 5.61MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [04:23<59:05, 5.81MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [04:23<43:36, 7.64MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [04:23<21:42, 15.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [04:23<27:17, 11.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [04:24<40:52, 8.15MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [04:24<20:17, 16.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:24<57:25, 5.73MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [04:24<25:28, 12.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [04:24<1:02:01, 5.54MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [04:24<18:56, 18.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [04:24<41:53, 7.95MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [04:24<18:04, 18.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:24<57:04, 5.77MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [04:24<24:28, 12.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [04:24<42:22, 7.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [04:24<17:23, 19.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [04:24<39:39, 8.40MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [04:24<16:59, 20.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [04:24<23:50, 13.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:24<56:09, 5.86MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [04:24<41:30, 8.03MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:24<52:50, 6.23MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [04:25<37:24, 8.43MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:25<56:18, 5.84MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [04:25<1:45:42, 3.25MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [04:25<32:19, 9.75MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:25<55:35, 5.92MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [04:25<29:11, 10.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.55G/22.2G [04:25<1:05:53, 5.21MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:25<52:53, 6.22MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [04:25<51:19, 6.66MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [04:25<26:17, 12.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [04:25<1:03:40, 5.39MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:25<55:43, 5.90MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [04:25<49:36, 6.89MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [04:25<2:07:21, 2.62MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [04:25<25:08, 12.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [04:26<1:01:07, 5.62MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [04:26<38:07, 8.96MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [04:26<24:23, 12.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [04:26<31:03, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [04:26<1:13:15, 4.55MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [04:26<26:15, 13.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [04:26<1:02:59, 5.45MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [04:26<23:52, 13.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [04:26<22:58, 14.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [04:26<1:07:02, 4.97MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [04:26<20:42, 16.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [04:26<1:01:44, 5.56MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [04:26<59:54, 5.56MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:26<1:34:32, 3.48MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [04:26<55:09, 6.04MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.56G/22.2G [04:26<1:01:01, 5.62MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [04:26<37:29, 8.40MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [04:27<52:13, 6.37MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [04:27<56:45, 5.80MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.66G/22.2G [04:27<35:02, 9.75MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [04:27<29:39, 10.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:27<1:00:24, 5.68MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [04:27<49:53, 6.67MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [04:27<26:26, 12.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [04:27<58:40, 5.61MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [04:27<44:59, 7.40MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [04:27<26:51, 11.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [04:27<23:21, 14.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:27<1:00:39, 5.66MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [04:27<26:49, 11.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [04:27<55:03, 5.97MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [04:27<45:22, 7.34MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [04:27<21:10, 16.1MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [04:27<19:42, 17.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [04:27<45:02, 7.39MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [04:27<24:51, 12.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:27<1:00:16, 5.69MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [04:27<57:04, 5.76MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [04:27<18:28, 18.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [04:27<41:31, 8.02MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [04:28<24:08, 13.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [04:28<17:44, 19.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [04:28<41:25, 8.03MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:28<59:20, 5.78MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:28<55:29, 5.93MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [04:28<23:33, 13.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [04:28<42:38, 7.80MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:28<53:53, 6.10MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:28<59:06, 5.80MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [04:28<23:15, 13.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [04:28<42:40, 7.80MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [04:28<32:45, 10.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:28<49:09, 6.69MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [04:28<25:12, 13.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:28<49:24, 6.65MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [04:28<22:23, 15.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [04:28<36:47, 8.55MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [04:29<20:47, 16.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:29<49:22, 6.66MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [04:29<32:15, 9.75MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [04:29<1:26:40, 3.84MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [04:29<19:07, 17.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:29<48:54, 6.72MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [04:29<30:07, 10.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.57G/22.2G [04:29<1:47:20, 3.20MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [04:29<18:02, 18.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [04:29<52:23, 6.35MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [04:29<17:24, 19.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:29<48:33, 6.77MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [04:29<27:08, 11.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [04:29<1:03:33, 5.40MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [04:29<48:30, 6.86MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:29<47:40, 6.89MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [04:29<25:36, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [04:29<46:54, 7.09MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [04:29<1:00:29, 5.67MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [04:29<29:01, 11.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [04:29<24:37, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [04:30<45:40, 7.28MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [04:30<59:07, 5.80MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [04:30<22:49, 14.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [04:30<41:49, 7.95MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [04:30<23:50, 13.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [04:30<20:56, 16.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [04:30<58:07, 5.90MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [04:30<43:18, 7.68MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [04:30<19:21, 17.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [04:30<23:18, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [04:30<56:55, 6.02MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [04:30<18:07, 18.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:30<1:42:15, 3.21MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [04:30<41:58, 7.92MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [04:30<17:10, 19.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [04:30<40:56, 8.12MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.58G/22.2G [04:30<56:15, 6.09MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [04:30<16:34, 20.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:30<56:37, 5.80MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [04:30<41:06, 8.09MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [04:30<38:40, 8.12MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [04:30<54:50, 6.25MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:31<51:59, 6.32MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [04:31<37:49, 8.79MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [04:31<30:39, 10.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [04:31<53:54, 6.36MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [04:31<29:14, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [04:31<39:49, 8.35MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:31<49:27, 6.64MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [04:31<31:49, 10.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [04:31<37:16, 8.92MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [04:31<52:43, 6.50MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [04:31<26:12, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [04:31<46:58, 6.99MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [04:31<24:28, 13.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [04:31<51:15, 6.69MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [04:31<24:49, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:31<21:51, 15.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [04:31<44:43, 7.34MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:31<19:59, 17.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [04:31<50:43, 6.76MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [04:31<23:37, 13.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [04:31<42:40, 7.70MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:31<18:39, 18.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [04:32<23:47, 13.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:32<17:52, 19.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.59G/22.2G [04:32<49:17, 6.95MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [04:32<41:12, 7.97MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [04:32<1:17:20, 4.30MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [04:32<17:10, 19.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [04:32<23:12, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [04:32<39:14, 8.37MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [04:32<44:09, 7.52MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [04:32<38:41, 8.49MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [04:32<44:22, 7.49MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [04:32<36:56, 8.89MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [04:32<42:31, 7.81MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [04:32<32:50, 9.99MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [04:32<32:49, 10.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.32G/22.2G [04:32<41:31, 7.56MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [04:32<41:11, 8.06MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [04:32<1:24:56, 4.03MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [04:32<25:19, 13.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [04:32<29:13, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [04:33<41:07, 8.08MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [04:33<50:18, 6.81MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [04:33<22:18, 15.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [04:33<29:20, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [04:33<38:22, 8.65MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [04:33<20:12, 16.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [04:33<28:32, 11.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [04:33<36:38, 9.06MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [04:33<51:02, 6.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [04:33<18:47, 18.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [04:33<34:51, 9.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [04:33<1:12:04, 4.55MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [04:33<25:16, 12.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [04:33<18:04, 18.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [04:33<48:14, 7.10MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [04:33<33:28, 9.92MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [04:33<17:20, 19.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [04:33<45:20, 7.24MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [04:33<23:51, 13.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [04:33<34:55, 9.51MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.60G/22.2G [04:33<45:21, 7.55MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [04:33<40:18, 8.14MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [04:33<33:48, 9.82MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [04:33<40:19, 8.49MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [04:33<23:05, 13.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [04:33<37:38, 8.72MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [04:34<38:56, 8.79MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [04:34<30:28, 10.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [04:34<35:27, 9.25MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [04:34<22:36, 13.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [04:34<38:19, 8.93MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [04:34<32:43, 10.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [04:34<33:24, 9.82MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [04:34<37:21, 9.17MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [04:34<25:05, 13.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [04:34<31:53, 10.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [04:34<36:34, 9.36MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [04:34<22:16, 15.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [04:34<30:25, 10.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [04:34<56:53, 5.83MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.61G/22.2G [04:34<35:57, 9.52MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [04:34<20:03, 16.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [04:34<29:21, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [04:34<40:21, 7.77MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [04:34<35:06, 9.75MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [04:34<18:35, 18.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [04:34<28:04, 11.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [04:34<38:01, 8.73MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [04:34<28:45, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [04:34<34:27, 9.93MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [04:34<17:36, 19.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [04:34<35:13, 9.42MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [04:34<27:26, 12.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [04:35<28:45, 10.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [04:35<17:02, 19.9MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [04:35<17:26, 19.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [04:35<41:09, 8.06MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [04:35<34:11, 9.59MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [04:35<26:48, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:35<37:38, 8.82MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:35<35:44, 9.28MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [04:35<24:55, 12.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [04:35<1:15:25, 4.54MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [04:35<24:37, 12.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:35<37:54, 8.75MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [04:35<32:57, 10.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [04:35<1:00:51, 5.39MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:35<35:29, 9.35MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [04:35<46:55, 7.29MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [04:35<30:03, 11.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [04:35<23:08, 13.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [04:36<29:52, 11.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [04:36<40:10, 8.16MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:36<37:40, 8.81MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [04:36<48:04, 7.12MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [04:36<22:47, 13.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [04:36<36:07, 9.18MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [04:36<36:54, 8.88MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [04:36<26:47, 12.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.62G/22.2G [04:36<46:03, 7.43MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:36<33:41, 9.84MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [04:36<35:41, 9.18MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [04:36<23:32, 14.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [04:36<34:42, 9.44MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:36<34:14, 9.68MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [04:36<44:03, 7.76MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [04:36<22:57, 14.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [04:36<33:33, 9.77MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:36<35:34, 9.32MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [04:36<37:47, 8.29MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [04:36<40:53, 8.37MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [04:36<20:40, 16.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.49G/22.2G [04:36<32:39, 10.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:36<34:20, 9.65MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [04:36<27:01, 11.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [04:36<21:14, 16.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.49G/22.2G [04:36<31:36, 10.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [04:36<41:18, 8.28MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [04:36<27:19, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [04:37<30:53, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [04:37<39:35, 8.64MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [04:37<25:28, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [04:37<30:21, 10.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [04:37<35:52, 9.53MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:37<1:02:57, 5.27MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [04:37<30:01, 10.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.63G/22.2G [04:37<35:14, 9.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [04:37<36:13, 9.36MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [04:37<24:00, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [04:37<41:25, 8.00MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [04:37<29:25, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [04:37<26:14, 12.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [04:37<35:01, 9.76MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:37<38:10, 8.69MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [04:37<28:58, 11.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [04:37<22:42, 13.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [04:37<26:03, 13.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [04:37<34:50, 9.81MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:37<36:23, 9.11MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [04:37<22:37, 15.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [04:37<34:48, 9.82MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [04:37<21:52, 14.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:37<34:36, 9.58MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [04:37<20:57, 16.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:37<33:39, 9.85MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [04:38<20:48, 16.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:38<32:54, 10.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [04:38<59:11, 5.53MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [04:38<19:33, 17.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:38<31:53, 10.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [04:38<18:46, 18.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [04:38<39:37, 8.26MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [04:38<31:16, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [04:38<32:55, 9.50MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:38<30:34, 10.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [04:38<34:37, 9.46MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [04:38<25:16, 12.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [04:38<1:26:59, 3.93MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:38<30:20, 10.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [04:38<32:43, 10.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [04:38<25:35, 12.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:38<29:48, 11.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [04:38<30:50, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.64G/22.2G [04:38<59:11, 5.78MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:38<29:42, 11.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [04:38<23:40, 13.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [04:38<29:24, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:39<31:49, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [04:39<20:52, 15.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [04:39<28:12, 11.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [04:39<53:53, 6.34MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [04:39<21:47, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [04:39<27:31, 11.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [04:39<51:41, 6.61MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [04:39<53:31, 6.33MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [04:39<19:49, 15.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [04:39<26:49, 12.2MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [04:39<26:34, 12.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [04:39<21:05, 14.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [04:39<49:35, 6.89MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [04:39<41:09, 8.23MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [04:39<26:05, 12.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [04:39<19:13, 16.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [04:39<1:08:19, 4.85MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [04:39<47:58, 7.12MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [04:39<33:09, 10.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:39<44:07, 7.50MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [04:39<27:49, 12.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:40<39:04, 8.47MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [04:40<48:10, 7.09MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [04:40<25:23, 13.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:40<36:38, 9.03MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [04:40<47:58, 6.82MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [04:40<22:45, 14.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.65G/22.2G [04:40<47:18, 7.22MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:40<34:21, 9.63MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [04:40<32:41, 10.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [04:40<44:31, 7.67MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:40<32:57, 10.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [04:40<25:03, 13.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [04:40<45:36, 6.85MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [04:40<29:17, 11.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [04:40<32:03, 10.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [04:40<21:40, 15.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [04:40<45:18, 7.54MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [04:40<40:04, 7.79MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [04:40<28:04, 11.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:40<31:20, 10.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [04:40<40:57, 8.34MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [04:40<35:32, 8.79MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [04:40<27:01, 12.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:40<28:36, 11.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [04:40<27:01, 12.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:40<28:11, 11.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [04:40<27:21, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   7%|▋         | 1.66G/22.2G [04:40<44:17, 7.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [04:41<36:57, 9.15MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:41<28:32, 11.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [04:41<26:09, 12.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [04:41<25:00, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:41<28:30, 11.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [04:41<30:26, 11.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [04:41<26:06, 12.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [04:41<22:03, 14.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:41<28:21, 11.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [04:41<30:08, 11.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [04:41<25:19, 12.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [04:41<19:56, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.55G/22.2G [04:41<25:15, 12.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [04:41<27:55, 12.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [04:41<18:35, 16.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  11%|█▏        | 2.55G/22.2G [04:41<24:53, 13.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.66G/22.2G [04:41<1:18:05, 4.37MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [04:41<24:05, 14.0MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [04:41<24:47, 13.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [04:41<56:23, 5.87MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [04:41<46:45, 7.30MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [04:42<24:44, 13.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [04:42<36:56, 8.95MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [04:42<30:31, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [04:42<43:36, 7.49MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [04:42<47:48, 7.14MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [04:42<22:15, 15.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [04:42<34:16, 9.65MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [04:42<26:52, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [04:42<32:43, 9.98MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [04:42<23:22, 14.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [04:42<32:28, 10.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [04:42<25:58, 12.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [04:42<45:25, 7.52MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [04:42<29:08, 11.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [04:42<31:04, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [04:42<21:30, 14.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [04:42<43:47, 7.80MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [04:42<19:06, 16.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [04:42<38:08, 8.67MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [04:42<34:05, 9.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [04:42<17:47, 17.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [04:42<44:19, 7.70MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [04:42<27:26, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:42<36:11, 9.14MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [04:42<16:59, 18.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.67G/22.2G [04:43<44:50, 7.61MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:43<34:46, 9.51MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.88G/22.2G [04:43<53:42, 6.29MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [04:43<31:23, 10.4MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [04:43<31:12, 10.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:43<37:06, 8.91MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [04:43<37:12, 9.08MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [04:43<47:11, 7.23MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [04:43<31:01, 10.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:43<35:14, 9.38MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [04:43<35:35, 9.49MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [04:43<48:58, 6.97MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [04:43<28:57, 11.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [04:43<34:39, 9.74MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:43<37:29, 8.82MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [04:43<30:03, 10.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [04:43<29:55, 10.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [04:43<35:55, 9.20MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [04:43<50:22, 6.77MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [04:43<23:22, 13.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [04:43<30:38, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [04:43<30:20, 10.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  10%|█         | 2.33G/22.2G [04:43<32:52, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [04:43<20:00, 15.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [04:43<30:47, 11.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [04:43<50:48, 6.72MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [04:44<18:23, 16.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [04:44<30:23, 11.1MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [04:44<30:25, 11.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [04:44<17:02, 18.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [04:44<1:01:06, 5.58MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [04:44<15:59, 19.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [04:44<27:51, 12.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [04:44<15:14, 20.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [04:44<28:27, 11.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [04:44<1:07:42, 4.82MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [04:44<28:40, 11.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [04:44<41:21, 7.89MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [04:44<1:36:36, 3.42MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [04:44<37:33, 8.69MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [04:44<26:36, 11.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [04:45<35:31, 9.19MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [04:45<23:43, 13.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [04:45<58:16, 5.67MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [04:45<51:50, 6.51MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [04:45<34:12, 9.54MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [04:45<19:10, 16.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [04:45<50:20, 6.56MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [04:45<32:52, 9.92MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [04:45<34:20, 9.83MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [04:45<17:19, 18.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.68G/22.2G [04:45<1:44:06, 3.28MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [04:45<48:48, 6.77MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [04:45<32:07, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [04:45<33:02, 10.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [04:45<16:00, 19.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [04:45<44:53, 7.36MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [04:45<31:15, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [04:45<15:17, 20.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:45<39:47, 8.30MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [04:45<29:13, 11.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [04:45<30:41, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [04:45<1:07:06, 5.08MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:45<36:48, 8.97MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [04:45<14:39, 21.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [04:45<29:15, 11.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [04:45<30:10, 10.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:45<34:52, 9.47MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [04:46<29:51, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [04:46<27:04, 12.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:46<33:10, 9.96MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [04:46<1:05:40, 5.19MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [04:46<27:29, 12.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:46<31:59, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [04:46<29:34, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [04:46<23:21, 13.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:46<31:32, 10.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [04:46<29:21, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [04:46<22:57, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [04:46<1:05:14, 5.23MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [04:46<26:15, 12.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [04:46<30:46, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [04:46<18:19, 17.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:46<30:42, 10.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [04:46<25:32, 13.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [04:46<16:37, 18.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [04:46<1:04:42, 5.27MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:46<30:35, 10.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [04:46<13:54, 22.3MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [04:46<13:42, 22.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.69G/22.2G [04:47<1:04:22, 5.30MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [04:47<1:00:31, 5.63MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [04:47<48:52, 6.90MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:47<1:14:33, 4.43MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [04:47<35:16, 9.56MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [04:47<24:19, 12.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [04:47<1:02:36, 5.45MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [04:47<20:15, 15.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [04:47<31:19, 10.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:47<46:58, 7.03MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [04:47<1:00:17, 5.65MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [04:47<16:24, 18.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▊         | 1.94G/22.2G [04:47<30:59, 10.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [04:47<44:59, 7.33MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [04:47<14:08, 21.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [04:48<28:23, 11.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:48<41:26, 7.96MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [04:48<13:47, 22.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [04:48<1:01:48, 5.52MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [04:48<12:19, 25.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [04:48<26:36, 12.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:48<41:29, 7.95MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [04:48<59:00, 5.78MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [04:48<27:48, 12.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:48<38:33, 8.56MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [04:48<25:44, 13.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:48<39:59, 8.25MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [04:48<1:00:53, 5.60MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [04:48<19:21, 16.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [04:48<26:30, 12.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [04:48<19:46, 15.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:48<39:38, 8.32MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.70G/22.2G [04:48<1:01:31, 5.54MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [04:48<16:08, 19.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:48<37:01, 8.91MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [04:49<13:34, 22.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [04:49<35:07, 9.39MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [04:49<12:01, 25.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:49<37:56, 8.69MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [04:49<48:52, 6.89MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:49<35:17, 9.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [04:49<34:30, 9.76MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:49<37:28, 8.80MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [04:49<30:45, 10.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [04:49<2:00:18, 2.83MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [04:49<28:38, 11.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [04:49<27:44, 11.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [04:50<28:38, 11.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [04:50<22:45, 13.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [04:50<27:14, 12.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:50<1:14:39, 4.42MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [04:50<19:17, 16.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [04:50<1:13:37, 4.63MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [04:50<15:37, 19.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [04:50<1:05:51, 5.17MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [04:50<26:08, 12.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [04:50<42:51, 7.69MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [04:50<13:12, 23.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [04:50<25:32, 13.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:50<42:54, 7.68MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:50<39:30, 8.34MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [04:50<1:11:26, 4.77MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [04:50<18:59, 16.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:50<39:32, 8.34MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:51<37:15, 8.84MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [04:51<16:45, 18.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.71G/22.2G [04:51<1:13:22, 4.64MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:51<35:31, 9.28MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [04:51<45:09, 7.45MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [04:51<13:36, 22.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [04:51<34:32, 9.54MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [04:51<11:49, 26.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [04:51<34:08, 9.85MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [04:51<1:11:55, 4.74MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:51<40:33, 8.12MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [04:51<30:43, 10.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [04:51<1:10:13, 4.85MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [04:51<6:28:02, 840kB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:51<44:09, 7.46MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [04:51<14:51, 20.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [04:51<28:47, 11.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:52<40:33, 8.12MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [04:52<28:12, 11.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [04:52<13:00, 23.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [04:52<1:09:05, 4.93MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:52<38:10, 8.63MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [04:52<11:40, 26.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:52<27:31, 12.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:52<48:10, 6.84MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [04:52<13:23, 23.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:52<32:46, 10.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [04:52<3:47:27, 1.43MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [04:52<1:18:45, 4.32MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [04:52<12:41, 24.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:52<31:52, 10.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [04:52<2:29:25, 2.18MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:52<31:49, 10.6MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:52<31:56, 10.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [04:52<2:02:44, 2.66MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [04:52<1:16:09, 4.47MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [04:53<20:10, 15.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:53<32:12, 10.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [04:53<1:39:58, 3.26MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [04:53<17:34, 17.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [04:53<1:28:03, 3.70MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [04:53<1:22:26, 4.00MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.72G/22.2G [04:53<1:18:49, 4.32MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [04:53<14:30, 21.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [04:53<1:11:04, 4.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [04:53<12:41, 24.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:53<50:35, 6.51MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [04:53<1:02:36, 5.37MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [04:53<11:42, 26.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [04:53<1:05:20, 4.99MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [04:53<1:20:51, 4.21MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:53<48:25, 6.80MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [04:53<56:10, 5.80MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:53<44:12, 7.60MB/s]  [A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:53<47:26, 6.94MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [04:54<51:34, 6.32MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:54<42:25, 7.92MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [04:54<1:25:34, 3.98MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [04:54<46:03, 7.07MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:54<40:03, 8.39MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:54<45:39, 7.21MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:54<37:20, 9.00MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [04:54<44:01, 7.40MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:54<45:09, 7.29MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [04:54<25:45, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [04:54<35:36, 9.43MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [04:54<41:14, 7.98MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [04:54<21:43, 14.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:54<31:05, 10.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [04:54<18:39, 16.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:54<41:37, 7.91MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:54<30:45, 10.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [04:54<15:27, 19.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:54<42:30, 7.74MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:54<30:44, 10.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [04:54<1:16:27, 4.26MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [04:54<13:18, 23.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:55<38:32, 8.54MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:55<30:29, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [04:55<43:42, 7.45MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:55<39:58, 8.23MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [04:55<28:05, 11.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:55<37:08, 8.86MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [04:55<43:56, 7.41MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:55<28:24, 11.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [04:55<40:36, 8.02MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [04:55<21:16, 14.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:55<38:57, 8.44MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [04:55<2:54:34, 1.95MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [04:55<18:34, 16.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [04:55<39:42, 8.20MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:55<48:11, 6.97MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [04:55<14:34, 21.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [04:55<35:22, 9.20MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [04:55<15:00, 20.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:56<37:10, 9.03MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [04:56<1:38:05, 3.47MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [04:56<37:50, 8.60MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [04:56<13:30, 22.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:56<34:43, 9.66MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [04:56<33:32, 9.70MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [04:56<1:14:34, 4.41MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [04:56<13:25, 22.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [04:56<30:22, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [04:56<36:29, 8.92MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.73G/22.2G [04:56<1:35:21, 3.57MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [04:56<42:59, 7.65MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [04:56<30:13, 11.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [04:56<32:46, 9.93MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [04:56<43:22, 7.58MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [04:56<27:50, 12.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [04:56<22:01, 14.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [04:56<39:56, 8.24MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [04:56<35:52, 9.07MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:56<1:30:23, 3.77MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [04:56<28:34, 11.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [04:56<20:40, 14.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [04:56<34:18, 9.48MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [04:56<40:16, 8.17MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [04:57<18:11, 16.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [04:57<26:39, 12.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [04:57<37:38, 8.74MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [04:57<14:44, 20.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:57<1:25:32, 3.98MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [04:57<27:32, 12.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:57<35:40, 9.22MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [04:57<14:25, 21.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:57<37:34, 8.75MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [04:57<14:05, 21.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:57<1:22:43, 4.11MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:57<35:02, 9.38MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [04:57<1:08:41, 4.74MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:57<34:09, 9.62MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [04:57<41:02, 7.93MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:57<1:17:40, 4.38MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [04:57<45:13, 7.41MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:57<36:47, 8.93MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [04:57<23:00, 13.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [04:57<38:38, 8.42MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [04:58<35:21, 9.29MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [04:58<18:57, 16.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [04:58<33:09, 10.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [04:58<37:21, 8.70MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:58<33:40, 9.76MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:58<1:19:32, 4.28MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [04:58<15:41, 19.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [04:58<29:58, 11.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:58<33:04, 9.93MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [04:58<40:07, 8.10MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [04:58<15:01, 20.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [04:58<29:49, 11.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [04:58<37:02, 8.78MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:58<1:15:41, 4.49MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [04:58<16:27, 18.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [04:58<27:39, 12.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [04:58<41:31, 7.83MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [04:58<14:15, 21.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [04:58<26:26, 12.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:58<1:06:16, 4.96MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.74G/22.2G [04:58<1:18:19, 4.34MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [04:58<41:37, 7.81MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [04:59<25:42, 13.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:59<40:49, 8.05MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [04:59<42:38, 7.62MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [04:59<26:15, 12.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [04:59<1:16:36, 4.44MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:59<41:38, 7.89MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [04:59<25:01, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [04:59<38:34, 8.52MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [04:59<43:19, 7.50MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [04:59<21:26, 14.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [04:59<1:15:17, 4.52MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [04:59<35:55, 9.14MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [04:59<18:45, 16.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [04:59<42:46, 7.60MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [04:59<37:25, 8.78MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [04:59<17:46, 17.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [04:59<43:00, 7.56MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [04:59<42:36, 7.86MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [04:59<35:19, 9.30MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [04:59<1:16:08, 4.47MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [04:59<16:59, 18.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [04:59<38:40, 8.40MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [05:00<16:20, 18.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [05:00<36:06, 9.09MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [05:00<32:22, 10.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [05:00<15:52, 19.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [05:00<34:55, 9.40MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [05:00<31:12, 10.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [05:00<33:06, 9.92MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [05:00<28:26, 11.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [05:00<35:08, 9.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [05:00<26:58, 12.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:00<1:16:38, 4.24MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [05:00<34:09, 9.61MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [05:00<30:47, 9.95MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [05:00<32:47, 10.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [05:00<25:51, 12.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:00<45:42, 7.11MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [05:00<23:35, 13.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [05:00<26:24, 12.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:01<46:24, 7.00MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [05:01<19:46, 15.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [05:01<25:03, 13.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [05:01<18:13, 16.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:01<44:07, 7.36MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [05:01<17:02, 18.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:01<41:37, 7.80MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [05:01<16:13, 18.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [05:01<1:13:23, 4.47MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:01<37:37, 8.63MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [05:01<15:34, 19.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [05:01<42:34, 7.71MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [05:01<42:25, 7.88MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:01<39:55, 8.13MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [05:01<42:39, 7.69MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [05:02<32:25, 10.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:02<38:36, 8.41MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [05:02<39:15, 8.36MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [05:02<28:54, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:02<34:49, 9.32MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [05:02<29:15, 11.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [05:02<39:22, 8.33MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [05:02<23:40, 12.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [05:02<29:13, 11.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [05:02<36:54, 8.89MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:02<37:29, 8.66MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [05:02<19:33, 15.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [05:02<34:41, 9.46MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:02<34:43, 9.35MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [05:02<27:18, 12.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [05:02<17:56, 17.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [05:02<36:08, 9.08MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:02<36:43, 8.84MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [05:02<16:36, 18.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [05:02<26:11, 12.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [05:02<34:02, 9.63MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [05:02<15:41, 19.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [05:03<32:59, 9.94MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [05:03<25:18, 13.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [05:03<15:04, 20.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [05:03<25:57, 12.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [05:03<35:06, 9.34MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [05:03<33:38, 9.75MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.10G/22.2G [05:03<25:27, 13.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [05:03<32:41, 10.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:03<1:14:16, 4.37MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [05:03<25:31, 12.0MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [05:03<22:39, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:03<43:27, 7.47MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [05:03<42:25, 7.88MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [05:03<19:43, 15.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:03<43:44, 7.42MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [05:03<17:39, 17.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [05:04<36:29, 9.15MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:04<40:32, 8.00MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [05:04<16:17, 18.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [05:04<6:22:04, 890kB/s] [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [05:04<1:13:53, 4.44MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [05:04<31:41, 10.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:04<39:11, 8.28MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [05:04<15:09, 20.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [05:04<30:49, 10.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [05:04<42:15, 7.76MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:04<36:56, 8.78MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [05:04<14:49, 20.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:04<34:23, 9.43MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [05:04<28:25, 11.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [05:04<42:22, 7.73MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.75G/22.2G [05:04<3:13:26, 1.76MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [05:04<39:24, 8.32MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:04<36:17, 8.94MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [05:04<26:54, 12.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [05:04<36:12, 9.05MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:04<34:39, 9.36MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [05:04<27:45, 11.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [05:04<24:07, 13.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [05:05<34:33, 9.48MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [05:05<2:47:03, 2.04MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:05<33:03, 9.81MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [05:05<25:18, 13.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [05:05<23:52, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.49G/22.2G [05:05<33:39, 9.74MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:05<31:51, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [05:05<19:19, 15.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [05:05<24:54, 13.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.49G/22.2G [05:05<36:23, 9.01MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [05:05<2:23:00, 2.38MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [05:05<17:27, 17.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:05<35:26, 9.15MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:05<34:27, 9.51MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [05:05<16:11, 18.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:05<33:15, 9.75MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:05<33:01, 9.92MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [05:05<2:05:10, 2.72MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [05:05<15:07, 20.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:05<31:20, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [05:05<42:20, 7.88MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [05:05<14:29, 21.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:05<30:43, 10.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [05:05<1:49:42, 3.10MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [05:05<34:23, 9.70MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:06<30:44, 10.7MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:06<30:48, 10.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:06<1:08:06, 4.76MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [05:06<29:57, 11.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [05:06<1:37:02, 3.50MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [05:06<22:45, 13.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [05:06<29:33, 11.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [05:06<22:33, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:06<44:49, 7.23MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [05:06<1:27:48, 3.87MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [05:06<18:14, 16.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [05:06<27:22, 12.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:06<40:44, 7.95MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [05:06<16:38, 18.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:06<1:03:17, 5.18MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:06<38:49, 8.35MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [05:06<25:18, 13.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.76G/22.2G [05:06<1:21:14, 4.18MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:06<37:18, 8.69MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [05:06<15:23, 19.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [05:06<40:58, 7.99MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [05:06<24:45, 13.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:06<35:42, 9.07MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [05:06<14:43, 20.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [05:07<37:01, 8.84MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [05:07<1:12:02, 4.72MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:07<35:01, 9.25MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [05:07<35:05, 9.33MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [05:07<24:13, 13.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  17%|█▋        | 3.88G/22.2G [05:07<18:05, 16.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:07<34:17, 9.45MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [05:07<33:32, 9.76MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [05:07<1:11:23, 4.76MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:07<33:49, 9.58MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [05:07<32:24, 10.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:07<33:29, 9.67MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [05:07<31:33, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [05:07<1:04:02, 5.31MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [05:07<30:52, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [05:07<43:57, 7.59MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:07<37:34, 8.62MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [05:07<1:00:35, 5.61MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [05:07<30:22, 10.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [05:07<37:12, 8.19MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [05:07<30:25, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [05:07<34:08, 9.76MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:07<36:11, 8.95MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [05:08<57:52, 5.87MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [05:08<29:20, 11.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [05:08<30:01, 10.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [05:08<30:37, 10.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:08<39:10, 8.27MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [05:08<29:30, 11.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [05:08<28:02, 10.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [05:08<27:51, 12.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [05:08<26:44, 12.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [05:08<24:21, 12.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [05:08<25:58, 12.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [05:08<20:39, 14.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [05:08<23:33, 14.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [05:08<21:21, 14.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.77G/22.2G [05:08<1:51:45, 3.04MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [05:08<24:06, 13.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [05:08<18:45, 16.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [05:09<57:38, 5.67MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:09<1:22:34, 3.92MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [05:09<24:03, 13.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [05:09<1:05:22, 5.19MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [05:09<19:31, 15.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [05:09<41:55, 7.80MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [05:09<1:00:17, 5.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:09<50:35, 6.40MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [05:09<36:40, 8.92MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [05:09<34:30, 9.48MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [05:09<30:08, 10.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [05:09<39:22, 8.46MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [05:09<56:18, 6.03MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:09<47:10, 6.86MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [05:09<31:09, 10.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [05:09<25:30, 11.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [05:09<29:55, 11.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [05:09<53:12, 6.38MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:09<44:56, 7.20MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [05:09<30:29, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [05:09<24:59, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [05:09<25:42, 13.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [05:09<20:12, 15.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [05:09<49:51, 6.81MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:09<42:50, 7.56MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [05:09<27:38, 11.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [05:10<25:49, 12.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [05:10<21:06, 14.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.78G/22.2G [05:10<45:20, 7.49MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [05:10<22:48, 14.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:10<41:12, 7.85MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [05:10<18:29, 16.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [05:10<28:25, 11.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [05:10<23:37, 14.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [05:10<16:37, 18.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [05:10<21:16, 15.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.55G/22.2G [05:10<26:38, 12.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:10<43:39, 7.41MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [05:10<54:52, 6.19MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [05:10<16:26, 18.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [05:10<22:16, 14.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  11%|█▏        | 2.55G/22.2G [05:10<27:28, 11.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:10<40:35, 7.97MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [05:10<20:35, 16.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [05:10<53:17, 6.37MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [05:10<52:54, 6.42MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:10<44:21, 7.29MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [05:11<51:40, 6.32MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [05:11<52:37, 6.45MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:11<45:02, 7.18MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [05:11<39:12, 8.48MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [05:11<34:32, 9.46MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [05:11<28:00, 11.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [05:11<52:19, 6.49MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:11<45:37, 7.09MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [05:11<32:59, 9.90MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [05:11<27:46, 12.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [05:11<31:57, 10.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:11<44:55, 7.20MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [05:11<50:47, 6.68MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [05:11<23:37, 14.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [05:11<30:45, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [05:11<23:53, 13.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [05:11<30:19, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [05:11<21:28, 15.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [05:12<29:56, 10.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [05:12<19:41, 16.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [05:12<29:20, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [05:12<18:29, 18.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [05:12<29:15, 11.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.79G/22.2G [05:12<1:38:50, 3.43MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [05:12<19:15, 17.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:12<1:28:28, 3.66MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [05:12<26:32, 12.3MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [05:12<26:59, 12.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:12<53:21, 6.06MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [05:12<59:39, 5.69MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [05:12<27:25, 11.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:12<49:55, 6.48MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [05:12<59:31, 5.70MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:13<48:40, 6.64MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [05:13<46:54, 7.08MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [05:13<1:03:35, 5.33MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:13<47:11, 6.85MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [05:13<37:10, 8.94MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [05:13<52:50, 6.18MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [05:13<29:31, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:13<46:36, 6.94MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [05:13<33:57, 9.61MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [05:13<1:01:44, 5.49MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [05:13<28:47, 11.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [05:13<31:50, 10.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:13<45:33, 7.09MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [05:13<24:17, 13.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.80G/22.2G [05:13<58:41, 5.78MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [05:13<28:36, 11.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [05:13<21:23, 15.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:13<44:51, 7.20MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [05:13<28:31, 11.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [05:13<55:28, 6.11MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [05:14<19:05, 17.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:14<44:26, 7.27MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [05:14<17:50, 18.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [05:14<26:38, 12.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [05:14<50:30, 6.71MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [05:14<26:52, 12.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:14<44:01, 7.34MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [05:14<47:57, 7.07MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [05:14<27:17, 12.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [05:14<24:53, 13.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [05:14<43:12, 7.48MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [05:14<25:15, 12.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [05:14<45:25, 7.46MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [05:14<24:04, 13.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [05:14<43:14, 7.47MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [05:14<43:10, 7.85MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [05:14<21:15, 15.6MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [05:14<19:08, 17.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.81G/22.2G [05:14<39:54, 8.50MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [05:15<43:12, 7.48MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [05:15<17:46, 18.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [05:15<42:23, 8.00MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [05:15<16:47, 19.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [05:15<44:49, 7.27MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [05:15<16:15, 20.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [05:15<32:32, 10.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [05:15<1:18:18, 4.13MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [05:15<31:10, 10.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [05:15<3:10:11, 1.60MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [05:15<1:18:19, 4.33MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [05:15<27:36, 11.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [05:15<49:39, 6.50MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [05:15<32:33, 10.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [05:16<27:35, 11.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [05:16<49:01, 6.91MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [05:16<24:36, 13.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:16<46:37, 6.93MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [05:16<2:20:05, 2.17MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [05:16<25:34, 12.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [05:16<49:21, 6.87MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [05:16<21:45, 15.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:16<45:44, 7.06MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [05:16<1:56:08, 2.62MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [05:16<19:31, 17.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [05:16<25:00, 13.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.82G/22.2G [05:16<45:30, 7.45MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [05:16<19:59, 16.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [05:16<24:52, 13.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [05:16<1:20:22, 3.78MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [05:16<17:53, 18.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [05:16<48:25, 7.00MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [05:16<26:00, 12.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [05:16<17:10, 19.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [05:16<26:43, 12.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [05:16<52:34, 5.77MB/s]  [A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:16<1:03:37, 5.08MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [05:16<48:11, 7.03MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [05:16<18:38, 17.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [05:17<44:05, 6.89MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [05:17<48:31, 6.98MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:17<1:02:49, 5.14MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [05:17<37:45, 8.04MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [05:17<49:22, 6.86MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [05:17<28:16, 11.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:17<1:00:36, 5.33MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [05:17<26:00, 12.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [05:17<1:02:21, 5.22MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [05:17<49:51, 6.79MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [05:17<24:08, 13.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:17<58:34, 5.51MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [05:17<39:29, 8.25MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [05:17<50:18, 6.73MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [05:17<21:27, 15.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [05:17<50:36, 6.00MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [05:17<36:47, 8.85MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [05:17<19:38, 16.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [05:18<35:04, 9.28MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.83G/22.2G [05:18<50:26, 6.71MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:18<56:47, 5.69MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [05:18<35:13, 8.61MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [05:18<18:19, 18.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [05:18<33:37, 9.68MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [05:18<33:53, 8.95MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:18<54:53, 5.88MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [05:18<50:00, 6.77MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [05:18<17:30, 18.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [05:18<32:48, 9.92MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [05:18<29:40, 10.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [05:18<31:44, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:18<52:01, 6.21MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [05:18<49:44, 6.81MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [05:18<31:01, 10.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [05:18<26:26, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:18<51:00, 6.33MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [05:18<30:33, 10.7MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [05:18<30:30, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [05:18<24:20, 12.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [05:18<33:02, 10.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:18<49:54, 6.47MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [05:18<29:24, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [05:19<24:58, 13.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [05:19<22:53, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [05:19<29:34, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [05:19<21:58, 15.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [05:19<29:19, 11.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [05:19<1:38:31, 3.44MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [05:19<20:11, 16.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [05:19<29:38, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [05:19<18:45, 17.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:19<1:31:13, 3.54MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [05:19<17:51, 18.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [05:19<57:25, 5.90MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [05:19<17:00, 19.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [05:19<36:21, 8.34MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [05:19<56:41, 5.74MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:19<53:08, 6.07MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.84G/22.2G [05:19<1:00:25, 5.60MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [05:19<27:24, 11.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [05:20<37:55, 8.57MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:20<50:03, 6.44MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [05:20<27:15, 11.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [05:20<34:43, 9.36MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [05:20<1:02:17, 5.43MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:20<48:46, 6.61MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [05:20<33:14, 9.78MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [05:20<24:53, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [05:20<32:13, 10.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:20<31:27, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:20<47:12, 6.83MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [05:20<24:34, 13.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [05:20<1:03:07, 5.36MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [05:20<23:18, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:20<30:27, 10.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [05:20<21:42, 15.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [05:20<20:31, 14.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:20<46:33, 6.93MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:20<30:05, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [05:20<19:34, 16.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [05:20<1:03:38, 5.32MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:20<28:47, 11.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [05:20<21:25, 14.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [05:20<18:06, 18.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:20<45:35, 7.07MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:20<28:31, 11.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [05:20<19:20, 15.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [05:21<18:28, 17.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:21<28:33, 11.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [05:21<1:05:36, 5.16MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:21<46:03, 7.00MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [05:21<18:01, 18.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:21<28:20, 11.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [05:21<18:28, 17.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:21<50:02, 6.44MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [05:21<1:13:27, 4.61MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:21<32:44, 9.92MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [05:21<38:18, 7.91MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:21<53:35, 6.02MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [05:21<33:24, 9.88MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [05:21<29:30, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [05:21<26:35, 12.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:21<55:27, 5.81MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [05:21<27:44, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.85G/22.2G [05:22<1:23:13, 4.07MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [05:22<26:29, 12.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [05:22<27:07, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:22<1:01:17, 5.30MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [05:22<22:57, 14.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:22<56:02, 5.75MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [05:22<26:48, 11.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [05:22<23:21, 14.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [05:22<26:30, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:22<42:01, 7.73MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [05:22<1:29:44, 3.77MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [05:22<20:30, 16.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:22<55:41, 5.79MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [05:22<26:19, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:22<42:00, 7.73MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [05:22<21:01, 15.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [05:22<26:11, 11.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:22<41:20, 7.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [05:22<19:26, 17.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [05:22<26:13, 11.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [05:22<1:34:41, 3.57MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:22<39:35, 8.20MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [05:23<24:27, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:23<37:10, 8.74MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [05:23<24:29, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:23<36:29, 8.90MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [05:23<1:36:47, 3.50MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:23<1:44:01, 3.10MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [05:23<34:55, 9.44MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:23<37:40, 8.62MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [05:23<25:50, 12.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:23<35:59, 9.02MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:23<57:47, 5.58MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [05:23<25:30, 12.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:23<34:48, 9.33MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [05:23<1:39:15, 3.41MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [05:23<22:42, 14.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:23<33:36, 9.66MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [05:23<52:11, 5.80MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:23<59:20, 5.43MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [05:23<20:24, 16.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:24<35:13, 9.22MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [05:24<20:47, 15.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [05:24<36:56, 8.18MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:24<34:06, 9.52MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:24<56:59, 5.65MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [05:24<1:39:52, 3.39MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [05:24<19:45, 16.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [05:24<33:10, 9.11MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:24<32:48, 9.89MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [05:24<18:20, 18.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [05:24<30:58, 9.76MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:24<55:17, 5.83MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [05:24<27:28, 11.0MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [05:24<27:26, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:24<52:35, 6.13MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [05:24<26:20, 11.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [05:24<31:10, 10.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:24<53:50, 5.98MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [05:25<24:33, 12.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [05:25<23:31, 14.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [05:25<24:32, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:25<1:30:21, 3.59MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [05:25<22:55, 14.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:25<53:05, 6.07MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [05:25<23:47, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [05:25<20:38, 15.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.86G/22.2G [05:25<3:17:06, 1.72MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:25<53:02, 6.12MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:25<51:30, 6.25MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [05:25<18:57, 17.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:25<49:09, 6.60MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [05:25<19:16, 17.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:25<51:54, 6.20MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:25<43:53, 7.39MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [05:25<18:47, 17.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [05:25<1:43:19, 3.27MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:25<40:15, 8.06MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [05:25<43:49, 6.89MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:26<50:37, 6.36MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:26<37:23, 8.68MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [05:26<31:12, 9.68MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:26<38:08, 8.51MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:26<51:13, 6.29MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [05:26<29:06, 10.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [05:26<1:42:00, 3.31MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:26<35:45, 9.07MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [05:26<32:35, 10.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:26<49:30, 6.50MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:26<33:55, 9.56MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [05:26<25:47, 11.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [05:26<24:45, 13.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:26<32:23, 10.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [05:26<25:39, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [05:26<23:39, 13.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:26<31:11, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [05:26<1:40:56, 3.35MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [05:26<23:41, 12.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:26<34:18, 9.45MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [05:27<23:44, 13.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [05:27<24:22, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:27<33:16, 9.74MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [05:27<21:24, 15.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:27<1:34:18, 3.41MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [05:27<22:46, 13.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [05:27<1:40:11, 3.37MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [05:27<22:58, 13.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [05:27<21:54, 15.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:27<58:09, 5.54MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [05:27<22:25, 14.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [05:27<1:38:25, 3.43MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:27<56:05, 5.74MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:27<1:11:07, 4.56MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:27<53:43, 5.99MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [05:27<41:34, 7.25MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:28<41:05, 7.89MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [05:28<1:36:14, 3.51MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [05:28<31:44, 9.50MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:28<56:25, 5.70MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:28<41:02, 7.90MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [05:28<28:54, 10.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:28<38:09, 8.49MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [05:28<44:29, 7.39MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:28<52:00, 6.19MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.87G/22.2G [05:28<1:33:16, 3.62MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:28<35:39, 9.09MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [05:28<25:50, 11.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [05:28<32:01, 10.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:28<34:00, 9.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:28<51:14, 6.28MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [05:28<25:31, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [05:28<31:01, 10.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:28<33:24, 9.70MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:28<50:45, 6.34MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [05:28<28:40, 11.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:28<32:17, 10.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [05:28<1:36:56, 3.49MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [05:28<23:51, 12.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:29<50:15, 6.40MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:29<34:35, 9.37MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [05:29<26:52, 12.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [05:29<23:14, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:29<33:26, 9.69MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [05:29<23:13, 13.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [05:29<1:33:28, 3.62MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [05:29<25:19, 13.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:29<31:55, 10.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:29<58:02, 5.54MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [05:29<22:45, 14.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:29<30:56, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [05:29<23:22, 14.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:29<31:04, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [05:29<1:32:17, 3.66MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:29<1:00:04, 5.36MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [05:29<21:23, 15.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:29<30:36, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [05:29<38:10, 7.89MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:30<1:01:25, 5.24MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [05:30<1:31:19, 3.70MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [05:30<29:19, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [05:30<34:35, 9.49MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [05:30<1:00:31, 5.31MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [05:30<27:33, 10.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [05:30<29:39, 11.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:30<1:07:10, 4.82MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [05:30<1:29:12, 3.79MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [05:30<29:36, 11.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [05:30<25:02, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [05:30<59:29, 5.41MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:30<43:05, 7.51MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [05:30<26:25, 12.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [05:30<25:04, 12.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   8%|▊         | 1.88G/22.2G [05:30<1:24:18, 4.01MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:30<42:28, 7.62MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [05:30<22:51, 14.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [05:30<23:34, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:30<39:25, 8.21MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [05:30<23:44, 13.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [05:31<23:55, 12.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:31<37:10, 8.71MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [05:31<20:59, 15.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:31<34:33, 9.37MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [05:31<22:03, 14.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.10G/22.2G [05:31<22:37, 13.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:31<33:31, 9.65MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [05:31<20:18, 16.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  18%|█▊        | 4.10G/22.2G [05:31<22:31, 13.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [05:31<1:59:48, 2.68MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:31<31:39, 10.2MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:31<30:56, 10.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.88G/22.2G [05:31<2:49:21, 1.99MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:31<34:06, 9.49MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [05:31<1:05:31, 4.91MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [05:31<37:33, 8.73MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:31<33:15, 9.73MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [05:31<41:38, 7.23MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [05:31<27:14, 12.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:32<32:28, 9.96MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [05:32<1:02:20, 5.16MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [05:32<27:09, 12.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [05:32<1:26:30, 3.90MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:32<31:59, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [05:32<33:14, 9.05MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [05:32<23:21, 14.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:32<30:54, 10.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [05:32<30:14, 9.95MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:32<59:48, 5.38MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [05:32<23:46, 13.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [05:32<1:22:10, 4.11MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.49G/22.2G [05:32<21:38, 15.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [05:32<26:14, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:32<57:44, 5.57MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:32<22:29, 14.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [05:32<1:19:32, 4.25MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [05:32<24:43, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:32<20:30, 16.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:32<55:40, 5.77MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:32<1:01:57, 5.22MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:32<21:35, 15.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [05:33<23:43, 12.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [05:33<1:15:35, 4.47MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:33<39:10, 8.25MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:33<51:35, 6.23MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [05:33<20:18, 16.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [05:33<24:01, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:33<39:04, 8.28MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [05:33<1:12:30, 4.66MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:33<54:16, 5.92MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [05:33<23:02, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:33<36:10, 8.94MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:33<34:22, 9.41MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.89G/22.2G [05:33<1:06:28, 5.08MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:33<50:31, 6.36MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [05:33<37:25, 8.75MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:33<33:02, 9.79MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [05:33<31:03, 10.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:33<53:20, 6.02MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:33<34:41, 9.32MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [05:33<1:05:53, 5.12MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [05:33<30:24, 10.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [05:33<38:02, 7.90MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [05:34<24:38, 13.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:34<33:10, 9.74MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:34<52:18, 6.14MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [05:34<1:00:20, 5.59MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [05:34<28:48, 10.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [05:34<25:12, 13.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:34<35:47, 9.03MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [05:34<27:07, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:34<48:55, 6.57MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [05:34<21:45, 15.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [05:34<58:13, 5.80MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [05:34<22:43, 14.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:34<37:15, 8.67MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [05:34<24:17, 12.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [05:34<55:26, 6.09MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:34<51:48, 6.20MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [05:34<20:17, 16.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [05:34<35:45, 9.04MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [05:34<21:25, 15.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [05:34<23:25, 12.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [05:34<53:25, 6.32MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [05:34<37:33, 8.61MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.90G/22.2G [05:34<48:06, 7.01MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [05:34<35:31, 9.10MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [05:34<22:36, 13.3MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [05:35<22:46, 13.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [05:35<48:50, 6.91MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [05:35<37:04, 8.82MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [05:35<21:37, 13.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:35<1:38:51, 3.25MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [05:35<26:04, 12.5MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [05:35<25:57, 12.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:35<55:58, 5.74MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [05:35<22:36, 14.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [05:35<1:23:47, 3.86MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [05:35<1:27:16, 3.87MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [05:35<23:09, 14.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:35<53:53, 5.96MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [05:35<46:46, 6.91MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [05:35<52:25, 6.44MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [05:35<21:16, 15.4MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [05:36<22:00, 14.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:36<56:25, 5.69MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [05:36<46:06, 7.01MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [05:36<51:28, 6.55MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [05:36<20:37, 15.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:36<42:29, 7.60MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:36<54:31, 5.89MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [05:36<43:04, 7.83MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [05:36<55:36, 5.40MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:36<38:14, 8.44MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  11%|█▏        | 2.55G/22.2G [05:36<21:10, 15.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.91G/22.2G [05:36<40:45, 8.28MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:36<49:27, 6.49MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:36<39:38, 8.15MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [05:36<39:24, 8.56MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [05:36<43:20, 6.92MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:36<36:48, 8.77MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [05:36<37:26, 9.01MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [05:36<39:17, 7.64MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:36<53:30, 6.00MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:36<33:56, 9.51MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [05:36<36:33, 8.94MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [05:36<36:06, 9.34MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [05:36<32:26, 9.25MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:37<36:08, 8.93MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [05:37<25:42, 12.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:37<49:12, 6.52MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [05:37<34:41, 9.72MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:37<34:33, 9.34MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [05:37<25:43, 12.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [05:37<33:16, 10.1MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [05:37<28:09, 10.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:37<48:34, 6.61MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:37<32:37, 9.89MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [05:37<22:38, 14.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.92G/22.2G [05:37<32:20, 10.4MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:37<31:39, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [05:37<23:10, 14.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [05:37<25:23, 11.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [05:37<32:06, 10.5MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [05:37<52:56, 6.06MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:37<30:40, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [05:37<21:36, 15.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [05:37<23:26, 12.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [05:37<31:52, 10.6MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [05:37<22:34, 14.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:37<33:19, 9.69MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [05:37<48:05, 6.67MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [05:37<20:22, 16.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [05:37<48:16, 6.65MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [05:38<21:05, 15.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [05:38<48:26, 6.62MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [05:38<1:03:05, 5.34MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [05:38<35:03, 8.55MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:38<1:06:02, 4.89MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [05:38<26:15, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [05:38<43:31, 7.74MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:38<41:59, 7.68MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [05:38<26:22, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.93G/22.2G [05:38<36:01, 9.35MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [05:38<40:21, 8.09MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:38<41:08, 7.84MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.94G/22.2G [05:38<33:31, 10.0MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [05:38<28:24, 11.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [05:38<24:17, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:38<38:04, 8.47MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▊         | 1.94G/22.2G [05:38<30:24, 11.1MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [05:38<27:57, 11.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [05:38<21:49, 13.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:38<35:20, 9.13MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [05:39<1:32:34, 3.47MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [05:39<23:44, 13.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:39<33:18, 9.68MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [05:39<26:50, 12.6MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [05:39<20:58, 14.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [05:39<24:05, 13.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:39<32:08, 10.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [05:39<18:56, 15.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [05:39<54:20, 5.90MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [05:39<21:24, 15.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:39<30:54, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.94G/22.2G [05:39<27:37, 12.2MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [05:39<19:58, 15.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [05:39<22:11, 14.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:39<30:02, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [05:39<51:38, 6.21MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [05:39<24:37, 13.7MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:39<29:44, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [05:39<22:42, 14.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [05:39<25:42, 13.1MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [05:39<50:26, 6.36MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:39<29:33, 10.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [05:39<23:02, 14.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:39<29:50, 10.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [05:39<49:28, 6.48MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:40<29:53, 10.8MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:40<29:37, 10.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [05:40<48:43, 6.58MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [05:40<52:04, 6.47MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [05:40<47:58, 6.68MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [05:40<51:37, 5.80MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [05:40<43:59, 7.41MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:40<47:35, 6.74MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.95G/22.2G [05:40<42:15, 7.97MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:40<57:00, 5.65MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [05:40<36:52, 8.12MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [05:40<32:53, 9.91MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:40<47:15, 6.79MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [05:40<33:52, 8.84MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [05:40<43:36, 7.72MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:40<39:33, 8.15MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [05:40<29:42, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:40<36:33, 8.81MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [05:41<29:32, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:41<47:04, 6.81MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [05:41<29:26, 10.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [05:41<42:42, 7.88MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:41<34:56, 9.22MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [05:41<29:04, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [05:41<27:31, 11.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:41<32:57, 9.78MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [05:41<44:44, 7.52MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:41<32:10, 10.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:41<54:34, 5.87MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [05:41<25:54, 11.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [05:41<25:59, 12.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [05:41<43:30, 7.73MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:41<31:16, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [05:41<25:40, 11.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [05:41<26:28, 12.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:41<31:02, 10.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:41<48:44, 6.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [05:41<22:03, 13.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [05:41<45:56, 7.33MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:41<30:27, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [05:41<24:56, 13.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [05:41<23:35, 12.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:41<29:59, 10.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.96G/22.2G [05:41<45:44, 7.36MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:41<51:38, 6.21MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:42<29:25, 10.9MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:42<29:26, 10.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [05:42<47:06, 7.14MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:42<29:47, 10.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [05:42<46:34, 7.22MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:42<29:34, 10.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [05:42<40:02, 8.13MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [05:42<42:38, 7.02MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [05:42<28:11, 10.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [05:42<47:10, 7.13MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [05:42<30:47, 10.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [05:42<28:11, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:42<1:44:40, 3.06MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [05:42<47:32, 7.08MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [05:42<28:12, 11.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [05:42<25:24, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [05:43<24:43, 13.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [05:43<47:21, 7.10MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:43<1:07:42, 4.76MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [05:43<58:34, 5.47MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [05:43<25:03, 13.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [05:43<23:50, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:43<42:01, 7.66MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [05:43<56:29, 5.67MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [05:43<23:57, 13.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [05:43<20:28, 14.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:43<37:47, 8.52MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [05:43<21:32, 13.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:43<35:58, 8.95MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [05:43<23:30, 13.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [05:43<58:19, 5.49MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [05:43<19:18, 15.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:43<34:10, 9.42MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [05:43<21:17, 15.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [05:43<20:35, 14.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.97G/22.2G [05:43<1:35:28, 3.52MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:43<33:04, 9.73MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [05:43<51:55, 6.17MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:43<31:59, 10.1MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:44<31:26, 10.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [05:44<1:01:05, 5.50MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [05:44<59:19, 5.40MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:44<30:40, 10.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [05:44<35:05, 8.51MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [05:44<39:07, 8.32MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:44<30:28, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [05:44<27:15, 11.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [05:44<29:02, 11.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:44<30:07, 10.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [05:44<1:00:53, 5.52MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [05:44<27:24, 10.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:44<54:58, 5.83MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [05:44<28:33, 11.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [05:44<30:10, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [05:44<21:56, 13.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [05:44<30:30, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:44<26:37, 12.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [05:44<1:02:39, 5.37MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [05:44<22:59, 13.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:44<56:46, 5.64MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [05:44<30:06, 10.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:44<23:29, 13.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [05:44<19:37, 15.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:45<24:01, 13.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [05:45<1:03:59, 5.25MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [05:45<20:57, 14.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:45<56:53, 5.63MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:45<21:15, 15.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [05:45<18:20, 16.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [05:45<22:05, 14.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [05:45<1:04:53, 5.18MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [05:45<19:46, 15.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:45<57:31, 5.57MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:45<20:33, 15.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [05:45<1:07:00, 4.80MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.98G/22.2G [05:45<1:04:52, 5.18MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [05:45<41:06, 7.82MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:45<58:00, 5.52MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [05:45<37:17, 8.62MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [05:45<1:04:54, 5.18MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:45<35:14, 9.12MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [05:45<38:07, 7.82MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:45<38:31, 8.44MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:45<58:25, 5.48MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:46<33:34, 9.58MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:46<26:44, 12.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [05:46<25:51, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:46<32:22, 9.93MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [05:46<26:25, 12.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [05:46<25:51, 11.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [05:46<1:05:17, 5.15MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [05:46<58:36, 5.46MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:46<31:22, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:46<23:10, 14.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [05:46<22:50, 13.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:46<30:18, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:46<23:34, 13.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [05:46<1:04:25, 5.22MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:46<29:55, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [05:46<19:44, 15.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:46<21:22, 15.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [05:46<29:38, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [05:46<22:10, 14.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [05:46<20:08, 14.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [05:46<1:04:00, 5.25MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:46<29:00, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:46<20:45, 15.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [05:46<18:15, 16.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:47<28:55, 11.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [05:47<1:51:39, 2.87MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:47<21:05, 15.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:47<28:32, 11.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [05:47<1:02:42, 5.36MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:47<28:51, 11.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [05:47<1:05:18, 4.90MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [05:47<1:02:39, 5.36MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:47<36:13, 8.96MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [05:47<32:04, 9.29MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [05:47<1:01:54, 5.17MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [05:47<25:31, 12.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [05:47<24:28, 12.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:47<1:00:04, 5.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:47<25:52, 12.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [05:47<24:23, 12.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [05:47<1:00:35, 5.28MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [05:48<40:56, 7.84MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:48<24:45, 13.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:48<37:05, 8.66MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [05:48<21:58, 13.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:48<21:01, 15.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 1.99G/22.2G [05:48<1:51:25, 3.02MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:48<58:55, 5.43MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [05:48<21:43, 14.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:48<37:12, 8.63MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [05:48<19:21, 15.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:48<20:22, 15.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:48<35:40, 9.00MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [05:48<19:39, 15.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:48<57:41, 5.55MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:48<21:14, 15.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:48<33:18, 9.64MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [05:48<1:06:01, 5.09MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [05:48<17:53, 16.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:48<19:51, 16.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:48<33:43, 9.52MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:48<56:59, 5.61MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:48<20:23, 15.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [05:48<1:03:52, 5.26MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [05:49<40:57, 7.84MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:49<56:17, 5.68MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [05:49<1:03:07, 5.32MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [05:49<33:01, 9.72MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [05:49<30:40, 9.70MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [05:49<38:30, 8.42MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [05:49<31:17, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:49<55:48, 5.73MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [05:49<23:36, 12.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [05:49<59:36, 5.63MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:49<30:34, 10.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [05:49<30:07, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [05:49<23:37, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:49<30:00, 10.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:49<56:06, 5.70MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [05:49<20:12, 14.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:49<24:29, 13.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.00G/22.2G [05:49<1:02:14, 5.40MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [05:49<24:58, 13.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [05:49<20:50, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [05:49<56:21, 5.67MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [05:49<1:00:01, 5.60MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:50<21:46, 14.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [05:50<18:35, 16.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [05:50<49:47, 6.44MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:50<21:50, 14.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [05:50<19:02, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [05:50<56:26, 5.67MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [05:50<1:01:26, 5.47MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [05:50<19:55, 16.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [05:50<34:11, 9.38MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [05:50<17:28, 17.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [05:50<32:55, 9.74MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [05:50<1:02:18, 5.39MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:50<20:30, 15.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [05:50<28:51, 11.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [05:50<59:14, 5.67MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [05:50<28:23, 11.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.32G/22.2G [05:50<31:43, 9.37MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:51<36:04, 8.98MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:51<25:40, 12.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.32G/22.2G [05:51<24:21, 12.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [05:51<1:00:51, 5.52MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:51<30:39, 10.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [05:51<24:27, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:51<24:18, 13.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [05:51<1:43:40, 3.08MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:51<30:06, 10.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [05:51<21:00, 14.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.01G/22.2G [05:51<1:00:32, 5.54MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:51<23:25, 13.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [05:51<24:35, 13.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [05:51<20:38, 14.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:51<24:56, 13.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [05:51<1:03:53, 5.00MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [05:51<1:01:33, 5.45MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [05:51<18:46, 15.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:51<22:13, 14.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [05:51<17:12, 17.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [05:51<1:01:17, 5.22MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:51<21:42, 14.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [05:52<18:09, 16.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [05:52<38:36, 8.30MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [05:52<20:06, 16.1MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:52<20:38, 15.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [05:52<59:43, 5.35MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [05:52<32:00, 10.0MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [05:52<26:11, 12.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [05:52<1:02:00, 5.15MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [05:52<26:14, 12.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [05:52<2:04:29, 2.70MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:52<39:07, 8.27MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [05:52<26:39, 12.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:52<27:46, 11.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:52<27:23, 11.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [05:52<46:59, 6.32MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [05:52<1:05:41, 4.87MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [05:53<27:30, 11.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:53<27:50, 11.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [05:53<1:12:05, 4.66MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [05:53<34:04, 8.71MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:53<27:43, 11.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:53<23:13, 13.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [05:53<1:04:51, 4.93MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [05:53<30:56, 9.59MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:53<27:35, 11.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:53<23:36, 13.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [05:53<1:08:30, 4.90MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [05:53<27:43, 11.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [05:53<24:47, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:53<21:55, 14.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [05:53<24:48, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [05:53<1:07:09, 4.76MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [05:53<25:19, 12.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [05:53<21:51, 14.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.02G/22.2G [05:53<1:05:38, 5.11MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [05:53<21:10, 14.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [05:53<1:02:31, 5.11MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [05:53<19:57, 16.2MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [05:53<20:48, 15.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [05:53<20:19, 14.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [05:54<1:03:23, 5.29MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [05:54<1:03:05, 5.06MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [05:54<19:04, 15.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [05:54<52:38, 6.08MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [05:54<1:01:04, 5.49MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [05:54<1:01:54, 5.16MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [05:54<34:36, 9.25MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [05:54<30:45, 9.64MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [05:54<32:30, 9.84MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [05:54<58:57, 5.69MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [05:54<45:43, 7.07MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [05:54<1:01:37, 5.18MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [05:54<23:03, 12.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [05:54<31:52, 10.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:54<28:33, 11.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [05:54<56:21, 5.95MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [05:54<23:19, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:54<30:50, 10.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:54<28:07, 11.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [05:54<1:01:36, 5.19MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [05:55<19:50, 14.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:55<25:15, 12.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [05:55<55:29, 6.04MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:55<25:26, 12.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:55<25:08, 12.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [05:55<20:00, 14.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:55<25:48, 12.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.03G/22.2G [05:55<52:58, 6.33MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [05:55<21:45, 14.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [05:55<17:53, 16.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [05:55<24:10, 13.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [05:55<51:13, 6.55MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:55<22:05, 14.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [05:55<18:31, 16.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [05:55<23:16, 13.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:55<19:38, 16.4MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:55<20:19, 15.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [05:55<54:51, 6.11MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 2.99G/22.2G [05:55<1:55:21, 2.77MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [05:56<54:59, 6.10MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [05:56<40:36, 7.87MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [05:56<34:13, 9.43MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [05:56<31:07, 9.51MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:56<1:06:35, 4.80MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:56<26:27, 12.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [05:56<29:45, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [05:56<23:53, 12.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:56<26:31, 12.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [05:56<23:53, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [05:56<26:52, 11.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:56<1:03:02, 5.06MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:56<22:14, 14.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [05:56<20:42, 14.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [05:56<25:07, 12.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [05:56<22:47, 14.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [05:56<1:51:13, 3.01MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [05:56<18:36, 15.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:56<1:00:53, 5.24MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:56<20:07, 16.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [05:56<23:51, 13.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [05:57<19:17, 15.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:57<20:29, 15.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [05:57<18:15, 16.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [05:57<23:00, 13.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:57<59:11, 5.39MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [05:57<1:09:59, 4.79MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:57<18:29, 17.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [05:57<16:41, 17.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [05:57<22:24, 14.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:57<57:50, 5.52MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.04G/22.2G [05:57<1:06:00, 5.08MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [05:57<1:01:02, 5.49MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:57<57:09, 5.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [05:57<33:27, 8.84MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [05:57<37:01, 8.71MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [05:57<58:23, 5.74MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [05:58<24:28, 12.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [05:58<56:24, 5.66MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:58<26:55, 12.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 2.99G/22.2G [05:58<40:36, 7.87MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [05:58<24:16, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:58<26:13, 12.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [05:58<1:01:19, 5.46MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [05:58<55:53, 5.71MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:58<31:17, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [05:58<23:05, 14.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [05:58<20:14, 14.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:58<29:45, 10.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [05:58<58:24, 5.74MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:58<22:52, 14.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [05:58<55:26, 5.76MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [05:58<22:20, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:58<26:20, 12.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:58<21:02, 15.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [05:58<55:35, 6.03MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:58<26:39, 12.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:58<21:45, 14.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.43G/22.2G [05:58<19:51, 14.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [05:58<55:48, 5.72MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [05:58<53:59, 6.20MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:58<27:21, 11.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [05:59<22:27, 14.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [05:59<27:40, 11.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|█▉        | 4.43G/22.2G [05:59<20:02, 14.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [05:59<52:04, 6.13MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.05G/22.2G [05:59<53:05, 6.31MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [05:59<28:09, 11.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:59<22:31, 14.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [05:59<28:15, 11.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [05:59<52:23, 6.39MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [05:59<27:56, 11.4MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [05:59<28:03, 11.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [05:59<51:49, 6.46MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:59<35:40, 9.02MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [05:59<27:30, 11.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.43G/22.2G [05:59<36:33, 8.08MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [05:59<51:24, 6.51MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [05:59<32:02, 10.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [05:59<27:10, 10.9MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [06:00<26:58, 10.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [06:00<28:21, 11.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [06:00<1:39:24, 3.21MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [06:00<24:39, 13.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [06:00<25:04, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [06:00<59:48, 5.33MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [06:00<25:12, 12.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:00<1:01:27, 5.19MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [06:00<23:17, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [06:00<22:25, 14.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:00<36:41, 8.69MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [06:00<23:04, 13.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:00<34:00, 9.38MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [06:00<1:39:46, 3.36MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [06:00<21:56, 13.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:00<58:53, 5.42MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:00<32:19, 9.87MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [06:00<22:35, 14.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:00<30:58, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [06:00<21:02, 14.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.06G/22.2G [06:01<58:24, 5.73MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:01<29:52, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:01<57:13, 5.57MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [06:01<55:12, 6.07MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:01<26:33, 12.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:01<55:56, 5.70MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [06:01<34:47, 9.24MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:01<26:55, 11.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [06:01<53:34, 6.25MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:01<26:54, 11.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [06:01<25:24, 12.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:01<54:19, 5.87MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [06:01<32:50, 8.98MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:01<27:07, 11.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [06:01<25:26, 12.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [06:01<51:44, 6.47MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [06:01<25:04, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [06:01<22:42, 14.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:01<55:21, 5.76MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:01<25:24, 12.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [06:01<25:12, 11.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [06:01<51:29, 6.50MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [06:01<23:22, 13.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:02<53:11, 5.99MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [06:02<20:41, 15.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [06:02<23:00, 12.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [06:02<50:36, 6.61MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [06:02<21:17, 15.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:02<52:24, 6.08MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [06:02<20:05, 14.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.07G/22.2G [06:02<50:02, 6.69MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [06:02<19:49, 16.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [06:02<20:53, 14.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:02<51:32, 6.18MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [06:02<18:19, 17.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:02<54:14, 5.88MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [06:02<18:12, 16.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [06:02<49:11, 6.80MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:02<34:10, 9.32MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [06:02<19:19, 15.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:02<52:12, 6.10MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [06:02<48:29, 6.90MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:02<32:25, 9.82MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:03<51:04, 6.24MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:03<29:17, 10.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [06:03<48:36, 6.88MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [06:03<40:01, 8.03MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▎        | 3.05G/22.2G [06:03<29:03, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [06:03<30:53, 9.54MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [06:03<48:37, 6.88MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:03<53:06, 6.00MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [06:03<29:21, 10.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:03<26:50, 11.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [06:03<22:53, 12.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [06:03<28:34, 11.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [06:03<47:57, 6.97MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:03<27:44, 11.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [06:03<23:12, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [06:03<24:08, 13.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:03<26:49, 11.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [06:03<19:55, 14.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:03<26:38, 12.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [06:03<23:11, 13.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [06:03<20:01, 14.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [06:03<20:48, 15.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:04<24:57, 12.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [06:04<17:51, 16.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:04<1:37:06, 3.28MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [06:04<19:04, 16.8MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [06:04<19:26, 16.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [06:04<18:23, 16.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.08G/22.2G [06:04<1:40:41, 3.32MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:04<55:04, 5.79MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:04<47:37, 6.68MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:04<52:05, 6.12MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [06:04<57:24, 5.83MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:04<32:33, 9.78MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:04<50:42, 6.28MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [06:04<55:13, 6.06MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [06:04<29:02, 10.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:04<31:15, 10.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:05<49:04, 6.49MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [06:05<22:34, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:05<29:40, 10.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [06:05<53:30, 6.25MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [06:05<48:55, 6.56MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [06:05<22:45, 12.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▎        | 3.05G/22.2G [06:05<48:08, 6.62MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:05<27:05, 11.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [06:05<51:36, 6.48MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [06:05<19:40, 15.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [06:05<35:38, 9.00MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:05<27:09, 11.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:05<46:46, 6.81MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [06:05<50:43, 6.59MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [06:05<30:51, 10.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [06:05<19:43, 14.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:05<25:16, 12.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [06:05<26:05, 12.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:05<25:38, 12.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [06:05<17:43, 16.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:05<50:15, 6.34MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.09G/22.2G [06:05<49:58, 6.69MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [06:05<22:47, 14.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:05<24:12, 13.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [06:05<17:59, 16.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [06:05<49:50, 6.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [06:05<22:39, 14.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:06<52:32, 6.06MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:06<24:56, 12.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [06:06<20:31, 15.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [06:06<49:22, 6.77MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:06<50:55, 6.25MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [06:06<19:05, 16.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [06:06<48:40, 6.87MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:06<54:25, 5.85MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [06:06<29:59, 9.80MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:06<46:11, 6.88MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [06:06<48:00, 6.96MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [06:06<23:03, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:06<52:04, 6.11MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [06:06<34:32, 9.28MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:06<31:37, 10.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [06:06<23:06, 12.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [06:06<47:32, 7.03MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:06<30:24, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [06:06<27:44, 11.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:06<49:50, 6.39MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [06:06<19:49, 14.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:   9%|▉         | 2.10G/22.2G [06:07<47:11, 7.08MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [06:07<17:58, 16.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:07<27:08, 11.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [06:07<25:41, 12.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:07<27:07, 11.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [06:07<18:42, 15.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [06:07<23:20, 13.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:07<27:06, 11.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.54G/22.2G [06:07<17:42, 16.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [06:07<21:11, 15.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  20%|██        | 4.54G/22.2G [06:07<16:23, 17.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [06:07<19:37, 16.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:07<24:51, 12.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [06:07<19:31, 16.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:07<25:16, 12.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [06:07<18:46, 17.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:07<25:28, 12.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:07<1:38:00, 3.25MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.10G/22.2G [06:07<1:43:19, 3.23MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:08<24:02, 13.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.54G/22.2G [06:08<31:44, 9.25MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:08<56:51, 5.60MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [06:08<23:47, 12.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [06:08<1:01:50, 5.40MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [06:08<36:59, 8.66MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:08<53:28, 5.95MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [06:08<23:45, 12.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [06:08<26:25, 12.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [06:08<57:22, 5.82MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [06:08<19:57, 14.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:08<51:12, 6.21MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [06:08<25:49, 12.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [06:08<17:37, 16.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:08<52:09, 6.09MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [06:08<56:13, 5.94MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [06:08<22:26, 14.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:08<49:27, 6.43MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [06:08<18:19, 16.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:08<35:14, 9.01MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [06:08<20:43, 15.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [06:09<56:00, 5.96MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [06:09<17:48, 16.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:09<48:38, 6.54MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [06:09<19:07, 16.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:09<36:11, 8.77MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [06:09<16:26, 17.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.11G/22.2G [06:09<55:16, 6.04MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [06:09<18:58, 16.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:09<47:23, 6.71MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:09<34:37, 9.17MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [06:09<19:06, 16.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:09<33:18, 9.53MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [06:09<54:38, 6.11MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:09<45:56, 6.92MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [06:09<18:18, 17.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:09<32:12, 9.86MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [06:09<28:00, 10.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:09<31:17, 10.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [06:09<53:28, 6.25MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:09<45:25, 7.00MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [06:09<21:37, 13.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:09<30:52, 10.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [06:09<51:57, 6.43MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [06:09<21:53, 13.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:09<45:01, 7.06MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:09<30:35, 10.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [06:09<32:18, 9.90MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [06:10<17:52, 16.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:10<30:23, 10.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:10<43:41, 7.28MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [06:10<51:37, 6.47MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [06:10<30:17, 10.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [06:10<18:36, 15.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:10<29:47, 10.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [06:10<28:56, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:10<43:52, 7.25MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:10<29:25, 10.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [06:10<17:38, 16.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [06:10<50:41, 6.59MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [06:10<29:06, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:10<29:26, 10.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [06:10<16:20, 17.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:10<43:24, 7.33MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.12G/22.2G [06:10<49:59, 6.68MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:10<28:46, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [06:10<28:59, 11.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [06:10<17:21, 16.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [06:10<48:53, 6.83MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [06:10<27:05, 11.8MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [06:10<27:26, 11.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:11<55:28, 5.72MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [06:11<27:49, 11.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [06:11<30:30, 9.60MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [06:11<28:16, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:11<1:31:01, 3.49MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:11<40:09, 7.90MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [06:11<24:46, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [06:11<26:16, 12.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [06:11<24:36, 11.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:11<35:54, 8.83MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [06:11<26:49, 11.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:11<52:18, 6.08MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [06:11<20:31, 14.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [06:11<1:34:52, 3.52MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:11<37:20, 8.49MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [06:11<18:11, 16.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:11<48:35, 6.54MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:11<35:02, 9.05MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [06:11<19:07, 15.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [06:12<57:04, 5.85MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [06:12<17:17, 16.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:12<50:36, 6.28MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:12<39:10, 8.09MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [06:12<53:37, 6.22MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [06:12<16:05, 18.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 2.99G/22.2G [06:12<1:00:08, 5.31MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:12<39:45, 7.97MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:12<51:04, 6.22MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.13G/22.2G [06:12<51:22, 6.49MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:12<37:30, 8.45MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [06:12<38:19, 8.33MB/s]  [A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [06:12<35:53, 8.90MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:12<38:06, 8.32MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [06:12<50:06, 6.66MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:12<53:03, 5.99MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [06:12<29:49, 9.81MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [06:12<30:57, 10.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:12<40:05, 7.91MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [06:12<49:08, 6.79MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:12<50:25, 6.30MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [06:12<23:06, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [06:13<30:20, 10.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [06:13<23:07, 12.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:13<42:07, 7.52MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [06:13<48:56, 6.82MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:13<53:39, 5.92MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [06:13<27:38, 11.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [06:13<19:27, 15.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:13<43:17, 7.32MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [06:13<27:57, 11.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [06:13<46:42, 7.14MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [06:13<17:17, 16.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:13<52:16, 6.08MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:13<41:19, 7.67MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [06:13<18:07, 16.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [06:13<25:52, 12.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [06:13<47:50, 6.97MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:13<51:11, 6.21MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [06:13<17:30, 16.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [06:13<26:35, 12.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.14G/22.2G [06:13<45:19, 7.36MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [06:13<16:26, 17.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:13<49:49, 6.37MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:14<48:06, 6.60MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [06:14<49:50, 6.69MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:14<1:21:38, 3.88MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:14<46:25, 6.84MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [06:14<29:43, 9.83MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [06:14<50:56, 6.55MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:14<44:25, 7.13MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [06:14<22:29, 13.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [06:14<47:27, 6.72MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:14<43:45, 7.24MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [06:14<22:30, 13.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [06:14<54:07, 6.16MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:14<34:13, 9.32MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:14<40:07, 7.89MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [06:14<18:50, 15.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:14<32:24, 9.84MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [06:14<16:59, 17.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:14<40:28, 7.82MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:15<29:12, 10.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:15<36:58, 8.56MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [06:15<17:51, 16.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:15<1:32:01, 3.45MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [06:15<27:44, 11.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:15<34:20, 9.22MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [06:15<17:03, 17.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:15<27:55, 11.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [06:15<1:39:51, 3.34MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [06:15<15:54, 18.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:15<52:23, 6.06MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:15<36:17, 8.72MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:15<34:15, 9.24MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:15<25:54, 12.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:15<49:44, 6.38MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [06:15<1:01:14, 5.44MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:15<32:42, 9.68MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:15<24:59, 12.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:15<31:38, 10.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:15<47:50, 6.64MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [06:15<25:39, 12.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.15G/22.2G [06:15<59:06, 5.64MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [06:15<29:48, 9.79MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:15<34:35, 9.15MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:15<46:20, 6.85MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [06:16<23:25, 12.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [06:16<57:37, 5.78MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:16<32:54, 9.62MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:16<45:16, 7.01MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [06:16<21:56, 13.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:16<31:46, 9.96MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [06:16<55:47, 5.97MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [06:16<19:06, 15.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:16<44:27, 7.14MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:16<44:49, 7.11MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [06:16<17:13, 16.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [06:16<54:24, 6.13MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:16<43:50, 7.24MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [06:16<16:04, 18.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:16<32:30, 9.80MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [06:16<53:09, 6.27MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:16<1:02:32, 5.06MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [06:16<15:43, 18.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:16<47:31, 6.68MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:16<29:24, 10.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:16<40:58, 7.72MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [06:16<53:59, 6.17MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:17<43:26, 7.30MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [06:17<29:15, 10.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:17<41:18, 7.66MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [06:17<51:38, 6.45MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:17<27:01, 11.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:17<38:12, 8.28MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [06:17<28:16, 10.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:17<49:08, 6.46MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.16G/22.2G [06:17<50:38, 6.58MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [06:17<21:42, 13.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:17<25:43, 12.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:17<36:57, 8.56MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:17<51:28, 6.16MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:17<26:10, 12.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [06:17<19:01, 15.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:17<35:09, 8.99MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [06:17<48:45, 6.83MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [06:17<17:46, 16.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:17<33:58, 9.31MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [06:17<24:58, 12.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [06:17<48:39, 6.85MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:17<52:17, 6.07MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:17<32:14, 9.81MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [06:17<16:45, 17.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:18<24:12, 13.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [06:18<16:01, 18.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [06:18<48:31, 6.86MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:18<34:07, 9.26MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [06:18<15:28, 18.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:18<32:40, 9.68MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:18<32:27, 9.74MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:18<31:29, 10.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:18<41:09, 7.73MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:18<33:37, 9.40MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:18<34:26, 9.24MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [06:18<28:01, 10.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:18<1:48:57, 2.91MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:18<32:14, 9.80MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [06:18<21:34, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.17G/22.2G [06:18<1:31:11, 3.65MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [06:18<30:17, 10.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [06:19<18:38, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:19<59:37, 5.32MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:19<30:00, 10.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██        | 4.71G/22.2G [06:19<17:21, 16.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [06:19<55:30, 6.00MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:19<27:23, 11.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:19<1:00:33, 5.24MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [06:19<16:34, 17.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:19<1:06:39, 4.74MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [06:19<52:52, 6.30MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:19<25:41, 12.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [06:19<15:37, 18.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:19<43:07, 7.33MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:19<57:37, 5.50MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [06:19<26:43, 11.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [06:19<15:05, 19.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [06:19<52:01, 6.40MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:19<42:42, 7.40MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:19<24:35, 12.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:19<39:34, 7.98MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [06:19<49:57, 6.66MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:19<56:01, 5.66MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:19<25:30, 12.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:20<36:13, 8.72MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:20<52:24, 6.05MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [06:20<49:14, 6.76MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [06:20<26:35, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:20<36:36, 8.63MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [06:20<20:44, 14.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.18G/22.2G [06:20<47:57, 6.94MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:20<34:51, 9.06MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:20<53:34, 5.92MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [06:20<17:43, 16.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:20<33:42, 9.37MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [06:20<46:57, 7.09MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [06:20<16:21, 17.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:20<48:52, 6.49MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:20<49:46, 6.39MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:20<32:15, 9.79MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [06:20<15:23, 18.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [06:20<46:21, 7.18MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:20<33:46, 9.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [06:20<34:32, 9.20MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [06:20<14:28, 20.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:20<52:35, 6.03MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:21<32:39, 9.67MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [06:21<13:53, 20.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [06:21<46:21, 7.18MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:21<30:22, 10.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:21<31:31, 10.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:21<48:08, 6.58MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [06:21<43:08, 7.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:21<29:43, 10.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:21<31:46, 9.94MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:21<48:20, 6.55MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [06:21<45:25, 7.33MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:21<26:57, 11.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:21<33:32, 9.41MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [06:21<27:06, 11.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:21<52:03, 6.09MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [06:21<46:44, 7.12MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [06:21<28:09, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:21<25:42, 12.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [06:21<24:44, 11.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:21<48:02, 6.60MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [06:21<19:34, 14.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:22<24:43, 12.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.21G/22.2G [06:22<1:09:10, 4.56MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [06:22<17:35, 16.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:22<43:04, 7.33MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [06:22<16:03, 18.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.19G/22.2G [06:22<1:30:52, 3.66MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [06:22<15:03, 19.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:22<42:03, 7.51MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [06:22<14:14, 20.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:22<1:31:36, 3.46MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:22<38:59, 8.10MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [06:22<53:30, 6.22MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:22<33:59, 9.28MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [06:22<43:36, 7.28MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [06:22<50:19, 6.61MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:22<55:03, 5.75MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:22<36:12, 8.72MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:23<32:54, 9.65MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.76G/22.2G [06:23<23:26, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:23<34:16, 9.21MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [06:23<48:47, 6.82MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [06:23<21:54, 13.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:23<55:54, 5.66MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:23<29:49, 10.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:23<32:40, 9.65MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [06:23<17:36, 16.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [06:23<48:06, 6.91MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:23<31:48, 9.92MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:23<50:45, 6.24MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:23<27:22, 11.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [06:23<16:03, 18.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:23<31:04, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [06:23<27:21, 11.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.20G/22.2G [06:23<47:09, 7.05MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [06:23<13:35, 21.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:23<30:41, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:23<53:13, 5.95MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [06:23<13:12, 21.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:23<30:22, 10.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:23<25:16, 12.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [06:23<46:55, 7.09MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:23<37:26, 8.42MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:24<57:11, 5.54MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:24<27:02, 11.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [06:24<50:39, 6.56MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:24<24:52, 12.7MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:24<25:51, 12.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [06:24<53:17, 6.24MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:24<59:17, 5.34MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [06:24<28:11, 10.3MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [06:24<22:11, 13.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [06:24<55:02, 6.04MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:24<59:11, 5.35MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [06:24<19:11, 15.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:24<1:11:44, 4.40MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [06:24<17:56, 16.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:24<59:45, 5.30MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [06:24<51:28, 6.16MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [06:24<56:47, 5.85MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:25<44:07, 7.15MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [06:25<17:01, 17.0MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [06:25<16:06, 18.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:25<38:33, 8.22MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|▉         | 2.21G/22.2G [06:25<55:03, 6.04MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:25<42:35, 7.40MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:25<1:00:57, 5.19MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [06:25<15:33, 18.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:25<36:16, 8.74MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:25<39:07, 8.06MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:25<34:39, 9.15MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:25<1:00:12, 5.26MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:25<39:16, 8.02MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:25<33:44, 9.40MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:25<36:19, 8.68MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [06:25<32:31, 9.75MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:25<56:05, 5.64MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:25<37:57, 8.30MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [06:25<28:12, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:25<28:40, 11.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:26<35:25, 8.90MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [06:26<1:31:31, 3.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:26<58:11, 5.44MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [06:26<21:41, 13.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:26<28:45, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:26<33:34, 9.38MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [06:26<18:20, 15.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:26<29:01, 10.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [06:26<53:44, 6.18MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:26<35:48, 8.80MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [06:26<17:09, 16.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:26<26:29, 12.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:26<32:56, 9.56MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [06:26<52:32, 6.32MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [06:26<15:50, 18.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [06:26<27:01, 11.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:26<31:48, 9.91MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [06:26<14:50, 19.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [06:26<51:31, 6.45MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [06:26<14:14, 20.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:26<34:27, 9.14MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:26<1:48:53, 2.91MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.22G/22.2G [06:26<50:42, 6.55MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [06:27<50:06, 6.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:27<59:13, 5.34MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:27<51:20, 6.17MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [06:27<23:14, 12.4MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [06:27<20:42, 13.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [06:27<48:56, 6.79MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:27<34:30, 9.18MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:27<1:00:17, 5.25MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:27<1:10:47, 4.45MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [06:27<18:18, 15.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:27<32:55, 9.62MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [06:27<48:08, 6.90MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [06:27<16:53, 17.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:27<40:37, 7.75MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:27<57:20, 5.52MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [06:27<29:18, 10.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [06:27<15:32, 18.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [06:27<48:11, 6.89MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:27<40:22, 7.80MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:27<28:43, 11.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [06:27<13:18, 21.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:28<37:11, 8.47MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [06:28<47:34, 6.98MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:28<55:14, 5.73MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [06:28<12:55, 22.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:28<26:11, 12.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:28<34:20, 9.17MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:28<26:28, 12.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.23G/22.2G [06:28<47:07, 7.04MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:28<53:36, 5.90MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:28<32:52, 9.58MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [06:28<43:30, 7.63MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:28<31:15, 10.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [06:28<25:03, 12.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:28<49:50, 6.34MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:28<30:26, 10.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [06:28<46:08, 7.19MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:28<24:01, 13.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:28<49:33, 6.38MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:28<30:58, 10.2MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:28<32:18, 9.74MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [06:28<31:33, 9.14MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:28<53:08, 5.95MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:29<31:43, 9.92MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [06:29<31:44, 9.08MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:29<30:45, 10.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:29<51:10, 6.18MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [06:29<24:54, 11.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:29<30:25, 10.3MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:29<29:50, 10.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:29<50:27, 6.27MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [06:29<20:45, 13.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [06:29<1:35:29, 3.48MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [06:29<15:58, 18.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:29<49:50, 6.34MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:29<1:01:25, 5.15MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [06:29<14:58, 19.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [06:29<55:39, 5.96MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:29<48:36, 6.50MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:29<41:41, 7.59MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [06:29<51:22, 6.46MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:29<1:01:51, 5.09MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [06:30<39:08, 8.08MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:30<40:10, 7.83MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.24G/22.2G [06:30<51:37, 6.43MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:30<40:20, 7.84MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [06:30<24:17, 11.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:30<36:11, 8.69MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [06:30<47:35, 6.97MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:30<37:16, 8.49MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [06:30<19:31, 14.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:30<34:13, 9.19MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:30<35:09, 9.00MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [06:30<17:46, 16.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [06:30<48:51, 6.79MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:30<33:00, 9.53MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:30<33:48, 9.35MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [06:30<14:29, 19.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:30<1:31:30, 3.45MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:30<31:42, 9.91MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [06:30<49:08, 6.75MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:30<32:41, 9.67MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [06:30<13:56, 20.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:30<30:49, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:30<32:11, 9.82MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [06:30<44:18, 7.49MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [06:30<13:33, 21.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:30<30:05, 10.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:31<53:23, 5.92MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [06:31<31:26, 10.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:31<29:54, 10.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [06:31<46:19, 7.16MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:31<31:02, 10.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:31<29:25, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:31<54:33, 5.79MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [06:31<43:18, 7.66MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:31<30:15, 10.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:31<28:59, 10.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [06:31<20:37, 13.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:31<30:01, 10.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:31<28:45, 10.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.25G/22.2G [06:31<45:50, 7.23MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [06:31<17:22, 16.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:31<52:45, 5.99MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:31<29:46, 10.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:31<28:41, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [06:31<14:56, 19.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [06:31<45:23, 7.31MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:31<28:34, 11.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:31<51:37, 6.12MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [06:31<14:14, 20.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [06:31<45:15, 7.33MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [06:32<13:44, 20.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:32<53:06, 5.95MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:32<49:11, 6.42MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [06:32<11:59, 24.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [06:32<45:05, 7.35MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [06:32<37:53, 8.34MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:32<48:21, 6.53MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:32<1:04:09, 4.90MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:32<35:10, 8.98MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:32<47:17, 6.68MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:32<33:42, 9.37MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:32<41:35, 7.55MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [06:32<21:33, 13.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:32<32:34, 9.70MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:32<37:35, 8.36MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:32<51:28, 6.14MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [06:32<1:21:43, 4.06MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [06:32<18:31, 15.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:32<31:22, 10.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:32<34:53, 9.00MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:32<30:51, 10.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [06:32<15:46, 18.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:32<48:43, 6.48MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:32<33:06, 9.49MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [06:33<29:33, 10.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [06:33<14:43, 19.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.26G/22.2G [06:33<54:51, 6.04MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:33<31:55, 9.84MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:33<47:57, 6.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [06:33<12:45, 22.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:33<30:56, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:33<33:42, 9.37MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [06:33<53:34, 6.19MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:33<30:09, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [06:33<12:37, 22.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:33<48:02, 6.57MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:33<32:05, 9.84MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:33<29:51, 10.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [06:33<51:05, 6.49MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:33<46:50, 6.74MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:33<28:02, 11.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:33<29:18, 10.7MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:33<29:00, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [06:33<28:16, 11.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [06:33<21:02, 13.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [06:33<53:18, 6.22MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:33<28:48, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [06:34<17:47, 16.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:34<28:41, 10.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [06:34<52:11, 6.35MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [06:34<15:59, 17.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:34<28:34, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [06:34<13:20, 21.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [06:34<56:29, 5.87MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [06:34<13:15, 21.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.21G/22.2G [06:34<59:15, 5.33MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.21G/22.2G [06:34<1:44:53, 3.01MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [06:34<11:42, 24.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [06:34<55:40, 5.95MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:34<37:31, 8.41MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.32G/22.2G [06:34<59:55, 5.24MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:34<34:56, 9.03MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:34<57:14, 5.51MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.27G/22.2G [06:34<53:30, 6.19MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:34<39:55, 7.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:34<33:44, 9.35MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:35<53:40, 5.88MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [06:35<19:54, 14.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:35<35:55, 8.74MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:35<32:22, 9.75MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [06:35<52:47, 6.28MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [06:35<18:37, 15.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:35<34:01, 9.22MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:35<31:33, 10.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:35<51:36, 6.12MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [06:35<51:53, 6.38MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [06:35<15:43, 18.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:35<32:29, 9.65MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:35<27:45, 11.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:35<50:07, 6.30MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:35<31:40, 9.90MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [06:35<14:43, 19.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [06:35<52:08, 6.35MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:35<28:03, 11.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:35<30:23, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [06:35<12:29, 22.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:35<48:22, 6.52MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:35<28:31, 11.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:35<29:49, 10.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [06:35<51:31, 6.43MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [06:35<12:33, 22.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:35<28:11, 11.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:35<29:18, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [06:35<47:36, 6.63MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:36<28:28, 11.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:36<28:56, 10.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:36<46:30, 6.78MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:36<28:42, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [06:36<19:38, 14.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:36<28:34, 11.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:36<45:44, 6.90MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [06:36<16:49, 17.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:36<28:27, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [06:36<14:37, 19.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:36<44:36, 7.07MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:36<27:58, 11.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [06:36<13:55, 20.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.28G/22.2G [06:36<1:43:10, 3.21MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:36<50:50, 6.20MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:36<43:48, 7.20MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [06:36<12:19, 23.2MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [06:36<12:13, 23.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:36<34:50, 9.05MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:36<42:54, 7.35MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [06:36<58:53, 5.62MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:37<33:18, 9.47MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:37<59:33, 5.26MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:37<42:11, 7.47MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [06:37<54:47, 6.04MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:37<32:27, 9.71MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:37<40:37, 7.72MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [06:37<20:41, 13.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:37<31:41, 9.94MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [06:37<52:49, 6.27MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:37<36:47, 8.52MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [06:37<17:32, 16.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:37<34:07, 9.18MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:37<29:00, 10.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [06:37<51:02, 6.49MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [06:37<15:06, 18.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:37<32:08, 9.75MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:37<28:37, 11.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [06:37<14:13, 20.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:37<30:40, 10.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [06:37<49:06, 6.74MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:37<28:33, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [06:37<1:22:44, 3.81MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [06:37<12:21, 23.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:37<30:14, 10.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:38<28:41, 11.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.29G/22.2G [06:38<47:44, 6.93MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [06:38<12:23, 23.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:38<29:42, 10.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:38<46:48, 6.74MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:38<28:01, 11.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:38<28:59, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:38<27:46, 11.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [06:38<47:17, 7.00MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:38<46:05, 6.84MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:38<26:18, 11.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:38<27:58, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:38<43:14, 7.29MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [06:38<48:00, 6.89MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:38<26:38, 11.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [06:38<44:49, 7.38MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:38<26:35, 11.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [06:38<41:12, 7.65MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:38<26:56, 11.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:38<37:23, 8.43MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [06:38<45:39, 7.25MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:38<52:59, 5.94MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:39<35:59, 8.76MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [06:39<46:44, 7.08MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:39<35:18, 8.92MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:39<35:24, 8.90MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [06:39<38:33, 7.40MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [06:39<43:39, 7.58MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:39<32:29, 9.69MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:39<53:00, 5.91MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:39<34:23, 9.16MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [06:39<29:12, 9.77MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:39<31:45, 9.92MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:39<33:37, 9.37MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [06:39<24:06, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:39<37:52, 8.27MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:39<31:04, 10.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:39<32:47, 9.61MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:39<34:21, 9.11MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [06:39<21:36, 13.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [06:39<30:52, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:39<30:21, 10.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:39<32:39, 9.58MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [06:39<19:36, 14.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:39<31:01, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:39<29:58, 10.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [06:40<17:59, 15.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:40<27:50, 11.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:40<29:31, 10.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.30G/22.2G [06:40<1:32:38, 3.57MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [06:40<16:43, 17.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:40<27:43, 11.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:40<26:53, 11.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:40<27:35, 11.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [06:40<53:03, 6.23MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:40<27:10, 11.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:40<1:02:51, 5.01MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:40<25:03, 12.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:40<27:39, 11.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [06:40<49:53, 6.63MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:40<25:26, 12.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:40<27:41, 11.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:40<36:36, 8.60MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [06:40<48:44, 6.78MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:40<24:03, 13.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:40<33:26, 9.41MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [06:40<34:15, 8.32MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [06:40<31:31, 9.99MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.31G/22.2G [06:41<47:32, 6.96MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [06:41<25:28, 11.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:41<29:59, 10.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [06:41<20:48, 13.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:41<57:12, 5.50MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [06:41<46:35, 7.10MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:41<28:35, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [06:41<18:37, 15.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:41<27:30, 11.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:41<45:32, 6.87MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:41<35:54, 8.76MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [06:41<46:26, 7.12MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [06:41<16:54, 16.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:41<27:00, 11.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:41<33:54, 9.28MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:41<31:44, 9.85MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [06:41<15:43, 18.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [06:41<26:56, 11.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [06:41<45:51, 7.21MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:41<32:43, 9.61MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [06:41<14:50, 19.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:41<25:49, 12.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [06:41<45:22, 7.28MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:41<32:28, 9.63MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:42<29:46, 10.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:42<34:46, 9.04MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:42<27:53, 11.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [06:42<44:44, 7.39MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:42<28:20, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:42<32:44, 9.60MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:42<27:58, 11.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:42<32:13, 9.76MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.32G/22.2G [06:42<44:34, 7.41MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:42<28:09, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:42<32:03, 9.80MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:42<28:16, 11.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  10%|█         | 2.33G/22.2G [06:42<43:58, 7.52MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:42<28:09, 11.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [06:42<37:26, 7.60MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:42<35:09, 8.94MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:42<53:05, 5.92MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:42<33:26, 9.40MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [06:42<29:58, 9.49MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:42<31:22, 9.96MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:42<38:27, 8.18MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:42<32:27, 9.68MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [06:43<28:39, 9.93MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [06:43<34:36, 9.09MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:43<30:55, 10.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:43<31:51, 9.86MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [06:43<23:10, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:43<33:15, 9.46MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:43<31:17, 10.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:43<33:07, 9.43MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [06:43<23:06, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:43<31:50, 9.87MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [06:43<1:28:44, 3.72MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [06:43<19:57, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:43<30:44, 10.2MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:43<29:55, 10.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [06:43<19:56, 14.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [06:43<53:55, 6.13MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:43<29:03, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [06:43<17:49, 16.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:43<1:04:36, 4.84MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:43<1:09:13, 4.54MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [06:43<28:27, 11.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [06:43<50:12, 6.58MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:44<27:57, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:44<45:19, 6.93MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.33G/22.2G [06:44<48:50, 6.76MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:44<27:23, 11.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:44<46:13, 6.76MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [06:44<28:52, 9.85MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:44<44:44, 7.02MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:44<26:55, 11.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [06:44<47:09, 7.00MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [06:44<21:24, 13.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:44<45:16, 6.90MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [06:44<21:41, 13.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:44<45:10, 6.95MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [06:44<46:20, 7.13MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:44<43:52, 7.12MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [06:44<18:43, 15.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:44<39:53, 7.83MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:44<44:53, 7.00MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [06:44<45:12, 7.30MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [06:44<16:59, 16.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [06:44<53:00, 5.93MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:44<40:09, 7.77MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:44<44:45, 7.02MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [06:45<17:51, 15.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [06:45<45:48, 7.21MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:45<37:07, 8.41MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:45<33:57, 9.25MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [06:45<17:09, 16.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:45<44:05, 7.12MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [06:45<42:45, 7.72MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:45<35:10, 8.88MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:45<30:26, 10.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [06:45<15:51, 17.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:45<33:45, 9.25MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:45<43:29, 7.22MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:45<29:01, 10.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.34G/22.2G [06:45<43:07, 7.66MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:45<36:02, 8.66MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:45<27:38, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:45<43:07, 7.28MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [06:45<43:49, 7.53MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:45<33:28, 9.32MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:45<26:56, 11.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [06:45<28:11, 10.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:45<42:40, 7.36MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [06:45<40:36, 8.13MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:45<32:15, 9.67MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [06:45<25:36, 12.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [06:45<21:01, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:46<25:26, 12.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:46<42:30, 7.39MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:46<34:56, 8.93MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [06:46<21:01, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:46<24:31, 12.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:46<40:36, 7.73MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:46<32:39, 9.55MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [06:46<18:11, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:46<24:37, 12.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:46<41:16, 7.61MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [06:46<16:36, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:46<24:16, 12.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [06:46<38:47, 8.09MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [06:46<1:15:13, 4.39MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [06:46<17:16, 16.4MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [06:46<16:01, 17.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [06:46<45:16, 7.29MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.43G/22.2G [06:46<1:13:27, 4.25MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [06:46<15:37, 18.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.32G/22.2G [06:46<45:54, 6.84MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.35G/22.2G [06:47<46:41, 7.07MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [06:47<44:37, 6.99MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.32G/22.2G [06:47<1:16:34, 4.10MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:47<32:37, 9.62MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [06:47<44:42, 7.38MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [06:47<43:34, 7.16MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:47<31:50, 9.85MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [06:47<27:31, 10.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [06:47<39:39, 7.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:47<47:58, 6.54MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [06:47<42:30, 7.76MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [06:47<36:21, 8.58MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [06:47<25:05, 11.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:47<43:45, 7.17MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:47<29:01, 10.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [06:47<39:36, 8.33MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [06:47<20:47, 13.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [06:47<37:28, 8.32MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:47<29:18, 10.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:47<43:46, 7.17MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [06:47<18:55, 15.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [06:47<40:18, 8.18MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [06:47<35:01, 8.90MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:47<29:18, 10.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:47<43:13, 7.26MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [06:47<17:34, 16.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [06:47<37:47, 8.73MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [06:47<32:23, 9.63MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:48<27:51, 11.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:48<39:54, 7.86MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [06:48<16:38, 17.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.36G/22.2G [06:48<35:10, 9.38MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:48<28:27, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [06:48<34:34, 9.02MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [06:48<15:58, 17.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [06:48<35:04, 9.40MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:48<29:15, 10.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [06:48<33:14, 9.38MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [06:48<40:32, 7.74MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [06:48<15:33, 18.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:48<29:12, 10.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [06:48<31:35, 9.87MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [06:48<36:03, 9.15MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:48<41:56, 7.48MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:48<29:31, 10.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [06:48<30:22, 10.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [06:48<36:18, 9.08MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:48<38:39, 8.11MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:48<29:47, 10.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [06:48<35:30, 9.29MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [06:48<33:09, 9.40MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:48<40:10, 7.80MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [06:48<31:33, 9.87MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [06:49<33:08, 8.55MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:49<40:45, 7.69MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [06:49<28:44, 9.85MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:49<37:16, 8.41MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [06:49<22:58, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:49<1:07:09, 4.67MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [06:49<1:15:07, 4.39MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [06:49<39:05, 8.02MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [06:49<1:02:00, 5.03MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [06:49<20:14, 14.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:49<40:47, 7.68MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.37G/22.2G [06:49<46:16, 7.12MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [06:49<41:25, 7.52MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [06:49<18:27, 15.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:49<36:13, 8.65MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [06:49<41:03, 8.03MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [06:49<17:09, 16.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [06:49<41:26, 7.52MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:49<34:27, 9.09MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [06:49<38:20, 8.60MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [06:49<16:22, 17.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [06:49<37:37, 8.28MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:50<32:44, 9.57MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [06:50<36:13, 9.10MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [06:50<15:42, 18.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [06:50<34:18, 9.08MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:50<31:27, 9.96MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [06:50<34:17, 9.61MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:50<1:18:41, 3.98MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:50<30:26, 10.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [06:50<36:28, 8.54MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [06:50<32:57, 10.0MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:50<29:41, 10.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [06:50<34:48, 8.95MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:50<47:12, 6.64MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.38G/22.2G [06:50<31:34, 10.4MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [06:50<32:14, 9.66MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:50<29:07, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.19G/22.2G [06:50<28:43, 9.84MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [06:50<30:32, 10.8MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:50<44:50, 6.99MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [06:50<31:33, 9.87MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:50<28:26, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.19G/22.2G [06:50<26:05, 10.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [06:50<29:32, 11.2MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:50<41:08, 7.62MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:50<28:23, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [06:50<33:52, 9.19MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.19G/22.2G [06:50<21:06, 13.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [06:50<28:54, 11.4MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:51<28:14, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [06:51<32:50, 9.48MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.19G/22.2G [06:51<19:03, 14.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [06:51<41:42, 7.51MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [06:51<28:05, 11.7MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:51<27:49, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [06:51<31:26, 9.90MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.20G/22.2G [06:51<17:32, 16.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:51<42:05, 7.45MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [06:51<30:34, 10.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.20G/22.2G [06:51<16:31, 17.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:51<39:27, 7.94MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [06:51<30:20, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  23%|██▎       | 5.20G/22.2G [06:51<15:53, 17.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:51<40:05, 7.81MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.39G/22.2G [06:51<54:11, 6.08MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.21G/22.2G [06:51<15:20, 18.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:51<52:55, 5.92MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:51<37:20, 8.39MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [06:51<37:21, 8.81MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:51<38:50, 8.06MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:51<38:45, 8.08MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [06:52<32:25, 10.2MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:52<36:49, 8.50MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:52<36:39, 8.54MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.21G/22.2G [06:52<27:45, 10.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [06:52<30:19, 10.9MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:52<36:29, 8.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.21G/22.2G [06:52<25:32, 11.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.40G/22.2G [06:52<27:57, 11.8MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [06:52<38:30, 8.13MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.21G/22.2G [06:52<20:52, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:52<36:45, 8.52MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:52<36:41, 8.54MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [06:52<25:48, 12.7MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.22G/22.2G [06:52<18:46, 15.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:52<34:02, 9.19MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.22G/22.2G [06:52<17:22, 16.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [06:52<24:26, 13.5MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:52<36:50, 8.49MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.22G/22.2G [06:52<16:27, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:52<33:14, 9.41MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [06:52<23:43, 13.9MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.23G/22.2G [06:53<15:53, 17.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:53<32:42, 9.57MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.23G/22.2G [06:53<15:15, 18.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:53<1:15:15, 4.16MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:53<36:09, 8.65MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:53<34:59, 8.94MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:53<45:46, 6.84MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.41G/22.2G [06:53<42:11, 7.80MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:53<32:51, 9.52MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.23G/22.2G [06:53<26:55, 10.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:53<41:48, 7.49MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:53<31:00, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.23G/22.2G [06:53<24:57, 11.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [06:53<32:21, 10.2MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [06:53<42:30, 7.36MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.24G/22.2G [06:53<20:21, 13.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [06:53<30:14, 10.9MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:54<39:03, 8.01MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.24G/22.2G [06:54<18:29, 15.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [06:54<29:37, 11.1MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:54<40:15, 7.77MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.24G/22.2G [06:54<17:10, 16.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.42G/22.2G [06:54<28:46, 11.4MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:54<1:00:05, 5.21MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:54<37:05, 8.44MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.24G/22.2G [06:54<16:15, 17.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [06:54<28:11, 11.7MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:54<39:18, 7.96MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.25G/22.2G [06:54<15:41, 18.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [06:54<27:39, 11.9MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:54<39:15, 7.97MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:54<35:43, 8.75MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [06:54<27:32, 11.9MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.25G/22.2G [06:54<15:47, 17.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:54<36:36, 8.55MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:54<34:01, 9.19MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [06:54<26:25, 12.4MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [06:54<38:31, 8.12MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:54<32:21, 9.66MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.43G/22.2G [06:55<27:05, 12.1MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:55<35:48, 8.74MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:55<31:29, 9.92MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.25G/22.2G [06:55<24:20, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:55<30:48, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.25G/22.2G [06:55<22:45, 12.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:55<37:29, 8.34MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:55<30:35, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.26G/22.2G [06:55<19:06, 14.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:55<36:11, 8.64MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:55<30:09, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▎       | 5.26G/22.2G [06:55<17:33, 16.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [06:55<43:13, 7.60MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:55<29:27, 10.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.26G/22.2G [06:55<16:30, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:55<28:59, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.27G/22.2G [06:55<15:45, 17.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [06:55<32:52, 10.0MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:55<28:38, 10.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.27G/22.2G [06:55<15:15, 18.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:56<28:22, 11.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.44G/22.2G [06:56<29:07, 11.3MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:56<1:08:54, 4.54MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [06:56<27:45, 11.8MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [06:56<40:34, 7.71MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [06:56<27:22, 12.0MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:56<41:12, 7.59MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.27G/22.2G [06:56<27:43, 10.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:56<54:18, 5.75MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.45G/22.2G [06:56<25:01, 13.1MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:56<38:29, 8.12MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.28G/22.2G [06:56<21:09, 13.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:56<36:10, 8.63MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:56<36:21, 8.60MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [06:56<24:09, 13.6MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.28G/22.2G [06:56<18:33, 15.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:56<32:51, 9.50MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:56<34:12, 9.14MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [06:57<6:50:56, 757kB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.28G/22.2G [06:57<17:20, 16.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [06:57<23:35, 13.9MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:57<31:56, 9.78MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:57<33:01, 9.47MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.29G/22.2G [06:57<16:27, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:57<30:14, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:57<32:07, 9.73MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.29G/22.2G [06:57<15:45, 17.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:57<29:51, 10.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [06:57<31:42, 9.85MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.29G/22.2G [06:57<15:10, 18.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [06:57<3:38:36, 1.42MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:57<28:54, 10.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:57<31:23, 9.95MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [06:57<41:37, 7.89MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [06:57<2:55:09, 1.78MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:57<27:56, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:57<30:47, 10.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [06:57<2:18:48, 2.24MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:57<30:18, 10.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:57<27:39, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.46G/22.2G [06:57<37:13, 8.82MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [06:57<1:47:28, 2.90MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:57<30:12, 10.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:57<27:21, 11.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [06:58<1:25:29, 3.64MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [06:58<31:28, 10.4MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:58<29:56, 10.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:58<27:08, 11.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.30G/22.2G [06:58<31:59, 8.78MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:58<26:59, 11.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [06:58<1:12:39, 4.28MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [06:58<33:59, 9.65MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.30G/22.2G [06:58<23:56, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [06:58<59:11, 5.25MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.30G/22.2G [06:58<20:43, 13.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [06:58<50:42, 6.13MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [06:58<36:32, 8.98MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [06:58<43:36, 7.13MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.31G/22.2G [06:58<18:47, 14.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [06:58<1:03:02, 4.96MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [06:58<38:16, 8.57MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.31G/22.2G [06:58<17:22, 16.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.43G/22.2G [06:58<54:52, 5.69MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [06:58<41:28, 7.50MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:58<40:12, 7.77MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.31G/22.2G [06:58<16:18, 17.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [06:58<39:21, 8.33MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:59<36:28, 8.56MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [06:59<34:39, 9.00MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [06:59<40:17, 7.72MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.31G/22.2G [06:59<15:29, 18.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.47G/22.2G [06:59<38:00, 8.63MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:59<34:24, 9.07MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [06:59<36:45, 8.46MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [06:59<31:38, 9.86MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [06:59<34:13, 9.08MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:59<32:21, 9.65MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [06:59<30:09, 10.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [06:59<41:16, 7.95MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [06:59<31:07, 10.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [06:59<29:20, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [06:59<38:55, 8.42MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [06:59<28:36, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:59<27:22, 11.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.32G/22.2G [06:59<28:42, 9.78MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [06:59<41:55, 7.82MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [06:59<27:45, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [06:59<27:26, 11.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.32G/22.2G [06:59<21:54, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [06:59<1:05:32, 4.74MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [06:59<40:17, 8.14MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [06:59<27:27, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [07:00<25:26, 12.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.32G/22.2G [07:00<18:42, 15.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [07:00<27:12, 11.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:00<42:25, 7.33MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [07:00<26:12, 11.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.33G/22.2G [07:00<17:14, 16.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [07:00<26:49, 11.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:00<38:27, 8.08MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.33G/22.2G [07:00<16:05, 17.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [07:00<26:44, 11.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:00<36:07, 8.60MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.33G/22.2G [07:00<15:13, 18.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [07:00<26:24, 11.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:00<33:54, 9.16MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.34G/22.2G [07:00<14:36, 19.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:00<32:08, 9.67MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.48G/22.2G [07:00<1:23:44, 3.92MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:00<31:18, 9.93MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.43G/22.2G [07:00<43:51, 7.11MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:00<30:25, 10.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [07:00<48:11, 6.80MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [07:01<31:41, 9.84MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:01<30:54, 10.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [07:01<54:05, 5.76MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.34G/22.2G [07:01<24:31, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [07:01<48:56, 6.70MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:01<30:13, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.34G/22.2G [07:01<19:14, 14.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [07:01<28:18, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [07:01<35:45, 8.71MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:01<29:17, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.35G/22.2G [07:01<16:47, 16.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [07:01<46:10, 7.10MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [07:01<24:22, 12.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [07:01<32:05, 9.71MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:01<28:30, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.35G/22.2G [07:01<15:41, 17.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [07:01<30:25, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:01<32:30, 9.55MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [07:01<29:53, 10.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.35G/22.2G [07:01<14:46, 19.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [07:01<29:29, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [07:01<51:54, 6.31MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:01<31:11, 9.95MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [07:01<24:56, 12.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.36G/22.2G [07:01<14:08, 19.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [07:01<28:40, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [07:01<25:41, 12.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.36G/22.2G [07:01<13:48, 20.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█         | 2.49G/22.2G [07:01<50:17, 6.52MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [07:01<27:50, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [07:02<26:27, 11.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [07:02<27:30, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.49G/22.2G [07:02<50:48, 6.45MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [07:02<27:37, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [07:02<26:58, 11.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [07:02<25:28, 12.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.36G/22.2G [07:02<23:13, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [07:02<26:55, 11.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:02<1:07:15, 4.62MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [07:02<51:14, 6.40MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.37G/22.2G [07:02<19:31, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [07:02<26:53, 11.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:02<42:44, 7.26MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [07:02<51:13, 6.40MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [07:02<26:29, 11.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.37G/22.2G [07:02<16:30, 17.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:02<42:48, 7.25MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.37G/22.2G [07:02<15:22, 18.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [07:02<51:20, 6.38MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:02<39:32, 7.85MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.38G/22.2G [07:02<14:18, 19.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [07:03<56:00, 5.56MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [07:03<51:25, 6.37MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.38G/22.2G [07:03<13:37, 20.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:03<38:48, 8.00MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [07:03<37:42, 8.26MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:03<35:57, 8.63MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [07:03<55:51, 5.57MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.38G/22.2G [07:03<13:21, 20.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [07:03<51:31, 6.36MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [07:03<34:55, 8.92MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:03<34:21, 9.03MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [07:03<47:02, 6.96MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [07:03<36:46, 8.46MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:03<34:46, 8.92MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [07:03<30:02, 10.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [07:03<32:31, 9.57MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [07:03<48:26, 6.76MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:03<33:34, 9.24MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.39G/22.2G [07:03<23:16, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [07:03<29:49, 10.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:03<30:47, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:03<32:04, 9.67MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.39G/22.2G [07:03<21:01, 13.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:03<29:28, 10.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [07:03<27:13, 11.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:03<31:04, 9.98MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.39G/22.2G [07:04<18:24, 15.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:04<28:23, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [07:04<27:26, 11.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:04<29:50, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.39G/22.2G [07:04<16:20, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:04<27:36, 11.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [07:04<25:06, 12.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.40G/22.2G [07:04<13:28, 20.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:04<27:19, 11.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.50G/22.2G [07:04<1:37:44, 3.35MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:04<37:18, 8.31MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [07:04<24:16, 12.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.40G/22.2G [07:04<13:05, 21.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:04<26:56, 11.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [07:04<24:04, 12.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:04<34:06, 9.09MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.41G/22.2G [07:04<12:50, 21.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:04<26:31, 11.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [07:04<59:45, 5.48MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [07:04<54:50, 5.97MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:04<33:47, 9.20MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.41G/22.2G [07:05<22:45, 12.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [07:05<51:43, 6.02MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:05<1:08:05, 4.56MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [07:05<52:29, 6.24MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.41G/22.2G [07:05<24:54, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:05<43:23, 7.15MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [07:05<33:50, 9.20MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [07:05<50:32, 6.48MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.41G/22.2G [07:05<20:34, 13.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:05<38:49, 7.99MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [07:05<32:14, 9.65MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.51G/22.2G [07:05<49:09, 6.66MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:05<35:37, 8.70MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.42G/22.2G [07:05<18:14, 15.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:05<31:52, 9.76MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:05<57:18, 5.43MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:05<33:33, 9.24MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.42G/22.2G [07:05<18:56, 14.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:05<31:35, 9.85MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [07:05<48:03, 6.81MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:05<37:31, 8.28MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:05<32:04, 9.66MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:05<31:00, 10.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.42G/22.2G [07:05<17:18, 16.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:06<31:00, 9.99MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [07:06<47:17, 6.92MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:06<30:33, 10.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.42G/22.2G [07:06<18:38, 15.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:06<38:37, 8.05MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:06<30:05, 10.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:06<30:14, 10.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:06<34:16, 9.07MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  24%|██▍       | 5.43G/22.2G [07:06<16:20, 17.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [07:06<46:47, 7.00MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:06<29:25, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:06<29:59, 10.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:06<33:40, 9.23MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:06<29:30, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [07:06<29:43, 10.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [07:06<48:00, 6.82MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:06<33:09, 9.37MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:06<28:46, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:06<29:11, 10.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [07:06<43:23, 7.54MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:06<32:36, 9.53MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:06<28:39, 10.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.43G/22.2G [07:06<27:46, 10.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:06<29:09, 10.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:06<32:17, 9.62MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:06<28:21, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.43G/22.2G [07:06<21:17, 13.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.52G/22.2G [07:06<45:47, 7.15MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:06<29:20, 10.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:06<28:09, 11.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:06<31:43, 9.79MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.43G/22.2G [07:06<21:32, 12.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:07<28:51, 10.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:07<31:15, 9.94MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [07:07<45:10, 7.24MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.44G/22.2G [07:07<18:10, 15.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:07<30:53, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.44G/22.2G [07:07<16:24, 17.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:07<30:19, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.44G/22.2G [07:07<17:35, 15.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:07<30:01, 10.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.45G/22.2G [07:07<16:41, 16.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:07<1:06:46, 4.66MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.45G/22.2G [07:07<15:54, 17.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [07:07<1:27:52, 3.72MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [07:07<44:27, 6.99MB/s]  [A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:08<39:31, 7.86MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [07:08<53:07, 6.16MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:08<1:03:23, 4.90MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:08<35:50, 8.67MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.45G/22.2G [07:08<30:42, 9.07MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [07:08<50:23, 6.49MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:08<33:23, 9.31MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:08<37:48, 8.21MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.45G/22.2G [07:08<22:16, 12.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:08<31:49, 9.76MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:08<34:27, 9.01MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [07:08<48:38, 6.72MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.46G/22.2G [07:08<19:07, 14.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:08<31:00, 10.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:08<32:32, 9.54MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.46G/22.2G [07:08<17:38, 15.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [07:08<30:02, 10.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.53G/22.2G [07:08<48:51, 6.69MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:08<31:15, 9.93MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.46G/22.2G [07:08<16:31, 16.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:08<29:22, 10.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:08<2:37:57, 1.96MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:08<30:19, 10.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [07:09<47:48, 6.84MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.47G/22.2G [07:09<15:41, 17.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:09<26:20, 11.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:09<29:14, 10.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.47G/22.2G [07:09<15:03, 18.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:09<26:40, 11.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [07:09<47:18, 6.91MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:09<28:25, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:09<26:55, 11.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:09<28:10, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:09<1:38:31, 3.14MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:09<1:19:37, 3.89MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:09<27:42, 11.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [07:09<52:57, 6.17MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:09<1:06:29, 4.66MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:09<27:25, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:09<56:09, 5.51MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:09<27:08, 11.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [07:09<55:58, 5.84MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.47G/22.2G [07:09<30:52, 9.01MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:09<48:18, 6.41MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [07:09<55:17, 5.62MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.48G/22.2G [07:10<23:14, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:10<42:00, 7.37MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [07:10<53:38, 6.09MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:10<35:38, 8.71MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.48G/22.2G [07:10<19:43, 14.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:10<37:04, 8.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:10<33:19, 9.32MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:10<33:56, 9.12MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.48G/22.2G [07:10<17:55, 15.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.54G/22.2G [07:10<57:40, 5.67MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:10<55:23, 5.60MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:10<32:10, 9.65MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:10<32:28, 9.53MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.49G/22.2G [07:10<16:29, 16.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:10<31:02, 9.97MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.55G/22.2G [07:10<56:51, 5.75MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.49G/22.2G [07:10<15:33, 17.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:10<35:01, 8.85MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:10<28:25, 10.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:10<30:01, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.49G/22.2G [07:10<14:44, 18.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:10<31:43, 9.77MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [07:10<28:28, 10.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  11%|█▏        | 2.55G/22.2G [07:10<52:38, 6.21MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:10<26:31, 11.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:10<28:14, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:10<30:09, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:11<28:16, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:11<28:55, 10.7MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:11<28:02, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:11<26:11, 11.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.49G/22.2G [07:11<26:40, 10.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:11<27:22, 11.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:11<26:07, 11.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.50G/22.2G [07:11<20:34, 13.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:11<50:38, 6.11MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:11<27:07, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:11<26:34, 11.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.50G/22.2G [07:11<17:29, 15.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:11<26:55, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [07:11<1:45:59, 3.08MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:11<38:05, 8.12MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.51G/22.2G [07:11<16:14, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:11<26:15, 11.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:11<34:59, 8.84MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.51G/22.2G [07:11<15:13, 18.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:11<26:05, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [07:11<32:48, 9.43MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [07:11<59:03, 5.53MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.51G/22.2G [07:12<14:28, 19.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [07:12<30:47, 10.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:12<25:55, 12.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.55G/22.2G [07:12<55:47, 5.86MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.52G/22.2G [07:12<13:56, 19.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [07:12<31:04, 9.95MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [07:12<58:30, 5.30MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [07:12<30:09, 10.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [07:12<53:43, 6.08MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:12<36:55, 8.40MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [07:12<28:47, 10.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:12<34:22, 9.02MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [07:12<52:00, 6.28MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [07:12<28:24, 10.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:12<54:10, 5.72MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.52G/22.2G [07:12<25:40, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:12<32:46, 9.46MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:12<28:09, 11.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [07:12<51:20, 6.36MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.52G/22.2G [07:12<21:06, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:12<37:30, 8.26MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [07:12<31:28, 9.85MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:12<27:59, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.52G/22.2G [07:13<17:36, 15.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:13<32:55, 9.41MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:13<27:34, 11.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [07:13<50:17, 6.49MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:13<28:06, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:13<27:11, 11.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.53G/22.2G [07:13<16:10, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:13<31:00, 9.99MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:13<27:43, 11.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [07:13<48:49, 6.69MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.53G/22.2G [07:13<15:06, 18.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:13<31:55, 9.71MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:13<36:21, 8.52MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.53G/22.2G [07:13<17:56, 15.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [07:13<48:25, 6.74MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:13<28:59, 10.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:13<31:20, 9.89MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▍       | 5.54G/22.2G [07:13<14:58, 18.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.56G/22.2G [07:13<47:33, 6.86MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:13<30:57, 10.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:13<32:44, 9.45MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:13<1:09:27, 4.45MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:13<31:32, 9.82MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [07:13<46:56, 6.96MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [07:13<34:20, 9.02MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:14<33:05, 9.36MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:14<34:08, 9.07MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:14<42:07, 7.33MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [07:14<46:02, 7.09MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.54G/22.2G [07:14<28:23, 9.76MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:14<32:25, 9.55MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:14<38:43, 7.98MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.54G/22.2G [07:14<22:55, 12.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [07:14<46:32, 7.01MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:14<30:22, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:14<38:56, 7.93MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.54G/22.2G [07:14<22:53, 12.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:14<30:40, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:14<36:32, 8.45MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.55G/22.2G [07:14<19:37, 14.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:14<1:09:05, 4.48MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:14<36:49, 8.39MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.55G/22.2G [07:14<20:12, 13.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:14<34:49, 8.87MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:14<39:45, 7.79MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.55G/22.2G [07:15<17:44, 15.6MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.55G/22.2G [07:15<18:43, 14.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:15<35:13, 8.77MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [07:15<1:33:14, 3.50MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:15<39:56, 7.75MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:15<33:37, 9.19MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:15<1:09:25, 4.46MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.56G/22.2G [07:15<17:07, 16.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [07:15<37:26, 8.27MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:15<32:26, 9.52MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.56G/22.2G [07:15<18:15, 15.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.57G/22.2G [07:15<53:57, 6.05MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:15<35:01, 8.84MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:15<43:40, 7.08MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:15<31:20, 9.85MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:15<33:44, 9.17MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:15<38:51, 7.96MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:15<30:27, 10.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [07:15<50:47, 6.43MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:15<32:28, 9.53MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [07:15<36:41, 8.43MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:15<29:52, 10.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:15<31:31, 9.82MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.56G/22.2G [07:15<32:35, 8.49MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [07:15<49:27, 6.60MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [07:15<34:33, 8.95MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:16<31:41, 9.74MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:16<30:54, 10.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.57G/22.2G [07:16<22:29, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [07:16<33:19, 9.28MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [07:16<47:49, 6.82MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:16<30:26, 10.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.57G/22.2G [07:16<22:26, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [07:16<32:03, 9.64MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [07:16<30:04, 10.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.57G/22.2G [07:16<18:46, 14.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [07:16<30:41, 10.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [07:16<46:40, 6.99MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:16<30:00, 10.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [07:16<30:04, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:16<29:19, 10.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.57G/22.2G [07:16<19:00, 14.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:16<1:03:37, 4.85MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [07:16<45:56, 7.10MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:16<29:15, 10.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.58G/22.2G [07:16<16:24, 16.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:16<41:57, 7.36MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:16<29:06, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.58G/22.2G [07:16<45:17, 7.20MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.58G/22.2G [07:16<17:17, 16.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:16<37:24, 8.25MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:16<28:35, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.58G/22.2G [07:17<16:26, 16.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:17<35:20, 8.73MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [07:17<47:03, 6.93MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:17<28:01, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:17<33:17, 9.27MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:17<58:30, 5.29MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [07:17<46:34, 7.00MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:17<31:57, 9.66MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:17<30:46, 10.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:17<36:08, 8.56MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [07:17<45:47, 7.12MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.58G/22.2G [07:17<30:13, 9.14MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:17<29:55, 10.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [07:17<34:58, 8.84MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:17<58:41, 5.27MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.59G/22.2G [07:17<22:35, 12.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:17<29:16, 10.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [07:17<32:52, 9.41MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [07:17<45:12, 7.21MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.59G/22.2G [07:17<22:27, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:17<28:45, 10.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [07:17<31:36, 9.78MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:17<37:34, 8.22MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:17<28:11, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.59G/22.2G [07:17<18:53, 14.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [07:18<30:32, 10.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:18<33:34, 9.20MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:18<28:21, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.60G/22.2G [07:18<16:56, 16.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:18<31:44, 9.73MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [07:18<28:00, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:18<28:16, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.60G/22.2G [07:18<17:44, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:18<30:18, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [07:18<27:51, 11.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  16%|█▋        | 3.66G/22.2G [07:18<28:06, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.60G/22.2G [07:18<16:25, 16.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:18<29:12, 10.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:18<28:00, 11.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.59G/22.2G [07:18<1:23:10, 3.92MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.60G/22.2G [07:18<15:22, 17.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:18<28:05, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:18<28:21, 10.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:18<27:37, 11.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [07:18<51:42, 6.30MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:18<26:43, 11.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:18<27:05, 11.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:18<58:27, 5.27MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:19<26:21, 11.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:19<26:52, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [07:19<49:04, 6.64MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.61G/22.2G [07:19<29:03, 9.49MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:19<38:49, 7.94MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:19<26:24, 11.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.61G/22.2G [07:19<21:49, 12.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:19<35:21, 8.72MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [07:19<48:18, 6.75MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:19<26:04, 11.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.61G/22.2G [07:19<21:49, 12.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:19<33:00, 9.34MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [07:19<47:05, 6.92MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:19<31:16, 9.85MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [07:19<51:22, 6.02MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.61G/22.2G [07:19<18:32, 14.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:19<30:16, 10.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.62G/22.2G [07:19<16:39, 16.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [07:19<46:09, 7.06MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:19<37:40, 8.20MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:19<29:09, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:19<34:59, 8.83MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.62G/22.2G [07:19<17:24, 15.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:19<28:10, 10.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.60G/22.2G [07:19<45:24, 7.18MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:20<33:05, 9.34MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.62G/22.2G [07:20<16:19, 16.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:20<27:26, 11.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:20<31:35, 9.78MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [07:20<44:57, 7.25MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:20<27:08, 11.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.63G/22.2G [07:20<15:30, 17.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:20<1:18:02, 3.96MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:20<27:17, 11.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:20<27:47, 11.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [07:20<44:32, 7.31MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:20<27:48, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [07:20<27:44, 11.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:20<50:07, 6.16MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:20<27:48, 11.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.63G/22.2G [07:20<24:25, 11.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [07:20<43:48, 7.44MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:20<27:38, 11.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:20<41:00, 7.52MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.63G/22.2G [07:20<21:52, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:20<27:46, 11.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [07:20<43:10, 7.54MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:20<38:31, 8.01MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.63G/22.2G [07:20<20:34, 13.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:21<25:28, 12.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:21<34:40, 8.90MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [07:21<42:44, 7.62MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.64G/22.2G [07:21<17:58, 15.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:21<26:06, 11.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:21<32:24, 9.52MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.64G/22.2G [07:21<16:17, 16.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:21<30:08, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:21<1:11:56, 4.28MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.64G/22.2G [07:21<15:15, 18.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:21<28:44, 10.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.65G/22.2G [07:21<16:08, 17.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:21<27:44, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:21<40:24, 7.62MB/s]  [A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:21<36:32, 8.43MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  25%|██▌       | 5.65G/22.2G [07:21<16:03, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:21<27:04, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [07:21<53:35, 5.76MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.61G/22.2G [07:21<1:19:22, 4.10MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:21<34:03, 9.04MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:21<26:54, 11.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:21<32:44, 9.40MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:22<35:12, 8.76MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  16%|█▋        | 3.66G/22.2G [07:22<26:25, 11.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [07:22<49:48, 6.54MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:22<31:13, 9.85MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:22<33:09, 9.31MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.65G/22.2G [07:22<27:56, 9.84MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:22<30:08, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:22<30:51, 10.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [07:22<46:08, 7.06MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.65G/22.2G [07:22<23:36, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:22<28:59, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:22<30:21, 10.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.66G/22.2G [07:22<23:00, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:22<29:18, 10.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [07:22<44:20, 7.34MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [07:22<29:35, 10.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:22<52:18, 5.89MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.66G/22.2G [07:22<19:07, 14.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:22<25:52, 11.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:22<36:56, 8.34MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.66G/22.2G [07:22<17:06, 16.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.62G/22.2G [07:22<42:25, 7.67MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:22<26:38, 11.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:22<26:20, 11.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:22<32:22, 9.52MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:22<26:54, 11.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.67G/22.2G [07:22<15:48, 17.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:23<26:19, 11.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [07:23<41:49, 7.78MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:23<27:00, 11.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:23<30:30, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.67G/22.2G [07:23<16:44, 16.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:23<27:01, 11.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:23<29:09, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [07:23<43:58, 7.40MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.67G/22.2G [07:23<19:23, 14.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:23<27:48, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [07:23<25:23, 12.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [07:23<44:44, 7.27MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:23<27:07, 11.4MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:23<26:37, 11.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [07:23<46:20, 7.02MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:23<58:22, 5.27MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:23<26:13, 11.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.67G/22.2G [07:23<31:47, 8.64MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [07:23<47:04, 6.91MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:23<36:10, 8.50MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:23<25:40, 12.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.68G/22.2G [07:23<22:56, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:24<50:17, 6.13MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:24<33:52, 9.08MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:24<25:33, 12.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [07:24<48:13, 6.75MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.68G/22.2G [07:24<23:09, 11.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:24<32:07, 9.57MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:24<25:19, 12.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:24<33:48, 9.12MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:24<30:38, 10.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.63G/22.2G [07:24<47:54, 6.79MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:24<31:58, 9.64MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.68G/22.2G [07:24<22:28, 12.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:24<29:50, 10.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:24<30:54, 9.97MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [07:24<47:03, 6.91MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:24<28:49, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.68G/22.2G [07:24<21:22, 12.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [07:24<30:05, 10.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:24<28:02, 11.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:24<51:42, 5.95MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.69G/22.2G [07:24<20:31, 13.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:24<27:45, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:24<26:44, 11.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.69G/22.2G [07:24<18:15, 15.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:24<27:35, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:24<27:09, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:24<38:27, 8.00MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:25<27:11, 11.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.69G/22.2G [07:25<19:16, 14.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:25<27:12, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:25<33:17, 9.25MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [07:25<1:29:02, 3.65MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:25<31:10, 9.88MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:25<24:47, 12.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:25<25:28, 12.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:25<29:41, 10.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [07:25<26:07, 11.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [07:25<56:30, 5.76MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:25<28:10, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:25<26:32, 11.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.69G/22.2G [07:25<30:59, 8.85MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:25<27:20, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [07:25<51:31, 6.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.70G/22.2G [07:25<22:25, 12.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:25<26:31, 11.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.70G/22.2G [07:25<22:46, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:25<52:12, 5.89MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:25<26:09, 11.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.64G/22.2G [07:26<55:33, 5.85MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.70G/22.2G [07:26<19:11, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:26<25:38, 12.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:26<33:47, 9.09MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.70G/22.2G [07:26<19:57, 13.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:26<54:12, 5.68MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:26<25:33, 12.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:26<31:25, 9.77MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.71G/22.2G [07:26<17:42, 15.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [07:26<57:27, 5.66MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:26<25:15, 12.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:26<30:04, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:26<35:49, 8.59MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.71G/22.2G [07:26<18:46, 14.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:26<29:27, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [07:26<55:22, 5.87MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [07:26<33:35, 9.16MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.71G/22.2G [07:26<16:43, 16.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:26<28:46, 10.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:26<31:52, 9.66MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:26<28:21, 10.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [07:26<52:26, 6.20MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.72G/22.2G [07:26<17:54, 15.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:26<49:15, 6.24MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:26<27:47, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:26<28:04, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:27<27:02, 11.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:27<28:10, 10.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [07:27<56:36, 5.74MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:27<37:06, 8.29MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:27<27:55, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:27<24:50, 12.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.72G/22.2G [07:27<28:34, 9.59MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:27<32:26, 9.48MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [07:27<55:12, 5.89MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:27<25:04, 12.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:27<30:12, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [07:27<25:42, 12.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.72G/22.2G [07:27<23:44, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:27<25:31, 12.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [07:27<54:04, 6.01MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:27<26:00, 11.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:27<28:36, 10.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.72G/22.2G [07:27<23:43, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:27<27:22, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.73G/22.2G [07:27<19:10, 14.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.65G/22.2G [07:27<53:20, 6.09MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:27<24:31, 12.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:27<26:15, 11.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.73G/22.2G [07:27<20:07, 13.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [07:27<48:03, 6.76MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:28<26:01, 11.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.73G/22.2G [07:28<17:29, 15.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:28<52:38, 5.83MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [07:28<49:02, 6.63MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:28<25:33, 12.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.73G/22.2G [07:28<17:36, 15.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:28<24:55, 12.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:28<33:15, 9.22MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [07:28<49:50, 6.52MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.74G/22.2G [07:28<16:03, 17.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:28<48:29, 6.34MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:28<24:49, 12.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:28<31:27, 9.75MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:28<30:27, 10.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [07:28<33:18, 9.23MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:28<26:19, 11.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:28<31:39, 9.71MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:28<26:27, 11.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.74G/22.2G [07:28<30:48, 8.88MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:29<27:56, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:29<49:00, 6.27MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:29<26:20, 11.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [07:29<1:35:53, 3.39MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:29<27:40, 11.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.74G/22.2G [07:29<22:55, 11.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:29<29:58, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:29<24:15, 12.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.74G/22.2G [07:29<22:49, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:29<28:24, 10.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [07:29<25:26, 12.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [07:29<58:27, 5.56MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:29<24:35, 12.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.75G/22.2G [07:29<19:00, 14.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:29<27:52, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:29<24:26, 12.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:29<23:12, 13.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.75G/22.2G [07:29<17:02, 16.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.66G/22.2G [07:29<54:59, 5.91MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:29<26:26, 11.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:29<24:43, 12.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:29<25:53, 11.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [07:29<52:54, 6.14MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.75G/22.2G [07:29<22:22, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:29<23:30, 13.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:29<23:43, 12.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.76G/22.2G [07:30<18:24, 14.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [07:30<51:55, 6.26MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:30<24:00, 12.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.76G/22.2G [07:30<18:56, 14.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:30<49:01, 6.25MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [07:30<51:43, 6.28MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:30<22:30, 13.6MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:30<22:34, 13.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:30<33:19, 9.20MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [07:30<51:39, 6.29MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [07:30<46:55, 6.55MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:30<34:05, 8.99MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [07:30<50:14, 6.46MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:30<33:20, 9.21MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:30<32:30, 9.42MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.76G/22.2G [07:30<34:07, 8.01MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:30<30:53, 9.92MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [07:30<49:52, 6.51MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:30<29:15, 10.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:30<41:04, 7.47MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.76G/22.2G [07:30<25:18, 10.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:31<30:00, 10.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.77G/22.2G [07:31<25:12, 10.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.67G/22.2G [07:31<49:05, 6.61MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.73G/22.2G [07:31<26:13, 11.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:31<31:52, 9.63MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:31<29:04, 10.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.77G/22.2G [07:31<25:30, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:31<28:39, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:31<27:45, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:31<24:21, 12.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [07:31<49:37, 6.54MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:31<28:21, 10.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:31<25:50, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.77G/22.2G [07:31<22:17, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:31<28:03, 10.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:31<23:22, 13.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [07:31<47:40, 6.81MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:31<24:48, 12.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.77G/22.2G [07:31<21:14, 12.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:31<27:51, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:31<20:57, 14.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:31<23:01, 13.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [07:31<48:04, 6.75MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:31<27:40, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:31<21:38, 14.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:31<22:31, 13.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.78G/22.2G [07:31<20:33, 13.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:31<27:36, 11.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:32<21:14, 14.4MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:32<21:47, 14.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.78G/22.2G [07:32<22:16, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:32<20:23, 15.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.74G/22.2G [07:32<44:04, 6.96MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:32<1:05:38, 4.66MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:32<33:26, 9.17MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:32<39:42, 7.72MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:32<31:55, 9.61MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.78G/22.2G [07:32<37:19, 7.31MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:32<40:32, 7.55MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:33<31:19, 9.79MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [07:33<1:51:25, 2.91MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:33<33:37, 9.11MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.79G/22.2G [07:33<28:25, 9.60MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:33<37:11, 8.23MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:33<28:27, 10.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.75G/22.2G [07:33<27:50, 11.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.79G/22.2G [07:33<27:05, 10.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:33<37:57, 8.06MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:33<27:39, 11.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:33<28:26, 10.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.68G/22.2G [07:33<1:07:21, 4.82MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.79G/22.2G [07:33<26:40, 10.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:33<35:09, 8.70MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:33<28:09, 10.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.79G/22.2G [07:33<26:17, 10.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:33<25:43, 11.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:33<36:24, 8.40MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:33<27:38, 11.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [07:33<1:06:15, 4.90MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.79G/22.2G [07:33<26:02, 10.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:33<25:59, 11.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:33<34:07, 8.96MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:33<27:14, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.79G/22.2G [07:33<25:48, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:33<26:53, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:33<26:05, 11.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:33<35:48, 8.54MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.80G/22.2G [07:34<25:02, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [07:34<1:07:48, 4.79MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:34<26:28, 11.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:34<33:33, 9.11MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.76G/22.2G [07:34<26:59, 11.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.80G/22.2G [07:34<25:05, 10.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:34<25:46, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:34<31:56, 9.58MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [07:34<1:01:19, 5.29MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.80G/22.2G [07:34<25:01, 10.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:34<25:34, 12.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:34<34:07, 8.96MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.80G/22.2G [07:34<25:01, 10.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:34<24:57, 12.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:34<32:19, 9.46MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [07:34<1:02:13, 5.21MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.80G/22.2G [07:34<25:01, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:34<31:15, 9.78MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:34<49:12, 6.23MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [07:34<57:32, 5.64MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:34<33:26, 9.14MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:34<33:16, 9.21MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:35<49:07, 6.23MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.69G/22.2G [07:35<58:47, 5.52MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.77G/22.2G [07:35<28:54, 10.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.80G/22.2G [07:35<49:55, 5.46MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:35<36:17, 8.43MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [07:35<56:10, 5.77MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:35<26:25, 11.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:35<31:16, 9.78MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.81G/22.2G [07:35<31:26, 8.66MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:35<29:04, 10.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [07:35<52:55, 6.13MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:35<1:10:12, 4.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:35<24:37, 12.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.81G/22.2G [07:35<30:15, 9.00MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:35<27:17, 11.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:35<44:22, 6.89MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.78G/22.2G [07:35<23:29, 13.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [07:35<50:34, 6.41MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:35<25:39, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.81G/22.2G [07:35<26:17, 10.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:35<43:20, 7.05MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:35<22:54, 13.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:35<24:33, 12.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▌       | 5.82G/22.2G [07:35<25:43, 10.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [07:36<50:54, 6.37MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:36<38:54, 7.85MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:36<24:11, 12.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.82G/22.2G [07:36<25:32, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:36<35:06, 8.70MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [07:36<51:06, 6.34MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:36<23:45, 12.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.82G/22.2G [07:36<25:04, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:36<36:24, 8.39MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:36<22:22, 13.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.82G/22.2G [07:36<22:59, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:36<33:12, 9.20MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:36<36:17, 8.43MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.82G/22.2G [07:36<23:21, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:36<31:23, 9.73MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:36<30:50, 9.91MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.79G/22.2G [07:36<28:28, 10.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.83G/22.2G [07:36<22:16, 12.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:37<33:10, 9.21MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:37<25:48, 11.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:37<43:38, 7.00MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.70G/22.2G [07:37<1:41:11, 3.20MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:37<31:10, 9.80MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:37<22:48, 13.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:37<31:53, 9.58MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:37<30:13, 10.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:37<23:09, 13.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.83G/22.2G [07:37<39:07, 6.96MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:37<28:05, 10.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:37<29:34, 10.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [07:37<57:55, 5.60MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.80G/22.2G [07:37<22:35, 13.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:37<26:07, 11.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.83G/22.2G [07:37<29:39, 9.17MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [07:37<54:23, 5.96MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:37<20:26, 15.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:37<24:13, 12.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.83G/22.2G [07:37<28:10, 9.66MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:37<21:11, 14.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [07:37<53:07, 6.10MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:37<23:17, 13.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.84G/22.2G [07:37<25:04, 10.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:38<22:21, 13.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [07:38<51:35, 6.28MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.84G/22.2G [07:38<24:59, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:38<1:07:59, 4.49MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:38<21:41, 14.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [07:38<51:23, 6.30MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.84G/22.2G [07:38<23:24, 11.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:38<39:12, 7.79MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:38<38:13, 8.00MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.84G/22.2G [07:38<23:39, 11.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.71G/22.2G [07:38<50:09, 6.46MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:38<39:02, 7.82MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.81G/22.2G [07:38<26:59, 11.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:38<36:03, 8.47MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.84G/22.2G [07:38<22:22, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:38<26:29, 11.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [07:38<49:56, 6.49MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:38<33:19, 9.16MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.85G/22.2G [07:38<22:51, 11.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:38<37:02, 8.24MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:38<24:57, 12.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [07:38<49:07, 6.59MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.85G/22.2G [07:39<23:04, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:39<34:30, 8.84MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:39<26:39, 11.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:39<21:31, 14.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:39<32:39, 9.34MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [07:39<49:02, 6.60MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:39<26:29, 11.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.82G/22.2G [07:39<21:52, 14.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:39<31:00, 9.84MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:39<25:28, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:39<20:02, 15.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:39<29:44, 10.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [07:39<48:25, 6.69MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:39<20:47, 14.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:39<23:57, 12.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:39<29:47, 10.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.85G/22.2G [07:39<44:05, 6.16MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:39<19:15, 15.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [07:39<48:36, 6.66MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:39<22:17, 13.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:39<32:27, 9.40MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.85G/22.2G [07:39<31:00, 8.76MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.72G/22.2G [07:39<48:02, 6.74MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:39<30:38, 9.96MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:39<21:22, 14.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.85G/22.2G [07:39<28:22, 9.58MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:40<29:45, 10.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.86G/22.2G [07:40<27:34, 9.85MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:40<20:46, 14.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.83G/22.2G [07:40<36:43, 8.32MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.86G/22.2G [07:40<26:05, 10.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:40<25:53, 11.8MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:40<25:32, 12.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.86G/22.2G [07:40<23:24, 11.6MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.86G/22.2G [07:40<23:18, 11.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:40<24:04, 12.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:40<33:04, 9.22MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.86G/22.2G [07:40<23:37, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:40<1:07:47, 4.50MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.84G/22.2G [07:40<21:08, 14.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [07:40<1:36:35, 3.35MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:40<24:20, 12.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.87G/22.2G [07:40<23:24, 11.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:40<21:42, 14.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:40<24:31, 12.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:40<38:59, 7.82MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:41<19:53, 15.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:41<24:44, 12.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [07:41<56:48, 5.70MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  26%|██▋       | 5.87G/22.2G [07:41<21:43, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:41<38:47, 7.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:41<20:19, 15.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:41<36:15, 8.41MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:41<22:14, 13.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [07:41<54:01, 5.99MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.85G/22.2G [07:41<18:50, 16.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:41<33:43, 9.04MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:41<21:19, 14.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [07:41<52:22, 6.18MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:41<32:13, 9.46MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:41<20:44, 14.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:41<33:31, 9.09MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.73G/22.2G [07:41<50:56, 6.35MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.87G/22.2G [07:41<44:00, 6.17MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:41<32:14, 9.45MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  17%|█▋        | 3.88G/22.2G [07:41<20:20, 15.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [07:42<50:24, 6.42MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.88G/22.2G [07:42<30:21, 8.94MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:42<31:02, 9.82MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:42<30:29, 9.99MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.88G/22.2G [07:42<28:59, 9.36MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [07:42<49:28, 6.54MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:42<47:13, 6.46MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:42<29:44, 10.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.88G/22.2G [07:42<25:26, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:42<29:03, 10.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [07:42<49:19, 6.56MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:42<40:07, 7.60MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [07:42<36:38, 8.32MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.88G/22.2G [07:42<25:15, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.88G/22.2G [07:42<28:39, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:42<36:38, 8.32MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [07:42<48:33, 6.66MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [07:42<27:39, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.88G/22.2G [07:42<24:17, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  17%|█▋        | 3.88G/22.2G [07:42<28:23, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [07:42<27:36, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.89G/22.2G [07:42<24:01, 11.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.86G/22.2G [07:42<30:07, 10.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [07:42<48:08, 6.72MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:42<25:09, 12.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:43<25:07, 12.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.89G/22.2G [07:43<22:09, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:43<25:04, 12.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [07:43<47:48, 6.77MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.89G/22.2G [07:43<22:38, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [07:43<57:43, 5.28MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:43<23:10, 13.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:43<21:44, 14.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.74G/22.2G [07:43<47:29, 6.81MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.87G/22.2G [07:43<22:20, 13.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.89G/22.2G [07:43<21:41, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [07:43<37:54, 8.03MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:43<21:59, 13.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  17%|█▋        | 3.88G/22.2G [07:43<19:41, 15.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [07:43<46:44, 6.92MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [07:43<35:04, 8.68MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:43<21:11, 14.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [07:43<36:24, 8.36MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.89G/22.2G [07:43<35:43, 7.59MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:43<34:25, 8.84MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:44<32:46, 9.29MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [07:44<35:54, 8.49MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.90G/22.2G [07:44<29:12, 9.28MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:44<31:21, 9.71MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [07:44<26:52, 11.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.90G/22.2G [07:44<27:05, 10.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.88G/22.2G [07:44<26:39, 11.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:44<33:50, 8.99MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:44<27:35, 11.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.90G/22.2G [07:44<26:31, 10.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [07:44<1:35:47, 3.38MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:44<22:17, 13.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:44<25:33, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:44<27:55, 10.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:44<22:39, 13.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.90G/22.2G [07:44<23:16, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:44<25:28, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:44<27:43, 11.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [07:44<55:53, 5.79MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:44<20:32, 14.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.91G/22.2G [07:44<23:32, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:44<27:38, 11.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:44<24:12, 12.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:44<21:25, 14.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:44<27:52, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.91G/22.2G [07:44<23:22, 11.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [07:44<51:50, 6.24MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.89G/22.2G [07:45<19:11, 15.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:45<27:39, 11.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:45<22:39, 13.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.91G/22.2G [07:45<21:26, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:45<20:14, 15.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:45<22:08, 13.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.75G/22.2G [07:45<49:37, 6.51MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.91G/22.2G [07:45<22:09, 12.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:45<20:58, 14.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [07:45<47:53, 6.75MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:45<20:31, 14.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [07:45<46:17, 6.98MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:45<37:27, 8.12MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [07:45<44:53, 7.20MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:45<1:23:38, 3.64MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:46<28:09, 10.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.92G/22.2G [07:46<40:43, 6.65MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [07:46<43:52, 7.37MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:46<27:36, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:46<35:14, 8.62MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.90G/22.2G [07:46<50:22, 6.04MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:46<27:04, 11.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.92G/22.2G [07:46<29:44, 9.10MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.76G/22.2G [07:46<43:17, 7.47MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:46<28:57, 10.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:46<27:02, 11.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.92G/22.2G [07:46<28:37, 9.45MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:46<46:42, 6.51MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:46<26:13, 11.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [07:46<41:04, 7.87MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:46<42:11, 7.21MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.92G/22.2G [07:46<28:18, 9.56MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:46<25:10, 12.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:46<26:58, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [07:46<37:20, 8.65MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.92G/22.2G [07:46<28:01, 9.65MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:46<25:24, 12.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:46<41:19, 7.36MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:46<27:17, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.93G/22.2G [07:46<27:46, 9.74MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  12%|█▏        | 2.77G/22.2G [07:46<40:17, 8.02MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:46<37:23, 8.13MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:46<27:46, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:46<24:08, 12.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.93G/22.2G [07:47<27:17, 9.91MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:47<27:51, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:47<24:32, 12.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:47<38:06, 7.98MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.93G/22.2G [07:47<26:54, 10.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:47<27:37, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:47<34:53, 8.71MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.93G/22.2G [07:47<26:17, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:47<24:14, 12.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:47<27:32, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.91G/22.2G [07:47<35:41, 8.52MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.93G/22.2G [07:47<25:49, 10.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:47<27:41, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:47<33:26, 9.09MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [07:47<1:19:19, 4.07MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.93G/22.2G [07:47<25:34, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:47<27:28, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:47<32:16, 9.42MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.94G/22.2G [07:47<25:25, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:47<27:18, 11.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.77G/22.2G [07:47<45:45, 7.06MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:47<30:57, 9.82MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.94G/22.2G [07:47<24:41, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:47<26:55, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [07:47<42:26, 7.61MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:48<49:52, 6.09MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:48<33:14, 9.14MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [07:48<40:06, 8.05MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:48<32:04, 9.48MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:48<34:52, 8.71MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [07:48<38:13, 8.45MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:48<50:21, 6.03MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [07:48<37:25, 8.63MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.94G/22.2G [07:48<47:25, 5.70MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:48<29:55, 10.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [07:48<36:06, 8.94MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:48<37:11, 8.16MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.94G/22.2G [07:48<31:36, 8.55MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:48<27:53, 10.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [07:48<34:14, 9.43MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:48<32:30, 9.34MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.94G/22.2G [07:48<28:39, 9.43MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.78G/22.2G [07:48<33:20, 9.68MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:48<25:25, 11.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:48<30:49, 9.84MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.92G/22.2G [07:48<1:16:52, 3.95MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.95G/22.2G [07:48<27:26, 9.85MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [07:49<32:17, 10.0MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:49<29:07, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.95G/22.2G [07:49<26:11, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:49<24:00, 12.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [07:49<31:13, 10.3MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:49<51:09, 5.94MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:49<28:12, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.95G/22.2G [07:49<25:44, 10.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [07:49<30:25, 10.6MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:49<22:56, 13.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:49<27:17, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:49<47:28, 6.40MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [07:49<29:46, 10.8MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.95G/22.2G [07:49<23:10, 11.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:49<26:39, 11.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:49<45:40, 6.65MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.95G/22.2G [07:49<23:30, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:49<26:08, 11.6MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:49<25:45, 11.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.95G/22.2G [07:49<23:33, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:49<43:49, 6.93MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:49<25:29, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.96G/22.2G [07:49<23:25, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:49<42:54, 7.08MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.79G/22.2G [07:50<58:06, 5.55MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:50<39:22, 7.71MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:50<25:24, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:50<42:11, 7.20MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [07:50<37:43, 8.55MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:50<30:12, 10.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.93G/22.2G [07:50<37:21, 8.13MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [07:50<33:02, 9.76MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:50<25:42, 11.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:50<38:39, 7.85MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:50<25:39, 11.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [07:50<31:58, 10.1MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:50<53:07, 5.71MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [07:50<29:40, 10.9MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.96G/22.2G [07:50<41:30, 6.50MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:50<39:11, 7.75MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:50<24:16, 12.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:50<36:15, 8.36MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:50<21:33, 14.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.80G/22.2G [07:50<26:43, 12.1MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:50<38:55, 7.80MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.96G/22.2G [07:50<29:33, 9.13MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:51<22:01, 13.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:51<31:53, 9.50MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:51<36:13, 8.38MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [07:51<26:23, 12.2MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.97G/22.2G [07:51<27:18, 9.88MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:51<30:04, 10.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:51<19:43, 15.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.97G/22.2G [07:51<26:45, 10.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:51<37:46, 8.04MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [07:51<24:43, 13.0MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:51<20:27, 14.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:51<28:27, 10.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:51<35:24, 8.57MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.97G/22.2G [07:51<23:36, 11.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:51<27:29, 11.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [07:51<22:57, 14.0MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:51<34:25, 8.80MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.97G/22.2G [07:51<29:15, 9.22MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:51<29:39, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.98G/22.2G [07:51<23:10, 11.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.81G/22.2G [07:51<35:18, 9.13MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:51<44:30, 6.81MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.98G/22.2G [07:52<24:06, 11.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [07:52<28:49, 11.2MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:52<31:42, 9.55MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:52<32:51, 9.22MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.94G/22.2G [07:52<1:25:34, 3.55MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.98G/22.2G [07:52<25:36, 10.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [07:52<29:08, 11.1MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:52<31:37, 9.58MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:52<31:27, 9.64MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.98G/22.2G [07:52<26:11, 10.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.82G/22.2G [07:52<25:05, 12.8MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:52<32:26, 9.33MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:52<30:43, 9.86MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.98G/22.2G [07:52<26:38, 10.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:52<55:07, 5.50MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:52<29:50, 10.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:52<32:07, 9.42MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [07:52<22:41, 14.2MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:52<50:44, 5.98MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:52<26:42, 11.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [07:52<19:53, 16.2MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:52<26:44, 11.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.83G/22.2G [07:52<20:55, 15.4MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:52<51:27, 5.90MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:53<27:19, 11.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.98G/22.2G [07:53<53:42, 5.02MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:53<59:39, 5.07MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:53<48:42, 6.23MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:53<25:04, 12.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.99G/22.2G [07:53<34:38, 7.78MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:53<38:40, 7.83MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.95G/22.2G [07:53<46:33, 6.52MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:53<25:56, 11.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.99G/22.2G [07:53<31:43, 8.50MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [07:53<31:33, 10.2MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:53<39:04, 7.75MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.99G/22.2G [07:53<30:35, 8.81MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:53<24:31, 12.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:53<48:50, 6.21MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [07:53<24:16, 13.3MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [07:53<34:35, 8.75MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.99G/22.2G [07:53<29:19, 9.18MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.84G/22.2G [07:53<21:05, 15.3MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:53<46:34, 6.51MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.99G/22.2G [07:53<28:40, 9.39MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [07:53<35:36, 8.50MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [07:53<19:39, 16.4MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:54<45:55, 6.60MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 5.99G/22.2G [07:54<27:52, 9.66MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [07:54<32:37, 9.28MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:54<42:30, 7.12MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [07:54<18:25, 17.5MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.00G/22.2G [07:54<27:16, 9.88MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [07:54<30:10, 10.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:54<44:33, 6.81MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.85G/22.2G [07:54<17:24, 18.5MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:54<33:18, 9.09MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.00G/22.2G [07:54<26:48, 10.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [07:54<30:10, 10.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [07:54<16:33, 19.4MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:54<42:59, 7.05MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:54<31:35, 9.58MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [07:54<29:56, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.00G/22.2G [07:54<26:31, 10.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:54<43:16, 7.01MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [07:54<29:49, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.00G/22.2G [07:54<26:17, 10.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [07:54<27:37, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [07:54<30:03, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.00G/22.2G [07:54<25:49, 10.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [07:54<27:24, 11.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [07:54<29:00, 11.1MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [07:54<29:48, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.96G/22.2G [07:54<48:35, 6.24MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.00G/22.2G [07:54<25:45, 10.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [07:55<24:53, 12.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.86G/22.2G [07:55<22:47, 14.1MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [07:55<29:37, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [07:55<25:13, 12.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [07:55<19:00, 16.9MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:55<53:40, 5.65MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [07:55<17:14, 18.6MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [07:55<24:08, 12.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.87G/22.2G [07:55<15:52, 20.2MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [07:55<23:23, 12.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.00G/22.2G [07:55<56:43, 4.74MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [07:55<14:52, 21.6MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [07:55<1:06:01, 4.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.01G/22.2G [07:55<33:39, 7.99MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [07:55<40:01, 7.56MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.01G/22.2G [07:55<30:30, 8.82MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:56<1:40:48, 3.01MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:56<36:01, 8.39MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.01G/22.2G [07:56<29:03, 9.26MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [07:56<25:23, 12.7MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [07:56<44:34, 6.78MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:56<34:22, 8.80MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.01G/22.2G [07:56<27:54, 9.64MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.88G/22.2G [07:56<20:50, 15.4MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:56<33:01, 9.15MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:56<1:01:24, 4.94MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.01G/22.2G [07:56<27:01, 9.95MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [07:56<31:41, 9.54MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [07:56<17:41, 18.2MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:56<32:00, 9.44MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:56<30:03, 10.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.02G/22.2G [07:56<27:00, 9.96MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.89G/22.2G [07:56<15:35, 20.6MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:56<30:53, 9.78MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.02G/22.2G [07:56<26:20, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:56<30:27, 9.92MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [07:56<16:30, 19.4MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:56<1:08:47, 4.41MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.02G/22.2G [07:56<25:50, 10.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:56<31:31, 9.59MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:56<29:43, 10.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [07:56<14:54, 21.5MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.02G/22.2G [07:57<25:42, 10.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:57<26:07, 11.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [07:57<29:31, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.02G/22.2G [07:57<24:50, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:57<26:28, 11.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:57<1:10:33, 4.29MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [07:57<29:08, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.02G/22.2G [07:57<24:33, 10.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [07:57<26:57, 11.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [07:57<29:08, 10.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.90G/22.2G [07:57<24:12, 13.3MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.03G/22.2G [07:57<25:08, 10.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [07:57<27:28, 11.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.97G/22.2G [07:57<1:06:49, 4.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [07:57<28:47, 10.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [07:57<21:19, 15.0MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [07:57<30:46, 9.82MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [07:57<18:29, 17.3MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:57<1:02:06, 4.88MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [07:57<30:34, 9.88MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.91G/22.2G [07:57<17:25, 18.4MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [07:57<30:03, 10.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:58<59:47, 5.07MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [07:58<16:32, 19.4MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.03G/22.2G [07:58<53:36, 5.01MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [07:58<15:53, 20.2MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [07:58<1:03:21, 4.77MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:58<55:46, 5.43MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.03G/22.2G [07:58<36:28, 7.37MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.92G/22.2G [07:58<15:23, 20.8MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:58<37:28, 8.06MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.03G/22.2G [07:58<32:09, 8.36MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:58<53:49, 5.63MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:58<34:28, 8.76MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.03G/22.2G [07:58<30:15, 8.88MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [07:58<58:44, 5.14MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:58<33:09, 9.11MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:58<52:05, 5.81MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.04G/22.2G [07:58<28:26, 9.45MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:58<36:00, 8.39MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [07:58<28:14, 11.3MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:58<31:51, 9.48MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.04G/22.2G [07:58<27:22, 9.81MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:58<52:00, 5.82MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:58<33:31, 9.01MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:58<30:48, 9.80MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [07:58<25:13, 12.7MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.04G/22.2G [07:59<26:42, 10.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:59<32:06, 9.41MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:59<30:24, 9.92MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.93G/22.2G [07:59<20:11, 15.9MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.98G/22.2G [07:59<50:45, 5.97MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.04G/22.2G [07:59<25:50, 10.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:59<31:15, 9.66MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [07:59<29:44, 10.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [07:59<18:01, 17.8MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.04G/22.2G [07:59<25:32, 10.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:59<30:30, 9.90MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:59<51:04, 5.93MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [07:59<29:39, 10.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [07:59<16:32, 19.4MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.04G/22.2G [07:59<25:16, 10.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [07:59<29:53, 10.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [07:59<29:08, 10.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.94G/22.2G [07:59<15:27, 20.7MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.05G/22.2G [07:59<25:03, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [07:59<49:58, 6.06MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [07:59<29:07, 10.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [07:59<29:12, 10.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.05G/22.2G [07:59<24:54, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [07:59<28:36, 10.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [07:59<28:52, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.05G/22.2G [07:59<24:54, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [07:59<28:13, 10.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:00<29:01, 10.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:00<27:56, 10.8MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:00<28:02, 10.8MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:00<27:45, 10.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [08:00<1:47:28, 2.82MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.05G/22.2G [08:00<51:03, 5.26MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [08:00<39:04, 8.19MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:00<1:06:33, 4.53MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.05G/22.2G [08:00<36:16, 7.40MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [08:00<29:45, 10.8MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [08:00<58:26, 5.18MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.05G/22.2G [08:00<31:54, 8.41MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.95G/22.2G [08:00<28:30, 11.2MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:00<38:39, 7.80MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:00<57:15, 5.27MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.06G/22.2G [08:01<29:47, 9.00MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:01<35:22, 8.53MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 3.99G/22.2G [08:01<55:43, 5.43MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [08:01<24:56, 12.8MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:01<34:28, 8.75MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.06G/22.2G [08:01<28:06, 9.54MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:01<33:38, 8.96MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [08:01<21:18, 15.0MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:01<32:37, 9.24MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.06G/22.2G [08:01<27:08, 9.89MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [08:01<54:46, 5.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:01<32:26, 9.29MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:01<31:20, 9.62MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.06G/22.2G [08:01<26:23, 10.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.96G/22.2G [08:01<21:24, 14.9MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:01<31:14, 9.65MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:01<29:55, 10.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.06G/22.2G [08:01<25:26, 10.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [08:01<19:53, 16.1MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [08:01<52:48, 5.73MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:01<30:18, 9.95MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:01<29:29, 10.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.06G/22.2G [08:01<25:29, 10.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:01<30:01, 10.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [08:01<49:51, 6.07MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:01<29:01, 10.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.07G/22.2G [08:01<24:54, 10.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:01<29:24, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:01<28:25, 10.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.07G/22.2G [08:02<25:04, 10.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:02<28:59, 10.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [08:02<51:39, 5.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:02<28:17, 10.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [08:02<32:26, 9.86MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.07G/22.2G [08:02<25:14, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:02<28:39, 10.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.97G/22.2G [08:02<24:44, 12.9MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:02<25:30, 11.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [08:02<50:59, 5.93MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:02<28:28, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [08:02<21:32, 14.8MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:02<26:08, 11.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:02<28:22, 10.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [08:02<49:00, 6.17MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [08:02<20:09, 15.9MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:02<26:39, 11.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.98G/22.2G [08:02<19:10, 16.7MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.00G/22.2G [08:02<50:22, 6.00MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.07G/22.2G [08:02<51:03, 5.25MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [08:02<18:16, 17.5MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [08:03<17:44, 18.0MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [08:03<50:39, 5.97MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.07G/22.2G [08:03<32:05, 8.35MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:03<1:01:13, 4.92MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:03<51:06, 5.89MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  13%|█▎        | 2.99G/22.2G [08:03<17:09, 18.6MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.08G/22.2G [08:03<29:25, 9.11MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [08:03<47:54, 6.31MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:03<36:12, 8.32MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.08G/22.2G [08:03<28:16, 9.48MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:03<32:47, 9.19MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:03<33:01, 9.12MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.08G/22.2G [08:03<26:47, 10.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:03<31:32, 9.55MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [08:03<50:04, 6.04MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 2.99G/22.2G [08:03<28:05, 11.4MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:03<32:28, 9.28MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:03<30:43, 9.80MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.08G/22.2G [08:03<26:12, 10.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [08:03<26:32, 12.0MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:03<30:56, 9.73MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:03<30:18, 9.94MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.08G/22.2G [08:03<25:44, 10.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:03<30:06, 10.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [08:03<21:45, 14.7MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.08G/22.2G [08:03<24:52, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:04<27:36, 10.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:04<29:08, 10.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [08:04<19:53, 16.1MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.09G/22.2G [08:04<24:59, 10.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:04<27:26, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:04<28:30, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.00G/22.2G [08:04<18:38, 17.1MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.09G/22.2G [08:04<24:47, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:04<27:39, 10.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:04<28:01, 10.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [08:04<17:39, 18.1MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [08:04<1:30:59, 3.32MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:04<27:51, 10.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:04<27:37, 10.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.09G/22.2G [08:04<22:42, 11.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [08:04<17:17, 18.4MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:04<28:05, 10.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.10G/22.2G [08:04<27:20, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  27%|██▋       | 6.09G/22.2G [08:04<23:09, 11.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.10G/22.2G [08:04<27:34, 10.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.01G/22.2G [08:04<54:10, 5.58MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.10G/22.2G [08:04<26:51, 11.2MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  18%|█▊        | 4.10G/22.2G [08:04<26:50, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  18%|█▊        | 4.10G/22.2G [08:04<25:57, 11.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [08:04<50:50, 5.95MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.01G/22.2G [08:05<31:53, 10.0MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [08:05<48:35, 6.22MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.09G/22.2G [08:05<45:51, 5.84MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [08:05<24:09, 13.2MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [08:05<20:39, 15.4MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [08:05<50:14, 6.02MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.10G/22.2G [08:05<34:07, 7.84MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [08:05<46:53, 6.42MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.02G/22.2G [08:05<19:09, 16.6MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.10G/22.2G [08:05<30:43, 8.71MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [08:05<1:03:10, 4.76MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [08:05<47:44, 6.33MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [08:05<34:16, 8.78MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [08:05<18:04, 17.6MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.10G/22.2G [08:05<28:50, 9.28MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [08:05<31:44, 9.48MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [08:05<46:28, 6.50MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [08:05<41:33, 7.24MB/s]  


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [08:05<17:12, 18.5MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.10G/22.2G [08:05<26:52, 9.95MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:05<30:59, 9.71MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [08:05<35:23, 8.50MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.03G/22.2G [08:05<16:42, 19.1MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [08:06<44:44, 6.75MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.10G/22.2G [08:06<26:16, 10.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:06<29:15, 10.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:06<32:34, 9.23MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.02G/22.2G [08:06<43:31, 6.94MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:06<28:42, 10.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:06<30:31, 9.85MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.11G/22.2G [08:06<23:09, 11.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:06<28:09, 10.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:06<28:51, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.11G/22.2G [08:06<23:37, 11.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [08:06<42:41, 7.08MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:06<27:56, 10.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:06<27:21, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [08:06<41:06, 7.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:06<28:08, 10.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [08:06<36:12, 8.80MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:06<26:24, 11.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.11G/22.2G [08:06<26:36, 10.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [08:06<40:32, 7.45MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [08:06<26:50, 11.9MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:06<25:40, 11.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:06<25:44, 11.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [08:06<26:27, 12.0MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:06<25:06, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:06<26:18, 11.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [08:06<40:03, 7.54MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:07<24:56, 12.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:07<26:24, 11.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▎        | 3.04G/22.2G [08:07<25:29, 12.5MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:07<24:38, 12.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [08:07<23:46, 13.4MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.11G/22.2G [08:07<36:59, 7.23MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [08:07<22:42, 14.0MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.12G/22.2G [08:07<28:02, 9.53MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:07<52:09, 5.76MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:07<46:43, 6.43MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.12G/22.2G [08:07<27:06, 9.86MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.05G/22.2G [08:07<22:00, 14.5MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.03G/22.2G [08:07<1:18:19, 3.86MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:07<32:55, 9.13MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.12G/22.2G [08:07<27:04, 9.87MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:07<35:50, 8.39MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:08<30:17, 9.92MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:08<31:51, 9.43MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.12G/22.2G [08:08<26:44, 9.99MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [08:08<46:56, 6.43MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:08<29:32, 10.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.13G/22.2G [08:08<26:41, 10.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [08:08<42:07, 7.17MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:08<28:36, 10.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:08<26:41, 11.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.13G/22.2G [08:08<26:06, 10.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:08<28:07, 10.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [08:08<41:58, 7.19MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [08:08<36:19, 8.76MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:08<24:36, 12.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.13G/22.2G [08:08<26:01, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:08<27:59, 10.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [08:08<38:03, 7.93MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [08:08<27:22, 11.6MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.13G/22.2G [08:08<25:36, 10.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:08<27:38, 10.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:08<23:10, 13.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [08:08<38:25, 7.86MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.06G/22.2G [08:08<27:26, 11.6MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.13G/22.2G [08:08<25:16, 10.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.04G/22.2G [08:08<36:02, 8.38MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:08<25:08, 11.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [08:08<25:30, 12.5MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:08<22:09, 13.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.13G/22.2G [08:08<24:43, 10.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:09<32:53, 9.18MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:09<25:33, 11.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.13G/22.2G [08:09<24:38, 10.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [08:09<23:10, 13.7MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:09<21:18, 14.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:09<30:59, 9.74MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:09<25:46, 11.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.14G/22.2G [08:09<24:20, 11.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:09<26:12, 11.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:09<33:15, 9.07MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [08:09<21:46, 14.6MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:09<31:48, 9.49MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.07G/22.2G [08:09<23:07, 13.8MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [08:09<24:50, 12.8MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:09<27:43, 10.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:09<40:30, 7.41MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.05G/22.2G [08:09<27:38, 10.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:09<53:45, 5.58MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [08:09<28:41, 11.1MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:10<34:53, 8.60MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [08:10<29:43, 10.7MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:10<34:18, 8.75MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:10<31:42, 9.47MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:10<30:36, 9.81MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:10<35:51, 8.37MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:10<29:21, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:10<34:03, 8.81MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.15G/22.2G [08:10<28:05, 10.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:10<32:06, 9.34MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [08:10<57:13, 5.55MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:10<27:08, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:10<34:16, 8.75MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.08G/22.2G [08:10<41:09, 7.72MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:10<26:16, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.15G/22.2G [08:10<32:48, 9.14MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:11<25:32, 11.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:11<31:15, 9.60MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [08:11<42:12, 7.53MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:11<24:58, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:11<30:03, 9.98MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [08:11<38:36, 8.23MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:11<24:53, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:11<29:35, 10.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:11<23:36, 12.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [08:11<39:11, 8.11MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:11<28:56, 10.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [08:11<36:17, 8.75MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:11<28:44, 10.4MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:11<28:53, 10.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [08:11<36:23, 8.73MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:11<41:59, 7.14MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:11<28:38, 10.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:11<2:21:21, 2.13MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [08:11<33:20, 9.53MB/s][A[A[A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.09G/22.2G [08:12<33:24, 9.51MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:12<33:57, 8.83MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [08:12<33:23, 9.51MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:12<28:58, 10.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [08:12<33:28, 9.49MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:12<1:33:56, 3.21MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:12<26:48, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:12<59:56, 5.00MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:12<1:16:50, 3.92MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [08:12<33:02, 9.61MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:12<25:02, 12.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:12<1:05:11, 4.63MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [08:12<33:12, 9.56MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:12<24:14, 12.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:12<40:11, 7.46MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.06G/22.2G [08:12<54:25, 5.54MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:12<22:59, 13.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:12<36:26, 8.22MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:12<22:32, 13.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:12<40:29, 7.45MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:13<33:56, 8.83MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:13<21:49, 13.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:13<31:38, 9.47MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:13<32:20, 9.32MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:13<30:38, 9.78MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:13<24:35, 12.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:13<28:06, 10.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:13<30:01, 9.98MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:13<28:49, 10.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.07G/22.2G [08:13<25:28, 11.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:13<28:03, 10.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.10G/22.2G [08:13<1:47:16, 2.96MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:13<25:34, 11.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:14<50:52, 5.89MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:14<29:19, 10.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [08:14<59:53, 5.30MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:14<43:19, 6.95MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:14<33:59, 8.81MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:14<29:08, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:14<31:26, 9.52MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [08:14<54:32, 5.82MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.08G/22.2G [08:14<28:03, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:14<30:38, 9.77MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [08:14<51:59, 6.11MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:14<25:18, 11.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:14<30:00, 9.97MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:14<21:43, 13.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.14G/22.2G [08:14<5:14:19, 849kB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:14<29:06, 10.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [08:14<50:59, 6.22MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:14<28:43, 10.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:14<53:51, 5.56MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:14<20:31, 14.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [08:15<48:32, 6.54MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:15<28:07, 10.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.09G/22.2G [08:15<18:33, 16.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:15<36:36, 8.18MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:15<27:43, 10.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  18%|█▊        | 4.10G/22.2G [08:15<17:11, 17.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.11G/22.2G [08:15<46:43, 6.79MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.14G/22.2G [08:15<2:54:45, 1.53MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:15<37:31, 7.98MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:15<27:05, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.14G/22.2G [08:15<2:17:55, 1.93MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [08:15<45:00, 7.05MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:15<35:39, 8.39MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:15<26:55, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.14G/22.2G [08:15<1:48:28, 2.46MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:15<26:30, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [08:15<27:50, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:15<36:29, 8.20MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [08:15<43:24, 7.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.15G/22.2G [08:15<1:24:46, 3.15MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.10G/22.2G [08:15<21:46, 13.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:15<26:18, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:15<34:40, 8.63MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.15G/22.2G [08:15<1:08:06, 3.92MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [08:15<42:34, 7.45MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:15<19:21, 15.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:16<32:47, 9.12MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:16<17:47, 16.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.15G/22.2G [08:16<46:52, 5.69MB/s]  [A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [08:16<41:32, 7.64MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:16<34:35, 8.65MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:16<16:36, 18.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.15G/22.2G [08:16<41:24, 6.44MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:16<32:53, 9.10MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.11G/22.2G [08:16<15:41, 19.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [08:16<41:24, 7.66MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:16<55:39, 5.37MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.15G/22.2G [08:16<35:55, 7.42MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:16<14:57, 20.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:16<31:44, 9.43MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:16<14:15, 21.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:16<39:17, 7.61MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:16<30:54, 9.68MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.16G/22.2G [08:16<29:43, 8.97MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:16<33:45, 8.86MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.16G/22.2G [08:16<27:43, 9.62MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:16<33:24, 8.95MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:16<31:29, 9.49MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:16<32:25, 9.23MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:17<23:42, 12.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:17<30:48, 9.70MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.12G/22.2G [08:17<1:14:29, 4.26MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:17<31:18, 9.55MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:17<29:20, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.12G/22.2G [08:17<27:27, 10.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [08:17<44:10, 7.18MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.16G/22.2G [08:17<45:03, 5.92MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:17<27:58, 10.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:17<20:18, 14.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:17<27:13, 11.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [08:17<45:16, 7.00MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.16G/22.2G [08:17<34:37, 7.70MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:17<18:27, 16.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:17<26:23, 11.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:17<1:00:06, 4.98MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.13G/22.2G [08:17<18:32, 16.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.17G/22.2G [08:17<31:18, 8.51MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [08:17<43:19, 7.32MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:17<25:52, 11.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:17<17:49, 16.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:17<38:39, 7.73MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.17G/22.2G [08:17<26:12, 10.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:17<18:09, 16.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [08:17<42:11, 7.51MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:17<23:47, 12.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:18<38:50, 7.70MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.17G/22.2G [08:18<25:57, 10.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:18<17:26, 17.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.13G/22.2G [08:18<41:01, 7.73MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:18<35:43, 8.37MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.17G/22.2G [08:18<23:17, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [08:18<37:44, 8.40MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:18<33:10, 9.01MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.17G/22.2G [08:18<23:52, 11.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:18<31:33, 9.47MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [08:18<40:07, 7.90MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:18<46:52, 6.37MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:18<30:58, 9.65MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.18G/22.2G [08:18<22:07, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.14G/22.2G [08:18<36:04, 8.32MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:18<29:28, 10.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:18<31:09, 9.59MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [08:18<39:49, 7.96MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:18<25:25, 11.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:18<28:21, 10.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.18G/22.2G [08:18<21:06, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:18<30:22, 9.84MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [08:18<36:39, 8.65MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:18<24:56, 12.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:19<27:16, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:19<30:07, 9.92MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▊        | 4.15G/22.2G [08:19<21:06, 14.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:19<26:22, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [08:19<39:32, 8.01MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:19<29:55, 9.98MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.15G/22.2G [08:19<18:29, 16.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:19<26:53, 11.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.14G/22.2G [08:19<37:35, 8.43MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:19<17:05, 17.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:19<32:42, 9.14MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.18G/22.2G [08:19<38:26, 6.92MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:19<17:06, 17.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:19<24:55, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:19<31:40, 9.43MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.16G/22.2G [08:19<17:21, 17.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:19<24:35, 12.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:19<31:10, 9.58MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.19G/22.2G [08:19<29:45, 8.94MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:19<24:34, 12.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.19G/22.2G [08:19<27:44, 9.59MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:19<24:21, 12.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.19G/22.2G [08:20<24:11, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:20<24:29, 12.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [08:20<1:13:52, 4.29MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:20<29:08, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.19G/22.2G [08:20<24:10, 11.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:20<1:06:31, 4.49MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:20<21:54, 13.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [08:20<44:08, 7.18MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.20G/22.2G [08:20<22:13, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.17G/22.2G [08:20<19:41, 15.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:20<39:59, 7.47MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [08:20<45:18, 6.99MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:20<17:57, 16.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.20G/22.2G [08:20<22:36, 11.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:20<48:39, 6.13MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:20<39:25, 7.58MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:20<16:50, 17.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.15G/22.2G [08:20<42:52, 7.39MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.20G/22.2G [08:20<21:06, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:20<36:11, 8.25MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:20<15:56, 18.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:20<31:48, 9.38MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.20G/22.2G [08:20<21:59, 12.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:20<34:37, 8.62MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.18G/22.2G [08:20<15:21, 19.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [08:20<41:52, 7.56MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:20<28:53, 10.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:21<14:54, 20.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:21<32:33, 9.17MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:21<27:25, 10.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [08:21<38:19, 8.26MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:21<26:34, 11.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [08:21<40:21, 7.85MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:21<39:30, 7.55MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:21<25:44, 11.6MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:21<25:07, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:21<27:14, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:21<34:54, 8.55MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [08:21<46:13, 6.85MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:21<24:44, 12.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:21<25:07, 11.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.20G/22.2G [08:21<46:59, 5.66MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:21<36:42, 8.13MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:21<24:27, 12.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [08:21<47:25, 6.68MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.19G/22.2G [08:21<21:05, 14.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.21G/22.2G [08:21<32:15, 8.24MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:22<24:13, 12.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:22<37:59, 7.86MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:22<18:30, 16.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [08:22<43:54, 7.21MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:22<24:05, 12.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:22<16:58, 17.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.21G/22.2G [08:22<31:10, 8.53MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:22<36:03, 8.28MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.16G/22.2G [08:22<45:32, 6.95MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.20G/22.2G [08:22<15:48, 18.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.21G/22.2G [08:22<30:32, 8.70MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:22<37:49, 7.89MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:22<15:07, 19.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.21G/22.2G [08:22<29:37, 8.97MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [08:22<46:08, 6.86MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:22<38:40, 7.71MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.21G/22.2G [08:22<28:47, 9.23MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [08:22<46:36, 6.79MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:22<47:02, 6.33MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.21G/22.2G [08:22<28:08, 9.44MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.22G/22.2G [08:22<27:43, 9.58MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:22<31:18, 9.51MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.22G/22.2G [08:23<27:08, 9.79MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:23<28:34, 10.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:23<30:51, 9.69MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.22G/22.2G [08:23<26:37, 9.97MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:23<27:19, 10.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.21G/22.2G [08:23<23:21, 12.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:23<1:17:59, 3.83MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.22G/22.2G [08:23<25:57, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:23<26:20, 11.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:23<19:59, 15.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [08:23<1:27:41, 3.61MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.22G/22.2G [08:23<26:38, 9.96MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:23<25:38, 11.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:23<17:51, 16.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:23<45:17, 6.59MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.22G/22.2G [08:23<25:56, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:23<25:05, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.22G/22.2G [08:23<16:27, 18.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:23<40:52, 7.30MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [08:23<51:02, 6.20MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:23<15:24, 19.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:23<24:41, 12.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:23<37:59, 7.85MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:23<14:50, 20.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:23<24:27, 12.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.17G/22.2G [08:23<47:40, 6.63MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:24<39:11, 7.61MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:24<24:25, 12.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:24<36:26, 8.18MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [08:24<45:51, 6.90MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:24<24:09, 12.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.23G/22.2G [08:24<54:03, 4.91MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:24<34:18, 8.69MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [08:24<44:33, 7.10MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.23G/22.2G [08:24<29:15, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:24<35:53, 8.31MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.23G/22.2G [08:24<32:51, 8.08MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:24<21:58, 13.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [08:24<43:29, 7.27MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:24<33:43, 8.84MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.23G/22.2G [08:24<30:08, 8.81MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:24<19:33, 15.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:24<32:47, 9.09MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.23G/22.2G [08:24<28:42, 9.24MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [08:24<42:20, 7.47MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:24<50:15, 5.92MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:24<17:39, 16.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:24<31:07, 9.57MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.23G/22.2G [08:24<26:45, 9.92MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.24G/22.2G [08:25<16:13, 18.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:25<33:01, 9.01MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [08:25<41:24, 7.64MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.24G/22.2G [08:25<26:07, 10.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:25<33:45, 8.83MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:25<15:15, 19.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:25<29:33, 10.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.18G/22.2G [08:25<38:12, 8.27MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:25<32:44, 9.10MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.24G/22.2G [08:25<23:15, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:25<18:19, 16.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:25<31:04, 9.59MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.24G/22.2G [08:25<23:27, 11.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:25<33:37, 8.84MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [08:25<45:24, 6.96MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.24G/22.2G [08:25<23:25, 11.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:25<28:15, 10.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [08:25<49:21, 6.41MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.25G/22.2G [08:25<21:54, 12.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:25<29:08, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.25G/22.2G [08:25<28:44, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.25G/22.2G [08:26<22:04, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:26<1:03:29, 4.69MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:26<28:57, 10.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:26<21:53, 13.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [08:26<50:22, 6.27MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:26<21:24, 13.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:26<30:38, 9.71MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:26<39:54, 7.46MB/s]  [Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:26<28:41, 10.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:26<20:08, 14.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:26<40:12, 7.41MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:26<28:59, 10.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.26G/22.2G [08:26<20:15, 14.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.25G/22.2G [08:26<40:21, 6.57MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:26<19:48, 15.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:26<37:12, 8.01MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:26<29:01, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:26<19:16, 15.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:26<34:56, 8.52MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:26<28:57, 10.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.25G/22.2G [08:26<31:25, 8.44MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:26<34:13, 8.70MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.25G/22.2G [08:26<28:01, 9.46MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:26<20:04, 14.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [08:27<1:38:18, 3.22MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:27<31:12, 9.54MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.26G/22.2G [08:27<25:51, 10.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.27G/22.2G [08:27<17:39, 16.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.26G/22.2G [08:27<24:33, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:27<33:30, 8.89MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.19G/22.2G [08:27<57:13, 5.52MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.26G/22.2G [08:27<24:12, 10.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.32G/22.2G [08:27<1:00:48, 4.89MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:27<31:38, 9.41MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.26G/22.2G [08:27<23:38, 11.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:27<30:41, 9.70MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [08:27<54:19, 5.82MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:27<31:12, 9.55MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.32G/22.2G [08:27<37:28, 7.93MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:27<29:15, 10.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:27<22:55, 13.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:27<33:38, 8.83MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.26G/22.2G [08:27<21:38, 12.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [08:27<52:43, 5.99MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:27<28:41, 10.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.28G/22.2G [08:27<22:43, 13.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:27<31:53, 9.31MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.27G/22.2G [08:27<22:21, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:27<19:15, 15.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:27<31:53, 9.33MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:28<30:25, 9.77MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [08:28<51:35, 6.12MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.27G/22.2G [08:28<21:08, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:28<19:19, 15.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:28<29:19, 10.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [08:28<50:39, 6.24MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:28<18:26, 16.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:28<28:12, 10.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.29G/22.2G [08:28<17:52, 16.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:28<27:08, 10.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [08:28<49:19, 6.40MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:28<26:44, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:28<17:06, 17.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:28<1:04:49, 4.59MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.20G/22.2G [08:28<49:03, 6.44MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:28<26:21, 11.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.27G/22.2G [08:28<42:33, 6.22MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:28<25:47, 11.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:28<37:37, 7.91MB/s]  [A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [08:28<48:07, 6.56MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.27G/22.2G [08:28<30:53, 8.57MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:29<25:22, 11.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:29<37:48, 7.87MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:29<29:51, 9.97MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.28G/22.2G [08:29<28:06, 9.42MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [08:29<47:31, 6.65MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:29<36:08, 8.23MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.30G/22.2G [08:29<21:58, 13.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.28G/22.2G [08:29<24:43, 10.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [08:29<47:00, 6.72MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:29<34:26, 8.64MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:29<21:03, 14.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.28G/22.2G [08:29<24:46, 10.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:29<32:57, 9.02MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:29<18:53, 15.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [08:29<46:37, 6.77MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:29<54:21, 5.46MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:29<31:56, 9.31MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.28G/22.2G [08:29<22:16, 11.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:29<31:22, 9.48MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  14%|█▍        | 3.21G/22.2G [08:29<46:22, 6.81MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:29<33:59, 8.73MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:29<20:56, 14.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.29G/22.2G [08:29<22:34, 11.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:29<30:47, 9.66MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:29<17:13, 17.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:29<30:21, 9.78MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.31G/22.2G [08:30<30:24, 9.78MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.29G/22.2G [08:30<21:20, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:30<17:33, 16.9MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:30<28:36, 10.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:30<29:49, 9.97MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.29G/22.2G [08:30<21:40, 12.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:30<27:03, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:30<19:08, 15.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:30<29:00, 10.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:30<26:05, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:30<28:51, 10.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:30<25:24, 11.7MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  19%|█▉        | 4.32G/22.2G [08:30<27:57, 10.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.21G/22.2G [08:30<1:26:26, 3.65MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:30<24:50, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.29G/22.2G [08:30<37:43, 7.01MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:30<25:44, 11.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.32G/22.2G [08:30<34:34, 8.60MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [08:30<51:54, 6.08MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:30<24:49, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.30G/22.2G [08:31<27:15, 9.70MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.32G/22.2G [08:31<30:26, 9.76MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [08:31<49:14, 6.41MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.30G/22.2G [08:31<25:51, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:31<23:17, 12.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:31<26:29, 11.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.32G/22.2G [08:31<1:04:03, 4.64MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [08:31<48:25, 6.52MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.30G/22.2G [08:31<23:10, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:31<24:07, 12.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.32G/22.2G [08:31<39:48, 7.46MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.30G/22.2G [08:31<23:07, 11.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [08:31<47:16, 6.67MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:31<36:47, 8.08MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:31<22:43, 13.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.31G/22.2G [08:31<21:42, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:31<35:03, 8.47MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:31<46:39, 6.35MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [08:31<46:22, 6.80MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:31<21:46, 13.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.31G/22.2G [08:31<22:10, 11.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:31<33:39, 8.83MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:32<34:17, 8.65MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.22G/22.2G [08:32<46:17, 6.82MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:32<20:57, 14.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:32<32:22, 9.18MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.31G/22.2G [08:32<20:45, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:32<30:22, 9.76MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:32<32:01, 9.28MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [08:32<45:38, 6.91MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  28%|██▊       | 6.31G/22.2G [08:32<21:25, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:32<28:22, 10.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:32<31:14, 9.51MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:32<26:54, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.33G/22.2G [08:32<30:38, 9.69MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [08:32<46:50, 6.73MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:32<26:06, 11.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:32<30:11, 9.83MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [08:32<44:24, 7.10MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:32<26:33, 11.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:32<36:03, 8.23MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:32<32:32, 9.13MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.31G/22.2G [08:32<36:13, 7.29MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:32<26:15, 11.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [08:32<44:48, 7.04MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:32<23:52, 12.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:32<25:35, 11.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:33<28:42, 10.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.32G/22.2G [08:33<26:28, 9.97MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:33<24:35, 12.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [08:33<44:33, 7.08MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:33<28:49, 10.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:33<24:10, 12.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.32G/22.2G [08:33<25:14, 10.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:33<24:19, 12.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:33<22:42, 13.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [08:33<44:52, 7.03MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.32G/22.2G [08:33<22:37, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:33<20:15, 14.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.33G/22.2G [08:33<22:52, 11.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:33<20:30, 14.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.34G/22.2G [08:33<56:14, 5.28MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:33<18:58, 15.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.33G/22.2G [08:33<21:14, 12.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:33<44:45, 6.62MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:33<19:26, 15.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.33G/22.2G [08:33<21:56, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:33<35:06, 8.45MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:34<32:43, 9.07MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.23G/22.2G [08:34<1:29:52, 3.51MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:34<31:22, 9.43MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.33G/22.2G [08:34<20:25, 12.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:34<31:54, 9.30MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:34<28:19, 10.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.33G/22.2G [08:34<21:08, 12.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:34<30:54, 9.60MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:34<27:10, 10.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [08:34<52:56, 5.96MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:34<40:00, 7.41MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:34<30:06, 9.86MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:34<25:49, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [08:34<49:45, 6.34MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:34<26:36, 11.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.35G/22.2G [08:34<29:34, 10.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:34<25:13, 11.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:34<25:34, 11.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:34<29:07, 10.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:34<24:20, 12.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [08:34<48:44, 6.47MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.34G/22.2G [08:34<38:00, 6.94MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:34<29:22, 10.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:34<24:08, 12.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:34<23:50, 12.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.24G/22.2G [08:35<47:22, 6.65MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:35<28:36, 10.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:35<20:52, 14.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:35<23:31, 12.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.34G/22.2G [08:35<27:22, 9.63MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:35<28:00, 10.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:35<21:02, 14.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:35<23:17, 12.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.34G/22.2G [08:35<25:52, 10.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [08:35<46:30, 6.78MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:35<27:57, 10.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:35<19:09, 15.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:35<23:05, 12.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.35G/22.2G [08:35<22:51, 11.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:35<27:56, 10.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:35<19:31, 15.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [08:35<45:15, 6.96MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:35<18:22, 16.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.35G/22.2G [08:35<21:50, 12.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [08:35<44:50, 7.03MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:35<18:55, 15.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.35G/22.2G [08:35<22:07, 11.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:35<46:22, 6.38MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [08:35<44:02, 7.15MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.35G/22.2G [08:36<20:35, 12.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.36G/22.2G [08:36<55:12, 5.37MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:36<32:12, 9.18MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [08:36<43:28, 7.25MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.36G/22.2G [08:36<21:13, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:36<35:23, 8.37MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:36<28:34, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:36<33:50, 8.76MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:36<28:20, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.25G/22.2G [08:36<43:04, 7.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.36G/22.2G [08:36<20:06, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:36<26:46, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:36<31:39, 9.36MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:36<27:41, 10.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:36<25:41, 11.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:36<30:42, 9.65MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [08:36<42:22, 7.43MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:36<24:22, 12.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:36<29:39, 9.99MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:36<24:35, 12.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:36<20:31, 14.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.37G/22.2G [08:36<28:29, 10.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:36<23:51, 12.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:36<20:54, 14.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.36G/22.2G [08:37<36:08, 7.29MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:37<23:31, 12.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:37<25:35, 11.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:37<18:46, 15.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.43G/22.2G [08:37<23:04, 12.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.36G/22.2G [08:37<30:10, 8.72MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:37<25:56, 11.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:37<19:24, 15.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.43G/22.2G [08:37<22:54, 12.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:37<26:11, 11.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.36G/22.2G [08:37<26:48, 9.81MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:37<17:54, 16.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|█▉        | 4.43G/22.2G [08:37<22:37, 13.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [08:37<1:17:24, 4.07MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.38G/22.2G [08:37<24:02, 12.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▊       | 6.37G/22.2G [08:37<23:34, 11.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [08:37<46:29, 6.77MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:37<24:32, 12.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.37G/22.2G [08:37<21:58, 12.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [08:37<47:11, 6.67MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.43G/22.2G [08:37<40:47, 7.24MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.37G/22.2G [08:38<20:52, 12.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:38<30:41, 9.62MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:38<44:27, 6.65MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.26G/22.2G [08:38<44:29, 7.08MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:38<45:26, 6.52MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.38G/22.2G [08:38<20:09, 13.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:38<27:23, 10.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:38<31:49, 9.29MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [08:38<42:36, 7.39MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:38<26:11, 11.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:38<30:18, 9.77MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:38<29:56, 9.87MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.38G/22.2G [08:38<19:28, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [08:38<39:43, 7.93MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:38<24:56, 11.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:38<24:31, 12.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.39G/22.2G [08:38<27:00, 11.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:38<24:18, 12.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [08:38<40:10, 7.84MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:38<24:10, 12.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:38<26:43, 11.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:38<23:35, 12.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:38<20:44, 14.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [08:38<37:16, 8.44MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:38<23:18, 12.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:38<20:53, 14.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:39<24:28, 12.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [08:39<35:03, 8.98MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:39<22:55, 12.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.43G/22.2G [08:39<19:04, 15.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.27G/22.2G [08:39<33:49, 9.30MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:39<22:56, 12.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|█▉        | 4.43G/22.2G [08:39<19:28, 15.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:39<22:49, 12.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.38G/22.2G [08:39<41:50, 6.28MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [08:39<33:40, 9.34MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.43G/22.2G [08:39<18:13, 16.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:39<22:33, 13.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.40G/22.2G [08:39<22:05, 13.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [08:39<33:31, 9.38MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.38G/22.2G [08:39<35:03, 7.50MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:39<19:59, 14.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [08:39<32:54, 9.56MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.39G/22.2G [08:39<32:10, 8.17MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.39G/22.2G [08:39<26:47, 9.81MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.43G/22.2G [08:39<34:19, 8.61MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:39<43:42, 6.75MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:40<28:36, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.39G/22.2G [08:40<23:43, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:40<31:46, 9.28MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:40<27:50, 10.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:40<39:45, 7.44MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.40G/22.2G [08:40<19:55, 13.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:40<27:43, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [08:40<1:12:18, 4.35MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:40<24:14, 12.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:40<28:06, 10.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.40G/22.2G [08:40<20:39, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:40<25:55, 11.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:40<20:24, 14.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.41G/22.2G [08:40<27:26, 10.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [08:40<43:02, 7.31MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.40G/22.2G [08:40<17:57, 14.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:40<24:49, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:40<20:47, 14.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.40G/22.2G [08:40<19:00, 13.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:40<24:50, 11.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:40<30:57, 9.55MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:40<18:59, 15.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.28G/22.2G [08:40<48:13, 6.52MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:40<25:47, 11.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:40<19:37, 15.0MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:41<30:32, 9.65MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:41<26:05, 11.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:41<17:55, 16.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [08:41<47:08, 6.67MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:41<22:52, 12.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.40G/22.2G [08:41<33:36, 7.81MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:41<24:25, 12.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [08:41<45:09, 6.96MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:41<23:51, 12.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.41G/22.2G [08:41<26:07, 10.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.42G/22.2G [08:41<25:00, 11.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:41<24:54, 11.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [08:41<43:17, 7.26MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.41G/22.2G [08:41<25:46, 10.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.43G/22.2G [08:41<24:55, 11.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:41<26:22, 11.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:41<37:31, 7.86MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.41G/22.2G [08:41<20:03, 13.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [08:41<40:33, 7.75MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.43G/22.2G [08:41<23:41, 12.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:41<25:38, 11.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.41G/22.2G [08:41<20:43, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|█▉        | 4.43G/22.2G [08:41<24:19, 12.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [08:41<40:42, 7.72MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:41<24:44, 11.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.42G/22.2G [08:41<16:55, 15.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.29G/22.2G [08:42<38:10, 8.23MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.42G/22.2G [08:42<17:57, 14.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:42<23:17, 12.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:42<48:48, 6.04MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.42G/22.2G [08:42<15:59, 16.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:42<20:09, 14.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [08:42<38:56, 8.07MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.43G/22.2G [08:42<42:58, 6.87MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:42<34:54, 8.44MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:42<20:21, 14.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [08:42<35:43, 8.80MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:42<31:36, 9.32MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:42<18:13, 16.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:42<32:01, 9.22MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [08:42<34:20, 9.15MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:42<30:15, 9.73MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:42<18:36, 15.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [08:42<33:36, 9.35MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:42<27:37, 10.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:42<28:52, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:42<17:53, 16.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.42G/22.2G [08:42<26:45, 9.80MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:42<28:07, 10.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:42<25:34, 11.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.43G/22.2G [08:42<20:03, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:43<26:59, 10.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.43G/22.2G [08:43<20:11, 13.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.44G/22.2G [08:43<23:56, 12.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:43<26:23, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.43G/22.2G [08:43<17:28, 15.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:43<25:51, 11.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:43<36:03, 8.17MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.30G/22.2G [08:43<1:11:44, 4.38MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:43<22:49, 12.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.44G/22.2G [08:43<15:45, 16.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:43<25:29, 11.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:43<25:20, 11.6MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.44G/22.2G [08:43<14:19, 18.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:43<22:00, 13.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [08:43<44:26, 7.07MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:43<24:58, 11.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:43<25:41, 11.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.44G/22.2G [08:43<13:59, 18.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:43<24:31, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:43<21:22, 13.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [08:43<47:35, 6.60MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:43<30:58, 9.51MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.45G/22.2G [08:43<17:15, 15.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:44<26:33, 11.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [08:44<46:05, 6.81MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.45G/22.2G [08:44<33:03, 8.92MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:44<47:47, 6.16MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:44<24:27, 12.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [08:44<45:55, 6.84MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:44<29:45, 9.91MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:44<24:45, 11.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.45G/22.2G [08:44<30:18, 8.64MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:44<31:59, 9.20MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [08:44<45:18, 6.93MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:44<24:46, 11.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.45G/22.2G [08:44<22:23, 11.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:44<26:06, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:44<28:03, 10.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.45G/22.2G [08:44<22:32, 11.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:44<23:01, 12.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [08:44<44:50, 7.00MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.46G/22.2G [08:44<24:10, 12.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:44<26:41, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:44<23:33, 12.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.46G/22.2G [08:44<20:53, 12.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:44<25:38, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.31G/22.2G [08:45<46:46, 6.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:45<23:52, 12.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.46G/22.2G [08:45<18:03, 14.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:45<24:46, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.46G/22.2G [08:45<18:53, 13.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:45<24:11, 12.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [08:45<48:59, 6.41MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:45<23:38, 12.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:45<42:03, 7.00MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.46G/22.2G [08:45<16:58, 15.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:45<23:40, 12.4MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [08:45<43:13, 7.26MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:45<20:47, 14.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.47G/22.2G [08:45<18:05, 14.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:45<22:57, 12.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:45<28:55, 10.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.47G/22.2G [08:45<21:25, 13.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [08:45<44:18, 7.09MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.47G/22.2G [08:45<16:13, 16.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:45<22:58, 12.8MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:45<25:37, 11.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [08:45<40:47, 7.70MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:45<23:11, 12.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:45<25:20, 11.6MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▍        | 3.32G/22.2G [08:46<43:17, 7.25MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:46<23:34, 12.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:46<38:35, 7.63MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.47G/22.2G [08:46<30:29, 8.58MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:46<22:18, 13.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.47G/22.2G [08:46<24:20, 10.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:46<30:55, 9.53MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:46<19:58, 14.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.48G/22.2G [08:46<23:58, 10.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:46<52:45, 5.57MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:46<25:25, 11.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:46<20:26, 14.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.48G/22.2G [08:46<25:15, 11.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:46<39:08, 7.51MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.48G/22.2G [08:46<20:24, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:46<20:40, 14.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.32G/22.2G [08:46<1:27:30, 3.59MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:46<21:21, 13.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:46<32:43, 8.98MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.48G/22.2G [08:46<17:49, 14.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:46<21:52, 13.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [08:47<47:29, 6.61MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:47<29:54, 9.82MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:47<21:06, 13.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:47<27:06, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.49G/22.2G [08:47<19:47, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:47<33:24, 8.80MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [08:47<47:53, 6.55MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.49G/22.2G [08:47<19:06, 15.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:47<24:41, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:47<28:13, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.49G/22.2G [08:47<17:41, 14.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [08:47<45:18, 6.92MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.49G/22.2G [08:47<18:29, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.54G/22.2G [08:47<23:28, 12.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:47<24:47, 11.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [08:47<43:19, 7.24MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:47<21:25, 13.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  20%|██        | 4.54G/22.2G [08:47<22:46, 12.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:47<21:48, 13.5MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.33G/22.2G [08:47<41:40, 7.53MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:47<34:30, 8.53MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.49G/22.2G [08:47<29:31, 8.84MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:48<21:02, 14.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:48<24:56, 11.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [08:48<40:25, 7.76MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.50G/22.2G [08:48<23:47, 11.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.54G/22.2G [08:48<18:58, 15.5MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.50G/22.2G [08:48<24:30, 12.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.50G/22.2G [08:48<24:19, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  20%|██        | 4.54G/22.2G [08:48<19:36, 15.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [08:48<39:42, 7.90MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:48<21:49, 13.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.50G/22.2G [08:48<24:10, 10.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.54G/22.2G [08:48<17:59, 16.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:48<21:58, 13.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [08:48<36:56, 8.49MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.54G/22.2G [08:48<38:43, 7.58MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:48<19:57, 14.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.50G/22.2G [08:48<21:26, 12.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [08:48<39:30, 7.94MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:48<29:39, 9.90MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.51G/22.2G [08:48<20:38, 14.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.50G/22.2G [08:48<20:29, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:48<26:56, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:48<18:31, 15.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.54G/22.2G [08:48<32:21, 9.07MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [08:48<39:40, 7.90MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:49<25:38, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:49<19:23, 15.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:49<27:17, 10.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.51G/22.2G [08:49<19:55, 13.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.34G/22.2G [08:49<35:20, 8.87MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:49<24:28, 12.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:49<18:19, 16.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:49<27:01, 10.9MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.51G/22.2G [08:49<20:11, 12.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:49<24:44, 11.9MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.51G/22.2G [08:49<19:23, 13.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:49<22:20, 13.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:49<28:29, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:49<21:59, 13.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:49<23:32, 12.5MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:49<22:37, 13.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:49<24:24, 12.0MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:50<23:41, 12.4MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:50<23:49, 12.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.51G/22.2G [08:50<42:40, 6.11MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:50<24:03, 12.2MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:50<24:48, 11.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:50<35:28, 8.27MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:50<1:05:55, 4.46MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.52G/22.2G [08:50<33:06, 7.87MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:50<27:41, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.52G/22.2G [08:50<30:53, 8.43MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:50<27:07, 10.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.52G/22.2G [08:50<54:14, 5.42MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.52G/22.2G [08:50<26:04, 9.99MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:50<28:20, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:50<48:24, 6.07MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:50<27:36, 10.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:51<42:53, 6.85MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:51<45:28, 6.45MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.53G/22.2G [08:51<23:16, 11.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:51<30:51, 9.50MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:51<33:25, 8.79MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:51<33:20, 8.79MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.53G/22.2G [08:51<21:16, 12.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:51<29:28, 9.94MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:51<26:05, 11.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:51<34:36, 8.47MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.53G/22.2G [08:51<25:43, 11.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.53G/22.2G [08:51<20:03, 13.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:51<32:11, 9.10MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:51<32:58, 8.89MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.54G/22.2G [08:51<21:18, 13.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:51<31:28, 9.31MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:51<29:58, 9.77MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  29%|██▉       | 6.54G/22.2G [08:51<19:34, 13.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  20%|██        | 4.54G/22.2G [08:51<18:38, 15.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:51<29:00, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:51<33:09, 8.83MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:51<29:10, 10.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:52<31:01, 9.44MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:52<29:29, 9.93MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:52<32:57, 8.89MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.54G/22.2G [08:52<32:03, 9.16MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.54G/22.2G [08:52<32:56, 7.90MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:52<30:38, 9.56MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:52<32:12, 9.10MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:52<24:19, 12.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.54G/22.2G [08:52<26:21, 9.88MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:52<31:00, 9.45MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:52<28:41, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:52<20:36, 14.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.54G/22.2G [08:52<26:14, 9.92MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:52<29:43, 9.85MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.55G/22.2G [08:52<18:08, 16.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:52<28:33, 10.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.54G/22.2G [08:52<22:59, 11.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:52<16:22, 17.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:52<28:06, 10.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:52<13:39, 21.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.55G/22.2G [08:53<21:03, 12.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:53<57:29, 5.09MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:53<31:00, 9.45MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.55G/22.2G [08:53<20:00, 13.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:53<38:57, 7.51MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.55G/22.2G [08:53<17:28, 14.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:53<35:02, 8.36MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.56G/22.2G [08:53<22:33, 13.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.56G/22.2G [08:53<18:25, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:53<33:45, 8.67MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:53<1:01:28, 4.76MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:53<18:32, 15.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:53<32:13, 9.08MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.57G/22.2G [08:53<15:19, 19.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:53<37:23, 7.83MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:53<31:09, 9.39MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:53<13:20, 22.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:53<34:26, 8.50MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:54<30:20, 9.64MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.58G/22.2G [08:54<11:50, 24.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:54<30:07, 9.71MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.56G/22.2G [08:54<34:41, 7.49MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:54<35:46, 8.18MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [08:54<5:49:13, 898kB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:54<29:39, 9.86MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:54<32:55, 8.89MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.56G/22.2G [08:54<23:57, 10.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:54<28:53, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:54<30:50, 9.49MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.56G/22.2G [08:54<23:52, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:54<29:18, 9.99MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:54<18:15, 16.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:54<28:42, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.57G/22.2G [08:54<21:47, 11.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:54<28:33, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.59G/22.2G [08:54<14:41, 19.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:54<30:33, 9.57MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [08:54<3:06:58, 1.68MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:54<28:30, 10.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.57G/22.2G [08:54<20:28, 12.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:54<29:42, 9.85MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [08:54<2:37:45, 1.99MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:55<15:44, 18.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:55<29:04, 10.1MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.57G/22.2G [08:55<23:07, 11.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:55<14:51, 19.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:55<37:21, 7.83MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:55<28:30, 10.3MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [08:55<2:01:59, 2.57MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.57G/22.2G [08:55<20:28, 12.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.60G/22.2G [08:55<13:03, 22.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:55<25:35, 11.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.58G/22.2G [08:55<20:44, 12.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:55<12:57, 22.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.35G/22.2G [08:55<1:37:47, 3.20MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.58G/22.2G [08:55<21:26, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:55<29:25, 9.94MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.58G/22.2G [08:55<21:37, 12.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [08:55<1:23:56, 3.73MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:55<1:14:17, 3.94MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [08:55<1:14:53, 4.18MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:56<47:20, 6.18MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:56<58:24, 5.01MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [08:56<1:10:04, 4.47MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.58G/22.2G [08:56<39:35, 6.56MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:56<43:19, 6.75MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:56<32:23, 9.03MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.58G/22.2G [08:56<29:33, 8.78MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [08:56<1:03:09, 4.96MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:56<41:01, 7.13MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:56<29:40, 9.85MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.59G/22.2G [08:56<27:01, 9.60MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:56<48:30, 6.03MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.61G/22.2G [08:56<40:25, 7.23MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [08:56<58:43, 5.33MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:56<24:44, 11.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.59G/22.2G [08:56<26:13, 9.89MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:56<39:15, 7.45MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:56<22:57, 12.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.36G/22.2G [08:56<55:26, 5.65MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:56<42:14, 6.92MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.59G/22.2G [08:56<22:53, 11.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<20:28, 14.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<38:24, 7.61MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.59G/22.2G [08:57<23:02, 11.3MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [08:57<52:58, 5.91MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<18:34, 15.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<44:43, 6.54MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.59G/22.2G [08:57<22:41, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<38:09, 7.66MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:57<17:03, 17.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [08:57<50:19, 6.22MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<34:42, 8.42MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.60G/22.2G [08:57<21:09, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<46:04, 6.34MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<35:48, 8.16MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.60G/22.2G [08:57<20:15, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<36:34, 7.99MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:57<27:03, 10.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<46:09, 6.33MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:57<33:43, 8.66MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:57<20:58, 13.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.62G/22.2G [08:58<46:17, 6.31MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [08:58<1:39:40, 3.14MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [08:58<18:44, 15.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:58<35:08, 8.31MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [08:58<17:12, 17.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:58<32:44, 8.92MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:58<44:45, 6.53MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [08:58<16:08, 18.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.60G/22.2G [08:58<38:02, 6.81MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [08:58<54:34, 5.74MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:58<34:47, 8.39MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:58<45:21, 6.44MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [08:58<15:32, 18.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.61G/22.2G [08:58<27:44, 9.34MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.37G/22.2G [08:58<50:02, 6.26MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [08:58<15:02, 19.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:58<45:04, 6.48MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.61G/22.2G [08:58<24:09, 10.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [08:58<47:52, 6.54MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.61G/22.2G [08:58<23:40, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [08:59<45:14, 6.92MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:59<1:10:14, 4.16MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.61G/22.2G [08:59<21:32, 12.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [08:59<43:49, 7.14MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.62G/22.2G [08:59<20:14, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:59<41:53, 6.97MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:59<1:28:48, 3.29MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [08:59<46:40, 6.70MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [08:59<38:00, 7.68MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.62G/22.2G [08:59<19:21, 13.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [08:59<40:03, 7.28MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [08:59<37:35, 7.77MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [08:59<43:06, 7.26MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.63G/22.2G [08:59<50:34, 5.77MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [08:59<30:39, 9.51MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [08:59<35:05, 8.32MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:00<25:40, 11.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [09:00<47:48, 6.11MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.38G/22.2G [09:00<47:57, 6.52MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [09:00<35:27, 8.23MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:00<22:05, 13.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.62G/22.2G [09:00<28:31, 9.07MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [09:00<33:37, 8.68MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [09:00<46:53, 6.23MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:00<19:33, 14.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [09:00<47:23, 6.60MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.63G/22.2G [09:00<21:34, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [09:00<33:58, 8.59MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:00<17:52, 16.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [09:00<45:43, 6.38MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.63G/22.2G [09:00<21:49, 11.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:00<16:23, 17.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:00<32:30, 8.98MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [09:00<50:12, 6.23MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [09:00<45:23, 6.43MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.63G/22.2G [09:00<20:58, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:00<33:18, 8.76MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [09:00<48:58, 6.39MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [09:00<44:31, 6.55MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:00<31:58, 9.13MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.64G/22.2G [09:00<19:53, 13.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:01<25:45, 11.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:01<32:53, 8.87MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [09:01<43:55, 6.64MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:01<20:01, 14.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.64G/22.2G [09:01<19:11, 13.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:01<31:43, 9.19MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:01<18:23, 15.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.64G/22.2G [09:01<43:32, 6.70MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.64G/22.2G [09:01<18:45, 13.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:01<17:02, 17.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:01<43:10, 6.76MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|██▉       | 6.65G/22.2G [09:01<16:49, 15.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:01<15:55, 18.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [09:01<1:33:16, 3.35MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:01<15:12, 19.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:01<43:29, 6.71MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:01<13:27, 21.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:01<43:13, 6.75MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.39G/22.2G [09:01<54:51, 5.70MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:02<1:08:07, 4.28MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:02<43:21, 6.73MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [09:02<52:43, 5.93MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:02<41:27, 7.03MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:02<24:20, 12.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:02<43:02, 6.78MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:02<37:34, 7.76MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [09:02<51:25, 6.08MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:02<20:01, 14.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:02<37:21, 7.80MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [09:02<50:22, 6.21MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:02<17:59, 16.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:02<34:29, 8.45MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██        | 4.71G/22.2G [09:02<16:39, 17.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [09:02<49:04, 6.37MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:02<35:04, 8.31MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:02<15:34, 18.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:03<33:15, 8.77MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:03<14:56, 19.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [09:03<48:34, 6.44MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.65G/22.2G [09:03<1:00:41, 4.26MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:03<14:26, 20.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:03<34:08, 8.54MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.65G/22.2G [09:03<1:39:53, 2.92MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [09:03<47:03, 6.64MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:03<32:32, 8.96MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:03<30:54, 9.43MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.40G/22.2G [09:03<47:20, 6.60MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:03<54:49, 5.32MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.65G/22.2G [09:03<53:40, 4.81MB/s]  [A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:03<33:17, 8.75MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [09:03<46:44, 6.68MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:03<26:50, 10.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.65G/22.2G [09:03<46:11, 5.59MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:03<50:49, 5.74MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:03<31:48, 9.16MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:03<20:42, 14.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [09:04<46:18, 6.75MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:04<48:43, 5.98MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.66G/22.2G [09:04<35:24, 7.30MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:04<33:29, 8.70MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:04<18:29, 15.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:04<31:42, 9.19MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:04<16:50, 17.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:04<46:28, 6.27MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [09:04<46:36, 6.70MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.66G/22.2G [09:04<29:09, 8.86MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:04<15:52, 18.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.66G/22.2G [09:04<23:56, 10.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:04<45:21, 6.43MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [09:04<46:45, 6.68MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:04<15:09, 19.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.66G/22.2G [09:04<23:55, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:04<14:33, 19.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.66G/22.2G [09:04<44:35, 6.54MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.67G/22.2G [09:04<20:26, 12.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:04<1:04:19, 4.53MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.67G/22.2G [09:04<21:00, 12.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:04<43:55, 6.64MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:05<42:39, 6.83MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:05<43:25, 6.71MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:05<26:32, 10.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:05<38:21, 7.59MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:05<43:08, 6.75MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:05<20:30, 14.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.67G/22.2G [09:05<35:37, 7.25MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:05<37:51, 7.69MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:05<18:13, 15.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.67G/22.2G [09:05<27:13, 9.48MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:05<42:52, 6.80MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:05<35:14, 8.26MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.41G/22.2G [09:05<1:45:03, 2.97MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:05<16:35, 17.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.67G/22.2G [09:05<26:50, 9.61MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:05<32:33, 8.94MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:05<42:07, 6.92MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:05<15:28, 18.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:05<34:04, 8.55MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.68G/22.2G [09:05<22:52, 11.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:05<14:55, 19.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:05<42:11, 6.91MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [09:05<1:03:30, 4.92MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:05<31:59, 9.10MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.68G/22.2G [09:06<19:22, 13.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:06<14:19, 20.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:06<30:32, 9.53MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.68G/22.2G [09:06<20:14, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.67G/22.2G [09:06<42:10, 6.91MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [09:06<1:01:36, 5.07MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:06<32:32, 8.94MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.69G/22.2G [09:06<19:32, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:06<30:36, 9.51MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.69G/22.2G [09:06<17:07, 15.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.76G/22.2G [09:06<26:35, 10.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [09:06<1:00:17, 5.18MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:06<32:26, 8.97MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.69G/22.2G [09:06<18:17, 14.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:06<20:35, 14.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:06<1:19:43, 3.65MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:06<35:34, 8.18MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:06<17:41, 16.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [09:06<58:22, 5.35MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:06<16:20, 17.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:07<49:14, 5.91MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:07<15:28, 18.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [09:07<59:42, 5.23MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.69G/22.2G [09:07<35:50, 7.19MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:07<14:51, 19.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:07<45:38, 6.38MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:07<14:18, 20.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.69G/22.2G [09:07<26:02, 9.89MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:07<44:17, 6.58MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.42G/22.2G [09:07<1:05:19, 4.78MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.70G/22.2G [09:07<25:53, 9.95MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.68G/22.2G [09:07<42:46, 6.81MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:07<1:13:50, 3.94MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.70G/22.2G [09:07<22:59, 11.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [09:07<1:07:22, 4.63MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.70G/22.2G [09:07<19:36, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:07<41:09, 7.07MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:07<27:42, 10.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:08<45:07, 6.45MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.70G/22.2G [09:08<20:25, 12.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:08<21:04, 13.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:08<40:29, 7.19MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:08<42:40, 6.81MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.71G/22.2G [09:08<17:49, 14.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:08<18:40, 15.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [09:08<1:07:06, 4.65MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.71G/22.2G [09:08<18:44, 13.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:08<39:31, 7.37MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:08<16:55, 17.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:08<41:49, 6.95MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.71G/22.2G [09:08<16:39, 15.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:08<15:45, 18.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [09:08<1:05:19, 4.78MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:08<38:23, 7.58MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.71G/22.2G [09:08<40:40, 7.15MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:08<14:56, 19.4MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:08<14:15, 20.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:08<38:12, 7.62MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██        | 4.71G/22.2G [09:08<40:08, 7.24MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [09:08<1:04:37, 4.83MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:08<36:15, 8.02MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.69G/22.2G [09:08<35:15, 8.26MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.71G/22.2G [09:09<32:12, 7.99MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:09<37:12, 7.82MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:09<37:22, 7.79MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [09:09<1:02:42, 4.98MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.72G/22.2G [09:09<23:06, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:09<23:39, 12.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:09<34:08, 8.52MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:09<37:34, 7.74MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.72G/22.2G [09:09<23:07, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:09<18:35, 15.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:09<34:25, 8.44MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  15%|█▌        | 3.43G/22.2G [09:09<1:02:47, 4.97MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:09<16:58, 17.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.72G/22.2G [09:09<21:14, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:09<35:11, 8.26MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:09<15:32, 18.6MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:09<14:54, 19.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.73G/22.2G [09:09<19:50, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:09<37:07, 7.83MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:09<1:04:59, 4.48MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:09<14:27, 20.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.73G/22.2G [09:10<18:54, 13.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:10<13:58, 20.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:10<40:34, 7.17MB/s]  

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:10<36:53, 7.88MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.73G/22.2G [09:10<16:38, 15.4MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.73G/22.2G [09:10<17:34, 14.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:10<40:46, 7.13MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.43G/22.2G [09:10<2:05:34, 2.48MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.70G/22.2G [09:10<37:48, 7.69MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:10<25:35, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.71G/22.2G [09:10<32:23, 8.98MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:10<19:59, 14.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██        | 4.71G/22.2G [09:10<31:40, 9.18MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [09:10<1:09:52, 4.46MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:10<17:56, 16.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:10<1:19:01, 3.68MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:10<31:02, 9.37MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.74G/22.2G [09:10<35:10, 7.31MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:11<16:20, 17.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:11<30:13, 9.62MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.74G/22.2G [09:11<24:10, 10.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [09:11<1:06:59, 4.66MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:11<15:20, 18.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:11<46:22, 6.26MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:11<29:37, 9.82MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.74G/22.2G [09:11<23:55, 10.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:11<14:43, 19.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.71G/22.2G [09:11<29:13, 9.95MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:11<14:09, 20.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [09:11<1:05:58, 4.73MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.74G/22.2G [09:11<21:19, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:11<46:57, 6.19MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:11<28:35, 10.2MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:11<27:46, 10.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:11<44:20, 6.55MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.75G/22.2G [09:11<19:42, 13.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [09:11<1:06:39, 4.68MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:11<27:09, 10.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:11<42:37, 6.81MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:11<24:34, 11.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.75G/22.2G [09:11<18:48, 13.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:12<19:09, 15.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.75G/22.2G [09:12<16:40, 15.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.44G/22.2G [09:12<1:04:52, 4.81MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:12<41:42, 6.96MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:12<17:28, 16.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  30%|███       | 6.76G/22.2G [09:12<17:36, 14.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:12<41:01, 7.08MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:12<51:43, 5.62MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:12<16:08, 17.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [09:12<1:03:33, 4.91MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:12<15:19, 18.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:12<40:30, 7.17MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.72G/22.2G [09:12<34:13, 8.49MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:12<14:35, 19.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:12<40:31, 7.17MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [09:12<1:02:38, 4.98MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:12<30:06, 9.65MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.76G/22.2G [09:12<31:05, 8.25MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:12<12:50, 22.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:12<28:25, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:12<40:05, 7.24MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.76G/22.2G [09:12<22:18, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:12<26:39, 10.9MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [09:12<1:02:40, 4.97MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.76G/22.2G [09:13<22:27, 11.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:13<40:15, 7.21MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:13<26:27, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.77G/22.2G [09:13<18:55, 13.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:13<24:08, 12.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:13<39:50, 7.29MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [09:13<1:03:21, 4.92MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:13<24:47, 11.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.77G/22.2G [09:13<19:35, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.73G/22.2G [09:13<23:35, 12.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:13<21:23, 13.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:13<39:32, 7.34MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.77G/22.2G [09:13<17:05, 15.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:13<22:41, 12.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [09:13<1:02:25, 4.99MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:13<18:51, 15.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.77G/22.2G [09:13<18:05, 14.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:13<39:37, 7.32MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:13<22:07, 13.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:13<17:04, 16.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.78G/22.2G [09:13<16:09, 15.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:13<15:45, 18.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.45G/22.2G [09:13<1:01:48, 5.04MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.78G/22.2G [09:14<17:20, 14.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:14<18:29, 15.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [09:14<1:00:05, 5.19MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:14<16:26, 17.5MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:14<15:42, 18.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:14<1:22:32, 3.52MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.78G/22.2G [09:14<25:58, 9.87MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.74G/22.2G [09:14<44:13, 6.56MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.78G/22.2G [09:14<20:37, 12.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:14<48:19, 6.00MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.79G/22.2G [09:14<20:55, 12.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:14<31:25, 9.23MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:14<29:08, 9.95MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:14<27:15, 10.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.79G/22.2G [09:14<17:28, 14.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:14<44:41, 6.49MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:14<28:29, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.79G/22.2G [09:14<18:28, 13.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:14<24:19, 11.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:15<43:15, 6.71MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:15<27:52, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.79G/22.2G [09:15<16:35, 15.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:15<22:48, 12.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [09:15<1:52:14, 2.78MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:15<41:44, 6.95MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:15<27:07, 10.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.80G/22.2G [09:15<14:43, 17.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:15<20:00, 14.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:15<26:28, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.80G/22.2G [09:15<15:44, 16.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:15<40:55, 7.09MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:15<17:54, 16.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:15<25:42, 11.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.80G/22.2G [09:15<15:09, 16.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [09:15<1:06:10, 4.71MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:15<16:33, 17.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:15<40:19, 7.19MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:15<25:10, 11.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:15<17:04, 16.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:15<24:55, 11.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.75G/22.2G [09:15<39:29, 7.34MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:15<17:05, 16.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [09:15<1:03:38, 4.90MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.80G/22.2G [09:15<24:41, 10.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:15<24:22, 11.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:16<39:19, 7.37MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:16<23:56, 12.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.81G/22.2G [09:16<21:30, 11.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [09:16<1:02:45, 4.96MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:16<39:39, 7.31MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.81G/22.2G [09:16<21:19, 12.0MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:16<31:24, 9.15MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.46G/22.2G [09:16<59:48, 5.21MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.81G/22.2G [09:16<17:33, 14.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:16<39:51, 7.27MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:16<22:29, 12.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.81G/22.2G [09:16<15:36, 16.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.76G/22.2G [09:16<45:26, 6.38MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:16<39:26, 7.35MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [09:16<1:01:04, 5.10MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:16<20:46, 13.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.82G/22.2G [09:16<14:23, 17.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:16<31:22, 9.23MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:16<39:31, 7.33MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:16<18:22, 15.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.82G/22.2G [09:16<14:06, 18.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [09:16<58:09, 5.35MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:16<28:02, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:16<16:52, 17.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.82G/22.2G [09:17<13:48, 18.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  21%|██▏       | 4.76G/22.2G [09:17<39:25, 7.35MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:17<26:15, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:17<15:57, 18.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [09:17<58:51, 5.29MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:17<24:56, 11.6MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:17<15:12, 18.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:17<24:34, 11.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [09:17<58:11, 5.35MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:17<25:54, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.82G/22.2G [09:17<30:58, 8.25MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.76G/22.2G [09:17<1:19:57, 3.62MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [09:17<52:28, 5.93MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:17<25:41, 11.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.83G/22.2G [09:17<25:07, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:17<25:15, 11.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.83G/22.2G [09:18<24:44, 10.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:18<32:42, 8.77MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.47G/22.2G [09:18<55:49, 5.58MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:18<25:59, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:18<53:10, 5.45MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:18<24:07, 11.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:18<26:47, 10.8MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [09:18<50:02, 6.22MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.83G/22.2G [09:18<22:23, 11.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:18<20:47, 13.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:18<27:07, 10.7MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:18<52:54, 5.48MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [09:18<49:50, 6.25MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:18<18:33, 15.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.84G/22.2G [09:18<20:25, 12.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:18<17:01, 16.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:18<52:00, 5.57MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.84G/22.2G [09:18<19:28, 13.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:18<16:07, 17.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.84G/22.2G [09:18<17:30, 14.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:18<51:30, 5.62MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:18<15:08, 18.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:18<51:12, 5.66MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.84G/22.2G [09:18<18:13, 14.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:19<34:45, 8.33MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:19<48:34, 5.96MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:19<31:33, 9.17MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [09:19<1:27:12, 3.57MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.77G/22.2G [09:19<46:44, 6.20MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:19<30:21, 9.53MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:19<28:21, 10.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:19<29:24, 9.84MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.85G/22.2G [09:19<31:48, 8.02MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [09:19<51:57, 5.99MB/s]  [A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:19<48:59, 5.91MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:19<21:32, 13.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:19<28:26, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.85G/22.2G [09:19<22:17, 11.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:19<19:03, 15.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.48G/22.2G [09:19<48:35, 6.40MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:19<28:02, 10.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:19<46:48, 6.19MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.85G/22.2G [09:19<22:22, 11.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:19<17:23, 16.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:19<27:22, 10.6MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:19<44:34, 6.50MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.85G/22.2G [09:19<18:49, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [09:19<47:19, 6.57MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:20<15:57, 17.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:20<26:51, 10.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:20<15:17, 18.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:20<42:29, 6.82MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:20<26:49, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.86G/22.2G [09:20<18:35, 13.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [09:20<45:39, 6.81MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:20<26:11, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:20<18:00, 15.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.86G/22.2G [09:20<21:47, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:20<45:17, 6.39MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [09:20<43:40, 7.12MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:20<26:03, 11.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.86G/22.2G [09:20<18:20, 13.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:20<25:57, 11.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [09:20<41:17, 7.53MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.78G/22.2G [09:20<47:21, 6.11MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.87G/22.2G [09:20<18:36, 13.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 4.98G/22.2G [09:20<25:56, 11.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.49G/22.2G [09:20<39:45, 7.82MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.87G/22.2G [09:20<19:14, 13.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:21<20:13, 14.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [09:21<35:08, 8.85MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:21<57:30, 5.03MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:21<20:21, 14.0MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [09:21<34:44, 8.95MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:21<55:55, 5.17MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:21<19:21, 14.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [09:21<34:13, 9.09MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:21<37:06, 7.79MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [09:21<33:28, 9.29MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:21<19:04, 15.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.87G/22.2G [09:21<35:32, 7.17MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:21<32:42, 8.83MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:21<18:01, 15.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.87G/22.2G [09:21<27:58, 9.11MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:21<30:37, 9.44MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:21<18:03, 15.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.87G/22.2G [09:21<25:52, 9.84MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:21<29:01, 9.96MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:21<1:46:15, 2.72MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:22<17:18, 16.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:22<27:47, 10.4MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.88G/22.2G [09:22<22:39, 11.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:22<26:58, 10.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [09:22<1:12:31, 4.29MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:22<1:00:51, 4.76MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:22<26:17, 11.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.88G/22.2G [09:22<21:21, 11.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.50G/22.2G [09:22<41:40, 7.46MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:22<25:49, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.88G/22.2G [09:22<21:20, 11.9MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:22<30:09, 9.47MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:22<57:29, 5.03MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:22<25:29, 11.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:22<22:01, 13.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.89G/22.2G [09:22<19:43, 12.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [09:22<43:29, 7.15MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:22<25:15, 11.4MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:22<21:50, 13.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:22<55:59, 5.17MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:22<25:04, 11.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.89G/22.2G [09:22<19:24, 13.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:22<19:34, 14.6MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [09:22<35:47, 8.68MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.89G/22.2G [09:23<19:50, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.79G/22.2G [09:23<54:45, 5.28MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:23<17:42, 16.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [09:23<34:38, 8.97MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:23<18:03, 15.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [09:23<36:34, 8.49MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:23<52:42, 5.49MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:23<17:19, 16.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.89G/22.2G [09:23<29:32, 8.61MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:23<51:24, 5.62MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.51G/22.2G [09:23<34:02, 9.13MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:23<16:12, 17.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.89G/22.2G [09:23<23:39, 10.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:23<52:20, 5.53MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:23<32:36, 8.86MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [09:23<36:38, 8.48MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.90G/22.2G [09:23<24:10, 10.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:23<48:57, 5.91MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:23<29:38, 9.74MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [09:23<35:22, 8.78MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.90G/22.2G [09:23<21:50, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:23<28:29, 10.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [09:23<33:30, 9.27MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:24<50:12, 5.76MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:24<29:02, 9.83MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [09:24<32:58, 9.42MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.90G/22.2G [09:24<20:23, 12.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:24<30:22, 9.50MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:24<21:42, 13.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [09:24<31:09, 9.97MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:24<49:33, 5.84MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:24<29:41, 9.72MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.91G/22.2G [09:24<19:24, 13.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:24<20:54, 13.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:24<48:14, 6.00MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:24<18:39, 15.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:24<30:59, 9.31MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.91G/22.2G [09:24<18:46, 13.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:24<17:05, 16.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:24<29:24, 9.81MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.80G/22.2G [09:24<48:22, 5.98MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.91G/22.2G [09:24<18:21, 13.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:24<16:07, 17.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:24<32:26, 8.89MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:24<16:30, 17.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.52G/22.2G [09:25<1:12:25, 4.29MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:25<48:57, 5.91MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:25<16:13, 17.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:25<32:19, 8.92MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:25<46:36, 6.20MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:25<29:33, 9.76MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [09:25<49:19, 6.29MB/s]  [A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.91G/22.2G [09:25<30:58, 8.20MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:25<29:22, 9.82MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [09:25<45:42, 6.79MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.92G/22.2G [09:25<23:32, 10.8MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.92G/22.2G [09:25<23:54, 10.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:25<32:01, 8.90MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [09:25<46:00, 6.75MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:25<24:12, 11.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███       | 6.92G/22.2G [09:25<21:38, 11.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [09:25<45:11, 6.87MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:26<1:33:15, 3.10MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:26<22:27, 12.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.92G/22.2G [09:26<20:11, 12.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:26<59:02, 4.88MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [09:26<44:04, 7.04MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:26<19:53, 14.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.93G/22.2G [09:26<19:47, 12.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:26<38:49, 7.43MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:26<17:53, 15.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.53G/22.2G [09:26<41:13, 7.53MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:26<56:30, 5.11MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:26<34:59, 8.24MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:26<16:27, 17.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.93G/22.2G [09:26<18:46, 13.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [09:26<43:13, 7.18MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:26<16:03, 17.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:26<33:17, 8.66MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.93G/22.2G [09:26<16:41, 15.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:26<53:40, 5.38MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [09:26<40:28, 7.67MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:26<16:44, 17.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:26<31:56, 9.03MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.93G/22.2G [09:26<17:49, 14.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.81G/22.2G [09:26<50:39, 5.70MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:26<30:46, 9.37MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [09:26<42:06, 7.37MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:27<29:36, 9.73MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [09:27<39:53, 7.78MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:27<50:44, 5.69MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:27<29:06, 9.90MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.94G/22.2G [09:27<32:10, 7.88MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [09:27<41:32, 7.47MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:27<28:28, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:27<50:58, 5.67MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.94G/22.2G [09:27<25:37, 9.90MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:27<27:57, 10.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [09:27<43:02, 7.21MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:27<48:19, 5.98MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.94G/22.2G [09:27<25:42, 9.86MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:27<27:34, 10.4MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:27<26:59, 10.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.54G/22.2G [09:27<44:59, 6.90MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.94G/22.2G [09:27<22:34, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:27<49:03, 5.89MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:27<26:55, 10.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.95G/22.2G [09:28<20:56, 12.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:28<48:33, 5.95MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:28<1:00:32, 4.70MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.95G/22.2G [09:28<19:25, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:28<47:37, 6.07MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [09:28<46:46, 6.09MB/s]  [A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.95G/22.2G [09:28<19:01, 13.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.82G/22.2G [09:28<46:46, 6.18MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [09:28<35:39, 7.98MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:28<1:01:31, 4.68MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [09:28<1:40:28, 3.09MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [09:28<28:54, 9.84MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.96G/22.2G [09:28<18:30, 13.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:28<47:36, 6.07MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:28<35:52, 8.03MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [09:28<27:07, 10.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [09:28<58:05, 5.34MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:28<32:14, 8.93MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:28<45:48, 6.30MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [09:28<23:34, 12.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:29<30:18, 9.50MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [09:29<20:11, 14.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [09:29<56:23, 5.50MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.96G/22.2G [09:29<29:27, 8.60MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:29<47:40, 6.06MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:29<29:11, 9.86MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [09:29<18:19, 15.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [09:29<52:22, 5.92MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.96G/22.2G [09:29<23:57, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:29<27:54, 10.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.96G/22.2G [09:29<24:20, 10.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:29<26:59, 10.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [09:29<51:28, 6.02MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:29<26:32, 10.8MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.97G/22.2G [09:29<21:36, 11.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [09:29<31:46, 8.95MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.55G/22.2G [09:29<50:16, 6.17MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:29<25:59, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [09:29<23:36, 12.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:29<25:36, 11.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.97G/22.2G [09:29<19:58, 12.7MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [09:29<49:24, 6.27MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:30<1:43:01, 2.80MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:30<25:33, 11.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [09:30<21:55, 13.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.97G/22.2G [09:30<19:01, 13.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:30<25:20, 11.3MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [09:30<48:30, 6.39MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [09:30<19:15, 14.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.98G/22.2G [09:30<18:23, 13.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.83G/22.2G [09:30<55:39, 5.19MB/s]  [A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [09:30<17:19, 16.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [09:30<47:34, 6.52MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [09:30<16:18, 17.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  31%|███▏      | 6.98G/22.2G [09:30<17:53, 14.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:30<50:50, 5.68MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [09:30<47:20, 6.55MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [09:30<16:23, 17.3MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [09:30<16:30, 17.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:30<50:46, 5.68MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [09:30<46:34, 6.65MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:30<56:58, 5.05MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 6.98G/22.2G [09:30<26:43, 9.46MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:31<47:54, 6.02MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:31<35:03, 8.20MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.56G/22.2G [09:31<46:10, 6.71MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 6.98G/22.2G [09:31<22:35, 11.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [09:31<27:19, 10.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:31<45:32, 6.34MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:31<31:14, 9.20MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 6.98G/22.2G [09:31<23:12, 10.9MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [09:31<45:49, 6.76MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [09:31<21:50, 13.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:31<29:36, 9.71MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:31<46:11, 6.25MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 6.99G/22.2G [09:31<20:55, 12.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:31<28:17, 10.2MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [09:31<45:31, 6.81MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:31<20:41, 13.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:31<27:18, 10.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:31<44:12, 6.53MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:31<18:19, 15.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 6.99G/22.2G [09:31<19:41, 12.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:31<26:31, 10.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:31<17:02, 16.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.84G/22.2G [09:31<43:07, 6.69MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 6.99G/22.2G [09:31<18:41, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:31<25:56, 11.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:32<15:49, 17.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:32<41:51, 6.89MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:32<25:36, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [09:32<15:11, 18.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.00G/22.2G [09:32<18:02, 14.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:32<38:09, 7.56MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:32<25:20, 11.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.00G/22.2G [09:32<17:56, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:32<25:07, 11.4MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:32<38:48, 7.43MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:32<24:57, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [09:32<1:52:15, 2.76MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:32<38:42, 7.45MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [09:32<27:02, 10.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:32<38:36, 7.47MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [09:32<20:43, 13.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.00G/22.2G [09:32<29:21, 8.60MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [09:32<1:09:52, 4.43MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.01G/22.2G [09:33<23:52, 10.6MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [09:33<20:00, 14.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:33<51:09, 5.61MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [09:33<1:01:06, 5.07MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.01G/22.2G [09:33<24:14, 10.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [09:33<18:21, 15.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:33<32:28, 8.84MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [09:33<16:50, 16.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.01G/22.2G [09:33<21:22, 11.8MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.57G/22.2G [09:33<57:03, 5.43MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:33<29:15, 9.82MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.85G/22.2G [09:33<1:15:44, 3.81MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [09:33<16:33, 17.1MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:33<28:09, 10.2MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.01G/22.2G [09:33<20:00, 12.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [09:33<53:49, 5.75MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [09:33<15:30, 18.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:33<45:17, 6.37MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:33<26:56, 10.7MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.02G/22.2G [09:33<18:53, 13.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [09:33<51:25, 6.02MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:33<26:17, 10.9MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:33<42:17, 6.82MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.02G/22.2G [09:33<16:38, 15.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:34<25:50, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:34<38:20, 7.52MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [09:34<49:01, 6.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.02G/22.2G [09:34<17:40, 14.3MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:34<26:01, 11.0MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:34<37:47, 7.63MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [09:34<47:51, 6.47MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [09:34<29:32, 9.59MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:34<25:40, 11.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:34<35:13, 8.18MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [09:34<22:20, 12.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:34<26:06, 11.0MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [09:34<47:02, 6.58MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.86G/22.2G [09:34<32:50, 8.78MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:34<25:42, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [09:34<21:06, 13.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:34<33:19, 8.65MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.02G/22.2G [09:34<32:31, 7.75MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.58G/22.2G [09:34<45:48, 6.76MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:34<26:12, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [09:34<18:50, 15.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:34<31:16, 9.21MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.03G/22.2G [09:34<22:45, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [09:34<17:13, 16.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [09:34<46:06, 6.71MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:34<29:46, 9.68MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.03G/22.2G [09:35<22:44, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [09:35<16:12, 17.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:35<29:07, 9.89MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [09:35<45:14, 6.84MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [09:35<15:20, 18.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.03G/22.2G [09:35<20:33, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:35<27:53, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:35<52:39, 5.45MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:35<26:58, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.04G/22.2G [09:35<17:43, 14.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [09:35<44:32, 6.95MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.04G/22.2G [09:35<18:30, 13.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:35<32:47, 8.75MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:35<29:29, 9.72MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [09:35<25:22, 11.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.04G/22.2G [09:35<18:08, 13.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:35<28:15, 10.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.19G/22.2G [09:35<19:43, 14.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.04G/22.2G [09:35<16:11, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:36<27:20, 10.5MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.19G/22.2G [09:36<17:46, 15.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.87G/22.2G [09:36<57:51, 4.98MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.05G/22.2G [09:36<17:04, 14.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:36<26:16, 10.9MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.19G/22.2G [09:36<16:49, 16.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [09:36<1:34:23, 3.28MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:36<34:04, 8.45MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.20G/22.2G [09:36<16:54, 16.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:36<25:49, 11.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:36<31:47, 9.05MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.20G/22.2G [09:36<16:54, 16.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:36<25:25, 11.3MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.05G/22.2G [09:36<27:31, 9.15MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:36<30:02, 9.58MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [09:36<54:07, 5.72MB/s]  [A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.20G/22.2G [09:36<15:48, 17.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:36<24:52, 11.5MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.05G/22.2G [09:36<22:49, 11.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.88G/22.2G [09:36<28:34, 10.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  23%|██▎       | 5.20G/22.2G [09:36<15:02, 18.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:36<24:49, 11.5MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.59G/22.2G [09:36<51:03, 6.06MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.05G/22.2G [09:36<22:48, 11.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:36<24:46, 11.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:36<24:29, 11.7MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [09:36<48:59, 6.31MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.05G/22.2G [09:36<20:00, 12.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:37<28:01, 10.3MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:37<28:42, 9.98MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.06G/22.2G [09:37<17:11, 14.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:37<27:34, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [09:37<47:52, 6.46MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:37<26:55, 10.7MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.06G/22.2G [09:37<17:18, 14.5MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▌        | 3.60G/22.2G [09:37<46:41, 6.62MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:37<27:11, 10.6MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.06G/22.2G [09:37<15:35, 16.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:37<29:13, 9.84MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [09:37<45:40, 6.77MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.07G/22.2G [09:37<16:06, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:37<1:02:15, 4.60MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.89G/22.2G [09:37<28:23, 10.1MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.21G/22.2G [09:37<47:06, 6.00MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [09:37<45:00, 6.87MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.21G/22.2G [09:37<32:32, 8.68MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:37<36:52, 7.77MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.21G/22.2G [09:38<30:06, 9.38MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.60G/22.2G [09:38<44:27, 6.95MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:38<37:30, 7.64MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.07G/22.2G [09:38<27:21, 9.19MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.22G/22.2G [09:38<27:26, 10.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [09:38<43:30, 7.10MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:38<53:00, 5.43MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.07G/22.2G [09:38<23:14, 10.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:38<35:30, 8.07MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.22G/22.2G [09:38<24:56, 11.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.07G/22.2G [09:38<22:41, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:38<31:12, 9.18MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [09:38<46:21, 6.67MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.22G/22.2G [09:38<21:23, 13.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:38<35:25, 8.12MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.08G/22.2G [09:38<18:27, 13.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:38<30:41, 9.33MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.22G/22.2G [09:38<21:21, 13.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:38<32:30, 8.85MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.08G/22.2G [09:38<16:07, 15.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:38<30:16, 9.46MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [09:38<47:50, 6.46MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.23G/22.2G [09:38<19:14, 14.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.90G/22.2G [09:38<30:56, 9.29MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.08G/22.2G [09:38<16:52, 14.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:38<29:54, 9.57MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.23G/22.2G [09:39<19:42, 14.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:39<29:22, 9.79MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.08G/22.2G [09:39<15:30, 16.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:39<29:39, 9.65MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [09:39<50:40, 6.10MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:39<28:21, 10.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:39<29:04, 9.84MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.09G/22.2G [09:39<14:14, 17.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:39<27:41, 10.4MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:39<28:38, 9.99MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:39<26:32, 10.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:39<28:20, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:39<26:32, 10.8MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.23G/22.2G [09:39<36:46, 7.67MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:39<27:48, 10.3MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.91G/22.2G [09:39<26:04, 11.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.23G/22.2G [09:39<25:30, 11.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.09G/22.2G [09:39<25:30, 9.84MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.23G/22.2G [09:39<25:01, 11.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.09G/22.2G [09:39<19:03, 13.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:39<23:33, 12.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.61G/22.2G [09:39<1:43:36, 2.98MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.24G/22.2G [09:40<20:50, 13.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:40<24:01, 12.0MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.10G/22.2G [09:40<16:33, 15.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.24G/22.2G [09:40<21:01, 13.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.10G/22.2G [09:40<15:31, 16.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.24G/22.2G [09:40<18:41, 15.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [09:40<58:32, 5.28MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:40<1:01:27, 4.66MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.10G/22.2G [09:40<14:39, 17.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.25G/22.2G [09:40<18:33, 15.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.11G/22.2G [09:40<13:56, 18.0MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [09:40<55:13, 5.60MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:40<35:58, 7.95MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.25G/22.2G [09:40<16:48, 16.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.11G/22.2G [09:40<13:32, 18.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:40<48:12, 5.96MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:40<32:33, 8.79MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [09:40<53:09, 5.81MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:40<30:48, 9.28MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:40<31:11, 9.21MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [09:40<52:05, 5.93MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:40<29:25, 9.72MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.92G/22.2G [09:40<29:31, 9.73MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:41<28:12, 10.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:41<28:20, 10.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.25G/22.2G [09:41<28:13, 9.98MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [09:41<50:39, 6.10MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.11G/22.2G [09:41<24:04, 10.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:41<27:47, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.25G/22.2G [09:41<21:09, 13.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:41<25:05, 11.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.12G/22.2G [09:41<18:32, 13.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:41<27:05, 10.6MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [09:41<49:06, 6.29MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.26G/22.2G [09:41<21:01, 13.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:41<24:55, 11.5MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.12G/22.2G [09:41<15:50, 15.8MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:41<26:21, 10.8MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▎       | 5.26G/22.2G [09:41<18:47, 15.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:41<26:05, 11.0MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.26G/22.2G [09:41<17:07, 16.4MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.62G/22.2G [09:41<54:18, 5.69MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.93G/22.2G [09:41<27:28, 10.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.12G/22.2G [09:41<19:48, 12.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:41<25:37, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.26G/22.2G [09:41<17:31, 16.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:41<25:05, 11.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.12G/22.2G [09:41<17:00, 14.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:41<25:36, 11.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.27G/22.2G [09:42<16:37, 16.9MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [09:42<48:22, 6.38MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:42<25:33, 11.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.13G/22.2G [09:42<15:59, 15.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:42<25:16, 11.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.27G/22.2G [09:42<15:35, 18.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:42<28:31, 10.1MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [09:42<52:54, 5.84MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.13G/22.2G [09:42<16:15, 15.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:42<30:44, 9.34MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [09:42<54:49, 5.63MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.27G/22.2G [09:42<30:29, 9.23MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [09:42<56:16, 5.49MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.13G/22.2G [09:42<27:01, 9.26MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.28G/22.2G [09:42<24:04, 11.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:43<1:10:19, 4.06MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.14G/22.2G [09:43<20:33, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.28G/22.2G [09:43<22:28, 12.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:43<1:06:46, 4.30MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [09:43<56:32, 5.46MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:43<47:28, 6.02MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.14G/22.2G [09:43<20:27, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.28G/22.2G [09:43<21:45, 12.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.14G/22.2G [09:43<17:37, 14.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:43<43:48, 6.52MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.94G/22.2G [09:43<41:09, 6.97MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:43<40:01, 7.14MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.29G/22.2G [09:43<21:07, 13.3MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.15G/22.2G [09:43<17:24, 14.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:43<39:54, 7.19MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.15G/22.2G [09:43<15:40, 16.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:43<39:33, 7.22MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.29G/22.2G [09:43<20:42, 13.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:43<39:35, 7.24MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.15G/22.2G [09:43<14:37, 17.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:43<37:01, 7.71MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.29G/22.2G [09:44<20:13, 13.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:44<39:18, 7.29MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:44<34:32, 8.27MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.16G/22.2G [09:44<15:11, 16.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:44<39:06, 7.33MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:44<35:36, 8.02MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.63G/22.2G [09:44<2:00:47, 2.56MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:44<33:47, 8.45MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:44<38:28, 7.45MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.16G/22.2G [09:44<23:03, 10.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.30G/22.2G [09:44<30:30, 9.21MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:44<32:13, 8.86MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.95G/22.2G [09:44<37:57, 7.55MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.16G/22.2G [09:44<20:33, 12.2MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.30G/22.2G [09:44<22:59, 12.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:44<30:36, 9.33MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [09:44<1:11:29, 4.32MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.16G/22.2G [09:44<17:29, 14.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:44<37:57, 7.55MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.30G/22.2G [09:44<22:53, 12.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:44<30:13, 9.44MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.17G/22.2G [09:44<16:05, 15.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:45<38:00, 7.54MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.30G/22.2G [09:45<22:12, 12.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:45<33:01, 8.64MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.17G/22.2G [09:45<15:05, 16.6MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [09:45<1:11:27, 4.32MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:45<31:33, 9.05MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:45<37:29, 7.64MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.17G/22.2G [09:45<14:22, 17.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.31G/22.2G [09:45<21:21, 13.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:45<34:56, 8.20MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.18G/22.2G [09:45<13:48, 18.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.31G/22.2G [09:45<19:02, 14.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [09:45<1:10:36, 4.37MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.31G/22.2G [09:45<19:42, 14.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.18G/22.2G [09:45<13:25, 18.6MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:45<35:52, 7.99MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:45<57:50, 4.93MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:45<36:25, 7.87MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.31G/22.2G [09:45<19:40, 14.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [09:45<1:10:47, 4.36MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.32G/22.2G [09:45<17:53, 15.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:45<37:38, 7.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.18G/22.2G [09:46<26:41, 9.35MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.64G/22.2G [09:46<1:10:06, 4.40MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:46<37:39, 7.58MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.18G/22.2G [09:46<23:32, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:46<32:37, 8.74MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.32G/22.2G [09:46<28:58, 9.69MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.96G/22.2G [09:46<1:10:43, 4.05MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.19G/22.2G [09:46<19:01, 13.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:46<31:02, 9.19MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [09:46<1:09:31, 4.44MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.32G/22.2G [09:46<25:04, 11.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.19G/22.2G [09:46<16:54, 14.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.32G/22.2G [09:46<25:23, 11.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:46<44:56, 6.37MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:46<33:57, 8.40MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.19G/22.2G [09:46<15:21, 16.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [09:46<1:08:31, 4.50MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:46<32:42, 8.72MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.33G/22.2G [09:46<22:52, 12.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:46<42:03, 6.81MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.19G/22.2G [09:46<14:17, 17.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:46<30:55, 9.22MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.33G/22.2G [09:46<19:42, 14.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  32%|███▏      | 7.20G/22.2G [09:47<13:34, 18.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:47<40:58, 6.99MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [09:47<1:07:31, 4.57MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:47<29:53, 9.54MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.33G/22.2G [09:47<20:22, 13.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:47<37:36, 7.61MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:47<27:56, 10.2MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.33G/22.2G [09:47<18:19, 15.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [09:47<1:05:03, 4.74MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:47<38:04, 7.52MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.33G/22.2G [09:47<19:17, 14.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:47<27:13, 10.5MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.97G/22.2G [09:47<34:54, 8.20MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:47<27:42, 10.3MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.34G/22.2G [09:47<17:21, 16.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [09:47<1:05:26, 4.71MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.20G/22.2G [09:47<26:28, 9.41MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:47<27:54, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:47<36:10, 7.91MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.20G/22.2G [09:47<19:53, 12.5MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:47<36:35, 7.82MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [09:47<1:04:27, 4.78MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.21G/22.2G [09:47<16:44, 14.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:48<34:11, 8.37MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.21G/22.2G [09:48<15:18, 16.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.34G/22.2G [09:48<36:43, 7.63MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.65G/22.2G [09:48<1:04:25, 4.79MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:48<35:20, 8.10MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.21G/22.2G [09:48<14:16, 17.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.34G/22.2G [09:48<25:15, 11.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.22G/22.2G [09:48<13:27, 18.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:48<1:07:28, 4.22MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.35G/22.2G [09:48<24:40, 11.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:48<35:58, 7.96MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  16%|█▋        | 3.66G/22.2G [09:48<1:03:44, 4.84MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.22G/22.2G [09:48<12:51, 19.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:48<33:46, 8.48MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:48<38:34, 7.39MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.35G/22.2G [09:48<23:02, 12.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:48<35:04, 8.12MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  22%|██▏       | 4.98G/22.2G [09:48<34:49, 8.22MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.35G/22.2G [09:48<20:12, 13.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:48<33:17, 8.56MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.35G/22.2G [09:48<19:47, 14.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:49<32:07, 8.87MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.36G/22.2G [09:49<19:10, 14.6MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:49<30:55, 9.21MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:49<30:01, 9.49MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.36G/22.2G [09:49<17:28, 16.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:49<1:13:49, 3.88MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:49<29:42, 9.59MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [09:49<2:11:55, 2.34MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:49<29:06, 9.78MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:49<44:18, 6.46MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:49<29:01, 9.81MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.22G/22.2G [09:49<43:55, 5.67MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:49<28:35, 9.96MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:49<41:59, 6.81MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.36G/22.2G [09:49<33:12, 8.43MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [09:49<1:13:49, 4.18MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:50<28:45, 9.90MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:50<38:29, 7.43MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.37G/22.2G [09:50<25:00, 11.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.23G/22.2G [09:50<35:48, 6.95MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:50<28:18, 10.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.37G/22.2G [09:50<24:33, 11.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:50<38:23, 7.45MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.23G/22.2G [09:50<29:33, 8.42MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:50<28:06, 10.1MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.37G/22.2G [09:50<21:16, 13.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [09:50<1:18:27, 3.93MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 4.99G/22.2G [09:50<38:11, 7.49MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.23G/22.2G [09:50<23:55, 10.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.37G/22.2G [09:50<21:24, 13.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:50<35:02, 8.16MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.24G/22.2G [09:50<20:05, 12.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.37G/22.2G [09:50<19:03, 14.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [09:50<1:18:23, 3.93MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.38G/22.2G [09:50<19:40, 14.2MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.24G/22.2G [09:50<17:26, 14.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:50<35:51, 7.98MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.38G/22.2G [09:50<17:53, 15.6MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.24G/22.2G [09:50<15:31, 16.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:50<36:28, 7.84MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.38G/22.2G [09:51<18:44, 14.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:51<33:39, 8.49MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.66G/22.2G [09:51<1:18:21, 3.93MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.38G/22.2G [09:51<17:14, 16.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:51<34:53, 8.19MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.24G/22.2G [09:51<23:09, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.00G/22.2G [09:51<36:07, 7.91MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [09:51<1:19:07, 3.89MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.25G/22.2G [09:51<19:28, 12.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:51<33:42, 8.48MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.39G/22.2G [09:51<33:05, 8.45MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.25G/22.2G [09:51<17:55, 13.9MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:51<1:55:56, 2.46MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:51<35:13, 8.11MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.39G/22.2G [09:51<27:42, 10.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [09:51<1:18:30, 3.92MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.25G/22.2G [09:51<17:38, 14.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.39G/22.2G [09:51<26:55, 10.4MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.39G/22.2G [09:52<21:36, 12.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.26G/22.2G [09:52<17:43, 14.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [09:52<1:12:31, 3.92MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [09:52<1:19:21, 3.88MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.39G/22.2G [09:52<21:58, 12.7MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.26G/22.2G [09:52<18:10, 13.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [09:52<58:18, 4.88MB/s]  
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.40G/22.2G [09:52<19:02, 14.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:52<1:12:02, 3.97MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [09:52<50:48, 5.60MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.40G/22.2G [09:52<19:52, 14.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.26G/22.2G [09:52<17:31, 14.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [09:52<1:18:50, 3.91MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.40G/22.2G [09:52<17:42, 15.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:52<43:08, 6.62MB/s]  [A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.27G/22.2G [09:52<17:32, 14.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [09:52<52:21, 5.43MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.40G/22.2G [09:52<18:42, 14.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:52<40:21, 7.08MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [09:53<1:17:31, 3.97MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [09:53<39:40, 7.17MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.01G/22.2G [09:53<36:59, 7.72MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [09:53<39:58, 7.12MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:53<37:08, 7.69MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [09:53<1:17:44, 3.96MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.27G/22.2G [09:53<32:36, 7.61MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:53<34:44, 8.22MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.41G/22.2G [09:53<34:08, 8.17MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [09:53<40:17, 7.06MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.27G/22.2G [09:53<25:12, 9.84MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.41G/22.2G [09:53<23:52, 11.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:53<35:26, 8.06MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [09:53<39:13, 7.25MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.27G/22.2G [09:53<24:49, 9.99MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.41G/22.2G [09:53<23:57, 11.6MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:53<32:56, 8.67MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.67G/22.2G [09:53<1:18:04, 3.95MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.41G/22.2G [09:53<20:36, 13.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [09:53<38:23, 7.41MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.28G/22.2G [09:53<21:51, 11.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:53<34:11, 8.35MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.42G/22.2G [09:54<20:48, 13.4MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:54<31:58, 8.93MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [09:54<37:46, 7.53MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.28G/22.2G [09:54<20:21, 12.2MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [09:54<1:17:01, 4.00MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.42G/22.2G [09:54<18:58, 14.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.02G/22.2G [09:54<33:10, 8.61MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.42G/22.2G [09:54<19:37, 14.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [09:54<36:40, 7.75MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.28G/22.2G [09:54<19:07, 13.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:54<31:09, 9.16MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.42G/22.2G [09:54<17:47, 15.7MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [09:54<1:17:12, 3.99MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.29G/22.2G [09:54<17:11, 14.4MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.43G/22.2G [09:54<18:34, 15.0MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:54<32:51, 8.69MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.29G/22.2G [09:54<18:03, 13.7MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  24%|██▍       | 5.43G/22.2G [09:54<17:07, 16.3MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:54<30:57, 9.22MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [09:54<1:09:09, 4.11MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.43G/22.2G [09:55<29:04, 9.59MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [09:55<40:10, 7.08MB/s]  


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.29G/22.2G [09:55<31:08, 7.96MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.43G/22.2G [09:55<24:56, 11.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:55<1:03:32, 4.49MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.43G/22.2G [09:55<24:43, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [09:55<40:21, 7.04MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.29G/22.2G [09:55<23:53, 10.4MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:55<40:24, 7.06MB/s]  [A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [09:55<2:33:28, 2.01MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.44G/22.2G [09:55<21:31, 12.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.30G/22.2G [09:55<21:25, 11.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [09:55<37:44, 7.53MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.44G/22.2G [09:55<21:06, 13.2MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.03G/22.2G [09:55<39:42, 7.19MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.30G/22.2G [09:55<21:51, 11.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.44G/22.2G [09:55<18:36, 15.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [09:55<36:25, 7.80MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:55<35:41, 7.99MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.30G/22.2G [09:55<22:03, 11.2MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:56<32:18, 8.83MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [09:56<1:22:02, 3.75MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.44G/22.2G [09:56<18:30, 15.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.30G/22.2G [09:56<22:14, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [09:56<34:54, 8.14MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:56<30:42, 9.29MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.45G/22.2G [09:56<17:23, 16.0MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.30G/22.2G [09:56<22:22, 11.1MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [09:56<31:15, 9.09MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:56<32:15, 8.84MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.30G/22.2G [09:56<22:15, 11.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.45G/22.2G [09:56<17:39, 15.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [09:56<1:20:15, 3.84MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [09:56<33:19, 8.53MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.04G/22.2G [09:56<30:46, 9.27MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.31G/22.2G [09:56<22:34, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [09:56<32:15, 8.80MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.31G/22.2G [09:56<22:36, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:56<26:37, 10.7MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.68G/22.2G [09:56<1:18:41, 3.91MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [09:56<31:36, 8.99MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.31G/22.2G [09:56<22:41, 10.9MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:56<26:27, 10.8MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [09:56<30:37, 9.27MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.45G/22.2G [09:56<33:44, 8.25MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.31G/22.2G [09:56<23:03, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:56<26:13, 10.9MB/s][A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:57<26:09, 10.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.45G/22.2G [09:57<26:55, 10.3MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [09:57<1:16:49, 4.01MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [09:57<30:59, 9.16MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:57<26:15, 10.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.46G/22.2G [09:57<26:19, 10.6MB/s][A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.46G/22.2G [09:57<21:29, 12.9MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.31G/22.2G [09:57<43:27, 5.69MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.46G/22.2G [09:57<21:43, 12.8MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [09:57<1:16:05, 4.05MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.46G/22.2G [09:57<18:39, 14.9MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [09:57<54:16, 5.23MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.32G/22.2G [09:57<27:25, 9.02MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [09:57<1:13:20, 4.20MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.47G/22.2G [09:57<18:48, 14.8MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.32G/22.2G [09:57<24:59, 9.89MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [09:57<34:55, 8.13MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.47G/22.2G [09:57<16:55, 16.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.32G/22.2G [09:58<24:17, 10.2MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [09:58<32:38, 8.70MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.05G/22.2G [09:58<1:16:52, 3.71MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.32G/22.2G [09:58<23:13, 10.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [09:58<31:44, 8.94MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [09:58<1:14:22, 4.14MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:58<31:01, 9.15MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.32G/22.2G [09:58<23:10, 10.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:58<48:34, 5.87MB/s]  [A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:58<30:24, 9.33MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.32G/22.2G [09:58<23:39, 10.4MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [09:58<1:11:58, 4.28MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:58<42:13, 6.75MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.47G/22.2G [09:58<28:51, 9.64MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:58<29:53, 9.49MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:58<37:10, 7.67MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.33G/22.2G [09:58<21:53, 11.3MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.48G/22.2G [09:58<21:27, 13.0MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:58<29:32, 9.60MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.48G/22.2G [09:58<21:14, 13.1MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.69G/22.2G [09:58<1:12:23, 4.25MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:58<29:30, 9.66MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:58<28:57, 9.80MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.33G/22.2G [09:58<20:59, 11.8MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.48G/22.2G [09:58<19:08, 14.5MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.06G/22.2G [09:59<27:50, 10.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:59<28:55, 9.81MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.33G/22.2G [09:59<21:08, 11.7MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:59<26:22, 10.8MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [09:59<1:09:18, 4.44MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [09:59<28:52, 9.82MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.48G/22.2G [09:59<18:46, 14.8MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [09:59<28:33, 9.93MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.49G/22.2G [09:59<17:44, 15.7MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:59<23:51, 11.9MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [09:59<1:08:05, 4.52MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [09:59<28:37, 9.91MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:59<23:38, 12.0MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.49G/22.2G [09:59<16:08, 17.2MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [09:59<28:28, 9.96MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.49G/22.2G [09:59<16:24, 16.9MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [09:59<21:57, 13.0MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [09:59<1:06:21, 4.64MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.33G/22.2G [09:59<40:05, 6.16MB/s][A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.34G/22.2G [09:59<27:50, 8.87MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [10:00<1:04:22, 4.78MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.34G/22.2G [10:00<25:47, 9.58MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.49G/22.2G [10:00<28:03, 9.90MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.07G/22.2G [10:00<38:12, 7.45MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [10:00<59:44, 4.75MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.34G/22.2G [10:00<24:25, 10.1MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.50G/22.2G [10:00<21:42, 12.8MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [10:00<28:04, 10.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.14G/22.2G [10:00<35:29, 7.99MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.34G/22.2G [10:00<23:32, 10.5MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.50G/22.2G [10:00<20:41, 13.4MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [10:00<32:22, 8.76MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [10:00<25:01, 11.4MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.50G/22.2G [10:00<18:25, 15.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.35G/22.2G [10:00<21:03, 11.7MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [10:00<31:31, 8.99MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.08G/22.2G [10:00<21:19, 13.3MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.51G/22.2G [10:00<16:50, 16.5MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.35G/22.2G [10:00<21:23, 11.5MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [10:00<30:25, 9.32MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [10:00<21:33, 13.2MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [10:00<2:01:08, 2.54MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.51G/22.2G [10:00<15:42, 17.7MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [10:00<29:38, 9.56MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.35G/22.2G [10:01<20:03, 12.3MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [10:01<20:31, 13.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.51G/22.2G [10:01<15:00, 18.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [10:01<29:00, 9.77MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.35G/22.2G [10:01<20:35, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [10:01<18:14, 15.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [10:01<28:56, 9.79MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.70G/22.2G [10:01<1:10:05, 4.39MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [10:01<18:40, 15.2MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.36G/22.2G [10:01<19:34, 12.6MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.15G/22.2G [10:01<28:32, 9.93MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.09G/22.2G [10:01<22:27, 12.7MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [10:01<28:26, 9.96MB/s]bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [10:01<28:09, 10.1MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [10:01<1:15:27, 4.07MB/s][A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [10:01<28:17, 10.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.36G/22.2G [10:01<31:41, 7.78MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [10:01<37:31, 7.58MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [10:01<28:01, 10.1MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.36G/22.2G [10:02<26:59, 9.14MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [10:02<27:48, 10.2MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [10:02<27:22, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [10:02<1:13:36, 4.18MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.36G/22.2G [10:02<25:02, 9.84MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.52G/22.2G [10:02<43:28, 6.38MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [10:02<26:02, 10.9MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.52G/22.2G [10:02<31:00, 8.94MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.36G/22.2G [10:02<22:09, 11.1MB/s][A[A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [10:02<1:08:25, 4.49MB/s][A[A[A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.10G/22.2G [10:02<23:21, 12.2MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.52G/22.2G [10:02<28:47, 9.63MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.37G/22.2G [10:02<22:11, 11.1MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [10:02<23:14, 12.2MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.16G/22.2G [10:02<57:18, 4.94MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.53G/22.2G [10:02<27:03, 10.2MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [10:02<1:07:32, 4.55MB/s][A[A[A[A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.37G/22.2G [10:02<20:35, 12.0MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [10:02<21:40, 13.1MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [10:02<37:07, 7.63MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.53G/22.2G [10:02<24:13, 11.4MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.37G/22.2G [10:02<20:57, 11.8MB/s][A[A[A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [10:03<20:57, 13.6MB/s][A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [10:03<33:41, 8.40MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [10:03<1:06:03, 4.65MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.53G/22.2G [10:03<21:13, 13.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.11G/22.2G [10:03<21:21, 13.3MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.37G/22.2G [10:03<19:48, 12.4MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [10:03<32:06, 8.82MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.53G/22.2G [10:03<21:09, 13.1MB/s][A

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [10:03<19:08, 14.8MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.38G/22.2G [10:03<20:32, 12.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [10:03<30:55, 9.16MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [10:03<1:04:55, 4.73MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.54G/22.2G [10:03<19:04, 14.5MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [10:03<30:17, 9.34MB/s]
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▍       | 5.54G/22.2G [10:03<19:25, 14.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.17G/22.2G [10:03<29:31, 9.59MB/s]


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.71G/22.2G [10:03<1:04:47, 4.74MB/s][A[A[A[A[AUpload 41 LFS files:   0%|          | 0/41 [10:03<?, ?it/s]
bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [10:03<28:55, 9.78MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [10:03<37:06, 7.65MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [10:03<59:37, 5.15MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [10:03<28:49, 9.82MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.38G/22.2G [10:03<36:59, 6.66MB/s][A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▌       | 5.54G/22.2G [10:04<36:08, 7.66MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [10:04<28:19, 9.99MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.12G/22.2G [10:04<27:14, 10.4MB/s][A[A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [10:04<1:00:17, 5.10MB/s][A[A[A[A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▌       | 5.54G/22.2G [10:04<24:55, 11.1MB/s][A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.38G/22.2G [10:04<26:23, 9.33MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [10:04<28:05, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [10:04<24:09, 11.7MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▌       | 5.54G/22.2G [10:04<24:26, 11.3MB/s][Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [10:04<28:12, 10.0MB/s]


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.38G/22.2G [10:04<24:38, 9.99MB/s][A[A[ATraceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/tqdm/contrib/concurrent.py", line 51, in _executor_map
    return list(tqdm_class(ex.map(fn, *iterables, chunksize=chunksize), **kwargs))
  File "/usr/local/lib/python3.10/dist-packages/tqdm/std.py", line 1181, in __iter__
    for obj in iterable:
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 621, in result_iterator
    yield _result_or_cancel(fs.pop())
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 319, in _result_or_cancel
    return fut.result(timeout)
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 453, in result
    self._condition.wait(timeout)
  File "/usr/lib/python3.10/threading.py", line 320, in wait
    waiter.acquire()
KeyboardInterrupt

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/bin/huggingface-cli", line 8, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/commands/huggingface_cli.py", line 52, in main
    service.run()
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/commands/upload.py", line 191, in run
    print(self._upload())
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/commands/upload.py", line 286, in _upload
    return self.api.upload_folder(
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
    return fn(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/hf_api.py", line 1398, in _inner
    return fn(self, *args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/hf_api.py", line 4857, in upload_folder
    commit_info = self.create_commit(
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
    return fn(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/hf_api.py", line 1398, in _inner
    return fn(self, *args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/hf_api.py", line 3770, in create_commit
    self.preupload_lfs_files(
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/hf_api.py", line 4317, in preupload_lfs_files
    _upload_lfs_files(
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
    return fn(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/_commit_api.py", line 446, in _upload_lfs_files
    thread_map(
  File "/usr/local/lib/python3.10/dist-packages/tqdm/contrib/concurrent.py", line 69, in thread_map
    return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs)
  File "/usr/local/lib/python3.10/dist-packages/tqdm/contrib/concurrent.py", line 49, in _executor_map
    with PoolExecutor(max_workers=max_workers, initializer=tqdm_class.set_lock,
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 649, in __exit__
    self.shutdown(wait=True)
  File "/usr/lib/python3.10/concurrent/futures/thread.py", line 235, in shutdown
    t.join()
  File "/usr/lib/python3.10/threading.py", line 1096, in join
    self._wait_for_tstate_lock()
  File "/usr/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
    if lock.acquire(block, timeout):
KeyboardInterrupt


bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [10:04<20:55, 13.6MB/s][A[A
bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt:  25%|██▌       | 5.55G/22.2G [10:04<20:32, 13.5MB/s][A


bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt:  17%|█▋        | 3.72G/22.2G [10:04<59:14, 5.19MB/s]  [A[A[A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [10:04<28:00, 10.1MB/s]

bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt:  23%|██▎       | 5.13G/22.2G [10:04<21:08, 13.4MB/s][A[A


bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt:  33%|███▎      | 7.39G/22.2G [10:04<22:17, 11.0MB/s][A[A[Abf16_zero_pp_rank_0_mp_rank_00_optim_states.pt:  23%|██▎       | 5.18G/22.2G [10:04<27:48, 10.2MB/s]Exception ignored in: <module 'threading' from '/usr/lib/python3.10/threading.py'>
Traceback (most recent call last):
  File "/usr/lib/python3.10/threading.py", line 1537, in _shutdown
    atexit_call()
  File "/usr/lib/python3.10/concurrent/futures/thread.py", line 31, in _python_exit
    t.join()
  File "/usr/lib/python3.10/threading.py", line 1096, in join
    self._wait_for_tstate_lock()
  File "/usr/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock
    if lock.acquire(block, timeout):
KeyboardInterrupt: