Checkpoint at step 78278
Browse files- checkpoint_77500.pth +3 -0
- checkpoint_78000.pth +3 -0
- events.out.tfevents.1766762704.eeafeeddd90e.1539.1 +2 -2
- trainer_0_log.txt +825 -0
checkpoint_77500.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:48c2bd9c12a44348e82ecc24489dfcb2dc9b65e6eed23b9115261d2055a93700
|
| 3 |
+
size 5607926741
|
checkpoint_78000.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:76155fe64a64c8ab23a1d63c9453b12069ce5e2883ea9b6b0142fefdafbbeb82
|
| 3 |
+
size 5607926741
|
events.out.tfevents.1766762704.eeafeeddd90e.1539.1
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:be745f8d381b2051ff297b68658780f9c647f9613bf14c92cf25aeebc444eacc
|
| 3 |
+
size 108326
|
trainer_0_log.txt
CHANGED
|
@@ -8220,3 +8220,828 @@
|
|
| 8220 |
| > avg_loss_mel_ce:[91m 3.7725630966392725 [0m(+0.02119288251206708)
|
| 8221 |
| > avg_loss:[91m 3.8002426769282365 [0m(+0.021157527292096745)
|
| 8222 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8220 |
| > avg_loss_mel_ce:[91m 3.7725630966392725 [0m(+0.02119288251206708)
|
| 8221 |
| > avg_loss:[91m 3.8002426769282365 [0m(+0.021157527292096745)
|
| 8222 |
|
| 8223 |
+
|
| 8224 |
+
[4m[1m > EPOCH: 11/12[0m
|
| 8225 |
+
--> /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000
|
| 8226 |
+
|
| 8227 |
+
[1m > TRAINING (2025-12-26 18:35:28) [0m
|
| 8228 |
+
|
| 8229 |
+
[1m --> TIME: 2025-12-26 18:35:33 -- STEP: 4/2207 -- GLOBAL_STEP: 76075[0m
|
| 8230 |
+
| > loss_text_ce: 0.02755327895283699 (0.027924106922000647)
|
| 8231 |
+
| > loss_mel_ce: 3.4745895862579346 (3.326134204864502)
|
| 8232 |
+
| > loss: 0.054720982909202576 (0.052407161332666874)
|
| 8233 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8234 |
+
| > step_time: 0.2918 (0.27978748083114624)
|
| 8235 |
+
| > loader_time: 0.0083 (0.009816348552703857)
|
| 8236 |
+
|
| 8237 |
+
|
| 8238 |
+
[1m --> TIME: 2025-12-26 18:35:42 -- STEP: 29/2207 -- GLOBAL_STEP: 76100[0m
|
| 8239 |
+
| > loss_text_ce: 0.030366934835910797 (0.028350009036989046)
|
| 8240 |
+
| > loss_mel_ce: 3.5372276306152344 (3.240666225038726)
|
| 8241 |
+
| > loss: 0.055743664503097534 (0.05107837889728875)
|
| 8242 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8243 |
+
| > step_time: 0.2977 (0.27683859035886565)
|
| 8244 |
+
| > loader_time: 0.0083 (0.008618215034747943)
|
| 8245 |
+
|
| 8246 |
+
|
| 8247 |
+
[1m --> TIME: 2025-12-26 18:35:52 -- STEP: 54/2207 -- GLOBAL_STEP: 76125[0m
|
| 8248 |
+
| > loss_text_ce: 0.028747759759426117 (0.028470003011601942)
|
| 8249 |
+
| > loss_mel_ce: 3.0890870094299316 (3.244285782178243)
|
| 8250 |
+
| > loss: 0.048716168850660324 (0.051136809297733836)
|
| 8251 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8252 |
+
| > step_time: 0.2895 (0.2778089708752103)
|
| 8253 |
+
| > loader_time: 0.008 (0.008555566823040997)
|
| 8254 |
+
|
| 8255 |
+
|
| 8256 |
+
[1m --> TIME: 2025-12-26 18:36:01 -- STEP: 79/2207 -- GLOBAL_STEP: 76150[0m
|
| 8257 |
+
| > loss_text_ce: 0.028700023889541626 (0.028404795741544493)
|
| 8258 |
+
| > loss_mel_ce: 3.1781933307647705 (3.244518319262734)
|
| 8259 |
+
| > loss: 0.05010771006345749 (0.05113942378872557)
|
| 8260 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8261 |
+
| > step_time: 0.2873 (0.27908754650550555)
|
| 8262 |
+
| > loader_time: 0.0087 (0.008504279052154928)
|
| 8263 |
+
|
| 8264 |
+
|
| 8265 |
+
[1m --> TIME: 2025-12-26 18:36:11 -- STEP: 104/2207 -- GLOBAL_STEP: 76175[0m
|
| 8266 |
+
| > loss_text_ce: 0.029501618817448616 (0.02840566312750945)
|
| 8267 |
+
| > loss_mel_ce: 3.1426446437835693 (3.2554910366351786)
|
| 8268 |
+
| > loss: 0.04956478625535965 (0.05131088598416401)
|
| 8269 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8270 |
+
| > step_time: 0.2988 (0.2794825022037212)
|
| 8271 |
+
| > loader_time: 0.0086 (0.008471491245123058)
|
| 8272 |
+
|
| 8273 |
+
|
| 8274 |
+
[1m --> TIME: 2025-12-26 18:36:21 -- STEP: 129/2207 -- GLOBAL_STEP: 76200[0m
|
| 8275 |
+
| > loss_text_ce: 0.02995274029672146 (0.028276232771517695)
|
| 8276 |
+
| > loss_mel_ce: 3.297744035720825 (3.2497549075488896)
|
| 8277 |
+
| > loss: 0.05199526250362396 (0.051219236683244844)
|
| 8278 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8279 |
+
| > step_time: 0.2666 (0.2792686159296553)
|
| 8280 |
+
| > loader_time: 0.0089 (0.008446734080943031)
|
| 8281 |
+
|
| 8282 |
+
|
| 8283 |
+
[1m --> TIME: 2025-12-26 18:36:30 -- STEP: 154/2207 -- GLOBAL_STEP: 76225[0m
|
| 8284 |
+
| > loss_text_ce: 0.026851488277316093 (0.028303810564631767)
|
| 8285 |
+
| > loss_mel_ce: 3.1961450576782227 (3.251576610973902)
|
| 8286 |
+
| > loss: 0.050359319895505905 (0.05124813168273343)
|
| 8287 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8288 |
+
| > step_time: 0.2918 (0.27900065694536463)
|
| 8289 |
+
| > loader_time: 0.0094 (0.008444730337564054)
|
| 8290 |
+
|
| 8291 |
+
|
| 8292 |
+
[1m --> TIME: 2025-12-26 18:36:40 -- STEP: 179/2207 -- GLOBAL_STEP: 76250[0m
|
| 8293 |
+
| > loss_text_ce: 0.027952909469604492 (0.028274590617784576)
|
| 8294 |
+
| > loss_mel_ce: 3.1654622554779053 (3.2503395493470078)
|
| 8295 |
+
| > loss: 0.04989711195230484 (0.05122834602214769)
|
| 8296 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8297 |
+
| > step_time: 0.2676 (0.2788836489842588)
|
| 8298 |
+
| > loader_time: 0.0082 (0.008482942368065183)
|
| 8299 |
+
|
| 8300 |
+
|
| 8301 |
+
[1m --> TIME: 2025-12-26 18:36:49 -- STEP: 204/2207 -- GLOBAL_STEP: 76275[0m
|
| 8302 |
+
| > loss_text_ce: 0.030396854504942894 (0.02835846817413089)
|
| 8303 |
+
| > loss_mel_ce: 3.077045440673828 (3.2514420373767017)
|
| 8304 |
+
| > loss: 0.04855378717184067 (0.051246882964144716)
|
| 8305 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8306 |
+
| > step_time: 0.2697 (0.27851230490441387)
|
| 8307 |
+
| > loader_time: 0.0082 (0.00850879327923644)
|
| 8308 |
+
|
| 8309 |
+
|
| 8310 |
+
[1m --> TIME: 2025-12-26 18:36:59 -- STEP: 229/2207 -- GLOBAL_STEP: 76300[0m
|
| 8311 |
+
| > loss_text_ce: 0.02872605249285698 (0.028386447627471525)
|
| 8312 |
+
| > loss_mel_ce: 3.29390025138855 (3.252538128190685)
|
| 8313 |
+
| > loss: 0.05191603675484657 (0.05126444657036309)
|
| 8314 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8315 |
+
| > step_time: 0.2889 (0.27850259027106256)
|
| 8316 |
+
| > loader_time: 0.0099 (0.008523956656976564)
|
| 8317 |
+
|
| 8318 |
+
|
| 8319 |
+
[1m --> TIME: 2025-12-26 18:37:09 -- STEP: 254/2207 -- GLOBAL_STEP: 76325[0m
|
| 8320 |
+
| > loss_text_ce: 0.02956364117562771 (0.028400950771793138)
|
| 8321 |
+
| > loss_mel_ce: 3.1540119647979736 (3.2519230025959756)
|
| 8322 |
+
| > loss: 0.04974336922168732 (0.05125506187048483)
|
| 8323 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8324 |
+
| > step_time: 0.2882 (0.2785647581881424)
|
| 8325 |
+
| > loader_time: 0.0084 (0.008553825964139209)
|
| 8326 |
+
|
| 8327 |
+
|
| 8328 |
+
[1m --> TIME: 2025-12-26 18:37:18 -- STEP: 279/2207 -- GLOBAL_STEP: 76350[0m
|
| 8329 |
+
| > loss_text_ce: 0.029421858489513397 (0.02839953949721697)
|
| 8330 |
+
| > loss_mel_ce: 3.1452901363372803 (3.247202940739183)
|
| 8331 |
+
| > loss: 0.0496048741042614 (0.051181288843300164)
|
| 8332 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8333 |
+
| > step_time: 0.2915 (0.278784552782667)
|
| 8334 |
+
| > loader_time: 0.0094 (0.008575576180625566)
|
| 8335 |
+
|
| 8336 |
+
|
| 8337 |
+
[1m --> TIME: 2025-12-26 18:37:28 -- STEP: 304/2207 -- GLOBAL_STEP: 76375[0m
|
| 8338 |
+
| > loss_text_ce: 0.030859777703881264 (0.028393915376478906)
|
| 8339 |
+
| > loss_mel_ce: 3.0349302291870117 (3.2475315850032)
|
| 8340 |
+
| > loss: 0.047902967780828476 (0.05118633602059591)
|
| 8341 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8342 |
+
| > step_time: 0.2682 (0.2791039120209843)
|
| 8343 |
+
| > loader_time: 0.0086 (0.00855911719171625)
|
| 8344 |
+
|
| 8345 |
+
|
| 8346 |
+
[1m --> TIME: 2025-12-26 18:37:38 -- STEP: 329/2207 -- GLOBAL_STEP: 76400[0m
|
| 8347 |
+
| > loss_text_ce: 0.026775214821100235 (0.028420852944481337)
|
| 8348 |
+
| > loss_mel_ce: 3.3600540161132812 (3.2447551115667927)
|
| 8349 |
+
| > loss: 0.05291920527815819 (0.05114337452050398)
|
| 8350 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8351 |
+
| > step_time: 0.2914 (0.27889997691006635)
|
| 8352 |
+
| > loader_time: 0.0085 (0.008571789982108727)
|
| 8353 |
+
|
| 8354 |
+
|
| 8355 |
+
[1m --> TIME: 2025-12-26 18:37:47 -- STEP: 354/2207 -- GLOBAL_STEP: 76425[0m
|
| 8356 |
+
| > loss_text_ce: 0.025896884500980377 (0.028459980268304968)
|
| 8357 |
+
| > loss_mel_ce: 2.9947152137756348 (3.2452200671373785)
|
| 8358 |
+
| > loss: 0.04719706252217293 (0.05115125079949695)
|
| 8359 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8360 |
+
| > step_time: 0.2878 (0.2793040248633778)
|
| 8361 |
+
| > loader_time: 0.0081 (0.008582097662370764)
|
| 8362 |
+
|
| 8363 |
+
|
| 8364 |
+
[1m --> TIME: 2025-12-26 18:37:57 -- STEP: 379/2207 -- GLOBAL_STEP: 76450[0m
|
| 8365 |
+
| > loss_text_ce: 0.0274538341909647 (0.028468755533363387)
|
| 8366 |
+
| > loss_mel_ce: 3.230142116546631 (3.2392202002391968)
|
| 8367 |
+
| > loss: 0.050899937748909 (0.051057640010967714)
|
| 8368 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8369 |
+
| > step_time: 0.291 (0.27927555886925354)
|
| 8370 |
+
| > loader_time: 0.0086 (0.008587208775542978)
|
| 8371 |
+
|
| 8372 |
+
|
| 8373 |
+
[1m --> TIME: 2025-12-26 18:38:06 -- STEP: 404/2207 -- GLOBAL_STEP: 76475[0m
|
| 8374 |
+
| > loss_text_ce: 0.030025817453861237 (0.028464864819578016)
|
| 8375 |
+
| > loss_mel_ce: 3.3585338592529297 (3.239105409914904)
|
| 8376 |
+
| > loss: 0.052946243435144424 (0.051055785600501696)
|
| 8377 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8378 |
+
| > step_time: 0.2908 (0.2790926671264195)
|
| 8379 |
+
| > loader_time: 0.0092 (0.008597329701527512)
|
| 8380 |
+
|
| 8381 |
+
|
| 8382 |
+
[1m --> TIME: 2025-12-26 18:38:16 -- STEP: 429/2207 -- GLOBAL_STEP: 76500[0m
|
| 8383 |
+
| > loss_text_ce: 0.029212504625320435 (0.028456895362525)
|
| 8384 |
+
| > loss_mel_ce: 3.600022315979004 (3.241620590081026)
|
| 8385 |
+
| > loss: 0.05670679360628128 (0.05109496074261919)
|
| 8386 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8387 |
+
| > step_time: 0.289 (0.27912182630080995)
|
| 8388 |
+
| > loader_time: 0.008 (0.008594295639536042)
|
| 8389 |
+
|
| 8390 |
+
|
| 8391 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000/checkpoint_76500.pth
|
| 8392 |
+
|
| 8393 |
+
[1m --> TIME: 2025-12-26 18:38:51 -- STEP: 454/2207 -- GLOBAL_STEP: 76525[0m
|
| 8394 |
+
| > loss_text_ce: 0.027661308646202087 (0.028445460776170706)
|
| 8395 |
+
| > loss_mel_ce: 3.078929901123047 (3.2396781938191554)
|
| 8396 |
+
| > loss: 0.04854048788547516 (0.0510644321514515)
|
| 8397 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8398 |
+
| > step_time: 0.288 (0.279223646361397)
|
| 8399 |
+
| > loader_time: 0.0079 (0.008611949004790861)
|
| 8400 |
+
|
| 8401 |
+
|
| 8402 |
+
[1m --> TIME: 2025-12-26 18:39:00 -- STEP: 479/2207 -- GLOBAL_STEP: 76550[0m
|
| 8403 |
+
| > loss_text_ce: 0.026601707562804222 (0.028443980211535925)
|
| 8404 |
+
| > loss_mel_ce: 3.4536221027374268 (3.2430626509831693)
|
| 8405 |
+
| > loss: 0.05437849834561348 (0.051117291156832494)
|
| 8406 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8407 |
+
| > step_time: 0.2908 (0.27900978817073663)
|
| 8408 |
+
| > loader_time: 0.0092 (0.0085996531245604)
|
| 8409 |
+
|
| 8410 |
+
|
| 8411 |
+
[1m --> TIME: 2025-12-26 18:39:09 -- STEP: 504/2207 -- GLOBAL_STEP: 76575[0m
|
| 8412 |
+
| > loss_text_ce: 0.02730691432952881 (0.028425538993721442)
|
| 8413 |
+
| > loss_mel_ce: 3.096026659011841 (3.2442016303539276)
|
| 8414 |
+
| > loss: 0.04880208522081375 (0.0511347995627494)
|
| 8415 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8416 |
+
| > step_time: 0.2677 (0.27886486715740605)
|
| 8417 |
+
| > loader_time: 0.0081 (0.008589617789737769)
|
| 8418 |
+
|
| 8419 |
+
|
| 8420 |
+
[1m --> TIME: 2025-12-26 18:39:19 -- STEP: 529/2207 -- GLOBAL_STEP: 76600[0m
|
| 8421 |
+
| > loss_text_ce: 0.02623615972697735 (0.028416229306327596)
|
| 8422 |
+
| > loss_mel_ce: 2.992464303970337 (3.242924647881088)
|
| 8423 |
+
| > loss: 0.04716719314455986 (0.05111470124869129)
|
| 8424 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8425 |
+
| > step_time: 0.3076 (0.27868463997579457)
|
| 8426 |
+
| > loader_time: 0.0086 (0.008577313900895297)
|
| 8427 |
+
|
| 8428 |
+
|
| 8429 |
+
[1m --> TIME: 2025-12-26 18:39:28 -- STEP: 554/2207 -- GLOBAL_STEP: 76625[0m
|
| 8430 |
+
| > loss_text_ce: 0.028271352872252464 (0.028413972901415743)
|
| 8431 |
+
| > loss_mel_ce: 3.4204750061035156 (3.239746762096667)
|
| 8432 |
+
| > loss: 0.05388666316866875 (0.0510650115349878)
|
| 8433 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8434 |
+
| > step_time: 0.2887 (0.278544981557109)
|
| 8435 |
+
| > loader_time: 0.0095 (0.008582736180577472)
|
| 8436 |
+
|
| 8437 |
+
|
| 8438 |
+
[1m --> TIME: 2025-12-26 18:39:38 -- STEP: 579/2207 -- GLOBAL_STEP: 76650[0m
|
| 8439 |
+
| > loss_text_ce: 0.02775268256664276 (0.028430289506835023)
|
| 8440 |
+
| > loss_mel_ce: 3.3831090927124023 (3.2377296136450897)
|
| 8441 |
+
| > loss: 0.05329471454024315 (0.051033748529323845)
|
| 8442 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8443 |
+
| > step_time: 0.2667 (0.27862886649545177)
|
| 8444 |
+
| > loader_time: 0.008 (0.008585101788113983)
|
| 8445 |
+
|
| 8446 |
+
|
| 8447 |
+
[1m --> TIME: 2025-12-26 18:39:48 -- STEP: 604/2207 -- GLOBAL_STEP: 76675[0m
|
| 8448 |
+
| > loss_text_ce: 0.026103440672159195 (0.028427416437138982)
|
| 8449 |
+
| > loss_mel_ce: 3.4148752689361572 (3.23761110629467)
|
| 8450 |
+
| > loss: 0.05376529321074486 (0.051031851968327116)
|
| 8451 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8452 |
+
| > step_time: 0.2884 (0.27852532800459684)
|
| 8453 |
+
| > loader_time: 0.0091 (0.008601390762834367)
|
| 8454 |
+
|
| 8455 |
+
|
| 8456 |
+
[1m --> TIME: 2025-12-26 18:39:57 -- STEP: 629/2207 -- GLOBAL_STEP: 76700[0m
|
| 8457 |
+
| > loss_text_ce: 0.02849736623466015 (0.028446617967113022)
|
| 8458 |
+
| > loss_mel_ce: 3.0908594131469727 (3.2379462089750835)
|
| 8459 |
+
| > loss: 0.048739951103925705 (0.051037387972747955)
|
| 8460 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8461 |
+
| > step_time: 0.2915 (0.27878278996114303)
|
| 8462 |
+
| > loader_time: 0.0086 (0.008600771711437435)
|
| 8463 |
+
|
| 8464 |
+
|
| 8465 |
+
[1m --> TIME: 2025-12-26 18:40:07 -- STEP: 654/2207 -- GLOBAL_STEP: 76725[0m
|
| 8466 |
+
| > loss_text_ce: 0.029301967471837997 (0.02845591907631549)
|
| 8467 |
+
| > loss_mel_ce: 3.3446786403656006 (3.238683279501189)
|
| 8468 |
+
| > loss: 0.05271844565868378 (0.05104905002912797)
|
| 8469 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8470 |
+
| > step_time: 0.2691 (0.2787951187256278)
|
| 8471 |
+
| > loader_time: 0.0081 (0.00859730579072912)
|
| 8472 |
+
|
| 8473 |
+
|
| 8474 |
+
[1m --> TIME: 2025-12-26 18:40:17 -- STEP: 679/2207 -- GLOBAL_STEP: 76750[0m
|
| 8475 |
+
| > loss_text_ce: 0.02736814133822918 (0.02844762038960724)
|
| 8476 |
+
| > loss_mel_ce: 3.2442069053649902 (3.2384552309544694)
|
| 8477 |
+
| > loss: 0.051118358969688416 (0.05104535710017179)
|
| 8478 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8479 |
+
| > step_time: 0.2893 (0.27882681784819424)
|
| 8480 |
+
| > loader_time: 0.0084 (0.008601977653110155)
|
| 8481 |
+
|
| 8482 |
+
|
| 8483 |
+
[1m --> TIME: 2025-12-26 18:40:26 -- STEP: 704/2207 -- GLOBAL_STEP: 76775[0m
|
| 8484 |
+
| > loss_text_ce: 0.029537783935666084 (0.028442053867779163)
|
| 8485 |
+
| > loss_mel_ce: 3.3125696182250977 (3.23888124153018)
|
| 8486 |
+
| > loss: 0.0522204264998436 (0.051051926539829845)
|
| 8487 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8488 |
+
| > step_time: 0.2594 (0.2787424921989441)
|
| 8489 |
+
| > loader_time: 0.0082 (0.008600711145184263)
|
| 8490 |
+
|
| 8491 |
+
|
| 8492 |
+
[1m --> TIME: 2025-12-26 18:40:36 -- STEP: 729/2207 -- GLOBAL_STEP: 76800[0m
|
| 8493 |
+
| > loss_text_ce: 0.028287993744015694 (0.028451998704030028)
|
| 8494 |
+
| > loss_mel_ce: 3.1804797649383545 (3.2418104627658964)
|
| 8495 |
+
| > loss: 0.0501369945704937 (0.051097851006418914)
|
| 8496 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8497 |
+
| > step_time: 0.2702 (0.27864748591094657)
|
| 8498 |
+
| > loader_time: 0.0084 (0.008595313881977441)
|
| 8499 |
+
|
| 8500 |
+
|
| 8501 |
+
[1m --> TIME: 2025-12-26 18:40:45 -- STEP: 754/2207 -- GLOBAL_STEP: 76825[0m
|
| 8502 |
+
| > loss_text_ce: 0.027993451803922653 (0.028455914993738306)
|
| 8503 |
+
| > loss_mel_ce: 3.1535685062408447 (3.243211562816913)
|
| 8504 |
+
| > loss: 0.04971190541982651 (0.05111980438726492)
|
| 8505 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8506 |
+
| > step_time: 0.2887 (0.27854996808960186)
|
| 8507 |
+
| > loader_time: 0.0088 (0.00859334424572852)
|
| 8508 |
+
|
| 8509 |
+
|
| 8510 |
+
[1m --> TIME: 2025-12-26 18:40:55 -- STEP: 779/2207 -- GLOBAL_STEP: 76850[0m
|
| 8511 |
+
| > loss_text_ce: 0.026838507503271103 (0.02846324702052923)
|
| 8512 |
+
| > loss_mel_ce: 3.1737918853759766 (3.2437810196956103)
|
| 8513 |
+
| > loss: 0.05000985041260719 (0.05112881671556)
|
| 8514 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8515 |
+
| > step_time: 0.2679 (0.27853429393988643)
|
| 8516 |
+
| > loader_time: 0.0094 (0.00859874479271788)
|
| 8517 |
+
|
| 8518 |
+
|
| 8519 |
+
[1m --> TIME: 2025-12-26 18:41:04 -- STEP: 804/2207 -- GLOBAL_STEP: 76875[0m
|
| 8520 |
+
| > loss_text_ce: 0.02945268712937832 (0.028465345112690284)
|
| 8521 |
+
| > loss_mel_ce: 2.991420269012451 (3.2438545075815117)
|
| 8522 |
+
| > loss: 0.047201141715049744 (0.05112999775423784)
|
| 8523 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8524 |
+
| > step_time: 0.2669 (0.2785900929080904)
|
| 8525 |
+
| > loader_time: 0.0082 (0.008601822366761934)
|
| 8526 |
+
|
| 8527 |
+
|
| 8528 |
+
[1m --> TIME: 2025-12-26 18:41:14 -- STEP: 829/2207 -- GLOBAL_STEP: 76900[0m
|
| 8529 |
+
| > loss_text_ce: 0.02894349955022335 (0.028472182679072075)
|
| 8530 |
+
| > loss_mel_ce: 3.075671911239624 (3.244518334097684)
|
| 8531 |
+
| > loss: 0.048509616404771805 (0.05114047687649008)
|
| 8532 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8533 |
+
| > step_time: 0.2911 (0.2787415515384281)
|
| 8534 |
+
| > loader_time: 0.0083 (0.008603618285911947)
|
| 8535 |
+
|
| 8536 |
+
|
| 8537 |
+
[1m --> TIME: 2025-12-26 18:41:24 -- STEP: 854/2207 -- GLOBAL_STEP: 76925[0m
|
| 8538 |
+
| > loss_text_ce: 0.02569418214261532 (0.028458917731598053)
|
| 8539 |
+
| > loss_mel_ce: 3.100703716278076 (3.2452502683398317)
|
| 8540 |
+
| > loss: 0.04884996637701988 (0.051151706075919595)
|
| 8541 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8542 |
+
| > step_time: 0.2669 (0.27883756160736073)
|
| 8543 |
+
| > loader_time: 0.0094 (0.008612368090091332)
|
| 8544 |
+
|
| 8545 |
+
|
| 8546 |
+
[1m --> TIME: 2025-12-26 18:41:34 -- STEP: 879/2207 -- GLOBAL_STEP: 76950[0m
|
| 8547 |
+
| > loss_text_ce: 0.028166333213448524 (0.02846194756459932)
|
| 8548 |
+
| > loss_mel_ce: 3.327000379562378 (3.2450256407328704)
|
| 8549 |
+
| > loss: 0.05242447927594185 (0.05114824361293931)
|
| 8550 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8551 |
+
| > step_time: 0.2683 (0.27882763740010086)
|
| 8552 |
+
| > loader_time: 0.0096 (0.008623503173984373)
|
| 8553 |
+
|
| 8554 |
+
|
| 8555 |
+
[1m --> TIME: 2025-12-26 18:41:43 -- STEP: 904/2207 -- GLOBAL_STEP: 76975[0m
|
| 8556 |
+
| > loss_text_ce: 0.03045610338449478 (0.02847093005409151)
|
| 8557 |
+
| > loss_mel_ce: 3.1821935176849365 (3.243620335000806)
|
| 8558 |
+
| > loss: 0.050197649747133255 (0.05112642606934615)
|
| 8559 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8560 |
+
| > step_time: 0.2629 (0.2789484127960374)
|
| 8561 |
+
| > loader_time: 0.0099 (0.008631740523650596)
|
| 8562 |
+
|
| 8563 |
+
|
| 8564 |
+
[1m --> TIME: 2025-12-26 18:41:53 -- STEP: 929/2207 -- GLOBAL_STEP: 77000[0m
|
| 8565 |
+
| > loss_text_ce: 0.024867841973900795 (0.028474090701075365)
|
| 8566 |
+
| > loss_mel_ce: 3.8527636528015137 (3.243895525311243)
|
| 8567 |
+
| > loss: 0.060587991029024124 (0.051130775294083185)
|
| 8568 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8569 |
+
| > step_time: 0.2912 (0.2789791723627065)
|
| 8570 |
+
| > loader_time: 0.0087 (0.008631090045103615)
|
| 8571 |
+
|
| 8572 |
+
|
| 8573 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000/checkpoint_77000.pth
|
| 8574 |
+
|
| 8575 |
+
[1m --> TIME: 2025-12-26 18:42:27 -- STEP: 954/2207 -- GLOBAL_STEP: 77025[0m
|
| 8576 |
+
| > loss_text_ce: 0.027607934549450874 (0.028477597073798284)
|
| 8577 |
+
| > loss_mel_ce: 2.8810653686523438 (3.2445704227223584)
|
| 8578 |
+
| > loss: 0.04544802010059357 (0.05114137534778448)
|
| 8579 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8580 |
+
| > step_time: 0.2877 (0.27899897673344976)
|
| 8581 |
+
| > loader_time: 0.009 (0.008655049016140805)
|
| 8582 |
+
|
| 8583 |
+
|
| 8584 |
+
[1m --> TIME: 2025-12-26 18:42:37 -- STEP: 979/2207 -- GLOBAL_STEP: 77050[0m
|
| 8585 |
+
| > loss_text_ce: 0.030392460525035858 (0.02848637126104581)
|
| 8586 |
+
| > loss_mel_ce: 3.156308889389038 (3.242028611916687)
|
| 8587 |
+
| > loss: 0.04979220777750015 (0.05110179664485301)
|
| 8588 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8589 |
+
| > step_time: 0.2881 (0.2789671274931366)
|
| 8590 |
+
| > loader_time: 0.0105 (0.008687782823369758)
|
| 8591 |
+
|
| 8592 |
+
|
| 8593 |
+
[1m --> TIME: 2025-12-26 18:42:47 -- STEP: 1004/2207 -- GLOBAL_STEP: 77075[0m
|
| 8594 |
+
| > loss_text_ce: 0.027945468202233315 (0.028469378564625503)
|
| 8595 |
+
| > loss_mel_ce: 3.1060526371002197 (3.241319824737382)
|
| 8596 |
+
| > loss: 0.04896872118115425 (0.051090456326436474)
|
| 8597 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8598 |
+
| > step_time: 0.2696 (0.27900366122979103)
|
| 8599 |
+
| > loader_time: 0.0093 (0.008723364883210093)
|
| 8600 |
+
|
| 8601 |
+
|
| 8602 |
+
[1m --> TIME: 2025-12-26 18:42:56 -- STEP: 1029/2207 -- GLOBAL_STEP: 77100[0m
|
| 8603 |
+
| > loss_text_ce: 0.030553221702575684 (0.02847575337650477)
|
| 8604 |
+
| > loss_mel_ce: 3.167560577392578 (3.241654769788099)
|
| 8605 |
+
| > loss: 0.04997052997350693 (0.0510957894486577)
|
| 8606 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8607 |
+
| > step_time: 0.2877 (0.27903047595010216)
|
| 8608 |
+
| > loader_time: 0.0083 (0.00873087678173426)
|
| 8609 |
+
|
| 8610 |
+
|
| 8611 |
+
[1m --> TIME: 2025-12-26 18:43:06 -- STEP: 1054/2207 -- GLOBAL_STEP: 77125[0m
|
| 8612 |
+
| > loss_text_ce: 0.027842404320836067 (0.02847978913629926)
|
| 8613 |
+
| > loss_mel_ce: 3.201298475265503 (3.240596338965183)
|
| 8614 |
+
| > loss: 0.050455328077077866 (0.05107931452723789)
|
| 8615 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8616 |
+
| > step_time: 0.2914 (0.2790217566987822)
|
| 8617 |
+
| > loader_time: 0.0089 (0.008733750973073547)
|
| 8618 |
+
|
| 8619 |
+
|
| 8620 |
+
[1m --> TIME: 2025-12-26 18:43:16 -- STEP: 1079/2207 -- GLOBAL_STEP: 77150[0m
|
| 8621 |
+
| > loss_text_ce: 0.030789831653237343 (0.028470158763402476)
|
| 8622 |
+
| > loss_mel_ce: 3.275624990463257 (3.240523535416456)
|
| 8623 |
+
| > loss: 0.05166273191571236 (0.051078026497993784)
|
| 8624 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8625 |
+
| > step_time: 0.2884 (0.27955182069313556)
|
| 8626 |
+
| > loader_time: 0.0095 (0.008735521730170202)
|
| 8627 |
+
|
| 8628 |
+
|
| 8629 |
+
[1m --> TIME: 2025-12-26 18:43:26 -- STEP: 1104/2207 -- GLOBAL_STEP: 77175[0m
|
| 8630 |
+
| > loss_text_ce: 0.027588538825511932 (0.02846544295839587)
|
| 8631 |
+
| > loss_mel_ce: 3.191793918609619 (3.2393217760583632)
|
| 8632 |
+
| > loss: 0.05030285194516182 (0.05105917531646032)
|
| 8633 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8634 |
+
| > step_time: 0.2887 (0.2796486223089524)
|
| 8635 |
+
| > loader_time: 0.0099 (0.008744112175443908)
|
| 8636 |
+
|
| 8637 |
+
|
| 8638 |
+
[1m --> TIME: 2025-12-26 18:43:35 -- STEP: 1129/2207 -- GLOBAL_STEP: 77200[0m
|
| 8639 |
+
| > loss_text_ce: 0.02866869419813156 (0.028470455691139186)
|
| 8640 |
+
| > loss_mel_ce: 3.0809102058410645 (3.2387034865582915)
|
| 8641 |
+
| > loss: 0.048587169498205185 (0.05104959286267615)
|
| 8642 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8643 |
+
| > step_time: 0.2701 (0.2796794527711017)
|
| 8644 |
+
| > loader_time: 0.0095 (0.008753021170968404)
|
| 8645 |
+
|
| 8646 |
+
|
| 8647 |
+
[1m --> TIME: 2025-12-26 18:43:45 -- STEP: 1154/2207 -- GLOBAL_STEP: 77225[0m
|
| 8648 |
+
| > loss_text_ce: 0.029066113755106926 (0.028465568183225612)
|
| 8649 |
+
| > loss_mel_ce: 3.4912214279174805 (3.2377236513372534)
|
| 8650 |
+
| > loss: 0.05500449240207672 (0.05103420656645734)
|
| 8651 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8652 |
+
| > step_time: 0.2667 (0.27969640937587015)
|
| 8653 |
+
| > loader_time: 0.0104 (0.008764816528796335)
|
| 8654 |
+
|
| 8655 |
+
|
| 8656 |
+
[1m --> TIME: 2025-12-26 18:43:55 -- STEP: 1179/2207 -- GLOBAL_STEP: 77250[0m
|
| 8657 |
+
| > loss_text_ce: 0.028696173802018166 (0.028470915461794817)
|
| 8658 |
+
| > loss_mel_ce: 2.9501895904541016 (3.237222062014645)
|
| 8659 |
+
| > loss: 0.04654508829116821 (0.051026452778253246)
|
| 8660 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8661 |
+
| > step_time: 0.2881 (0.2797064210925696)
|
| 8662 |
+
| > loader_time: 0.0092 (0.008788517918801099)
|
| 8663 |
+
|
| 8664 |
+
|
| 8665 |
+
[1m --> TIME: 2025-12-26 18:44:04 -- STEP: 1204/2207 -- GLOBAL_STEP: 77275[0m
|
| 8666 |
+
| > loss_text_ce: 0.028850233182311058 (0.028467667522682403)
|
| 8667 |
+
| > loss_mel_ce: 3.1064953804016113 (3.2362481003187833)
|
| 8668 |
+
| > loss: 0.04898977652192116 (0.051011183877719996)
|
| 8669 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8670 |
+
| > step_time: 0.268 (0.2797042830441875)
|
| 8671 |
+
| > loader_time: 0.0095 (0.00880344266511278)
|
| 8672 |
+
|
| 8673 |
+
|
| 8674 |
+
[1m --> TIME: 2025-12-26 18:44:14 -- STEP: 1229/2207 -- GLOBAL_STEP: 77300[0m
|
| 8675 |
+
| > loss_text_ce: 0.02607744373381138 (0.028472046989968774)
|
| 8676 |
+
| > loss_mel_ce: 3.2960610389709473 (3.235376294790196)
|
| 8677 |
+
| > loss: 0.05190841481089592 (0.050997630346708914)
|
| 8678 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8679 |
+
| > step_time: 0.2886 (0.2798562787042776)
|
| 8680 |
+
| > loader_time: 0.0097 (0.008814152127072726)
|
| 8681 |
+
|
| 8682 |
+
|
| 8683 |
+
[1m --> TIME: 2025-12-26 18:44:24 -- STEP: 1254/2207 -- GLOBAL_STEP: 77325[0m
|
| 8684 |
+
| > loss_text_ce: 0.03019668348133564 (0.028481317190913874)
|
| 8685 |
+
| > loss_mel_ce: 3.0860512256622314 (3.236033199696639)
|
| 8686 |
+
| > loss: 0.04869137331843376 (0.051008039336812)
|
| 8687 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8688 |
+
| > step_time: 0.2694 (0.279818185398643)
|
| 8689 |
+
| > loader_time: 0.0088 (0.008826872378445136)
|
| 8690 |
+
|
| 8691 |
+
|
| 8692 |
+
[1m --> TIME: 2025-12-26 18:44:34 -- STEP: 1279/2207 -- GLOBAL_STEP: 77350[0m
|
| 8693 |
+
| > loss_text_ce: 0.029982032254338264 (0.02847985074978447)
|
| 8694 |
+
| > loss_mel_ce: 3.1099467277526855 (3.2367318237638734)
|
| 8695 |
+
| > loss: 0.04906138777732849 (0.05101893242795177)
|
| 8696 |
+
| > grad_norm: 0 (0.0)
|
| 8697 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8698 |
+
| > step_time: 0.2872 (0.2798618064251049)
|
| 8699 |
+
| > loader_time: 0.0096 (0.00883020452450178)
|
| 8700 |
+
|
| 8701 |
+
|
| 8702 |
+
[1m --> TIME: 2025-12-26 18:44:43 -- STEP: 1304/2207 -- GLOBAL_STEP: 77375[0m
|
| 8703 |
+
| > loss_text_ce: 0.029672548174858093 (0.02847488940567738)
|
| 8704 |
+
| > loss_mel_ce: 3.260866641998291 (3.2372209385860184)
|
| 8705 |
+
| > loss: 0.05141467601060867 (0.05102649732761407)
|
| 8706 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8707 |
+
| > step_time: 0.2962 (0.279778135334787)
|
| 8708 |
+
| > loader_time: 0.0086 (0.008838409477947689)
|
| 8709 |
+
|
| 8710 |
+
|
| 8711 |
+
[1m --> TIME: 2025-12-26 18:44:53 -- STEP: 1329/2207 -- GLOBAL_STEP: 77400[0m
|
| 8712 |
+
| > loss_text_ce: 0.029069386422634125 (0.02847405265458909)
|
| 8713 |
+
| > loss_mel_ce: 2.9507675170898438 (3.2364250382953696)
|
| 8714 |
+
| > loss: 0.046559952199459076 (0.05101404831015231)
|
| 8715 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8716 |
+
| > step_time: 0.2672 (0.279800325878766)
|
| 8717 |
+
| > loader_time: 0.0099 (0.008849558030150731)
|
| 8718 |
+
|
| 8719 |
+
|
| 8720 |
+
[1m --> TIME: 2025-12-26 18:45:02 -- STEP: 1354/2207 -- GLOBAL_STEP: 77425[0m
|
| 8721 |
+
| > loss_text_ce: 0.028717035427689552 (0.028470665241207824)
|
| 8722 |
+
| > loss_mel_ce: 3.238875150680542 (3.2357179181790596)
|
| 8723 |
+
| > loss: 0.05105612799525261 (0.051002946631374904)
|
| 8724 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8725 |
+
| > step_time: 0.2666 (0.2797469576845491)
|
| 8726 |
+
| > loader_time: 0.0095 (0.008861168624731436)
|
| 8727 |
+
|
| 8728 |
+
|
| 8729 |
+
[1m --> TIME: 2025-12-26 18:45:12 -- STEP: 1379/2207 -- GLOBAL_STEP: 77450[0m
|
| 8730 |
+
| > loss_text_ce: 0.030025554820895195 (0.028469713828901264)
|
| 8731 |
+
| > loss_mel_ce: 3.659090757369995 (3.2366211742832665)
|
| 8732 |
+
| > loss: 0.05764244124293327 (0.05101704513998549)
|
| 8733 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8734 |
+
| > step_time: 0.2673 (0.2797339705992123)
|
| 8735 |
+
| > loader_time: 0.009 (0.008868956756038552)
|
| 8736 |
+
|
| 8737 |
+
|
| 8738 |
+
[1m --> TIME: 2025-12-26 18:45:22 -- STEP: 1404/2207 -- GLOBAL_STEP: 77475[0m
|
| 8739 |
+
| > loss_text_ce: 0.02893776074051857 (0.028476415865356767)
|
| 8740 |
+
| > loss_mel_ce: 3.5774776935577393 (3.238175418811646)
|
| 8741 |
+
| > loss: 0.05635024234652519 (0.051041434930260726)
|
| 8742 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8743 |
+
| > step_time: 0.2909 (0.27964550800133264)
|
| 8744 |
+
| > loader_time: 0.0082 (0.008871728538447981)
|
| 8745 |
+
|
| 8746 |
+
|
| 8747 |
+
[1m --> TIME: 2025-12-26 18:45:31 -- STEP: 1429/2207 -- GLOBAL_STEP: 77500[0m
|
| 8748 |
+
| > loss_text_ce: 0.024822553619742393 (0.02847343137744551)
|
| 8749 |
+
| > loss_mel_ce: 3.4797487258911133 (3.2376295372353643)
|
| 8750 |
+
| > loss: 0.05475892499089241 (0.05103285889877067)
|
| 8751 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8752 |
+
| > step_time: 0.2906 (0.2795997456790181)
|
| 8753 |
+
| > loader_time: 0.0091 (0.008865504435011365)
|
| 8754 |
+
|
| 8755 |
+
|
| 8756 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000/checkpoint_77500.pth
|
| 8757 |
+
|
| 8758 |
+
[1m --> TIME: 2025-12-26 18:46:06 -- STEP: 1454/2207 -- GLOBAL_STEP: 77525[0m
|
| 8759 |
+
| > loss_text_ce: 0.030959313735365868 (0.028473818549140776)
|
| 8760 |
+
| > loss_mel_ce: 3.4375522136688232 (3.2375380102509155)
|
| 8761 |
+
| > loss: 0.05419549345970154 (0.05103143484189473)
|
| 8762 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8763 |
+
| > step_time: 0.2588 (0.27963094796570276)
|
| 8764 |
+
| > loader_time: 0.0094 (0.008877688129797605)
|
| 8765 |
+
|
| 8766 |
+
|
| 8767 |
+
[1m --> TIME: 2025-12-26 18:46:15 -- STEP: 1479/2207 -- GLOBAL_STEP: 77550[0m
|
| 8768 |
+
| > loss_text_ce: 0.026863934472203255 (0.02846340398840037)
|
| 8769 |
+
| > loss_mel_ce: 3.1678366661071777 (3.2365651225785776)
|
| 8770 |
+
| > loss: 0.04991719871759415 (0.051016070747129674)
|
| 8771 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8772 |
+
| > step_time: 0.291 (0.2796170735697714)
|
| 8773 |
+
| > loader_time: 0.009 (0.008888224800991974)
|
| 8774 |
+
|
| 8775 |
+
|
| 8776 |
+
[1m --> TIME: 2025-12-26 18:46:25 -- STEP: 1504/2207 -- GLOBAL_STEP: 77575[0m
|
| 8777 |
+
| > loss_text_ce: 0.029093941673636436 (0.02846291351894987)
|
| 8778 |
+
| > loss_mel_ce: 3.371005058288574 (3.236263193031575)
|
| 8779 |
+
| > loss: 0.05312654748558998 (0.0510113454359445)
|
| 8780 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8781 |
+
| > step_time: 0.2636 (0.2796023041009899)
|
| 8782 |
+
| > loader_time: 0.0102 (0.008891621168623588)
|
| 8783 |
+
|
| 8784 |
+
|
| 8785 |
+
[1m --> TIME: 2025-12-26 18:46:34 -- STEP: 1529/2207 -- GLOBAL_STEP: 77600[0m
|
| 8786 |
+
| > loss_text_ce: 0.028175119310617447 (0.028469184900836823)
|
| 8787 |
+
| > loss_mel_ce: 3.323124408721924 (3.2365292490815087)
|
| 8788 |
+
| > loss: 0.052364055067300797 (0.05101560055478639)
|
| 8789 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8790 |
+
| > step_time: 0.3023 (0.2795772098730867)
|
| 8791 |
+
| > loader_time: 0.0093 (0.008892629098237311)
|
| 8792 |
+
|
| 8793 |
+
|
| 8794 |
+
[1m --> TIME: 2025-12-26 18:46:44 -- STEP: 1554/2207 -- GLOBAL_STEP: 77625[0m
|
| 8795 |
+
| > loss_text_ce: 0.026646001264452934 (0.02846265981450888)
|
| 8796 |
+
| > loss_mel_ce: 3.1285500526428223 (3.2368098070876172)
|
| 8797 |
+
| > loss: 0.0492999367415905 (0.05101988231823176)
|
| 8798 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8799 |
+
| > step_time: 0.2906 (0.27959725295254584)
|
| 8800 |
+
| > loader_time: 0.0083 (0.008892723185368653)
|
| 8801 |
+
|
| 8802 |
+
|
| 8803 |
+
[1m --> TIME: 2025-12-26 18:46:54 -- STEP: 1579/2207 -- GLOBAL_STEP: 77650[0m
|
| 8804 |
+
| > loss_text_ce: 0.029372259974479675 (0.028453044253599048)
|
| 8805 |
+
| > loss_mel_ce: 3.0140576362609863 (3.237355043949999)
|
| 8806 |
+
| > loss: 0.047553591430187225 (0.05102825139786642)
|
| 8807 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8808 |
+
| > step_time: 0.2703 (0.2796294865840741)
|
| 8809 |
+
| > loader_time: 0.0096 (0.008892020520868007)
|
| 8810 |
+
|
| 8811 |
+
|
| 8812 |
+
[1m --> TIME: 2025-12-26 18:47:04 -- STEP: 1604/2207 -- GLOBAL_STEP: 77675[0m
|
| 8813 |
+
| > loss_text_ce: 0.03028406761586666 (0.028451028135965145)
|
| 8814 |
+
| > loss_mel_ce: 3.193387031555176 (3.2366834065860646)
|
| 8815 |
+
| > loss: 0.05036986246705055 (0.05101772556151091)
|
| 8816 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8817 |
+
| > step_time: 0.2674 (0.27967758829754163)
|
| 8818 |
+
| > loader_time: 0.0086 (0.008887217526423975)
|
| 8819 |
+
|
| 8820 |
+
|
| 8821 |
+
[1m --> TIME: 2025-12-26 18:47:13 -- STEP: 1629/2207 -- GLOBAL_STEP: 77700[0m
|
| 8822 |
+
| > loss_text_ce: 0.030793659389019012 (0.028445894979146305)
|
| 8823 |
+
| > loss_mel_ce: 3.0936965942382812 (3.2359032778742858)
|
| 8824 |
+
| > loss: 0.04882016032934189 (0.05100545583966323)
|
| 8825 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8826 |
+
| > step_time: 0.2673 (0.279610905492137)
|
| 8827 |
+
| > loader_time: 0.0085 (0.008885889861245635)
|
| 8828 |
+
|
| 8829 |
+
|
| 8830 |
+
[1m --> TIME: 2025-12-26 18:47:23 -- STEP: 1654/2207 -- GLOBAL_STEP: 77725[0m
|
| 8831 |
+
| > loss_text_ce: 0.029394833371043205 (0.028442977578519983)
|
| 8832 |
+
| > loss_mel_ce: 3.099526882171631 (3.2358772330878005)
|
| 8833 |
+
| > loss: 0.0488894023001194 (0.05100500330491108)
|
| 8834 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8835 |
+
| > step_time: 0.2893 (0.27959459047075286)
|
| 8836 |
+
| > loader_time: 0.0079 (0.008882686045322646)
|
| 8837 |
+
|
| 8838 |
+
|
| 8839 |
+
[1m --> TIME: 2025-12-26 18:47:32 -- STEP: 1679/2207 -- GLOBAL_STEP: 77750[0m
|
| 8840 |
+
| > loss_text_ce: 0.027710091322660446 (0.028438831288467355)
|
| 8841 |
+
| > loss_mel_ce: 3.0219175815582275 (3.2352714179313344)
|
| 8842 |
+
| > loss: 0.04765043407678604 (0.05099547266108144)
|
| 8843 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8844 |
+
| > step_time: 0.2906 (0.27958717638712055)
|
| 8845 |
+
| > loader_time: 0.0099 (0.00888115568768775)
|
| 8846 |
+
|
| 8847 |
+
|
| 8848 |
+
[1m --> TIME: 2025-12-26 18:47:42 -- STEP: 1704/2207 -- GLOBAL_STEP: 77775[0m
|
| 8849 |
+
| > loss_text_ce: 0.029714345932006836 (0.02843475295121436)
|
| 8850 |
+
| > loss_mel_ce: 3.222139596939087 (3.2360454702601187)
|
| 8851 |
+
| > loss: 0.05081021785736084 (0.05100750350371495)
|
| 8852 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8853 |
+
| > step_time: 0.2889 (0.2795923071008326)
|
| 8854 |
+
| > loader_time: 0.0091 (0.008880259145593442)
|
| 8855 |
+
|
| 8856 |
+
|
| 8857 |
+
[1m --> TIME: 2025-12-26 18:47:52 -- STEP: 1729/2207 -- GLOBAL_STEP: 77800[0m
|
| 8858 |
+
| > loss_text_ce: 0.029237335547804832 (0.02843356854295786)
|
| 8859 |
+
| > loss_mel_ce: 3.056673288345337 (3.2353470348911935)
|
| 8860 |
+
| > loss: 0.048217352479696274 (0.050996571938690956)
|
| 8861 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8862 |
+
| > step_time: 0.306 (0.27959724504041633)
|
| 8863 |
+
| > loader_time: 0.0093 (0.008878413482233605)
|
| 8864 |
+
|
| 8865 |
+
|
| 8866 |
+
[1m --> TIME: 2025-12-26 18:48:01 -- STEP: 1754/2207 -- GLOBAL_STEP: 77825[0m
|
| 8867 |
+
| > loss_text_ce: 0.02727603167295456 (0.028432496410275267)
|
| 8868 |
+
| > loss_mel_ce: 3.1551461219787598 (3.2345479620767157)
|
| 8869 |
+
| > loss: 0.04972534626722336 (0.050984069673151236)
|
| 8870 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8871 |
+
| > step_time: 0.2698 (0.2796093362766897)
|
| 8872 |
+
| > loader_time: 0.0088 (0.008877903732769598)
|
| 8873 |
+
|
| 8874 |
+
|
| 8875 |
+
[1m --> TIME: 2025-12-26 18:48:11 -- STEP: 1779/2207 -- GLOBAL_STEP: 77850[0m
|
| 8876 |
+
| > loss_text_ce: 0.029735876247286797 (0.02843661024797961)
|
| 8877 |
+
| > loss_mel_ce: 3.30867338180542 (3.2344959966917934)
|
| 8878 |
+
| > loss: 0.052162643522024155 (0.05098332199590003)
|
| 8879 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8880 |
+
| > step_time: 0.2673 (0.2795423123861972)
|
| 8881 |
+
| > loader_time: 0.0086 (0.008874698191830947)
|
| 8882 |
+
|
| 8883 |
+
|
| 8884 |
+
[1m --> TIME: 2025-12-26 18:48:20 -- STEP: 1804/2207 -- GLOBAL_STEP: 77875[0m
|
| 8885 |
+
| > loss_text_ce: 0.026523202657699585 (0.02843507734269829)
|
| 8886 |
+
| > loss_mel_ce: 3.1697516441345215 (3.2338974404493617)
|
| 8887 |
+
| > loss: 0.04994179308414459 (0.05097394560705335)
|
| 8888 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8889 |
+
| > step_time: 0.3066 (0.2795383051335141)
|
| 8890 |
+
| > loader_time: 0.0086 (0.00887205912638665)
|
| 8891 |
+
|
| 8892 |
+
|
| 8893 |
+
[1m --> TIME: 2025-12-26 18:48:30 -- STEP: 1829/2207 -- GLOBAL_STEP: 77900[0m
|
| 8894 |
+
| > loss_text_ce: 0.025550249963998795 (0.028435324428184493)
|
| 8895 |
+
| > loss_mel_ce: 3.010570526123047 (3.2335961909161166)
|
| 8896 |
+
| > loss: 0.04743938893079758 (0.050969242444694744)
|
| 8897 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8898 |
+
| > step_time: 0.3069 (0.27956504415070854)
|
| 8899 |
+
| > loader_time: 0.0099 (0.008871132458018918)
|
| 8900 |
+
|
| 8901 |
+
|
| 8902 |
+
[1m --> TIME: 2025-12-26 18:48:40 -- STEP: 1854/2207 -- GLOBAL_STEP: 77925[0m
|
| 8903 |
+
| > loss_text_ce: 0.026484891772270203 (0.02843378770678528)
|
| 8904 |
+
| > loss_mel_ce: 3.518221855163574 (3.23343328506301)
|
| 8905 |
+
| > loss: 0.05538604408502579 (0.05096667302730208)
|
| 8906 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8907 |
+
| > step_time: 0.3069 (0.2796063115810231)
|
| 8908 |
+
| > loader_time: 0.0089 (0.008874466761287647)
|
| 8909 |
+
|
| 8910 |
+
|
| 8911 |
+
[1m --> TIME: 2025-12-26 18:48:49 -- STEP: 1879/2207 -- GLOBAL_STEP: 77950[0m
|
| 8912 |
+
| > loss_text_ce: 0.024520978331565857 (0.028436370573624643)
|
| 8913 |
+
| > loss_mel_ce: 3.084439754486084 (3.232797743602913)
|
| 8914 |
+
| > loss: 0.04857750982046127 (0.05095678304598141)
|
| 8915 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8916 |
+
| > step_time: 0.2874 (0.2795722913462906)
|
| 8917 |
+
| > loader_time: 0.0094 (0.008871439862213242)
|
| 8918 |
+
|
| 8919 |
+
|
| 8920 |
+
[1m --> TIME: 2025-12-26 18:48:59 -- STEP: 1904/2207 -- GLOBAL_STEP: 77975[0m
|
| 8921 |
+
| > loss_text_ce: 0.02873215824365616 (0.02843987077627206)
|
| 8922 |
+
| > loss_mel_ce: 3.192657232284546 (3.232237666720102)
|
| 8923 |
+
| > loss: 0.05033420771360397 (0.0509480865351355)
|
| 8924 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8925 |
+
| > step_time: 0.2874 (0.2795992810435645)
|
| 8926 |
+
| > loader_time: 0.0081 (0.00886446913751236)
|
| 8927 |
+
|
| 8928 |
+
|
| 8929 |
+
[1m --> TIME: 2025-12-26 18:49:09 -- STEP: 1929/2207 -- GLOBAL_STEP: 78000[0m
|
| 8930 |
+
| > loss_text_ce: 0.029951870441436768 (0.028444559522409586)
|
| 8931 |
+
| > loss_mel_ce: 3.4030110836029053 (3.23245055617416)
|
| 8932 |
+
| > loss: 0.05364004522562027 (0.050951486195598396)
|
| 8933 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8934 |
+
| > step_time: 0.288 (0.2796167390466406)
|
| 8935 |
+
| > loader_time: 0.0083 (0.008858825580761086)
|
| 8936 |
+
|
| 8937 |
+
|
| 8938 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000/checkpoint_78000.pth
|
| 8939 |
+
|
| 8940 |
+
[1m --> TIME: 2025-12-26 18:49:42 -- STEP: 1954/2207 -- GLOBAL_STEP: 78025[0m
|
| 8941 |
+
| > loss_text_ce: 0.029546713456511497 (0.02843851972363714)
|
| 8942 |
+
| > loss_mel_ce: 3.1352412700653076 (3.232223406784854)
|
| 8943 |
+
| > loss: 0.04944981262087822 (0.05094784261237454)
|
| 8944 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8945 |
+
| > step_time: 0.2701 (0.27964911490160116)
|
| 8946 |
+
| > loader_time: 0.0094 (0.008869848085401515)
|
| 8947 |
+
|
| 8948 |
+
|
| 8949 |
+
[1m --> TIME: 2025-12-26 18:49:52 -- STEP: 1979/2207 -- GLOBAL_STEP: 78050[0m
|
| 8950 |
+
| > loss_text_ce: 0.03002789616584778 (0.028433832826074958)
|
| 8951 |
+
| > loss_mel_ce: 3.0979385375976562 (3.2321176462669814)
|
| 8952 |
+
| > loss: 0.04887447506189346 (0.05094611687287351)
|
| 8953 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8954 |
+
| > step_time: 0.2923 (0.27966890918178366)
|
| 8955 |
+
| > loader_time: 0.0104 (0.008872493461504823)
|
| 8956 |
+
|
| 8957 |
+
|
| 8958 |
+
[1m --> TIME: 2025-12-26 18:50:02 -- STEP: 2004/2207 -- GLOBAL_STEP: 78075[0m
|
| 8959 |
+
| > loss_text_ce: 0.02730989083647728 (0.028432220326524652)
|
| 8960 |
+
| > loss_mel_ce: 3.790254831314087 (3.232776851949102)
|
| 8961 |
+
| > loss: 0.059649448841810226 (0.0509563917647341)
|
| 8962 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8963 |
+
| > step_time: 0.3214 (0.2796655737473331)
|
| 8964 |
+
| > loader_time: 0.0087 (0.00887368670004811)
|
| 8965 |
+
|
| 8966 |
+
|
| 8967 |
+
[1m --> TIME: 2025-12-26 18:50:11 -- STEP: 2029/2207 -- GLOBAL_STEP: 78100[0m
|
| 8968 |
+
| > loss_text_ce: 0.028728673234581947 (0.028431812606057903)
|
| 8969 |
+
| > loss_mel_ce: 3.3652422428131104 (3.2335624242428196)
|
| 8970 |
+
| > loss: 0.053030796349048615 (0.05096865996182289)
|
| 8971 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8972 |
+
| > step_time: 0.2968 (0.27972913046903025)
|
| 8973 |
+
| > loader_time: 0.0095 (0.008878110017959823)
|
| 8974 |
+
|
| 8975 |
+
|
| 8976 |
+
[1m --> TIME: 2025-12-26 18:50:21 -- STEP: 2054/2207 -- GLOBAL_STEP: 78125[0m
|
| 8977 |
+
| > loss_text_ce: 0.030516687780618668 (0.02843844334793503)
|
| 8978 |
+
| > loss_mel_ce: 3.24049711227417 (3.2341606211871228)
|
| 8979 |
+
| > loss: 0.051109589636325836 (0.05097811039765716)
|
| 8980 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8981 |
+
| > step_time: 0.2891 (0.27969028445970834)
|
| 8982 |
+
| > loader_time: 0.0081 (0.008880642860286333)
|
| 8983 |
+
|
| 8984 |
+
|
| 8985 |
+
[1m --> TIME: 2025-12-26 18:50:31 -- STEP: 2079/2207 -- GLOBAL_STEP: 78150[0m
|
| 8986 |
+
| > loss_text_ce: 0.02688022330403328 (0.028439415424359772)
|
| 8987 |
+
| > loss_mel_ce: 3.3593859672546387 (3.234638448860164)
|
| 8988 |
+
| > loss: 0.05291040986776352 (0.05098559164467658)
|
| 8989 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8990 |
+
| > step_time: 0.271 (0.279701241740473)
|
| 8991 |
+
| > loader_time: 0.0086 (0.008882290701157454)
|
| 8992 |
+
|
| 8993 |
+
|
| 8994 |
+
[1m --> TIME: 2025-12-26 18:50:40 -- STEP: 2104/2207 -- GLOBAL_STEP: 78175[0m
|
| 8995 |
+
| > loss_text_ce: 0.027825672179460526 (0.02844047521244551)
|
| 8996 |
+
| > loss_mel_ce: 3.1415913105010986 (3.235117570075698)
|
| 8997 |
+
| > loss: 0.04952213913202286 (0.050993094472179194)
|
| 8998 |
+
| > current_lr: 1.9999803595518558e-05
|
| 8999 |
+
| > step_time: 0.2695 (0.2797646286822987)
|
| 9000 |
+
| > loader_time: 0.0078 (0.008877920107243203)
|
| 9001 |
+
|
| 9002 |
+
|
| 9003 |
+
[1m --> TIME: 2025-12-26 18:50:50 -- STEP: 2129/2207 -- GLOBAL_STEP: 78200[0m
|
| 9004 |
+
| > loss_text_ce: 0.02734287455677986 (0.02843843409402262)
|
| 9005 |
+
| > loss_mel_ce: 3.2411725521087646 (3.2353584740631107)
|
| 9006 |
+
| > loss: 0.051070552319288254 (0.050996826705293194)
|
| 9007 |
+
| > current_lr: 1.9999803595518558e-05
|
| 9008 |
+
| > step_time: 0.2698 (0.2797999458169636)
|
| 9009 |
+
| > loader_time: 0.0101 (0.008878890647189412)
|
| 9010 |
+
|
| 9011 |
+
|
| 9012 |
+
[1m --> TIME: 2025-12-26 18:51:00 -- STEP: 2154/2207 -- GLOBAL_STEP: 78225[0m
|
| 9013 |
+
| > loss_text_ce: 0.027265088632702827 (0.02843906189174069)
|
| 9014 |
+
| > loss_mel_ce: 3.172513484954834 (3.23655556614131)
|
| 9015 |
+
| > loss: 0.04999653995037079 (0.05101554108024365)
|
| 9016 |
+
| > current_lr: 1.9999803595518558e-05
|
| 9017 |
+
| > step_time: 0.2914 (0.27979934126996414)
|
| 9018 |
+
| > loader_time: 0.0089 (0.008885474036773732)
|
| 9019 |
+
|
| 9020 |
+
|
| 9021 |
+
[1m --> TIME: 2025-12-26 18:51:09 -- STEP: 2179/2207 -- GLOBAL_STEP: 78250[0m
|
| 9022 |
+
| > loss_text_ce: 0.027238352224230766 (0.028440236646943604)
|
| 9023 |
+
| > loss_mel_ce: 3.1119444370269775 (3.2369716716720833)
|
| 9024 |
+
| > loss: 0.049049731343984604 (0.05102206108396676)
|
| 9025 |
+
| > current_lr: 1.9999803595518558e-05
|
| 9026 |
+
| > step_time: 0.2919 (0.279789123473882)
|
| 9027 |
+
| > loader_time: 0.0082 (0.008886995748841166)
|
| 9028 |
+
|
| 9029 |
+
|
| 9030 |
+
[1m --> TIME: 2025-12-26 18:51:19 -- STEP: 2204/2207 -- GLOBAL_STEP: 78275[0m
|
| 9031 |
+
| > loss_text_ce: 0.027152754366397858 (0.028441662196164615)
|
| 9032 |
+
| > loss_mel_ce: 3.2825725078582764 (3.236936985797327)
|
| 9033 |
+
| > loss: 0.0517144575715065 (0.05102154139273829)
|
| 9034 |
+
| > current_lr: 1.9999803595518558e-05
|
| 9035 |
+
| > step_time: 0.2882 (0.2797760436625739)
|
| 9036 |
+
| > loader_time: 0.0086 (0.00888729279357165)
|
| 9037 |
+
|
| 9038 |
+
|
| 9039 |
+
[1m > EVALUATION [0m
|
| 9040 |
+
|
| 9041 |
+
|
| 9042 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 9043 |
+
| > avg_loader_time:[92m 0.0819671556756303 [0m(-0.025489058043505708)
|
| 9044 |
+
| > avg_loss_text_ce:[92m 0.027631033576924254 [0m(-4.8548637612444434e-05)
|
| 9045 |
+
| > avg_loss_mel_ce:[92m 3.7634712054922774 [0m(-0.009091891146995046)
|
| 9046 |
+
| > avg_loss:[92m 3.7911022402144767 [0m(-0.009140436713759748)
|
| 9047 |
+
|