Checkpoint at step 76071
Browse files- checkpoint_75500.pth +3 -0
- checkpoint_76000.pth +3 -0
- events.out.tfevents.1766762704.eeafeeddd90e.1539.1 +2 -2
- trainer_0_log.txt +819 -0
checkpoint_75500.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5fab63043d3f01e2dd80c6610b40a080eab6781182d494eb39acea8ebbcec071
|
| 3 |
+
size 5607926741
|
checkpoint_76000.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b8f0afdaa80a359071b118bfeaec8e8a70607a9cd2b7b095435ddffe61b3b811
|
| 3 |
+
size 5607926741
|
events.out.tfevents.1766762704.eeafeeddd90e.1539.1
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:99b655170eca90dbd8d57467c196fd7860eb1460e351c5ae9be72538ce73c86b
|
| 3 |
+
size 99076
|
trainer_0_log.txt
CHANGED
|
@@ -7401,3 +7401,822 @@
|
|
| 7401 |
| > avg_loss_mel_ce:[92m 3.7513702141272054 [0m(-0.0014747397319689881)
|
| 7402 |
| > avg_loss:[92m 3.7790851496361397 [0m(-0.0015209123894974752)
|
| 7403 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7401 |
| > avg_loss_mel_ce:[92m 3.7513702141272054 [0m(-0.0014747397319689881)
|
| 7402 |
| > avg_loss:[92m 3.7790851496361397 [0m(-0.0015209123894974752)
|
| 7403 |
|
| 7404 |
+
|
| 7405 |
+
[4m[1m > EPOCH: 10/12[0m
|
| 7406 |
+
--> /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000
|
| 7407 |
+
|
| 7408 |
+
[1m > TRAINING (2025-12-26 18:16:28) [0m
|
| 7409 |
+
|
| 7410 |
+
[1m --> TIME: 2025-12-26 18:16:35 -- STEP: 11/2207 -- GLOBAL_STEP: 73875[0m
|
| 7411 |
+
| > loss_text_ce: 0.02931404486298561 (0.02885142070325938)
|
| 7412 |
+
| > loss_mel_ce: 3.1618640422821045 (3.26498066295277)
|
| 7413 |
+
| > loss: 0.04986215755343437 (0.05146612633358349)
|
| 7414 |
+
| > current_lr: 1.999984091227059e-05
|
| 7415 |
+
| > step_time: 0.292 (0.27310297705910425)
|
| 7416 |
+
| > loader_time: 0.0084 (0.008719444274902344)
|
| 7417 |
+
|
| 7418 |
+
|
| 7419 |
+
[1m --> TIME: 2025-12-26 18:16:45 -- STEP: 36/2207 -- GLOBAL_STEP: 73900[0m
|
| 7420 |
+
| > loss_text_ce: 0.028648190200328827 (0.02846097159716818)
|
| 7421 |
+
| > loss_mel_ce: 3.208458423614502 (3.2754179173045688)
|
| 7422 |
+
| > loss: 0.05057979002594948 (0.05162310776197248)
|
| 7423 |
+
| > current_lr: 1.999984091227059e-05
|
| 7424 |
+
| > step_time: 0.2918 (0.2792581783400643)
|
| 7425 |
+
| > loader_time: 0.0079 (0.00843783219655355)
|
| 7426 |
+
|
| 7427 |
+
|
| 7428 |
+
[1m --> TIME: 2025-12-26 18:16:54 -- STEP: 61/2207 -- GLOBAL_STEP: 73925[0m
|
| 7429 |
+
| > loss_text_ce: 0.028761612251400948 (0.028470638009612678)
|
| 7430 |
+
| > loss_mel_ce: 3.1754279136657715 (3.2484507482559954)
|
| 7431 |
+
| > loss: 0.05006546154618263 (0.05120189677252144)
|
| 7432 |
+
| > current_lr: 1.999984091227059e-05
|
| 7433 |
+
| > step_time: 0.2518 (0.27700352668762207)
|
| 7434 |
+
| > loader_time: 0.0077 (0.008458129695204436)
|
| 7435 |
+
|
| 7436 |
+
|
| 7437 |
+
[1m --> TIME: 2025-12-26 18:17:04 -- STEP: 86/2207 -- GLOBAL_STEP: 73950[0m
|
| 7438 |
+
| > loss_text_ce: 0.026422196999192238 (0.02859570571156435)
|
| 7439 |
+
| > loss_mel_ce: 3.068089485168457 (3.2590017401894857)
|
| 7440 |
+
| > loss: 0.048351746052503586 (0.05136871021674125)
|
| 7441 |
+
| > current_lr: 1.999984091227059e-05
|
| 7442 |
+
| > step_time: 0.2881 (0.27841086443080465)
|
| 7443 |
+
| > loader_time: 0.0092 (0.008417478827543037)
|
| 7444 |
+
|
| 7445 |
+
|
| 7446 |
+
[1m --> TIME: 2025-12-26 18:17:14 -- STEP: 111/2207 -- GLOBAL_STEP: 73975[0m
|
| 7447 |
+
| > loss_text_ce: 0.028263196349143982 (0.02850044889560154)
|
| 7448 |
+
| > loss_mel_ce: 3.2740654945373535 (3.252161401886124)
|
| 7449 |
+
| > loss: 0.05159888416528702 (0.05126034159649601)
|
| 7450 |
+
| > current_lr: 1.999984091227059e-05
|
| 7451 |
+
| > step_time: 0.269 (0.27870178007864754)
|
| 7452 |
+
| > loader_time: 0.0077 (0.008368468499398452)
|
| 7453 |
+
|
| 7454 |
+
|
| 7455 |
+
[1m --> TIME: 2025-12-26 18:17:24 -- STEP: 136/2207 -- GLOBAL_STEP: 74000[0m
|
| 7456 |
+
| > loss_text_ce: 0.028226157650351524 (0.028573325451682594)
|
| 7457 |
+
| > loss_mel_ce: 3.406977653503418 (3.2592761849655822)
|
| 7458 |
+
| > loss: 0.05367505922913551 (0.051372648792012664)
|
| 7459 |
+
| > current_lr: 1.999984091227059e-05
|
| 7460 |
+
| > step_time: 0.2695 (0.2799173803890454)
|
| 7461 |
+
| > loader_time: 0.0084 (0.008350130389718451)
|
| 7462 |
+
|
| 7463 |
+
|
| 7464 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000/checkpoint_74000.pth
|
| 7465 |
+
|
| 7466 |
+
[1m --> TIME: 2025-12-26 18:17:58 -- STEP: 161/2207 -- GLOBAL_STEP: 74025[0m
|
| 7467 |
+
| > loss_text_ce: 0.029249321669340134 (0.02849935289032711)
|
| 7468 |
+
| > loss_mel_ce: 3.5447945594787598 (3.2507156081821607)
|
| 7469 |
+
| > loss: 0.05584443733096123 (0.05123773397681137)
|
| 7470 |
+
| > current_lr: 1.999984091227059e-05
|
| 7471 |
+
| > step_time: 0.291 (0.2802099144977074)
|
| 7472 |
+
| > loader_time: 0.0086 (0.008394358321006254)
|
| 7473 |
+
|
| 7474 |
+
|
| 7475 |
+
[1m --> TIME: 2025-12-26 18:18:08 -- STEP: 186/2207 -- GLOBAL_STEP: 74050[0m
|
| 7476 |
+
| > loss_text_ce: 0.028975125402212143 (0.028526905052844557)
|
| 7477 |
+
| > loss_mel_ce: 3.2237963676452637 (3.2556246557543354)
|
| 7478 |
+
| > loss: 0.05082455277442932 (0.051314868233216716)
|
| 7479 |
+
| > current_lr: 1.999984091227059e-05
|
| 7480 |
+
| > step_time: 0.2886 (0.2807401816050212)
|
| 7481 |
+
| > loader_time: 0.0081 (0.008409028412193377)
|
| 7482 |
+
|
| 7483 |
+
|
| 7484 |
+
[1m --> TIME: 2025-12-26 18:18:18 -- STEP: 211/2207 -- GLOBAL_STEP: 74075[0m
|
| 7485 |
+
| > loss_text_ce: 0.02888643741607666 (0.028575007137246606)
|
| 7486 |
+
| > loss_mel_ce: 3.639608144760132 (3.263264731773268)
|
| 7487 |
+
| > loss: 0.05732022970914841 (0.05143499603909905)
|
| 7488 |
+
| > current_lr: 1.999984091227059e-05
|
| 7489 |
+
| > step_time: 0.2681 (0.28082428837274487)
|
| 7490 |
+
| > loader_time: 0.0094 (0.00840538598914847)
|
| 7491 |
+
|
| 7492 |
+
|
| 7493 |
+
[1m --> TIME: 2025-12-26 18:18:27 -- STEP: 236/2207 -- GLOBAL_STEP: 74100[0m
|
| 7494 |
+
| > loss_text_ce: 0.02803642302751541 (0.028562113429593333)
|
| 7495 |
+
| > loss_mel_ce: 3.221923828125 (3.267612328973867)
|
| 7496 |
+
| > loss: 0.05078062787652016 (0.0515027257882185)
|
| 7497 |
+
| > current_lr: 1.999984091227059e-05
|
| 7498 |
+
| > step_time: 0.2677 (0.2807379637734365)
|
| 7499 |
+
| > loader_time: 0.0081 (0.008390740822937531)
|
| 7500 |
+
|
| 7501 |
+
|
| 7502 |
+
[1m --> TIME: 2025-12-26 18:18:37 -- STEP: 261/2207 -- GLOBAL_STEP: 74125[0m
|
| 7503 |
+
| > loss_text_ce: 0.029173938557505608 (0.028531189172946173)
|
| 7504 |
+
| > loss_mel_ce: 3.0340840816497803 (3.26981389933619)
|
| 7505 |
+
| > loss: 0.04786340519785881 (0.05153664214343861)
|
| 7506 |
+
| > current_lr: 1.999984091227059e-05
|
| 7507 |
+
| > step_time: 0.2884 (0.28085184279986275)
|
| 7508 |
+
| > loader_time: 0.0085 (0.008426647076661563)
|
| 7509 |
+
|
| 7510 |
+
|
| 7511 |
+
[1m --> TIME: 2025-12-26 18:18:47 -- STEP: 286/2207 -- GLOBAL_STEP: 74150[0m
|
| 7512 |
+
| > loss_text_ce: 0.02778639644384384 (0.02855568390886684)
|
| 7513 |
+
| > loss_mel_ce: 3.538912773132324 (3.2746368304832822)
|
| 7514 |
+
| > loss: 0.05572967603802681 (0.05161238313242273)
|
| 7515 |
+
| > current_lr: 1.999984091227059e-05
|
| 7516 |
+
| > step_time: 0.2523 (0.2806482681861292)
|
| 7517 |
+
| > loader_time: 0.0092 (0.008437162512665866)
|
| 7518 |
+
|
| 7519 |
+
|
| 7520 |
+
[1m --> TIME: 2025-12-26 18:18:56 -- STEP: 311/2207 -- GLOBAL_STEP: 74175[0m
|
| 7521 |
+
| > loss_text_ce: 0.027154598385095596 (0.028555353756071676)
|
| 7522 |
+
| > loss_mel_ce: 2.9619128704071045 (3.2718175461821217)
|
| 7523 |
+
| > loss: 0.04670418053865433 (0.051568326639113315)
|
| 7524 |
+
| > current_lr: 1.999984091227059e-05
|
| 7525 |
+
| > step_time: 0.292 (0.2803449124768616)
|
| 7526 |
+
| > loader_time: 0.0078 (0.008417594854471393)
|
| 7527 |
+
|
| 7528 |
+
|
| 7529 |
+
[1m --> TIME: 2025-12-26 18:19:06 -- STEP: 336/2207 -- GLOBAL_STEP: 74200[0m
|
| 7530 |
+
| > loss_text_ce: 0.028324538841843605 (0.028555009480831884)
|
| 7531 |
+
| > loss_mel_ce: 3.0626823902130127 (3.2685026774803783)
|
| 7532 |
+
| > loss: 0.048296984285116196 (0.05151652643412706)
|
| 7533 |
+
| > current_lr: 1.999984091227059e-05
|
| 7534 |
+
| > step_time: 0.2668 (0.2801310895454314)
|
| 7535 |
+
| > loader_time: 0.0085 (0.008413537627174754)
|
| 7536 |
+
|
| 7537 |
+
|
| 7538 |
+
[1m --> TIME: 2025-12-26 18:19:15 -- STEP: 361/2207 -- GLOBAL_STEP: 74225[0m
|
| 7539 |
+
| > loss_text_ce: 0.026577481999993324 (0.02854036339597359)
|
| 7540 |
+
| > loss_mel_ce: 3.2600584030151367 (3.2624515133221053)
|
| 7541 |
+
| > loss: 0.051353685557842255 (0.05142174816981907)
|
| 7542 |
+
| > current_lr: 1.999984091227059e-05
|
| 7543 |
+
| > step_time: 0.2913 (0.280033250924953)
|
| 7544 |
+
| > loader_time: 0.0079 (0.008433530205174506)
|
| 7545 |
+
|
| 7546 |
+
|
| 7547 |
+
[1m --> TIME: 2025-12-26 18:19:25 -- STEP: 386/2207 -- GLOBAL_STEP: 74250[0m
|
| 7548 |
+
| > loss_text_ce: 0.03021833300590515 (0.0285452227904389)
|
| 7549 |
+
| > loss_mel_ce: 3.130411148071289 (3.2604886719599895)
|
| 7550 |
+
| > loss: 0.04938483610749245 (0.051391154691731375)
|
| 7551 |
+
| > current_lr: 1.999984091227059e-05
|
| 7552 |
+
| > step_time: 0.2671 (0.27984027356063756)
|
| 7553 |
+
| > loader_time: 0.0083 (0.008418812652943678)
|
| 7554 |
+
|
| 7555 |
+
|
| 7556 |
+
[1m --> TIME: 2025-12-26 18:19:35 -- STEP: 411/2207 -- GLOBAL_STEP: 74275[0m
|
| 7557 |
+
| > loss_text_ce: 0.026307642459869385 (0.02853110368270654)
|
| 7558 |
+
| > loss_mel_ce: 3.247427463531494 (3.26317499095796)
|
| 7559 |
+
| > loss: 0.05115211009979248 (0.05143290781699248)
|
| 7560 |
+
| > current_lr: 1.999984091227059e-05
|
| 7561 |
+
| > step_time: 0.2904 (0.27997605469975173)
|
| 7562 |
+
| > loader_time: 0.0088 (0.008403961965927546)
|
| 7563 |
+
|
| 7564 |
+
|
| 7565 |
+
[1m --> TIME: 2025-12-26 18:19:44 -- STEP: 436/2207 -- GLOBAL_STEP: 74300[0m
|
| 7566 |
+
| > loss_text_ce: 0.025788558647036552 (0.028545141074840633)
|
| 7567 |
+
| > loss_mel_ce: 3.185185670852661 (3.2575094322545803)
|
| 7568 |
+
| > loss: 0.05017147213220596 (0.05134460279598422)
|
| 7569 |
+
| > current_lr: 1.999984091227059e-05
|
| 7570 |
+
| > step_time: 0.2881 (0.2798000114773392)
|
| 7571 |
+
| > loader_time: 0.0085 (0.008397690746762351)
|
| 7572 |
+
|
| 7573 |
+
|
| 7574 |
+
[1m --> TIME: 2025-12-26 18:19:54 -- STEP: 461/2207 -- GLOBAL_STEP: 74325[0m
|
| 7575 |
+
| > loss_text_ce: 0.030558591708540916 (0.028514605290266545)
|
| 7576 |
+
| > loss_mel_ce: 3.0433194637298584 (3.255582687394478)
|
| 7577 |
+
| > loss: 0.0480293445289135 (0.051314020291003126)
|
| 7578 |
+
| > current_lr: 1.999984091227059e-05
|
| 7579 |
+
| > step_time: 0.271 (0.27971078103121333)
|
| 7580 |
+
| > loader_time: 0.0073 (0.008390337681304878)
|
| 7581 |
+
|
| 7582 |
+
|
| 7583 |
+
[1m --> TIME: 2025-12-26 18:20:03 -- STEP: 486/2207 -- GLOBAL_STEP: 74350[0m
|
| 7584 |
+
| > loss_text_ce: 0.02780057117342949 (0.028508765408347667)
|
| 7585 |
+
| > loss_mel_ce: 3.0256259441375732 (3.2565455220854322)
|
| 7586 |
+
| > loss: 0.047709789127111435 (0.05132897334640899)
|
| 7587 |
+
| > current_lr: 1.999984091227059e-05
|
| 7588 |
+
| > step_time: 0.2884 (0.2795696410622615)
|
| 7589 |
+
| > loader_time: 0.0087 (0.00839708665761439)
|
| 7590 |
+
|
| 7591 |
+
|
| 7592 |
+
[1m --> TIME: 2025-12-26 18:20:13 -- STEP: 511/2207 -- GLOBAL_STEP: 74375[0m
|
| 7593 |
+
| > loss_text_ce: 0.027800854295492172 (0.028493233546323042)
|
| 7594 |
+
| > loss_mel_ce: 3.0532400608062744 (3.2537110673004856)
|
| 7595 |
+
| > loss: 0.04814126342535019 (0.051284442296700004)
|
| 7596 |
+
| > grad_norm: 0 (0.0)
|
| 7597 |
+
| > current_lr: 1.999984091227059e-05
|
| 7598 |
+
| > step_time: 0.3095 (0.279713921817548)
|
| 7599 |
+
| > loader_time: 0.008 (0.00840214787164081)
|
| 7600 |
+
|
| 7601 |
+
|
| 7602 |
+
[1m --> TIME: 2025-12-26 18:20:23 -- STEP: 536/2207 -- GLOBAL_STEP: 74400[0m
|
| 7603 |
+
| > loss_text_ce: 0.02901184931397438 (0.028484716928049703)
|
| 7604 |
+
| > loss_mel_ce: 3.1887760162353516 (3.25022281372725)
|
| 7605 |
+
| > loss: 0.050277937203645706 (0.051229805261619514)
|
| 7606 |
+
| > current_lr: 1.999984091227059e-05
|
| 7607 |
+
| > step_time: 0.2885 (0.27946200477543154)
|
| 7608 |
+
| > loader_time: 0.008 (0.008407234255947298)
|
| 7609 |
+
|
| 7610 |
+
|
| 7611 |
+
[1m --> TIME: 2025-12-26 18:20:32 -- STEP: 561/2207 -- GLOBAL_STEP: 74425[0m
|
| 7612 |
+
| > loss_text_ce: 0.030291983857750893 (0.02847657897534537)
|
| 7613 |
+
| > loss_mel_ce: 3.2776145935058594 (3.250958143496044)
|
| 7614 |
+
| > loss: 0.05168604105710983 (0.05124116763339749)
|
| 7615 |
+
| > current_lr: 1.999984091227059e-05
|
| 7616 |
+
| > step_time: 0.2703 (0.27949799743353154)
|
| 7617 |
+
| > loader_time: 0.0088 (0.00843662926644991)
|
| 7618 |
+
|
| 7619 |
+
|
| 7620 |
+
[1m --> TIME: 2025-12-26 18:20:42 -- STEP: 586/2207 -- GLOBAL_STEP: 74450[0m
|
| 7621 |
+
| > loss_text_ce: 0.027369234710931778 (0.028482291650395758)
|
| 7622 |
+
| > loss_mel_ce: 3.397357225418091 (3.248793753340788)
|
| 7623 |
+
| > loss: 0.053511351346969604 (0.05120743829566261)
|
| 7624 |
+
| > current_lr: 1.999984091227059e-05
|
| 7625 |
+
| > step_time: 0.263 (0.2794164333734088)
|
| 7626 |
+
| > loader_time: 0.0085 (0.008433569820260847)
|
| 7627 |
+
|
| 7628 |
+
|
| 7629 |
+
[1m --> TIME: 2025-12-26 18:20:52 -- STEP: 611/2207 -- GLOBAL_STEP: 74475[0m
|
| 7630 |
+
| > loss_text_ce: 0.029731908813118935 (0.02849143695653266)
|
| 7631 |
+
| > loss_mel_ce: 3.2096495628356934 (3.2470133300693824)
|
| 7632 |
+
| > loss: 0.0506153367459774 (0.051179762076092636)
|
| 7633 |
+
| > current_lr: 1.999984091227059e-05
|
| 7634 |
+
| > step_time: 0.2697 (0.2795231201839915)
|
| 7635 |
+
| > loader_time: 0.0095 (0.008446359400273162)
|
| 7636 |
+
|
| 7637 |
+
|
| 7638 |
+
[1m --> TIME: 2025-12-26 18:21:01 -- STEP: 636/2207 -- GLOBAL_STEP: 74500[0m
|
| 7639 |
+
| > loss_text_ce: 0.029000531882047653 (0.028497067830706924)
|
| 7640 |
+
| > loss_mel_ce: 3.4186525344848633 (3.247042080141461)
|
| 7641 |
+
| > loss: 0.05386957898736 (0.051180299266048196)
|
| 7642 |
+
| > current_lr: 1.999984091227059e-05
|
| 7643 |
+
| > step_time: 0.2879 (0.27966207753187444)
|
| 7644 |
+
| > loader_time: 0.0082 (0.008453684782831938)
|
| 7645 |
+
|
| 7646 |
+
|
| 7647 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000/checkpoint_74500.pth
|
| 7648 |
+
|
| 7649 |
+
[1m --> TIME: 2025-12-26 18:21:36 -- STEP: 661/2207 -- GLOBAL_STEP: 74525[0m
|
| 7650 |
+
| > loss_text_ce: 0.028211338445544243 (0.028492541656844922)
|
| 7651 |
+
| > loss_mel_ce: 3.265098810195923 (3.2474094836686613)
|
| 7652 |
+
| > loss: 0.0514579713344574 (0.05118596921259264)
|
| 7653 |
+
| > current_lr: 1.999984091227059e-05
|
| 7654 |
+
| > step_time: 0.3025 (0.2796782127126442)
|
| 7655 |
+
| > loader_time: 0.0083 (0.008468544609427642)
|
| 7656 |
+
|
| 7657 |
+
|
| 7658 |
+
[1m --> TIME: 2025-12-26 18:21:46 -- STEP: 686/2207 -- GLOBAL_STEP: 74550[0m
|
| 7659 |
+
| > loss_text_ce: 0.030026793479919434 (0.028498524639744633)
|
| 7660 |
+
| > loss_mel_ce: 3.258890151977539 (3.246499229798162)
|
| 7661 |
+
| > loss: 0.05138932913541794 (0.0511718399811851)
|
| 7662 |
+
| > current_lr: 1.999984091227059e-05
|
| 7663 |
+
| > step_time: 0.2692 (0.27953379842352244)
|
| 7664 |
+
| > loader_time: 0.0088 (0.008469232317310041)
|
| 7665 |
+
|
| 7666 |
+
|
| 7667 |
+
[1m --> TIME: 2025-12-26 18:21:55 -- STEP: 711/2207 -- GLOBAL_STEP: 74575[0m
|
| 7668 |
+
| > loss_text_ce: 0.029767654836177826 (0.028518196273956627)
|
| 7669 |
+
| > loss_mel_ce: 3.7501890659332275 (3.247847998360204)
|
| 7670 |
+
| > loss: 0.05906182527542114 (0.05119322185861746)
|
| 7671 |
+
| > current_lr: 1.999984091227059e-05
|
| 7672 |
+
| > step_time: 0.2663 (0.2792770943058192)
|
| 7673 |
+
| > loader_time: 0.0079 (0.008461982053711112)
|
| 7674 |
+
|
| 7675 |
+
|
| 7676 |
+
[1m --> TIME: 2025-12-26 18:22:04 -- STEP: 736/2207 -- GLOBAL_STEP: 74600[0m
|
| 7677 |
+
| > loss_text_ce: 0.030024683102965355 (0.02851345529020562)
|
| 7678 |
+
| > loss_mel_ce: 3.3412322998046875 (3.2481558397412282)
|
| 7679 |
+
| > loss: 0.052675891667604446 (0.05119795779414151)
|
| 7680 |
+
| > current_lr: 1.999984091227059e-05
|
| 7681 |
+
| > step_time: 0.2671 (0.2792360128267955)
|
| 7682 |
+
| > loader_time: 0.0079 (0.008451994346535732)
|
| 7683 |
+
|
| 7684 |
+
|
| 7685 |
+
[1m --> TIME: 2025-12-26 18:22:14 -- STEP: 761/2207 -- GLOBAL_STEP: 74625[0m
|
| 7686 |
+
| > loss_text_ce: 0.028660498559474945 (0.02850976625014386)
|
| 7687 |
+
| > loss_mel_ce: 3.3593590259552 (3.248885530993125)
|
| 7688 |
+
| > loss: 0.05293780565261841 (0.051209301578857586)
|
| 7689 |
+
| > current_lr: 1.999984091227059e-05
|
| 7690 |
+
| > step_time: 0.267 (0.27923876477917026)
|
| 7691 |
+
| > loader_time: 0.0082 (0.008452663910373285)
|
| 7692 |
+
|
| 7693 |
+
|
| 7694 |
+
[1m --> TIME: 2025-12-26 18:22:24 -- STEP: 786/2207 -- GLOBAL_STEP: 74650[0m
|
| 7695 |
+
| > loss_text_ce: 0.028472738340497017 (0.02850659105111549)
|
| 7696 |
+
| > loss_mel_ce: 2.9979076385498047 (3.248227439157227)
|
| 7697 |
+
| > loss: 0.047287192195653915 (0.05119896927521429)
|
| 7698 |
+
| > current_lr: 1.999984091227059e-05
|
| 7699 |
+
| > step_time: 0.2622 (0.27928544668144284)
|
| 7700 |
+
| > loader_time: 0.008 (0.008444565246426729)
|
| 7701 |
+
|
| 7702 |
+
|
| 7703 |
+
[1m --> TIME: 2025-12-26 18:22:33 -- STEP: 811/2207 -- GLOBAL_STEP: 74675[0m
|
| 7704 |
+
| > loss_text_ce: 0.028025662526488304 (0.028505885773757535)
|
| 7705 |
+
| > loss_mel_ce: 3.466132164001465 (3.248201712433123)
|
| 7706 |
+
| > loss: 0.05459621548652649 (0.051198556256720526)
|
| 7707 |
+
| > current_lr: 1.999984091227059e-05
|
| 7708 |
+
| > step_time: 0.2656 (0.2792220327327935)
|
| 7709 |
+
| > loader_time: 0.0094 (0.008440357542214356)
|
| 7710 |
+
|
| 7711 |
+
|
| 7712 |
+
[1m --> TIME: 2025-12-26 18:22:43 -- STEP: 836/2207 -- GLOBAL_STEP: 74700[0m
|
| 7713 |
+
| > loss_text_ce: 0.027990423142910004 (0.02850610865101789)
|
| 7714 |
+
| > loss_mel_ce: 3.285581111907959 (3.2507606801804156)
|
| 7715 |
+
| > loss: 0.05177455395460129 (0.051238543612427974)
|
| 7716 |
+
| > current_lr: 1.999984091227059e-05
|
| 7717 |
+
| > step_time: 0.2672 (0.2791602263610332)
|
| 7718 |
+
| > loader_time: 0.008 (0.008439996310968718)
|
| 7719 |
+
|
| 7720 |
+
|
| 7721 |
+
[1m --> TIME: 2025-12-26 18:22:53 -- STEP: 861/2207 -- GLOBAL_STEP: 74725[0m
|
| 7722 |
+
| > loss_text_ce: 0.02950947917997837 (0.028505381497443073)
|
| 7723 |
+
| > loss_mel_ce: 3.052945613861084 (3.2509025788057957)
|
| 7724 |
+
| > loss: 0.048163361847400665 (0.05124074941485878)
|
| 7725 |
+
| > current_lr: 1.999984091227059e-05
|
| 7726 |
+
| > step_time: 0.2875 (0.2792332377306401)
|
| 7727 |
+
| > loader_time: 0.0086 (0.008440960851972804)
|
| 7728 |
+
|
| 7729 |
+
|
| 7730 |
+
[1m --> TIME: 2025-12-26 18:23:02 -- STEP: 886/2207 -- GLOBAL_STEP: 74750[0m
|
| 7731 |
+
| > loss_text_ce: 0.02824772708117962 (0.02850462071614395)
|
| 7732 |
+
| > loss_mel_ce: 3.073819637298584 (3.249344847272264)
|
| 7733 |
+
| > loss: 0.04846980422735214 (0.05121639797266396)
|
| 7734 |
+
| > current_lr: 1.999984091227059e-05
|
| 7735 |
+
| > step_time: 0.2663 (0.27921595315093695)
|
| 7736 |
+
| > loader_time: 0.0089 (0.008437878660369946)
|
| 7737 |
+
|
| 7738 |
+
|
| 7739 |
+
[1m --> TIME: 2025-12-26 18:23:12 -- STEP: 911/2207 -- GLOBAL_STEP: 74775[0m
|
| 7740 |
+
| > loss_text_ce: 0.029228346422314644 (0.02849460373562071)
|
| 7741 |
+
| > loss_mel_ce: 3.3150622844696045 (3.250134963235529)
|
| 7742 |
+
| > loss: 0.05225454270839691 (0.05122858702187742)
|
| 7743 |
+
| > current_lr: 1.999984091227059e-05
|
| 7744 |
+
| > step_time: 0.2879 (0.27916180498644555)
|
| 7745 |
+
| > loader_time: 0.0089 (0.00843504807297406)
|
| 7746 |
+
|
| 7747 |
+
|
| 7748 |
+
[1m --> TIME: 2025-12-26 18:23:21 -- STEP: 936/2207 -- GLOBAL_STEP: 74800[0m
|
| 7749 |
+
| > loss_text_ce: 0.02941373735666275 (0.028505913775939595)
|
| 7750 |
+
| > loss_mel_ce: 3.2976481914520264 (3.251064061099646)
|
| 7751 |
+
| > loss: 0.05198534205555916 (0.05124328089639162)
|
| 7752 |
+
| > current_lr: 1.999984091227059e-05
|
| 7753 |
+
| > step_time: 0.2552 (0.27905258103313657)
|
| 7754 |
+
| > loader_time: 0.0083 (0.008432946908168303)
|
| 7755 |
+
|
| 7756 |
+
|
| 7757 |
+
[1m --> TIME: 2025-12-26 18:23:31 -- STEP: 961/2207 -- GLOBAL_STEP: 74825[0m
|
| 7758 |
+
| > loss_text_ce: 0.029734348878264427 (0.028512772052518787)
|
| 7759 |
+
| > loss_mel_ce: 3.103614330291748 (3.248170571074152)
|
| 7760 |
+
| > loss: 0.04895857349038124 (0.05119817727996457)
|
| 7761 |
+
| > current_lr: 1.999984091227059e-05
|
| 7762 |
+
| > step_time: 0.2883 (0.27908311872651054)
|
| 7763 |
+
| > loader_time: 0.0085 (0.008458515359757468)
|
| 7764 |
+
|
| 7765 |
+
|
| 7766 |
+
[1m --> TIME: 2025-12-26 18:23:41 -- STEP: 986/2207 -- GLOBAL_STEP: 74850[0m
|
| 7767 |
+
| > loss_text_ce: 0.026694251224398613 (0.028514242874605786)
|
| 7768 |
+
| > loss_mel_ce: 3.1833887100219727 (3.2479093875169256)
|
| 7769 |
+
| > loss: 0.05015754699707031 (0.051194119269054754)
|
| 7770 |
+
| > current_lr: 1.999984091227059e-05
|
| 7771 |
+
| > step_time: 0.2903 (0.27946496227450157)
|
| 7772 |
+
| > loader_time: 0.0083 (0.00846503980261317)
|
| 7773 |
+
|
| 7774 |
+
|
| 7775 |
+
[1m --> TIME: 2025-12-26 18:23:51 -- STEP: 1011/2207 -- GLOBAL_STEP: 74875[0m
|
| 7776 |
+
| > loss_text_ce: 0.027927419170737267 (0.028508078078458385)
|
| 7777 |
+
| > loss_mel_ce: 2.8743011951446533 (3.2472512877659545)
|
| 7778 |
+
| > loss: 0.04534732177853584 (0.05118374012845442)
|
| 7779 |
+
| > current_lr: 1.999984091227059e-05
|
| 7780 |
+
| > step_time: 0.2663 (0.2794911328456291)
|
| 7781 |
+
| > loader_time: 0.0091 (0.008463348998042406)
|
| 7782 |
+
|
| 7783 |
+
|
| 7784 |
+
[1m --> TIME: 2025-12-26 18:24:00 -- STEP: 1036/2207 -- GLOBAL_STEP: 74900[0m
|
| 7785 |
+
| > loss_text_ce: 0.029365403577685356 (0.02851001790303982)
|
| 7786 |
+
| > loss_mel_ce: 3.1445960998535156 (3.24751362446192)
|
| 7787 |
+
| > loss: 0.04959314689040184 (0.05118786944543878)
|
| 7788 |
+
| > current_lr: 1.999984091227059e-05
|
| 7789 |
+
| > step_time: 0.2563 (0.2795207355473497)
|
| 7790 |
+
| > loader_time: 0.008 (0.008463664864941451)
|
| 7791 |
+
|
| 7792 |
+
|
| 7793 |
+
[1m --> TIME: 2025-12-26 18:24:10 -- STEP: 1061/2207 -- GLOBAL_STEP: 74925[0m
|
| 7794 |
+
| > loss_text_ce: 0.02604135125875473 (0.02850490811471385)
|
| 7795 |
+
| > loss_mel_ce: 3.7946648597717285 (3.2470732575649355)
|
| 7796 |
+
| > loss: 0.05969853326678276 (0.051180908886556006)
|
| 7797 |
+
| > current_lr: 1.999984091227059e-05
|
| 7798 |
+
| > step_time: 0.2701 (0.27953320623680106)
|
| 7799 |
+
| > loader_time: 0.0079 (0.00846684012741103)
|
| 7800 |
+
|
| 7801 |
+
|
| 7802 |
+
[1m --> TIME: 2025-12-26 18:24:20 -- STEP: 1086/2207 -- GLOBAL_STEP: 74950[0m
|
| 7803 |
+
| > loss_text_ce: 0.028227047994732857 (0.028503323876012537)
|
| 7804 |
+
| > loss_mel_ce: 3.2368884086608887 (3.248324147465039)
|
| 7805 |
+
| > loss: 0.05101742967963219 (0.051200429286334385)
|
| 7806 |
+
| > current_lr: 1.999984091227059e-05
|
| 7807 |
+
| > step_time: 0.2868 (0.2795374713989037)
|
| 7808 |
+
| > loader_time: 0.0092 (0.008465507192945391)
|
| 7809 |
+
|
| 7810 |
+
|
| 7811 |
+
[1m --> TIME: 2025-12-26 18:24:29 -- STEP: 1111/2207 -- GLOBAL_STEP: 74975[0m
|
| 7812 |
+
| > loss_text_ce: 0.028481852263212204 (0.028502181502650936)
|
| 7813 |
+
| > loss_mel_ce: 3.348447322845459 (3.2486578810869506)
|
| 7814 |
+
| > loss: 0.052764520049095154 (0.051205626030852394)
|
| 7815 |
+
| > current_lr: 1.999984091227059e-05
|
| 7816 |
+
| > step_time: 0.2674 (0.27946565968833714)
|
| 7817 |
+
| > loader_time: 0.0078 (0.008462362950391107)
|
| 7818 |
+
|
| 7819 |
+
|
| 7820 |
+
[1m --> TIME: 2025-12-26 18:24:39 -- STEP: 1136/2207 -- GLOBAL_STEP: 75000[0m
|
| 7821 |
+
| > loss_text_ce: 0.02799856849014759 (0.02850347028350013)
|
| 7822 |
+
| > loss_mel_ce: 3.1449999809265137 (3.2487302307091954)
|
| 7823 |
+
| > loss: 0.04957810416817665 (0.051206776636167295)
|
| 7824 |
+
| > current_lr: 1.999984091227059e-05
|
| 7825 |
+
| > step_time: 0.2867 (0.2794874938021249)
|
| 7826 |
+
| > loader_time: 0.0082 (0.008456668383638621)
|
| 7827 |
+
|
| 7828 |
+
|
| 7829 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000/checkpoint_75000.pth
|
| 7830 |
+
|
| 7831 |
+
[1m --> TIME: 2025-12-26 18:25:13 -- STEP: 1161/2207 -- GLOBAL_STEP: 75025[0m
|
| 7832 |
+
| > loss_text_ce: 0.031178312376141548 (0.0285026312827371)
|
| 7833 |
+
| > loss_mel_ce: 3.4044547080993652 (3.2484742258254253)
|
| 7834 |
+
| > loss: 0.053681764751672745 (0.05120276345275372)
|
| 7835 |
+
| > current_lr: 1.999984091227059e-05
|
| 7836 |
+
| > step_time: 0.2669 (0.27946598133478223)
|
| 7837 |
+
| > loader_time: 0.009 (0.008484057575130542)
|
| 7838 |
+
|
| 7839 |
+
|
| 7840 |
+
[1m --> TIME: 2025-12-26 18:25:23 -- STEP: 1186/2207 -- GLOBAL_STEP: 75050[0m
|
| 7841 |
+
| > loss_text_ce: 0.02873758040368557 (0.028504357894672122)
|
| 7842 |
+
| > loss_mel_ce: 3.43912410736084 (3.2488398157927074)
|
| 7843 |
+
| > loss: 0.05418533831834793 (0.05120850277163123)
|
| 7844 |
+
| > current_lr: 1.999984091227059e-05
|
| 7845 |
+
| > step_time: 0.2959 (0.27950240568608176)
|
| 7846 |
+
| > loader_time: 0.0083 (0.008488001147972993)
|
| 7847 |
+
|
| 7848 |
+
|
| 7849 |
+
[1m --> TIME: 2025-12-26 18:25:33 -- STEP: 1211/2207 -- GLOBAL_STEP: 75075[0m
|
| 7850 |
+
| > loss_text_ce: 0.02830764465034008 (0.028505403642303497)
|
| 7851 |
+
| > loss_mel_ce: 3.2454051971435547 (3.2498908997763287)
|
| 7852 |
+
| > loss: 0.051151763647794724 (0.051224942302945)
|
| 7853 |
+
| > current_lr: 1.999984091227059e-05
|
| 7854 |
+
| > step_time: 0.2869 (0.2795413026723074)
|
| 7855 |
+
| > loader_time: 0.0083 (0.008492246331703947)
|
| 7856 |
+
|
| 7857 |
+
|
| 7858 |
+
[1m --> TIME: 2025-12-26 18:25:42 -- STEP: 1236/2207 -- GLOBAL_STEP: 75100[0m
|
| 7859 |
+
| > loss_text_ce: 0.028239984065294266 (0.028504135709521178)
|
| 7860 |
+
| > loss_mel_ce: 3.017456293106079 (3.249391433490517)
|
| 7861 |
+
| > loss: 0.047589004039764404 (0.05121711832938357)
|
| 7862 |
+
| > current_lr: 1.999984091227059e-05
|
| 7863 |
+
| > step_time: 0.2921 (0.27952771163681267)
|
| 7864 |
+
| > loader_time: 0.0087 (0.00849609351852565)
|
| 7865 |
+
|
| 7866 |
+
|
| 7867 |
+
[1m --> TIME: 2025-12-26 18:25:52 -- STEP: 1261/2207 -- GLOBAL_STEP: 75125[0m
|
| 7868 |
+
| > loss_text_ce: 0.02467103861272335 (0.02850648735832309)
|
| 7869 |
+
| > loss_mel_ce: 3.2651073932647705 (3.249703339600166)
|
| 7870 |
+
| > loss: 0.05140278860926628 (0.05122202860514951)
|
| 7871 |
+
| > current_lr: 1.999984091227059e-05
|
| 7872 |
+
| > step_time: 0.2868 (0.2796193524828805)
|
| 7873 |
+
| > loader_time: 0.0082 (0.008493337623662738)
|
| 7874 |
+
|
| 7875 |
+
|
| 7876 |
+
[1m --> TIME: 2025-12-26 18:26:01 -- STEP: 1286/2207 -- GLOBAL_STEP: 75150[0m
|
| 7877 |
+
| > loss_text_ce: 0.02739032171666622 (0.028504704903017503)
|
| 7878 |
+
| > loss_mel_ce: 3.558516025543213 (3.2493977455848113)
|
| 7879 |
+
| > loss: 0.0560297854244709 (0.05121722584228213)
|
| 7880 |
+
| > current_lr: 1.999984091227059e-05
|
| 7881 |
+
| > step_time: 0.2669 (0.2795482896164092)
|
| 7882 |
+
| > loader_time: 0.0094 (0.008496152854076993)
|
| 7883 |
+
|
| 7884 |
+
|
| 7885 |
+
[1m --> TIME: 2025-12-26 18:26:11 -- STEP: 1311/2207 -- GLOBAL_STEP: 75175[0m
|
| 7886 |
+
| > loss_text_ce: 0.027565591037273407 (0.02849649669905141)
|
| 7887 |
+
| > loss_mel_ce: 3.055180311203003 (3.249360157823308)
|
| 7888 |
+
| > loss: 0.04816790297627449 (0.05121651027865939)
|
| 7889 |
+
| > current_lr: 1.999984091227059e-05
|
| 7890 |
+
| > step_time: 0.2662 (0.27959944618102994)
|
| 7891 |
+
| > loader_time: 0.0081 (0.008499165512422491)
|
| 7892 |
+
|
| 7893 |
+
|
| 7894 |
+
[1m --> TIME: 2025-12-26 18:26:21 -- STEP: 1336/2207 -- GLOBAL_STEP: 75200[0m
|
| 7895 |
+
| > loss_text_ce: 0.02885270304977894 (0.02849596772554839)
|
| 7896 |
+
| > loss_mel_ce: 3.042893648147583 (3.249249959063387)
|
| 7897 |
+
| > loss: 0.04799603670835495 (0.051214780150996335)
|
| 7898 |
+
| > current_lr: 1.999984091227059e-05
|
| 7899 |
+
| > step_time: 0.2695 (0.2795518602796669)
|
| 7900 |
+
| > loader_time: 0.0089 (0.008499139618730825)
|
| 7901 |
+
|
| 7902 |
+
|
| 7903 |
+
[1m --> TIME: 2025-12-26 18:26:30 -- STEP: 1361/2207 -- GLOBAL_STEP: 75225[0m
|
| 7904 |
+
| > loss_text_ce: 0.029902134090662003 (0.028498609441494707)
|
| 7905 |
+
| > loss_mel_ce: 3.500291585922241 (3.249806580168635)
|
| 7906 |
+
| > loss: 0.055159278213977814 (0.05122351863754748)
|
| 7907 |
+
| > current_lr: 1.999984091227059e-05
|
| 7908 |
+
| > step_time: 0.2872 (0.27954379028471765)
|
| 7909 |
+
| > loader_time: 0.0081 (0.008502271226186412)
|
| 7910 |
+
|
| 7911 |
+
|
| 7912 |
+
[1m --> TIME: 2025-12-26 18:26:40 -- STEP: 1386/2207 -- GLOBAL_STEP: 75250[0m
|
| 7913 |
+
| > loss_text_ce: 0.027866052463650703 (0.028492165234848373)
|
| 7914 |
+
| > loss_mel_ce: 3.4271719455718994 (3.2505726422066297)
|
| 7915 |
+
| > loss: 0.05398496985435486 (0.05123538767158295)
|
| 7916 |
+
| > current_lr: 1.999984091227059e-05
|
| 7917 |
+
| > step_time: 0.2696 (0.27954005455386)
|
| 7918 |
+
| > loader_time: 0.0087 (0.008503962904859922)
|
| 7919 |
+
|
| 7920 |
+
|
| 7921 |
+
[1m --> TIME: 2025-12-26 18:26:50 -- STEP: 1411/2207 -- GLOBAL_STEP: 75275[0m
|
| 7922 |
+
| > loss_text_ce: 0.02856244333088398 (0.028493994955187323)
|
| 7923 |
+
| > loss_mel_ce: 3.334455966949463 (3.2507683347084266)
|
| 7924 |
+
| > loss: 0.052547164261341095 (0.05123847396417257)
|
| 7925 |
+
| > current_lr: 1.999984091227059e-05
|
| 7926 |
+
| > step_time: 0.2873 (0.2797294683611887)
|
| 7927 |
+
| > loader_time: 0.0088 (0.00850119884769431)
|
| 7928 |
+
|
| 7929 |
+
|
| 7930 |
+
[1m --> TIME: 2025-12-26 18:27:00 -- STEP: 1436/2207 -- GLOBAL_STEP: 75300[0m
|
| 7931 |
+
| > loss_text_ce: 0.031019749119877815 (0.02849155733709466)
|
| 7932 |
+
| > loss_mel_ce: 3.231387138366699 (3.2496931204224695)
|
| 7933 |
+
| > loss: 0.05097510665655136 (0.0512216356492142)
|
| 7934 |
+
| > current_lr: 1.999984091227059e-05
|
| 7935 |
+
| > step_time: 0.267 (0.2797590027612577)
|
| 7936 |
+
| > loader_time: 0.0086 (0.008497197149855838)
|
| 7937 |
+
|
| 7938 |
+
|
| 7939 |
+
[1m --> TIME: 2025-12-26 18:27:09 -- STEP: 1461/2207 -- GLOBAL_STEP: 75325[0m
|
| 7940 |
+
| > loss_text_ce: 0.028467165306210518 (0.028490685061650447)
|
| 7941 |
+
| > loss_mel_ce: 3.072627067565918 (3.2495388646879726)
|
| 7942 |
+
| > loss: 0.04845459759235382 (0.05121921177272677)
|
| 7943 |
+
| > current_lr: 1.999984091227059e-05
|
| 7944 |
+
| > step_time: 0.2903 (0.27972408976153557)
|
| 7945 |
+
| > loader_time: 0.008 (0.008494847445843718)
|
| 7946 |
+
|
| 7947 |
+
|
| 7948 |
+
[1m --> TIME: 2025-12-26 18:27:19 -- STEP: 1486/2207 -- GLOBAL_STEP: 75350[0m
|
| 7949 |
+
| > loss_text_ce: 0.0268696378916502 (0.028487788925691947)
|
| 7950 |
+
| > loss_mel_ce: 3.315735340118408 (3.2493359156925616)
|
| 7951 |
+
| > loss: 0.05222820118069649 (0.051215995439048714)
|
| 7952 |
+
| > current_lr: 1.999984091227059e-05
|
| 7953 |
+
| > step_time: 0.2877 (0.2796719696768853)
|
| 7954 |
+
| > loader_time: 0.0087 (0.008492415878686087)
|
| 7955 |
+
|
| 7956 |
+
|
| 7957 |
+
[1m --> TIME: 2025-12-26 18:27:28 -- STEP: 1511/2207 -- GLOBAL_STEP: 75375[0m
|
| 7958 |
+
| > loss_text_ce: 0.02885393239557743 (0.028485856744418243)
|
| 7959 |
+
| > loss_mel_ce: 3.4055657386779785 (3.247923921544689)
|
| 7960 |
+
| > loss: 0.05366280674934387 (0.05119390283729127)
|
| 7961 |
+
| > current_lr: 1.999984091227059e-05
|
| 7962 |
+
| > step_time: 0.2629 (0.2795496962072054)
|
| 7963 |
+
| > loader_time: 0.0083 (0.008491528420792269)
|
| 7964 |
+
|
| 7965 |
+
|
| 7966 |
+
[1m --> TIME: 2025-12-26 18:27:38 -- STEP: 1536/2207 -- GLOBAL_STEP: 75400[0m
|
| 7967 |
+
| > loss_text_ce: 0.02771391160786152 (0.02848677139991197)
|
| 7968 |
+
| > loss_mel_ce: 3.2657253742218018 (3.248063920686642)
|
| 7969 |
+
| > loss: 0.051459990441799164 (0.05119610461406408)
|
| 7970 |
+
| > current_lr: 1.999984091227059e-05
|
| 7971 |
+
| > step_time: 0.2622 (0.27953057658548275)
|
| 7972 |
+
| > loader_time: 0.0084 (0.008489634065578367)
|
| 7973 |
+
|
| 7974 |
+
|
| 7975 |
+
[1m --> TIME: 2025-12-26 18:27:47 -- STEP: 1561/2207 -- GLOBAL_STEP: 75425[0m
|
| 7976 |
+
| > loss_text_ce: 0.027523530647158623 (0.028492182392977822)
|
| 7977 |
+
| > loss_mel_ce: 3.0877363681793213 (3.247403208526719)
|
| 7978 |
+
| > loss: 0.048675935715436935 (0.05118586552794126)
|
| 7979 |
+
| > current_lr: 1.999984091227059e-05
|
| 7980 |
+
| > step_time: 0.2957 (0.2795353154201383)
|
| 7981 |
+
| > loader_time: 0.0081 (0.008489545990762444)
|
| 7982 |
+
|
| 7983 |
+
|
| 7984 |
+
[1m --> TIME: 2025-12-26 18:27:57 -- STEP: 1586/2207 -- GLOBAL_STEP: 75450[0m
|
| 7985 |
+
| > loss_text_ce: 0.026772622019052505 (0.028483436796414907)
|
| 7986 |
+
| > loss_mel_ce: 3.0845935344696045 (3.2463299783151056)
|
| 7987 |
+
| > loss: 0.048615097999572754 (0.051168959658278175)
|
| 7988 |
+
| > current_lr: 1.999984091227059e-05
|
| 7989 |
+
| > step_time: 0.2869 (0.27954902534677395)
|
| 7990 |
+
| > loader_time: 0.0076 (0.008484295758443387)
|
| 7991 |
+
|
| 7992 |
+
|
| 7993 |
+
[1m --> TIME: 2025-12-26 18:28:06 -- STEP: 1611/2207 -- GLOBAL_STEP: 75475[0m
|
| 7994 |
+
| > loss_text_ce: 0.028487715870141983 (0.028479746312208246)
|
| 7995 |
+
| > loss_mel_ce: 3.512401580810547 (3.246348404721545)
|
| 7996 |
+
| > loss: 0.055326394736766815 (0.05116918990139927)
|
| 7997 |
+
| > current_lr: 1.999984091227059e-05
|
| 7998 |
+
| > step_time: 0.268 (0.2795010862107764)
|
| 7999 |
+
| > loader_time: 0.0081 (0.00847852548259447)
|
| 8000 |
+
|
| 8001 |
+
|
| 8002 |
+
[1m --> TIME: 2025-12-26 18:28:16 -- STEP: 1636/2207 -- GLOBAL_STEP: 75500[0m
|
| 8003 |
+
| > loss_text_ce: 0.027832217514514923 (0.028475546470472103)
|
| 8004 |
+
| > loss_mel_ce: 3.2282586097717285 (3.2459789495596385)
|
| 8005 |
+
| > loss: 0.050876419991254807 (0.051163351534430856)
|
| 8006 |
+
| > current_lr: 1.999984091227059e-05
|
| 8007 |
+
| > step_time: 0.2915 (0.2794943039458083)
|
| 8008 |
+
| > loader_time: 0.008 (0.008476090168894636)
|
| 8009 |
+
|
| 8010 |
+
|
| 8011 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000/checkpoint_75500.pth
|
| 8012 |
+
|
| 8013 |
+
[1m --> TIME: 2025-12-26 18:28:50 -- STEP: 1661/2207 -- GLOBAL_STEP: 75525[0m
|
| 8014 |
+
| > loss_text_ce: 0.029250947758555412 (0.028478463021685108)
|
| 8015 |
+
| > loss_mel_ce: 3.173936128616333 (3.2460534045812524)
|
| 8016 |
+
| > loss: 0.0500497967004776 (0.05116456046628706)
|
| 8017 |
+
| > current_lr: 1.999984091227059e-05
|
| 8018 |
+
| > step_time: 0.2669 (0.27949498605469897)
|
| 8019 |
+
| > loader_time: 0.0084 (0.008480541681683532)
|
| 8020 |
+
|
| 8021 |
+
|
| 8022 |
+
[1m --> TIME: 2025-12-26 18:29:00 -- STEP: 1686/2207 -- GLOBAL_STEP: 75550[0m
|
| 8023 |
+
| > loss_text_ce: 0.0273405984044075 (0.028482569666625775)
|
| 8024 |
+
| > loss_mel_ce: 2.909013032913208 (3.2455735882547545)
|
| 8025 |
+
| > loss: 0.045880526304244995 (0.0511571275023909)
|
| 8026 |
+
| > current_lr: 1.999984091227059e-05
|
| 8027 |
+
| > step_time: 0.267 (0.2793562683770676)
|
| 8028 |
+
| > loader_time: 0.009 (0.008485708938091804)
|
| 8029 |
+
|
| 8030 |
+
|
| 8031 |
+
[1m --> TIME: 2025-12-26 18:29:09 -- STEP: 1711/2207 -- GLOBAL_STEP: 75575[0m
|
| 8032 |
+
| > loss_text_ce: 0.027768777683377266 (0.02849392427326399)
|
| 8033 |
+
| > loss_mel_ce: 3.1333703994750977 (3.246333797238571)
|
| 8034 |
+
| > loss: 0.04939280077815056 (0.0511691831857889)
|
| 8035 |
+
| > current_lr: 1.999984091227059e-05
|
| 8036 |
+
| > step_time: 0.2882 (0.2794417204876391)
|
| 8037 |
+
| > loader_time: 0.0084 (0.00848877868897589)
|
| 8038 |
+
|
| 8039 |
+
|
| 8040 |
+
[1m --> TIME: 2025-12-26 18:29:19 -- STEP: 1736/2207 -- GLOBAL_STEP: 75600[0m
|
| 8041 |
+
| > loss_text_ce: 0.0292130708694458 (0.028489619417304893)
|
| 8042 |
+
| > loss_mel_ce: 2.97172474861145 (3.2461739904869535)
|
| 8043 |
+
| > loss: 0.04688965529203415 (0.051166618942424705)
|
| 8044 |
+
| > current_lr: 1.999984091227059e-05
|
| 8045 |
+
| > step_time: 0.2673 (0.2794845523098109)
|
| 8046 |
+
| > loader_time: 0.0088 (0.008489435306891856)
|
| 8047 |
+
|
| 8048 |
+
|
| 8049 |
+
[1m --> TIME: 2025-12-26 18:29:29 -- STEP: 1761/2207 -- GLOBAL_STEP: 75625[0m
|
| 8050 |
+
| > loss_text_ce: 0.028141936287283897 (0.028484461147894522)
|
| 8051 |
+
| > loss_mel_ce: 3.5913853645324707 (3.246223164444143)
|
| 8052 |
+
| > loss: 0.056555114686489105 (0.05116730668644955)
|
| 8053 |
+
| > current_lr: 1.999984091227059e-05
|
| 8054 |
+
| > step_time: 0.2545 (0.27944986120264187)
|
| 8055 |
+
| > loader_time: 0.0087 (0.008490164533113619)
|
| 8056 |
+
|
| 8057 |
+
|
| 8058 |
+
[1m --> TIME: 2025-12-26 18:29:38 -- STEP: 1786/2207 -- GLOBAL_STEP: 75650[0m
|
| 8059 |
+
| > loss_text_ce: 0.02974388748407364 (0.028484418511615962)
|
| 8060 |
+
| > loss_mel_ce: 3.3168411254882812 (3.2452136124635462)
|
| 8061 |
+
| > loss: 0.05229039117693901 (0.05115153177037483)
|
| 8062 |
+
| > current_lr: 1.999984091227059e-05
|
| 8063 |
+
| > step_time: 0.2671 (0.2794028866331107)
|
| 8064 |
+
| > loader_time: 0.0082 (0.008494956229061436)
|
| 8065 |
+
|
| 8066 |
+
|
| 8067 |
+
[1m --> TIME: 2025-12-26 18:29:48 -- STEP: 1811/2207 -- GLOBAL_STEP: 75675[0m
|
| 8068 |
+
| > loss_text_ce: 0.027714816853404045 (0.028487538787202577)
|
| 8069 |
+
| > loss_mel_ce: 3.2287769317626953 (3.2454761732091963)
|
| 8070 |
+
| > loss: 0.05088268220424652 (0.05115568303579719)
|
| 8071 |
+
| > current_lr: 1.999984091227059e-05
|
| 8072 |
+
| > step_time: 0.2687 (0.27941483953656193)
|
| 8073 |
+
| > loader_time: 0.0089 (0.008497222756101949)
|
| 8074 |
+
|
| 8075 |
+
|
| 8076 |
+
[1m --> TIME: 2025-12-26 18:29:57 -- STEP: 1836/2207 -- GLOBAL_STEP: 75700[0m
|
| 8077 |
+
| > loss_text_ce: 0.028238102793693542 (0.028487433283741963)
|
| 8078 |
+
| > loss_mel_ce: 3.370675563812256 (3.2448902718381945)
|
| 8079 |
+
| > loss: 0.05310802534222603 (0.051146526676896856)
|
| 8080 |
+
| > current_lr: 1.999984091227059e-05
|
| 8081 |
+
| > step_time: 0.2879 (0.2794151569763316)
|
| 8082 |
+
| > loader_time: 0.0081 (0.008499674028301015)
|
| 8083 |
+
|
| 8084 |
+
|
| 8085 |
+
[1m --> TIME: 2025-12-26 18:30:07 -- STEP: 1861/2207 -- GLOBAL_STEP: 75725[0m
|
| 8086 |
+
| > loss_text_ce: 0.0263417549431324 (0.028487750449683776)
|
| 8087 |
+
| > loss_mel_ce: 3.0168616771698 (3.244934764618133)
|
| 8088 |
+
| > loss: 0.047550052404403687 (0.05114722683473275)
|
| 8089 |
+
| > current_lr: 1.999984091227059e-05
|
| 8090 |
+
| > step_time: 0.2698 (0.27941999648092797)
|
| 8091 |
+
| > loader_time: 0.008 (0.008503748997765653)
|
| 8092 |
+
|
| 8093 |
+
|
| 8094 |
+
[1m --> TIME: 2025-12-26 18:30:17 -- STEP: 1886/2207 -- GLOBAL_STEP: 75750[0m
|
| 8095 |
+
| > loss_text_ce: 0.02867002971470356 (0.02848391155766626)
|
| 8096 |
+
| > loss_mel_ce: 3.23088002204895 (3.245022951608498)
|
| 8097 |
+
| > loss: 0.05093047022819519 (0.05114854476784975)
|
| 8098 |
+
| > current_lr: 1.999984091227059e-05
|
| 8099 |
+
| > step_time: 0.2906 (0.2794254748970444)
|
| 8100 |
+
| > loader_time: 0.0086 (0.008503753206004248)
|
| 8101 |
+
|
| 8102 |
+
|
| 8103 |
+
[1m --> TIME: 2025-12-26 18:30:26 -- STEP: 1911/2207 -- GLOBAL_STEP: 75775[0m
|
| 8104 |
+
| > loss_text_ce: 0.02809242159128189 (0.02848403037865396)
|
| 8105 |
+
| > loss_mel_ce: 3.1280157566070557 (3.2450999522446216)
|
| 8106 |
+
| > loss: 0.049314189702272415 (0.05114974975897839)
|
| 8107 |
+
| > current_lr: 1.999984091227059e-05
|
| 8108 |
+
| > step_time: 0.2709 (0.2793799021804242)
|
| 8109 |
+
| > loader_time: 0.0085 (0.008503707898717329)
|
| 8110 |
+
|
| 8111 |
+
|
| 8112 |
+
[1m --> TIME: 2025-12-26 18:30:36 -- STEP: 1936/2207 -- GLOBAL_STEP: 75800[0m
|
| 8113 |
+
| > loss_text_ce: 0.027230270206928253 (0.02848551394841211)
|
| 8114 |
+
| > loss_mel_ce: 3.2575595378875732 (3.244851880576)
|
| 8115 |
+
| > loss: 0.051324840635061264 (0.05114589681972032)
|
| 8116 |
+
| > current_lr: 1.999984091227059e-05
|
| 8117 |
+
| > step_time: 0.2912 (0.2794019534321871)
|
| 8118 |
+
| > loader_time: 0.009 (0.008506559882282208)
|
| 8119 |
+
|
| 8120 |
+
|
| 8121 |
+
[1m --> TIME: 2025-12-26 18:30:46 -- STEP: 1961/2207 -- GLOBAL_STEP: 75825[0m
|
| 8122 |
+
| > loss_text_ce: 0.027990741655230522 (0.028492964965646097)
|
| 8123 |
+
| > loss_mel_ce: 3.0958635807037354 (3.2445254125259533)
|
| 8124 |
+
| > loss: 0.04881022498011589 (0.051140912181105544)
|
| 8125 |
+
| > current_lr: 1.999984091227059e-05
|
| 8126 |
+
| > step_time: 0.2629 (0.2794302359702103)
|
| 8127 |
+
| > loader_time: 0.0078 (0.008514100468201257)
|
| 8128 |
+
|
| 8129 |
+
|
| 8130 |
+
[1m --> TIME: 2025-12-26 18:30:55 -- STEP: 1986/2207 -- GLOBAL_STEP: 75850[0m
|
| 8131 |
+
| > loss_text_ce: 0.028785699978470802 (0.028497259314514307)
|
| 8132 |
+
| > loss_mel_ce: 3.7493860721588135 (3.244384605238685)
|
| 8133 |
+
| > loss: 0.05903393402695656 (0.05113877916721717)
|
| 8134 |
+
| > current_lr: 1.999984091227059e-05
|
| 8135 |
+
| > step_time: 0.2905 (0.27950177403975496)
|
| 8136 |
+
| > loader_time: 0.0085 (0.008512716278928986)
|
| 8137 |
+
|
| 8138 |
+
|
| 8139 |
+
[1m --> TIME: 2025-12-26 18:31:05 -- STEP: 2011/2207 -- GLOBAL_STEP: 75875[0m
|
| 8140 |
+
| > loss_text_ce: 0.028859876096248627 (0.02849667675215205)
|
| 8141 |
+
| > loss_mel_ce: 3.260416269302368 (3.2444291361406155)
|
| 8142 |
+
| > loss: 0.05139493942260742 (0.05113946585506125)
|
| 8143 |
+
| > current_lr: 1.999984091227059e-05
|
| 8144 |
+
| > step_time: 0.2914 (0.279517530624338)
|
| 8145 |
+
| > loader_time: 0.0089 (0.008512763583440778)
|
| 8146 |
+
|
| 8147 |
+
|
| 8148 |
+
[1m --> TIME: 2025-12-26 18:31:15 -- STEP: 2036/2207 -- GLOBAL_STEP: 75900[0m
|
| 8149 |
+
| > loss_text_ce: 0.028545038774609566 (0.028498647736352536)
|
| 8150 |
+
| > loss_mel_ce: 3.0662310123443604 (3.245162755894989)
|
| 8151 |
+
| > loss: 0.04835587739944458 (0.05115095945898523)
|
| 8152 |
+
| > current_lr: 1.999984091227059e-05
|
| 8153 |
+
| > step_time: 0.2871 (0.2794664471697494)
|
| 8154 |
+
| > loader_time: 0.0082 (0.008509920247645632)
|
| 8155 |
+
|
| 8156 |
+
|
| 8157 |
+
[1m --> TIME: 2025-12-26 18:31:24 -- STEP: 2061/2207 -- GLOBAL_STEP: 75925[0m
|
| 8158 |
+
| > loss_text_ce: 0.030773261561989784 (0.028498401776794847)
|
| 8159 |
+
| > loss_mel_ce: 3.3690013885498047 (3.2448222467125416)
|
| 8160 |
+
| > loss: 0.053121477365493774 (0.05114563516206037)
|
| 8161 |
+
| > current_lr: 1.999984091227059e-05
|
| 8162 |
+
| > step_time: 0.2882 (0.2795090836150634)
|
| 8163 |
+
| > loader_time: 0.0082 (0.008508745181450603)
|
| 8164 |
+
|
| 8165 |
+
|
| 8166 |
+
[1m --> TIME: 2025-12-26 18:31:34 -- STEP: 2086/2207 -- GLOBAL_STEP: 75950[0m
|
| 8167 |
+
| > loss_text_ce: 0.028117209672927856 (0.028497100534490814)
|
| 8168 |
+
| > loss_mel_ce: 3.2481982707977295 (3.2445746143620853)
|
| 8169 |
+
| > loss: 0.051192428916692734 (0.05114174557069345)
|
| 8170 |
+
| > current_lr: 1.999984091227059e-05
|
| 8171 |
+
| > step_time: 0.2669 (0.2794372803412842)
|
| 8172 |
+
| > loader_time: 0.0082 (0.008509008676413705)
|
| 8173 |
+
|
| 8174 |
+
|
| 8175 |
+
[1m --> TIME: 2025-12-26 18:31:44 -- STEP: 2111/2207 -- GLOBAL_STEP: 75975[0m
|
| 8176 |
+
| > loss_text_ce: 0.027265293523669243 (0.02850110717611904)
|
| 8177 |
+
| > loss_mel_ce: 3.5624451637268066 (3.2446720435234218)
|
| 8178 |
+
| > loss: 0.05608922615647316 (0.051143330504675705)
|
| 8179 |
+
| > grad_norm: 0 (0.0)
|
| 8180 |
+
| > current_lr: 1.999984091227059e-05
|
| 8181 |
+
| > step_time: 0.3236 (0.2795087932015654)
|
| 8182 |
+
| > loader_time: 0.0079 (0.008508292830199311)
|
| 8183 |
+
|
| 8184 |
+
|
| 8185 |
+
[1m --> TIME: 2025-12-26 18:31:53 -- STEP: 2136/2207 -- GLOBAL_STEP: 76000[0m
|
| 8186 |
+
| > loss_text_ce: 0.02771497517824173 (0.02849889295315197)
|
| 8187 |
+
| > loss_mel_ce: 3.373234272003174 (3.244728252459107)
|
| 8188 |
+
| > loss: 0.053139831870794296 (0.05114417417107449)
|
| 8189 |
+
| > current_lr: 1.999984091227059e-05
|
| 8190 |
+
| > step_time: 0.2878 (0.27947930104277247)
|
| 8191 |
+
| > loader_time: 0.0088 (0.008507525541362674)
|
| 8192 |
+
|
| 8193 |
+
|
| 8194 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-26-2025_03+25PM-0000000/checkpoint_76000.pth
|
| 8195 |
+
|
| 8196 |
+
[1m --> TIME: 2025-12-26 18:32:28 -- STEP: 2161/2207 -- GLOBAL_STEP: 76025[0m
|
| 8197 |
+
| > loss_text_ce: 0.03056650422513485 (0.028498186415672477)
|
| 8198 |
+
| > loss_mel_ce: 2.941533327102661 (3.2445256654888532)
|
| 8199 |
+
| > loss: 0.04643905907869339 (0.051140997707085266)
|
| 8200 |
+
| > current_lr: 1.999984091227059e-05
|
| 8201 |
+
| > step_time: 0.267 (0.2795397049296198)
|
| 8202 |
+
| > loader_time: 0.0095 (0.008513532044544405)
|
| 8203 |
+
|
| 8204 |
+
|
| 8205 |
+
[1m --> TIME: 2025-12-26 18:32:37 -- STEP: 2186/2207 -- GLOBAL_STEP: 76050[0m
|
| 8206 |
+
| > loss_text_ce: 0.029442545026540756 (0.02849358015277059)
|
| 8207 |
+
| > loss_mel_ce: 2.75170636177063 (3.2438647735042507)
|
| 8208 |
+
| > loss: 0.04345545172691345 (0.05113059929542223)
|
| 8209 |
+
| > current_lr: 1.999984091227059e-05
|
| 8210 |
+
| > step_time: 0.2668 (0.27955343793099596)
|
| 8211 |
+
| > loader_time: 0.0083 (0.00851503088184156)
|
| 8212 |
+
|
| 8213 |
+
|
| 8214 |
+
[1m > EVALUATION [0m
|
| 8215 |
+
|
| 8216 |
+
|
| 8217 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 8218 |
+
| > avg_loader_time:[91m 0.10745621371913601 [0m(+0.02693082996316852)
|
| 8219 |
+
| > avg_loss_text_ce:[92m 0.0276795822145367 [0m(-3.5353974011300554e-05)
|
| 8220 |
+
| > avg_loss_mel_ce:[91m 3.7725630966392725 [0m(+0.02119288251206708)
|
| 8221 |
+
| > avg_loss:[91m 3.8002426769282365 [0m(+0.021157527292096745)
|
| 8222 |
+
|