Checkpoint at step 49864
Browse files- best_model.pth +1 -1
- best_model_49864.pth +3 -0
- checkpoint_49000.pth +3 -0
- checkpoint_49500.pth +3 -0
- events.out.tfevents.1766698157.0f9434978106.1562.0 +2 -2
- trainer_0_log.txt +817 -0
best_model.pth
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 5607926805
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:935ce866742c679c88777e1fe1878df7b39faf551567fc595620152ea8d974af
|
| 3 |
size 5607926805
|
best_model_49864.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:935ce866742c679c88777e1fe1878df7b39faf551567fc595620152ea8d974af
|
| 3 |
+
size 5607926805
|
checkpoint_49000.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8c5c6dbc7eafea8d9211ebf7e070376f42b62ee295c914962578ca992b15daef
|
| 3 |
+
size 5607926741
|
checkpoint_49500.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0f80713bf8413d3ff9dcc7e5378994699e67b8edb0ca162c5d2f11b3a4825a52
|
| 3 |
+
size 5607926741
|
events.out.tfevents.1766698157.0f9434978106.1562.0
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8c749defc10cc7c89e5f17a244c31c681b6156be45cafe1c876c183ebed4e966
|
| 3 |
+
size 89826
|
trainer_0_log.txt
CHANGED
|
@@ -6591,3 +6591,820 @@
|
|
| 6591 |
| > avg_loss:[92m 3.7301667123227507 [0m(-0.00996698559941489)
|
| 6592 |
|
| 6593 |
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_47657.pth
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6591 |
| > avg_loss:[92m 3.7301667123227507 [0m(-0.00996698559941489)
|
| 6592 |
|
| 6593 |
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_47657.pth
|
| 6594 |
+
|
| 6595 |
+
[4m[1m > EPOCH: 9/12[0m
|
| 6596 |
+
--> /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000
|
| 6597 |
+
|
| 6598 |
+
[1m > TRAINING (2025-12-26 00:15:44) [0m
|
| 6599 |
+
|
| 6600 |
+
[1m --> TIME: 2025-12-26 00:15:55 -- STEP: 18/2207 -- GLOBAL_STEP: 47675[0m
|
| 6601 |
+
| > loss_text_ce: 0.029418371617794037 (0.029337260975605912)
|
| 6602 |
+
| > loss_mel_ce: 3.2751619815826416 (3.578384863005744)
|
| 6603 |
+
| > loss: 0.05163406953215599 (0.05637065880000591)
|
| 6604 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6605 |
+
| > step_time: 0.2695 (0.27630746364593506)
|
| 6606 |
+
| > loader_time: 0.0088 (0.009911338488260904)
|
| 6607 |
+
|
| 6608 |
+
|
| 6609 |
+
[1m --> TIME: 2025-12-26 00:16:04 -- STEP: 43/2207 -- GLOBAL_STEP: 47700[0m
|
| 6610 |
+
| > loss_text_ce: 0.031197141855955124 (0.029337143222259922)
|
| 6611 |
+
| > loss_mel_ce: 3.616787910461426 (3.537957790286042)
|
| 6612 |
+
| > loss: 0.05699976533651352 (0.055738983372616215)
|
| 6613 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6614 |
+
| > step_time: 0.2675 (0.27608265987662384)
|
| 6615 |
+
| > loader_time: 0.0101 (0.009777207707249842)
|
| 6616 |
+
|
| 6617 |
+
|
| 6618 |
+
[1m --> TIME: 2025-12-26 00:16:14 -- STEP: 68/2207 -- GLOBAL_STEP: 47725[0m
|
| 6619 |
+
| > loss_text_ce: 0.030151059851050377 (0.02929872071699184)
|
| 6620 |
+
| > loss_mel_ce: 3.4499993324279785 (3.5283713200513054)
|
| 6621 |
+
| > loss: 0.05437735095620155 (0.05558859436389278)
|
| 6622 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6623 |
+
| > step_time: 0.2653 (0.2779338114401873)
|
| 6624 |
+
| > loader_time: 0.0086 (0.00965500228545245)
|
| 6625 |
+
|
| 6626 |
+
|
| 6627 |
+
[1m --> TIME: 2025-12-26 00:16:24 -- STEP: 93/2207 -- GLOBAL_STEP: 47750[0m
|
| 6628 |
+
| > loss_text_ce: 0.029510756954550743 (0.02932930715702554)
|
| 6629 |
+
| > loss_mel_ce: 3.543421745300293 (3.520310325007285)
|
| 6630 |
+
| > loss: 0.0558270700275898 (0.05546311925976507)
|
| 6631 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6632 |
+
| > step_time: 0.2899 (0.27759668903966084)
|
| 6633 |
+
| > loader_time: 0.0086 (0.009501357232370687)
|
| 6634 |
+
|
| 6635 |
+
|
| 6636 |
+
[1m --> TIME: 2025-12-26 00:16:33 -- STEP: 118/2207 -- GLOBAL_STEP: 47775[0m
|
| 6637 |
+
| > loss_text_ce: 0.029739664867520332 (0.029205490913936646)
|
| 6638 |
+
| > loss_mel_ce: 3.1920604705810547 (3.5151568873453947)
|
| 6639 |
+
| > loss: 0.05034062638878822 (0.055380662145503495)
|
| 6640 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6641 |
+
| > step_time: 0.266 (0.2783842389866455)
|
| 6642 |
+
| > loader_time: 0.0093 (0.009578607850155588)
|
| 6643 |
+
|
| 6644 |
+
|
| 6645 |
+
[1m --> TIME: 2025-12-26 00:16:43 -- STEP: 143/2207 -- GLOBAL_STEP: 47800[0m
|
| 6646 |
+
| > loss_text_ce: 0.02857828326523304 (0.0292954282066622)
|
| 6647 |
+
| > loss_mel_ce: 3.7616395950317383 (3.527209101857005)
|
| 6648 |
+
| > loss: 0.059222154319286346 (0.055570383250088125)
|
| 6649 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6650 |
+
| > step_time: 0.2691 (0.2784933577050694)
|
| 6651 |
+
| > loader_time: 0.0095 (0.009565453429322142)
|
| 6652 |
+
|
| 6653 |
+
|
| 6654 |
+
[1m --> TIME: 2025-12-26 00:16:53 -- STEP: 168/2207 -- GLOBAL_STEP: 47825[0m
|
| 6655 |
+
| > loss_text_ce: 0.028904659673571587 (0.029404565747383805)
|
| 6656 |
+
| > loss_mel_ce: 3.3494248390197754 (3.5232873260974875)
|
| 6657 |
+
| > loss: 0.05278639867901802 (0.055510810820297116)
|
| 6658 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6659 |
+
| > step_time: 0.2657 (0.27931223596845334)
|
| 6660 |
+
| > loader_time: 0.0095 (0.009504812104361398)
|
| 6661 |
+
|
| 6662 |
+
|
| 6663 |
+
[1m --> TIME: 2025-12-26 00:17:03 -- STEP: 193/2207 -- GLOBAL_STEP: 47850[0m
|
| 6664 |
+
| > loss_text_ce: 0.029446255415678024 (0.029366924856919698)
|
| 6665 |
+
| > loss_mel_ce: 3.6124072074890137 (3.5217363043770264)
|
| 6666 |
+
| > loss: 0.056903962045907974 (0.05548598798754302)
|
| 6667 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6668 |
+
| > step_time: 0.2868 (0.28001919682161774)
|
| 6669 |
+
| > loader_time: 0.0089 (0.009466457861075132)
|
| 6670 |
+
|
| 6671 |
+
|
| 6672 |
+
[1m --> TIME: 2025-12-26 00:17:12 -- STEP: 218/2207 -- GLOBAL_STEP: 47875[0m
|
| 6673 |
+
| > loss_text_ce: 0.030121125280857086 (0.029366841625183)
|
| 6674 |
+
| > loss_mel_ce: 3.7393665313720703 (3.52055007711463)
|
| 6675 |
+
| > loss: 0.05889824405312538 (0.05546745187987428)
|
| 6676 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6677 |
+
| > step_time: 0.2874 (0.2799459531766558)
|
| 6678 |
+
| > loader_time: 0.0082 (0.00941190150899625)
|
| 6679 |
+
|
| 6680 |
+
|
| 6681 |
+
[1m --> TIME: 2025-12-26 00:17:22 -- STEP: 243/2207 -- GLOBAL_STEP: 47900[0m
|
| 6682 |
+
| > loss_text_ce: 0.03047184832394123 (0.02936066894834179)
|
| 6683 |
+
| > loss_mel_ce: 3.392866373062134 (3.5205143891244264)
|
| 6684 |
+
| > loss: 0.05348965898156166 (0.0554667977547204)
|
| 6685 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6686 |
+
| > step_time: 0.267 (0.2796755690633514)
|
| 6687 |
+
| > loader_time: 0.0087 (0.00935785270031588)
|
| 6688 |
+
|
| 6689 |
+
|
| 6690 |
+
[1m --> TIME: 2025-12-26 00:17:31 -- STEP: 268/2207 -- GLOBAL_STEP: 47925[0m
|
| 6691 |
+
| > loss_text_ce: 0.030860671773552895 (0.02936461309332456)
|
| 6692 |
+
| > loss_mel_ce: 3.3977065086364746 (3.5199855868496113)
|
| 6693 |
+
| > loss: 0.05357136204838753 (0.055458596838054375)
|
| 6694 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6695 |
+
| > step_time: 0.2668 (0.27957216216557057)
|
| 6696 |
+
| > loader_time: 0.0099 (0.009311930457157879)
|
| 6697 |
+
|
| 6698 |
+
|
| 6699 |
+
[1m --> TIME: 2025-12-26 00:17:41 -- STEP: 293/2207 -- GLOBAL_STEP: 47950[0m
|
| 6700 |
+
| > loss_text_ce: 0.02825525403022766 (0.02935487548772996)
|
| 6701 |
+
| > loss_mel_ce: 3.450655221939087 (3.523250059056201)
|
| 6702 |
+
| > loss: 0.05435797572135925 (0.055509452081886165)
|
| 6703 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6704 |
+
| > step_time: 0.2891 (0.2792232134236409)
|
| 6705 |
+
| > loader_time: 0.0092 (0.009268456351634997)
|
| 6706 |
+
|
| 6707 |
+
|
| 6708 |
+
[1m --> TIME: 2025-12-26 00:17:50 -- STEP: 318/2207 -- GLOBAL_STEP: 47975[0m
|
| 6709 |
+
| > loss_text_ce: 0.02891305647790432 (0.029368630493258906)
|
| 6710 |
+
| > loss_mel_ce: 3.3904759883880615 (3.523923786931068)
|
| 6711 |
+
| > loss: 0.05342795327305794 (0.05552019400276103)
|
| 6712 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6713 |
+
| > step_time: 0.2923 (0.2791316906611123)
|
| 6714 |
+
| > loader_time: 0.0086 (0.009217228529588234)
|
| 6715 |
+
|
| 6716 |
+
|
| 6717 |
+
[1m --> TIME: 2025-12-26 00:18:00 -- STEP: 343/2207 -- GLOBAL_STEP: 48000[0m
|
| 6718 |
+
| > loss_text_ce: 0.028865251690149307 (0.029384233942662662)
|
| 6719 |
+
| > loss_mel_ce: 3.5737030506134033 (3.5222103623537557)
|
| 6720 |
+
| > loss: 0.05629013106226921 (0.05549366552784561)
|
| 6721 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6722 |
+
| > step_time: 0.2873 (0.27917185816750917)
|
| 6723 |
+
| > loader_time: 0.0087 (0.009179042310130843)
|
| 6724 |
+
|
| 6725 |
+
|
| 6726 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_48000.pth
|
| 6727 |
+
|
| 6728 |
+
[1m --> TIME: 2025-12-26 00:18:35 -- STEP: 368/2207 -- GLOBAL_STEP: 48025[0m
|
| 6729 |
+
| > loss_text_ce: 0.027616219595074654 (0.02936074347741416)
|
| 6730 |
+
| > loss_mel_ce: 3.479339361190796 (3.5230949994014655)
|
| 6731 |
+
| > loss: 0.05479618161916733 (0.055507120959784675)
|
| 6732 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6733 |
+
| > step_time: 0.2904 (0.2794709542523259)
|
| 6734 |
+
| > loader_time: 0.0094 (0.009185502710549728)
|
| 6735 |
+
|
| 6736 |
+
|
| 6737 |
+
[1m --> TIME: 2025-12-26 00:18:44 -- STEP: 393/2207 -- GLOBAL_STEP: 48050[0m
|
| 6738 |
+
| > loss_text_ce: 0.028366126120090485 (0.029351240350999903)
|
| 6739 |
+
| > loss_mel_ce: 3.72489857673645 (3.5279312060989496)
|
| 6740 |
+
| > loss: 0.05864476040005684 (0.05558253818075469)
|
| 6741 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6742 |
+
| > step_time: 0.2691 (0.2794228309893424)
|
| 6743 |
+
| > loader_time: 0.0085 (0.009148347165445036)
|
| 6744 |
+
|
| 6745 |
+
|
| 6746 |
+
[1m --> TIME: 2025-12-26 00:18:54 -- STEP: 418/2207 -- GLOBAL_STEP: 48075[0m
|
| 6747 |
+
| > loss_text_ce: 0.029229113832116127 (0.029338157668923646)
|
| 6748 |
+
| > loss_mel_ce: 3.4752635955810547 (3.5311573355962214)
|
| 6749 |
+
| > loss: 0.054757699370384216 (0.055632742023781726)
|
| 6750 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6751 |
+
| > step_time: 0.2688 (0.279522615186335)
|
| 6752 |
+
| > loader_time: 0.0085 (0.009127754343753798)
|
| 6753 |
+
|
| 6754 |
+
|
| 6755 |
+
[1m --> TIME: 2025-12-26 00:19:03 -- STEP: 443/2207 -- GLOBAL_STEP: 48100[0m
|
| 6756 |
+
| > loss_text_ce: 0.029086370021104813 (0.029361252178927846)
|
| 6757 |
+
| > loss_mel_ce: 3.548222064971924 (3.5310001933009576)
|
| 6758 |
+
| > loss: 0.05589544400572777 (0.05563064753740151)
|
| 6759 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6760 |
+
| > step_time: 0.2917 (0.27939991219049076)
|
| 6761 |
+
| > loader_time: 0.0101 (0.009104808350987146)
|
| 6762 |
+
|
| 6763 |
+
|
| 6764 |
+
[1m --> TIME: 2025-12-26 00:19:13 -- STEP: 468/2207 -- GLOBAL_STEP: 48125[0m
|
| 6765 |
+
| > loss_text_ce: 0.028658054769039154 (0.029331425037712622)
|
| 6766 |
+
| > loss_mel_ce: 3.938842296600342 (3.5328668432357984)
|
| 6767 |
+
| > loss: 0.06199219450354576 (0.055659347898366615)
|
| 6768 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6769 |
+
| > step_time: 0.2689 (0.2792936324054356)
|
| 6770 |
+
| > loader_time: 0.0098 (0.009152314092358975)
|
| 6771 |
+
|
| 6772 |
+
|
| 6773 |
+
[1m --> TIME: 2025-12-26 00:19:23 -- STEP: 493/2207 -- GLOBAL_STEP: 48150[0m
|
| 6774 |
+
| > loss_text_ce: 0.028402332216501236 (0.029329976029165133)
|
| 6775 |
+
| > loss_mel_ce: 3.3788788318634033 (3.5353780779345283)
|
| 6776 |
+
| > loss: 0.05323876813054085 (0.05569856328999295)
|
| 6777 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6778 |
+
| > step_time: 0.2918 (0.27950640140634003)
|
| 6779 |
+
| > loader_time: 0.0096 (0.009174796678957061)
|
| 6780 |
+
|
| 6781 |
+
|
| 6782 |
+
[1m --> TIME: 2025-12-26 00:19:32 -- STEP: 518/2207 -- GLOBAL_STEP: 48175[0m
|
| 6783 |
+
| > loss_text_ce: 0.0302241463214159 (0.0293439691454805)
|
| 6784 |
+
| > loss_mel_ce: 3.3003196716308594 (3.533381391676236)
|
| 6785 |
+
| > loss: 0.052039746195077896 (0.055667583704800225)
|
| 6786 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6787 |
+
| > step_time: 0.2912 (0.2791888400854748)
|
| 6788 |
+
| > loader_time: 0.008 (0.009155466289593909)
|
| 6789 |
+
|
| 6790 |
+
|
| 6791 |
+
[1m --> TIME: 2025-12-26 00:19:42 -- STEP: 543/2207 -- GLOBAL_STEP: 48200[0m
|
| 6792 |
+
| > loss_text_ce: 0.032884132117033005 (0.029345624807460533)
|
| 6793 |
+
| > loss_mel_ce: 3.3753268718719482 (3.532526935661695)
|
| 6794 |
+
| > loss: 0.05325329676270485 (0.05565425869537103)
|
| 6795 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6796 |
+
| > step_time: 0.2674 (0.279142244964234)
|
| 6797 |
+
| > loader_time: 0.0089 (0.009141420571624028)
|
| 6798 |
+
|
| 6799 |
+
|
| 6800 |
+
[1m --> TIME: 2025-12-26 00:19:51 -- STEP: 568/2207 -- GLOBAL_STEP: 48225[0m
|
| 6801 |
+
| > loss_text_ce: 0.030816098675131798 (0.029360686596089477)
|
| 6802 |
+
| > loss_mel_ce: 3.6690351963043213 (3.531509289019544)
|
| 6803 |
+
| > loss: 0.05781017616391182 (0.05563859330018014)
|
| 6804 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6805 |
+
| > step_time: 0.2671 (0.2790685220503468)
|
| 6806 |
+
| > loader_time: 0.0092 (0.009112248118494598)
|
| 6807 |
+
|
| 6808 |
+
|
| 6809 |
+
[1m --> TIME: 2025-12-26 00:20:01 -- STEP: 593/2207 -- GLOBAL_STEP: 48250[0m
|
| 6810 |
+
| > loss_text_ce: 0.028091544285416603 (0.029352619282863594)
|
| 6811 |
+
| > loss_mel_ce: 3.6105382442474365 (3.5331841938411204)
|
| 6812 |
+
| > loss: 0.05685358867049217 (0.0556646376270267)
|
| 6813 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6814 |
+
| > step_time: 0.2896 (0.27867310445184024)
|
| 6815 |
+
| > loader_time: 0.0098 (0.009123186236891335)
|
| 6816 |
+
|
| 6817 |
+
|
| 6818 |
+
[1m --> TIME: 2025-12-26 00:20:10 -- STEP: 618/2207 -- GLOBAL_STEP: 48275[0m
|
| 6819 |
+
| > loss_text_ce: 0.029470719397068024 (0.029335720982993304)
|
| 6820 |
+
| > loss_mel_ce: 3.523622989654541 (3.534324496695139)
|
| 6821 |
+
| > loss: 0.05551708862185478 (0.055682190817488444)
|
| 6822 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6823 |
+
| > step_time: 0.2659 (0.27863609790801963)
|
| 6824 |
+
| > loader_time: 0.0083 (0.009117634551039025)
|
| 6825 |
+
|
| 6826 |
+
|
| 6827 |
+
[1m --> TIME: 2025-12-26 00:20:20 -- STEP: 643/2207 -- GLOBAL_STEP: 48300[0m
|
| 6828 |
+
| > loss_text_ce: 0.026869436725974083 (0.029338628803937467)
|
| 6829 |
+
| > loss_mel_ce: 3.489090919494629 (3.535885809368887)
|
| 6830 |
+
| > loss: 0.054936882108449936 (0.05570663176882509)
|
| 6831 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6832 |
+
| > step_time: 0.2903 (0.2786020287088061)
|
| 6833 |
+
| > loader_time: 0.0082 (0.009105004567195249)
|
| 6834 |
+
|
| 6835 |
+
|
| 6836 |
+
[1m --> TIME: 2025-12-26 00:20:30 -- STEP: 668/2207 -- GLOBAL_STEP: 48325[0m
|
| 6837 |
+
| > loss_text_ce: 0.02878863736987114 (0.029337540369280084)
|
| 6838 |
+
| > loss_mel_ce: 3.3784992694854736 (3.5357628682416355)
|
| 6839 |
+
| > loss: 0.053238872438669205 (0.05570469381187313)
|
| 6840 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6841 |
+
| > step_time: 0.267 (0.2788361821345936)
|
| 6842 |
+
| > loader_time: 0.0087 (0.009089192944372484)
|
| 6843 |
+
|
| 6844 |
+
|
| 6845 |
+
[1m --> TIME: 2025-12-26 00:20:39 -- STEP: 693/2207 -- GLOBAL_STEP: 48350[0m
|
| 6846 |
+
| > loss_text_ce: 0.030137674883008003 (0.029343412358523446)
|
| 6847 |
+
| > loss_mel_ce: 3.6369497776031494 (3.5347576389065036)
|
| 6848 |
+
| > loss: 0.057298243045806885 (0.055689078856469246)
|
| 6849 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6850 |
+
| > step_time: 0.2692 (0.278762661877476)
|
| 6851 |
+
| > loader_time: 0.0085 (0.009064095360892163)
|
| 6852 |
+
|
| 6853 |
+
|
| 6854 |
+
[1m --> TIME: 2025-12-26 00:20:49 -- STEP: 718/2207 -- GLOBAL_STEP: 48375[0m
|
| 6855 |
+
| > loss_text_ce: 0.027695555239915848 (0.029333187547359293)
|
| 6856 |
+
| > loss_mel_ce: 3.7420976161956787 (3.535637532436084)
|
| 6857 |
+
| > loss: 0.05890301987528801 (0.055702667434318495)
|
| 6858 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6859 |
+
| > step_time: 0.2899 (0.2786425262440539)
|
| 6860 |
+
| > loader_time: 0.0097 (0.009048368937457843)
|
| 6861 |
+
|
| 6862 |
+
|
| 6863 |
+
[1m --> TIME: 2025-12-26 00:20:58 -- STEP: 743/2207 -- GLOBAL_STEP: 48400[0m
|
| 6864 |
+
| > loss_text_ce: 0.028989030048251152 (0.02933341932318814)
|
| 6865 |
+
| > loss_mel_ce: 3.4565181732177734 (3.5344377318108866)
|
| 6866 |
+
| > loss: 0.054461050778627396 (0.055683924173522616)
|
| 6867 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6868 |
+
| > step_time: 0.2969 (0.27862956065027655)
|
| 6869 |
+
| > loader_time: 0.0089 (0.009030916167781624)
|
| 6870 |
+
|
| 6871 |
+
|
| 6872 |
+
[1m --> TIME: 2025-12-26 00:21:08 -- STEP: 768/2207 -- GLOBAL_STEP: 48425[0m
|
| 6873 |
+
| > loss_text_ce: 0.02854948304593563 (0.029327480510498088)
|
| 6874 |
+
| > loss_mel_ce: 3.528278112411499 (3.534732522753378)
|
| 6875 |
+
| > loss: 0.055575430393218994 (0.05568843748187647)
|
| 6876 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6877 |
+
| > step_time: 0.2596 (0.27864373009651827)
|
| 6878 |
+
| > loader_time: 0.0091 (0.009014113495747251)
|
| 6879 |
+
|
| 6880 |
+
|
| 6881 |
+
[1m --> TIME: 2025-12-26 00:21:17 -- STEP: 793/2207 -- GLOBAL_STEP: 48450[0m
|
| 6882 |
+
| > loss_text_ce: 0.02960527315735817 (0.02931367445262654)
|
| 6883 |
+
| > loss_mel_ce: 3.5533089637756348 (3.532809943298975)
|
| 6884 |
+
| > loss: 0.0559830367565155 (0.05565818145514137)
|
| 6885 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6886 |
+
| > step_time: 0.2865 (0.2787981330972593)
|
| 6887 |
+
| > loader_time: 0.0088 (0.00900097512657676)
|
| 6888 |
+
|
| 6889 |
+
|
| 6890 |
+
[1m --> TIME: 2025-12-26 00:21:27 -- STEP: 818/2207 -- GLOBAL_STEP: 48475[0m
|
| 6891 |
+
| > loss_text_ce: 0.03152763843536377 (0.029312814941087825)
|
| 6892 |
+
| > loss_mel_ce: 3.5348360538482666 (3.5319294926589744)
|
| 6893 |
+
| > loss: 0.055724434554576874 (0.05564441099708268)
|
| 6894 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6895 |
+
| > step_time: 0.2668 (0.27869906810210127)
|
| 6896 |
+
| > loader_time: 0.0084 (0.008985621713484422)
|
| 6897 |
+
|
| 6898 |
+
|
| 6899 |
+
[1m --> TIME: 2025-12-26 00:21:37 -- STEP: 843/2207 -- GLOBAL_STEP: 48500[0m
|
| 6900 |
+
| > loss_text_ce: 0.028120897710323334 (0.029303611288220573)
|
| 6901 |
+
| > loss_mel_ce: 3.424769163131714 (3.529253675420089)
|
| 6902 |
+
| > loss: 0.053951408714056015 (0.055602457559935994)
|
| 6903 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6904 |
+
| > step_time: 0.2865 (0.27886576952317654)
|
| 6905 |
+
| > loader_time: 0.008 (0.008974683949390213)
|
| 6906 |
+
|
| 6907 |
+
|
| 6908 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_48500.pth
|
| 6909 |
+
|
| 6910 |
+
[1m --> TIME: 2025-12-26 00:22:11 -- STEP: 868/2207 -- GLOBAL_STEP: 48525[0m
|
| 6911 |
+
| > loss_text_ce: 0.02893184684216976 (0.029300793563337646)
|
| 6912 |
+
| > loss_mel_ce: 3.3138558864593506 (3.52770526865111)
|
| 6913 |
+
| > loss: 0.05223105847835541 (0.05557821968287093)
|
| 6914 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6915 |
+
| > step_time: 0.2664 (0.27891961907461443)
|
| 6916 |
+
| > loader_time: 0.0083 (0.008978857148078194)
|
| 6917 |
+
|
| 6918 |
+
|
| 6919 |
+
[1m --> TIME: 2025-12-26 00:22:21 -- STEP: 893/2207 -- GLOBAL_STEP: 48550[0m
|
| 6920 |
+
| > loss_text_ce: 0.029021603986620903 (0.029284384109972573)
|
| 6921 |
+
| > loss_mel_ce: 3.358370780944824 (3.527961206863335)
|
| 6922 |
+
| > loss: 0.052928004413843155 (0.05558196231639399)
|
| 6923 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6924 |
+
| > step_time: 0.2875 (0.2790070877630521)
|
| 6925 |
+
| > loader_time: 0.0097 (0.008972994290701097)
|
| 6926 |
+
|
| 6927 |
+
|
| 6928 |
+
[1m --> TIME: 2025-12-26 00:22:30 -- STEP: 918/2207 -- GLOBAL_STEP: 48575[0m
|
| 6929 |
+
| > loss_text_ce: 0.028209052979946136 (0.02927916262436796)
|
| 6930 |
+
| > loss_mel_ce: 3.667708158493042 (3.5301187087767523)
|
| 6931 |
+
| > loss: 0.0577487051486969 (0.05561559170094971)
|
| 6932 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6933 |
+
| > step_time: 0.2875 (0.27894246006842993)
|
| 6934 |
+
| > loader_time: 0.0086 (0.00898068195334708)
|
| 6935 |
+
|
| 6936 |
+
|
| 6937 |
+
[1m --> TIME: 2025-12-26 00:22:40 -- STEP: 943/2207 -- GLOBAL_STEP: 48600[0m
|
| 6938 |
+
| > loss_text_ce: 0.02978096716105938 (0.029297658962257847)
|
| 6939 |
+
| > loss_mel_ce: 3.6340153217315674 (3.5313417522171524)
|
| 6940 |
+
| > loss: 0.05724681541323662 (0.05563499074535937)
|
| 6941 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6942 |
+
| > step_time: 0.2911 (0.2790985744553173)
|
| 6943 |
+
| > loader_time: 0.0092 (0.008968164660392117)
|
| 6944 |
+
|
| 6945 |
+
|
| 6946 |
+
[1m --> TIME: 2025-12-26 00:22:50 -- STEP: 968/2207 -- GLOBAL_STEP: 48625[0m
|
| 6947 |
+
| > loss_text_ce: 0.028105368837714195 (0.02928751844466348)
|
| 6948 |
+
| > loss_mel_ce: 3.585841417312622 (3.5323595133694736)
|
| 6949 |
+
| > loss: 0.05646791681647301 (0.055650734812144415)
|
| 6950 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6951 |
+
| > step_time: 0.2691 (0.2790430015768881)
|
| 6952 |
+
| > loader_time: 0.0088 (0.008960308614841189)
|
| 6953 |
+
|
| 6954 |
+
|
| 6955 |
+
[1m --> TIME: 2025-12-26 00:22:59 -- STEP: 993/2207 -- GLOBAL_STEP: 48650[0m
|
| 6956 |
+
| > loss_text_ce: 0.028939399868249893 (0.029285248061351306)
|
| 6957 |
+
| > loss_mel_ce: 3.8304171562194824 (3.5317947201858475)
|
| 6958 |
+
| > loss: 0.06030244752764702 (0.0556418744458893)
|
| 6959 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6960 |
+
| > step_time: 0.2871 (0.2789606277315993)
|
| 6961 |
+
| > loader_time: 0.0088 (0.008949504037878196)
|
| 6962 |
+
|
| 6963 |
+
|
| 6964 |
+
[1m --> TIME: 2025-12-26 00:23:09 -- STEP: 1018/2207 -- GLOBAL_STEP: 48675[0m
|
| 6965 |
+
| > loss_text_ce: 0.028835400938987732 (0.029284588880983342)
|
| 6966 |
+
| > loss_mel_ce: 3.776682138442993 (3.5329056959489473)
|
| 6967 |
+
| > loss: 0.059461209923028946 (0.055659223135655904)
|
| 6968 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6969 |
+
| > step_time: 0.287 (0.2788718568786886)
|
| 6970 |
+
| > loader_time: 0.0081 (0.008942195849240657)
|
| 6971 |
+
|
| 6972 |
+
|
| 6973 |
+
[1m --> TIME: 2025-12-26 00:23:18 -- STEP: 1043/2207 -- GLOBAL_STEP: 48700[0m
|
| 6974 |
+
| > loss_text_ce: 0.028841625899076462 (0.0292808989907614)
|
| 6975 |
+
| > loss_mel_ce: 3.2719061374664307 (3.53266237588994)
|
| 6976 |
+
| > loss: 0.051574185490608215 (0.05565536360784094)
|
| 6977 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6978 |
+
| > step_time: 0.266 (0.27875049093654214)
|
| 6979 |
+
| > loader_time: 0.008 (0.008932224970421659)
|
| 6980 |
+
|
| 6981 |
+
|
| 6982 |
+
[1m --> TIME: 2025-12-26 00:23:28 -- STEP: 1068/2207 -- GLOBAL_STEP: 48725[0m
|
| 6983 |
+
| > loss_text_ce: 0.030237771570682526 (0.029289020877680302)
|
| 6984 |
+
| > loss_mel_ce: 3.424987316131592 (3.533322357059864)
|
| 6985 |
+
| > loss: 0.05398789048194885 (0.05566580271634381)
|
| 6986 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6987 |
+
| > step_time: 0.2868 (0.27876465709021875)
|
| 6988 |
+
| > loader_time: 0.0088 (0.00892386900798213)
|
| 6989 |
+
|
| 6990 |
+
|
| 6991 |
+
[1m --> TIME: 2025-12-26 00:23:37 -- STEP: 1093/2207 -- GLOBAL_STEP: 48750[0m
|
| 6992 |
+
| > loss_text_ce: 0.03019256331026554 (0.02928981457113511)
|
| 6993 |
+
| > loss_mel_ce: 3.312772035598755 (3.5338105391932535)
|
| 6994 |
+
| > loss: 0.052233822643756866 (0.055673442968428966)
|
| 6995 |
+
| > current_lr: 1.9999874300983004e-05
|
| 6996 |
+
| > step_time: 0.2667 (0.27874415551209114)
|
| 6997 |
+
| > loader_time: 0.0083 (0.008919887551449911)
|
| 6998 |
+
|
| 6999 |
+
|
| 7000 |
+
[1m --> TIME: 2025-12-26 00:23:47 -- STEP: 1118/2207 -- GLOBAL_STEP: 48775[0m
|
| 7001 |
+
| > loss_text_ce: 0.028912438079714775 (0.029295768290558206)
|
| 7002 |
+
| > loss_mel_ce: 3.4293384552001953 (3.534097795622932)
|
| 7003 |
+
| > loss: 0.05403517186641693 (0.05567802438155911)
|
| 7004 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7005 |
+
| > step_time: 0.2909 (0.27876945919555457)
|
| 7006 |
+
| > loader_time: 0.0088 (0.008910295788418642)
|
| 7007 |
+
|
| 7008 |
+
|
| 7009 |
+
[1m --> TIME: 2025-12-26 00:23:56 -- STEP: 1143/2207 -- GLOBAL_STEP: 48800[0m
|
| 7010 |
+
| > loss_text_ce: 0.027648460119962692 (0.029287028445007503)
|
| 7011 |
+
| > loss_mel_ce: 3.998156785964966 (3.534006881588713)
|
| 7012 |
+
| > loss: 0.06290321052074432 (0.05567646729299509)
|
| 7013 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7014 |
+
| > step_time: 0.2871 (0.2787349837345709)
|
| 7015 |
+
| > loader_time: 0.0085 (0.008903150900559592)
|
| 7016 |
+
|
| 7017 |
+
|
| 7018 |
+
[1m --> TIME: 2025-12-26 00:24:06 -- STEP: 1168/2207 -- GLOBAL_STEP: 48825[0m
|
| 7019 |
+
| > loss_text_ce: 0.028647933155298233 (0.029291847470369548)
|
| 7020 |
+
| > loss_mel_ce: 3.8476994037628174 (3.5354226992963103)
|
| 7021 |
+
| > loss: 0.06056792661547661 (0.055698664747306156)
|
| 7022 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7023 |
+
| > step_time: 0.2894 (0.27871692690947236)
|
| 7024 |
+
| > loader_time: 0.0086 (0.00889557600021363)
|
| 7025 |
+
|
| 7026 |
+
|
| 7027 |
+
[1m --> TIME: 2025-12-26 00:24:16 -- STEP: 1193/2207 -- GLOBAL_STEP: 48850[0m
|
| 7028 |
+
| > loss_text_ce: 0.030165335163474083 (0.029285404484559786)
|
| 7029 |
+
| > loss_mel_ce: 3.9288363456726074 (3.536148180206183)
|
| 7030 |
+
| > loss: 0.06185940280556679 (0.055709899720881854)
|
| 7031 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7032 |
+
| > step_time: 0.2968 (0.27919701718524587)
|
| 7033 |
+
| > loader_time: 0.0083 (0.008891248742972528)
|
| 7034 |
+
|
| 7035 |
+
|
| 7036 |
+
[1m --> TIME: 2025-12-26 00:24:26 -- STEP: 1218/2207 -- GLOBAL_STEP: 48875[0m
|
| 7037 |
+
| > loss_text_ce: 0.03023318387567997 (0.02928950920034267)
|
| 7038 |
+
| > loss_mel_ce: 3.396355152130127 (3.5363632865335743)
|
| 7039 |
+
| > loss: 0.053540442138910294 (0.055713324903928024)
|
| 7040 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7041 |
+
| > step_time: 0.2865 (0.27919077990677515)
|
| 7042 |
+
| > loader_time: 0.0085 (0.008882498702000726)
|
| 7043 |
+
|
| 7044 |
+
|
| 7045 |
+
[1m --> TIME: 2025-12-26 00:24:35 -- STEP: 1243/2207 -- GLOBAL_STEP: 48900[0m
|
| 7046 |
+
| > loss_text_ce: 0.028449326753616333 (0.029293267033868323)
|
| 7047 |
+
| > loss_mel_ce: 3.468842029571533 (3.5362754621912367)
|
| 7048 |
+
| > loss: 0.054645176976919174 (0.05571201135878688)
|
| 7049 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7050 |
+
| > step_time: 0.2863 (0.2791867655127256)
|
| 7051 |
+
| > loader_time: 0.0081 (0.008881237462750395)
|
| 7052 |
+
|
| 7053 |
+
|
| 7054 |
+
[1m --> TIME: 2025-12-26 00:24:45 -- STEP: 1268/2207 -- GLOBAL_STEP: 48925[0m
|
| 7055 |
+
| > loss_text_ce: 0.0302909966558218 (0.029289726704088765)
|
| 7056 |
+
| > loss_mel_ce: 3.5192136764526367 (3.5366688686590457)
|
| 7057 |
+
| > loss: 0.055461011826992035 (0.055718103012405006)
|
| 7058 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7059 |
+
| > step_time: 0.2873 (0.2791778388835652)
|
| 7060 |
+
| > loader_time: 0.0086 (0.008875313805480867)
|
| 7061 |
+
|
| 7062 |
+
|
| 7063 |
+
[1m --> TIME: 2025-12-26 00:24:54 -- STEP: 1293/2207 -- GLOBAL_STEP: 48950[0m
|
| 7064 |
+
| > loss_text_ce: 0.031137434765696526 (0.029290191016728254)
|
| 7065 |
+
| > loss_mel_ce: 3.480175018310547 (3.5356127551979357)
|
| 7066 |
+
| > loss: 0.05486425757408142 (0.05570160849660937)
|
| 7067 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7068 |
+
| > step_time: 0.2665 (0.2792023279274145)
|
| 7069 |
+
| > loader_time: 0.0086 (0.008868485020742993)
|
| 7070 |
+
|
| 7071 |
+
|
| 7072 |
+
[1m --> TIME: 2025-12-26 00:25:04 -- STEP: 1318/2207 -- GLOBAL_STEP: 48975[0m
|
| 7073 |
+
| > loss_text_ce: 0.02967890165746212 (0.02928918455790994)
|
| 7074 |
+
| > loss_mel_ce: 3.466586112976074 (3.534825266764268)
|
| 7075 |
+
| > loss: 0.054629139602184296 (0.055689288263685445)
|
| 7076 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7077 |
+
| > step_time: 0.2876 (0.27921023282006213)
|
| 7078 |
+
| > loader_time: 0.0092 (0.008864214460115335)
|
| 7079 |
+
|
| 7080 |
+
|
| 7081 |
+
[1m --> TIME: 2025-12-26 00:25:14 -- STEP: 1343/2207 -- GLOBAL_STEP: 49000[0m
|
| 7082 |
+
| > loss_text_ce: 0.02837091125547886 (0.0292836484094986)
|
| 7083 |
+
| > loss_mel_ce: 3.546309471130371 (3.5342379622910323)
|
| 7084 |
+
| > loss: 0.05585438013076782 (0.05568002513592492)
|
| 7085 |
+
| > grad_norm: 0 (0.0)
|
| 7086 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7087 |
+
| > step_time: 0.288 (0.279212004376807)
|
| 7088 |
+
| > loader_time: 0.0086 (0.008861630689006026)
|
| 7089 |
+
|
| 7090 |
+
|
| 7091 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_49000.pth
|
| 7092 |
+
|
| 7093 |
+
[1m --> TIME: 2025-12-26 00:25:48 -- STEP: 1368/2207 -- GLOBAL_STEP: 49025[0m
|
| 7094 |
+
| > loss_text_ce: 0.029342636466026306 (0.02927872222407084)
|
| 7095 |
+
| > loss_mel_ce: 3.2338552474975586 (3.5330283221794145)
|
| 7096 |
+
| > loss: 0.05098746716976166 (0.05566104754195578)
|
| 7097 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7098 |
+
| > step_time: 0.2872 (0.27924510331181757)
|
| 7099 |
+
| > loader_time: 0.0084 (0.0088677099573682)
|
| 7100 |
+
|
| 7101 |
+
|
| 7102 |
+
[1m --> TIME: 2025-12-26 00:25:58 -- STEP: 1393/2207 -- GLOBAL_STEP: 49050[0m
|
| 7103 |
+
| > loss_text_ce: 0.02886904589831829 (0.029279158663937967)
|
| 7104 |
+
| > loss_mel_ce: 3.512120246887207 (3.5328940381613543)
|
| 7105 |
+
| > loss: 0.05532795935869217 (0.05565895617799802)
|
| 7106 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7107 |
+
| > step_time: 0.2688 (0.2792546651556778)
|
| 7108 |
+
| > loader_time: 0.0085 (0.008864190025630835)
|
| 7109 |
+
|
| 7110 |
+
|
| 7111 |
+
[1m --> TIME: 2025-12-26 00:26:07 -- STEP: 1418/2207 -- GLOBAL_STEP: 49075[0m
|
| 7112 |
+
| > loss_text_ce: 0.02991265244781971 (0.029279885342021503)
|
| 7113 |
+
| > loss_mel_ce: 3.6669247150421143 (3.532428619051181)
|
| 7114 |
+
| > loss: 0.05776308476924896 (0.05565169536113237)
|
| 7115 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7116 |
+
| > step_time: 0.287 (0.2792644929482335)
|
| 7117 |
+
| > loader_time: 0.0081 (0.008859781519482279)
|
| 7118 |
+
|
| 7119 |
+
|
| 7120 |
+
[1m --> TIME: 2025-12-26 00:26:17 -- STEP: 1443/2207 -- GLOBAL_STEP: 49100[0m
|
| 7121 |
+
| > loss_text_ce: 0.030668942257761955 (0.029281644346818524)
|
| 7122 |
+
| > loss_mel_ce: 3.5339293479919434 (3.5308717075355824)
|
| 7123 |
+
| > loss: 0.05569684877991676 (0.05562739610176326)
|
| 7124 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7125 |
+
| > step_time: 0.2684 (0.279248800694075)
|
| 7126 |
+
| > loader_time: 0.0082 (0.008864022259570142)
|
| 7127 |
+
|
| 7128 |
+
|
| 7129 |
+
[1m --> TIME: 2025-12-26 00:26:27 -- STEP: 1468/2207 -- GLOBAL_STEP: 49125[0m
|
| 7130 |
+
| > loss_text_ce: 0.031275175511837006 (0.02927860492911464)
|
| 7131 |
+
| > loss_mel_ce: 3.719550371170044 (3.531023667358897)
|
| 7132 |
+
| > loss: 0.058606650680303574 (0.05562972298719255)
|
| 7133 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7134 |
+
| > step_time: 0.2625 (0.2791822287951894)
|
| 7135 |
+
| > loader_time: 0.0081 (0.00886073408074861)
|
| 7136 |
+
|
| 7137 |
+
|
| 7138 |
+
[1m --> TIME: 2025-12-26 00:26:36 -- STEP: 1493/2207 -- GLOBAL_STEP: 49150[0m
|
| 7139 |
+
| > loss_text_ce: 0.028919557109475136 (0.029281677966419676)
|
| 7140 |
+
| > loss_mel_ce: 3.571729898452759 (3.530998170894819)
|
| 7141 |
+
| > loss: 0.05626014620065689 (0.05562937261919728)
|
| 7142 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7143 |
+
| > step_time: 0.2911 (0.2792261917959983)
|
| 7144 |
+
| > loader_time: 0.0084 (0.008859436063261865)
|
| 7145 |
+
|
| 7146 |
+
|
| 7147 |
+
[1m --> TIME: 2025-12-26 00:26:46 -- STEP: 1518/2207 -- GLOBAL_STEP: 49175[0m
|
| 7148 |
+
| > loss_text_ce: 0.029365617781877518 (0.029290837618584926)
|
| 7149 |
+
| > loss_mel_ce: 3.380483865737915 (3.5309175829328248)
|
| 7150 |
+
| > loss: 0.05327889695763588 (0.0556282565557203)
|
| 7151 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7152 |
+
| > step_time: 0.2883 (0.2791583011901112)
|
| 7153 |
+
| > loader_time: 0.0083 (0.008854857389477725)
|
| 7154 |
+
|
| 7155 |
+
|
| 7156 |
+
[1m --> TIME: 2025-12-26 00:26:55 -- STEP: 1543/2207 -- GLOBAL_STEP: 49200[0m
|
| 7157 |
+
| > loss_text_ce: 0.02942664548754692 (0.029290345664620784)
|
| 7158 |
+
| > loss_mel_ce: 3.4598019123077393 (3.531020222896889)
|
| 7159 |
+
| > loss: 0.05451919510960579 (0.05562985262079667)
|
| 7160 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7161 |
+
| > step_time: 0.2872 (0.2792523942515977)
|
| 7162 |
+
| > loader_time: 0.0093 (0.008851493710600095)
|
| 7163 |
+
|
| 7164 |
+
|
| 7165 |
+
[1m --> TIME: 2025-12-26 00:27:05 -- STEP: 1568/2207 -- GLOBAL_STEP: 49225[0m
|
| 7166 |
+
| > loss_text_ce: 0.02992861717939377 (0.029280272703285197)
|
| 7167 |
+
| > loss_mel_ce: 3.521158456802368 (3.530688367479917)
|
| 7168 |
+
| > loss: 0.05548573657870293 (0.055624509990519426)
|
| 7169 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7170 |
+
| > step_time: 0.2663 (0.2792522580343848)
|
| 7171 |
+
| > loader_time: 0.0087 (0.008850105410935937)
|
| 7172 |
+
|
| 7173 |
+
|
| 7174 |
+
[1m --> TIME: 2025-12-26 00:27:15 -- STEP: 1593/2207 -- GLOBAL_STEP: 49250[0m
|
| 7175 |
+
| > loss_text_ce: 0.031545139849185944 (0.02927884012367167)
|
| 7176 |
+
| > loss_mel_ce: 3.602458953857422 (3.5317120857382442)
|
| 7177 |
+
| > loss: 0.05678131431341171 (0.05564048320373051)
|
| 7178 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7179 |
+
| > step_time: 0.2616 (0.27927979626748345)
|
| 7180 |
+
| > loader_time: 0.008 (0.008844235223266071)
|
| 7181 |
+
|
| 7182 |
+
|
| 7183 |
+
[1m --> TIME: 2025-12-26 00:27:24 -- STEP: 1618/2207 -- GLOBAL_STEP: 49275[0m
|
| 7184 |
+
| > loss_text_ce: 0.029052192345261574 (0.029285956286953022)
|
| 7185 |
+
| > loss_mel_ce: 3.3742847442626953 (3.5316004449563794)
|
| 7186 |
+
| > loss: 0.05317714065313339 (0.05563885000694375)
|
| 7187 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7188 |
+
| > step_time: 0.2627 (0.27926888837213115)
|
| 7189 |
+
| > loader_time: 0.0082 (0.00883913496958164)
|
| 7190 |
+
|
| 7191 |
+
|
| 7192 |
+
[1m --> TIME: 2025-12-26 00:27:34 -- STEP: 1643/2207 -- GLOBAL_STEP: 49300[0m
|
| 7193 |
+
| > loss_text_ce: 0.030780388042330742 (0.029290274508882424)
|
| 7194 |
+
| > loss_mel_ce: 3.5452957153320312 (3.5311114770642007)
|
| 7195 |
+
| > loss: 0.05587618798017502 (0.05563127735150131)
|
| 7196 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7197 |
+
| > step_time: 0.2875 (0.2792822023172314)
|
| 7198 |
+
| > loader_time: 0.0077 (0.00883303919146139)
|
| 7199 |
+
|
| 7200 |
+
|
| 7201 |
+
[1m --> TIME: 2025-12-26 00:27:44 -- STEP: 1668/2207 -- GLOBAL_STEP: 49325[0m
|
| 7202 |
+
| > loss_text_ce: 0.02695401757955551 (0.02928536019245569)
|
| 7203 |
+
| > loss_mel_ce: 3.4183266162872314 (3.530636592162884)
|
| 7204 |
+
| > loss: 0.053832508623600006 (0.055623780487091865)
|
| 7205 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7206 |
+
| > step_time: 0.2875 (0.27926705738337493)
|
| 7207 |
+
| > loader_time: 0.0084 (0.008832700675625887)
|
| 7208 |
+
|
| 7209 |
+
|
| 7210 |
+
[1m --> TIME: 2025-12-26 00:27:53 -- STEP: 1693/2207 -- GLOBAL_STEP: 49350[0m
|
| 7211 |
+
| > loss_text_ce: 0.030460162088274956 (0.029284733569285817)
|
| 7212 |
+
| > loss_mel_ce: 3.2584149837493896 (3.5301527022188814)
|
| 7213 |
+
| > loss: 0.05138867348432541 (0.05561620991608835)
|
| 7214 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7215 |
+
| > step_time: 0.2909 (0.2792756644776779)
|
| 7216 |
+
| > loader_time: 0.009 (0.008831116129362025)
|
| 7217 |
+
|
| 7218 |
+
|
| 7219 |
+
[1m --> TIME: 2025-12-26 00:28:03 -- STEP: 1718/2207 -- GLOBAL_STEP: 49375[0m
|
| 7220 |
+
| > loss_text_ce: 0.029574880376458168 (0.029287565046351088)
|
| 7221 |
+
| > loss_mel_ce: 3.7472646236419678 (3.5295653375119347)
|
| 7222 |
+
| > loss: 0.05901311710476875 (0.05560707657384374)
|
| 7223 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7224 |
+
| > step_time: 0.2875 (0.27925428439352373)
|
| 7225 |
+
| > loader_time: 0.008 (0.008826621197710504)
|
| 7226 |
+
|
| 7227 |
+
|
| 7228 |
+
[1m --> TIME: 2025-12-26 00:28:12 -- STEP: 1743/2207 -- GLOBAL_STEP: 49400[0m
|
| 7229 |
+
| > loss_text_ce: 0.02976164221763611 (0.029280318447393518)
|
| 7230 |
+
| > loss_mel_ce: 3.493147134780884 (3.528186239870676)
|
| 7231 |
+
| > loss: 0.05504544824361801 (0.055585414938504725)
|
| 7232 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7233 |
+
| > step_time: 0.2898 (0.27930400840728664)
|
| 7234 |
+
| > loader_time: 0.0085 (0.008825995630733743)
|
| 7235 |
+
|
| 7236 |
+
|
| 7237 |
+
[1m --> TIME: 2025-12-26 00:28:22 -- STEP: 1768/2207 -- GLOBAL_STEP: 49425[0m
|
| 7238 |
+
| > loss_text_ce: 0.027403345331549644 (0.029284290933551695)
|
| 7239 |
+
| > loss_mel_ce: 3.831237554550171 (3.528357426235578)
|
| 7240 |
+
| > loss: 0.060291264206171036 (0.055588151791819904)
|
| 7241 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7242 |
+
| > step_time: 0.2864 (0.2793336800590362)
|
| 7243 |
+
| > loader_time: 0.009 (0.00882073562609125)
|
| 7244 |
+
|
| 7245 |
+
|
| 7246 |
+
[1m --> TIME: 2025-12-26 00:28:32 -- STEP: 1793/2207 -- GLOBAL_STEP: 49450[0m
|
| 7247 |
+
| > loss_text_ce: 0.027127325534820557 (0.029281279874152133)
|
| 7248 |
+
| > loss_mel_ce: 3.4323582649230957 (3.5287216447680203)
|
| 7249 |
+
| > loss: 0.05405446141958237 (0.05559379565885331)
|
| 7250 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7251 |
+
| > step_time: 0.269 (0.2792892055809334)
|
| 7252 |
+
| > loader_time: 0.0082 (0.008815802480013627)
|
| 7253 |
+
|
| 7254 |
+
|
| 7255 |
+
[1m --> TIME: 2025-12-26 00:28:41 -- STEP: 1818/2207 -- GLOBAL_STEP: 49475[0m
|
| 7256 |
+
| > loss_text_ce: 0.028301801532506943 (0.029284429853234945)
|
| 7257 |
+
| > loss_mel_ce: 3.5522849559783936 (3.52881371856916)
|
| 7258 |
+
| > loss: 0.05594666674733162 (0.05559528353075909)
|
| 7259 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7260 |
+
| > step_time: 0.2905 (0.2792574144957223)
|
| 7261 |
+
| > loader_time: 0.0083 (0.008811705970134675)
|
| 7262 |
+
|
| 7263 |
+
|
| 7264 |
+
[1m --> TIME: 2025-12-26 00:28:51 -- STEP: 1843/2207 -- GLOBAL_STEP: 49500[0m
|
| 7265 |
+
| > loss_text_ce: 0.028978843241930008 (0.029284514132048276)
|
| 7266 |
+
| > loss_mel_ce: 3.288996458053589 (3.528862168849711)
|
| 7267 |
+
| > loss: 0.05184336379170418 (0.055596041882726994)
|
| 7268 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7269 |
+
| > step_time: 0.2657 (0.2792416361448111)
|
| 7270 |
+
| > loader_time: 0.0086 (0.008809764295961961)
|
| 7271 |
+
|
| 7272 |
+
|
| 7273 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_49500.pth
|
| 7274 |
+
|
| 7275 |
+
[1m --> TIME: 2025-12-26 00:29:25 -- STEP: 1868/2207 -- GLOBAL_STEP: 49525[0m
|
| 7276 |
+
| > loss_text_ce: 0.029728876426815987 (0.029288909654282592)
|
| 7277 |
+
| > loss_mel_ce: 3.811872959136963 (3.528998979664461)
|
| 7278 |
+
| > loss: 0.06002502888441086 (0.05559824823104724)
|
| 7279 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7280 |
+
| > step_time: 0.2907 (0.27921627790565434)
|
| 7281 |
+
| > loader_time: 0.0088 (0.008822701089642547)
|
| 7282 |
+
|
| 7283 |
+
|
| 7284 |
+
[1m --> TIME: 2025-12-26 00:29:35 -- STEP: 1893/2207 -- GLOBAL_STEP: 49550[0m
|
| 7285 |
+
| > loss_text_ce: 0.027736319229006767 (0.029291810145098823)
|
| 7286 |
+
| > loss_mel_ce: 3.253326416015625 (3.5288457234448622)
|
| 7287 |
+
| > loss: 0.05126660689711571 (0.05559589892696656)
|
| 7288 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7289 |
+
| > step_time: 0.2673 (0.2791516822415942)
|
| 7290 |
+
| > loader_time: 0.0088 (0.00882631247470572)
|
| 7291 |
+
|
| 7292 |
+
|
| 7293 |
+
[1m --> TIME: 2025-12-26 00:29:45 -- STEP: 1918/2207 -- GLOBAL_STEP: 49575[0m
|
| 7294 |
+
| > loss_text_ce: 0.02999100089073181 (0.029293836300320557)
|
| 7295 |
+
| > loss_mel_ce: 3.299464702606201 (3.528215643138905)
|
| 7296 |
+
| > loss: 0.052022743970155716 (0.05558608558195974)
|
| 7297 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7298 |
+
| > step_time: 0.2864 (0.27914959061754907)
|
| 7299 |
+
| > loader_time: 0.009 (0.008834924389597525)
|
| 7300 |
+
|
| 7301 |
+
|
| 7302 |
+
[1m --> TIME: 2025-12-26 00:29:54 -- STEP: 1943/2207 -- GLOBAL_STEP: 49600[0m
|
| 7303 |
+
| > loss_text_ce: 0.03641633316874504 (0.029297151401968757)
|
| 7304 |
+
| > loss_mel_ce: 4.2173004150390625 (3.5288311095196008)
|
| 7305 |
+
| > loss: 0.06646432727575302 (0.055595754049008995)
|
| 7306 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7307 |
+
| > step_time: 0.4055 (0.2792835408611761)
|
| 7308 |
+
| > loader_time: 0.0093 (0.008832232828184437)
|
| 7309 |
+
|
| 7310 |
+
|
| 7311 |
+
[1m --> TIME: 2025-12-26 00:30:04 -- STEP: 1968/2207 -- GLOBAL_STEP: 49625[0m
|
| 7312 |
+
| > loss_text_ce: 0.030885135754942894 (0.02929815154038067)
|
| 7313 |
+
| > loss_mel_ce: 3.70080828666687 (3.529286628210447)
|
| 7314 |
+
| > loss: 0.0583077110350132 (0.05560288715698735)
|
| 7315 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7316 |
+
| > step_time: 0.2613 (0.27926650086069515)
|
| 7317 |
+
| > loader_time: 0.0089 (0.008833444336565533)
|
| 7318 |
+
|
| 7319 |
+
|
| 7320 |
+
[1m --> TIME: 2025-12-26 00:30:14 -- STEP: 1993/2207 -- GLOBAL_STEP: 49650[0m
|
| 7321 |
+
| > loss_text_ce: 0.028021905571222305 (0.02929666750338648)
|
| 7322 |
+
| > loss_mel_ce: 3.745973825454712 (3.529080218067733)
|
| 7323 |
+
| > loss: 0.0589686818420887 (0.05559963881371313)
|
| 7324 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7325 |
+
| > step_time: 0.2873 (0.2792207260438086)
|
| 7326 |
+
| > loader_time: 0.0088 (0.008830059370202634)
|
| 7327 |
+
|
| 7328 |
+
|
| 7329 |
+
[1m --> TIME: 2025-12-26 00:30:23 -- STEP: 2018/2207 -- GLOBAL_STEP: 49675[0m
|
| 7330 |
+
| > loss_text_ce: 0.02862389013171196 (0.029291055835340436)
|
| 7331 |
+
| > loss_mel_ce: 3.3028266429901123 (3.528107822709324)
|
| 7332 |
+
| > loss: 0.05205391347408295 (0.05558435745145924)
|
| 7333 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7334 |
+
| > step_time: 0.2906 (0.2792263589836327)
|
| 7335 |
+
| > loader_time: 0.0078 (0.008828179805555243)
|
| 7336 |
+
|
| 7337 |
+
|
| 7338 |
+
[1m --> TIME: 2025-12-26 00:30:33 -- STEP: 2043/2207 -- GLOBAL_STEP: 49700[0m
|
| 7339 |
+
| > loss_text_ce: 0.02743091993033886 (0.02928782776326335)
|
| 7340 |
+
| > loss_mel_ce: 3.333369731903076 (3.527617666456434)
|
| 7341 |
+
| > loss: 0.0525125116109848 (0.05557664831698205)
|
| 7342 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7343 |
+
| > step_time: 0.2686 (0.27920264539564604)
|
| 7344 |
+
| > loader_time: 0.0085 (0.008824594803902084)
|
| 7345 |
+
|
| 7346 |
+
|
| 7347 |
+
[1m --> TIME: 2025-12-26 00:30:42 -- STEP: 2068/2207 -- GLOBAL_STEP: 49725[0m
|
| 7348 |
+
| > loss_text_ce: 0.026749692857265472 (0.029290284030139446)
|
| 7349 |
+
| > loss_mel_ce: 3.5555551052093506 (3.5282165382770776)
|
| 7350 |
+
| > loss: 0.055973511189222336 (0.055586044069530896)
|
| 7351 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7352 |
+
| > step_time: 0.2681 (0.27914865485012436)
|
| 7353 |
+
| > loader_time: 0.0108 (0.008822636401399657)
|
| 7354 |
+
|
| 7355 |
+
|
| 7356 |
+
[1m --> TIME: 2025-12-26 00:30:52 -- STEP: 2093/2207 -- GLOBAL_STEP: 49750[0m
|
| 7357 |
+
| > loss_text_ce: 0.02878699079155922 (0.029290017926851018)
|
| 7358 |
+
| > loss_mel_ce: 3.9079430103302 (3.52845656433233)
|
| 7359 |
+
| > loss: 0.06151140481233597 (0.055589790315831396)
|
| 7360 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7361 |
+
| > step_time: 0.2903 (0.27913072812061723)
|
| 7362 |
+
| > loader_time: 0.0081 (0.00881970367304059)
|
| 7363 |
+
|
| 7364 |
+
|
| 7365 |
+
[1m --> TIME: 2025-12-26 00:31:01 -- STEP: 2118/2207 -- GLOBAL_STEP: 49775[0m
|
| 7366 |
+
| > loss_text_ce: 0.030014643445611 (0.029291489748134714)
|
| 7367 |
+
| > loss_mel_ce: 3.3642873764038086 (3.5284556447615363)
|
| 7368 |
+
| > loss: 0.05303597077727318 (0.055589798943432474)
|
| 7369 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7370 |
+
| > step_time: 0.266 (0.279127352411056)
|
| 7371 |
+
| > loader_time: 0.009 (0.008820228468595079)
|
| 7372 |
+
|
| 7373 |
+
|
| 7374 |
+
[1m --> TIME: 2025-12-26 00:31:11 -- STEP: 2143/2207 -- GLOBAL_STEP: 49800[0m
|
| 7375 |
+
| > loss_text_ce: 0.029396342113614082 (0.029298320038020742)
|
| 7376 |
+
| > loss_mel_ce: 3.693934440612793 (3.5277143841973535)
|
| 7377 |
+
| > loss: 0.05817704275250435 (0.055578323469657624)
|
| 7378 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7379 |
+
| > step_time: 0.2692 (0.2791458252040351)
|
| 7380 |
+
| > loader_time: 0.0091 (0.00881999806160689)
|
| 7381 |
+
|
| 7382 |
+
|
| 7383 |
+
[1m --> TIME: 2025-12-26 00:31:20 -- STEP: 2168/2207 -- GLOBAL_STEP: 49825[0m
|
| 7384 |
+
| > loss_text_ce: 0.030312787741422653 (0.02929681303182348)
|
| 7385 |
+
| > loss_mel_ce: 3.676762342453003 (3.5274594746172636)
|
| 7386 |
+
| > loss: 0.05792304873466492 (0.05557431695927826)
|
| 7387 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7388 |
+
| > step_time: 0.2895 (0.27911710563181097)
|
| 7389 |
+
| > loader_time: 0.0094 (0.008821861660348542)
|
| 7390 |
+
|
| 7391 |
+
|
| 7392 |
+
[1m --> TIME: 2025-12-26 00:31:30 -- STEP: 2193/2207 -- GLOBAL_STEP: 49850[0m
|
| 7393 |
+
| > loss_text_ce: 0.0289970263838768 (0.029293665595308483)
|
| 7394 |
+
| > loss_mel_ce: 3.426485538482666 (3.5278314381932963)
|
| 7395 |
+
| > loss: 0.05399191379547119 (0.05558007971009486)
|
| 7396 |
+
| > current_lr: 1.9999874300983004e-05
|
| 7397 |
+
| > step_time: 0.2663 (0.27909805923632414)
|
| 7398 |
+
| > loader_time: 0.0081 (0.008820031366552666)
|
| 7399 |
+
|
| 7400 |
+
|
| 7401 |
+
[1m > EVALUATION [0m
|
| 7402 |
+
|
| 7403 |
+
|
| 7404 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 7405 |
+
| > avg_loader_time:[92m 0.10445866713652738 [0m(-0.029388253753249688)
|
| 7406 |
+
| > avg_loss_text_ce:[92m 0.028338571091660776 [0m(-8.815265185124072e-05)
|
| 7407 |
+
| > avg_loss_mel_ce:[92m 3.69927747990634 [0m(-0.002462499850504951)
|
| 7408 |
+
| > avg_loss:[92m 3.727616055591686 [0m(-0.002550656731064471)
|
| 7409 |
+
|
| 7410 |
+
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_49864.pth
|