Checkpoint at step 41036
Browse files- best_model.pth +1 -1
- best_model_41036.pth +3 -0
- checkpoint_40500.pth +3 -0
- checkpoint_41000.pth +3 -0
- events.out.tfevents.1766698157.0f9434978106.1562.0 +2 -2
- trainer_0_log.txt +819 -0
best_model.pth
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 5607926805
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b7966a308e71fe9bfde918ec57fdf479a28954542c0ab19fe965a85c87a3c094
|
| 3 |
size 5607926805
|
best_model_41036.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b7966a308e71fe9bfde918ec57fdf479a28954542c0ab19fe965a85c87a3c094
|
| 3 |
+
size 5607926805
|
checkpoint_40500.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:603f308ff9de64778722edb25aab31bc6ad869f2427d8f948d7a381113de8ba4
|
| 3 |
+
size 5607926741
|
checkpoint_41000.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ceaf88fb4daa3d7f52ef5d312d4818c1262a9fcee61e37d1c089f590ee42669b
|
| 3 |
+
size 5607926741
|
events.out.tfevents.1766698157.0f9434978106.1562.0
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f6600e443a6819e2e78347df4e1ccaa4073ee7c70f3a49578dc1cff689b51ae6
|
| 3 |
+
size 52762
|
trainer_0_log.txt
CHANGED
|
@@ -3310,3 +3310,822 @@
|
|
| 3310 |
| > avg_loss:[92m 3.7595141610583744 [0m(-0.005462854295163577)
|
| 3311 |
|
| 3312 |
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_38829.pth
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3310 |
| > avg_loss:[92m 3.7595141610583744 [0m(-0.005462854295163577)
|
| 3311 |
|
| 3312 |
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_38829.pth
|
| 3313 |
+
|
| 3314 |
+
[4m[1m > EPOCH: 5/12[0m
|
| 3315 |
+
--> /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000
|
| 3316 |
+
|
| 3317 |
+
[1m > TRAINING (2025-12-25 22:54:18) [0m
|
| 3318 |
+
|
| 3319 |
+
[1m --> TIME: 2025-12-25 22:54:30 -- STEP: 21/2207 -- GLOBAL_STEP: 38850[0m
|
| 3320 |
+
| > loss_text_ce: 0.031177401542663574 (0.029689010676173938)
|
| 3321 |
+
| > loss_mel_ce: 3.6057448387145996 (3.588645174389794)
|
| 3322 |
+
| > loss: 0.056826911866664886 (0.056536471914677394)
|
| 3323 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3324 |
+
| > step_time: 0.2691 (0.28241264252435594)
|
| 3325 |
+
| > loader_time: 0.0088 (0.008961870556785947)
|
| 3326 |
+
|
| 3327 |
+
|
| 3328 |
+
[1m --> TIME: 2025-12-25 22:54:39 -- STEP: 46/2207 -- GLOBAL_STEP: 38875[0m
|
| 3329 |
+
| > loss_text_ce: 0.0286884605884552 (0.02955744431718536)
|
| 3330 |
+
| > loss_mel_ce: 3.3129050731658936 (3.5703886643699976)
|
| 3331 |
+
| > loss: 0.05221239849925041 (0.05624915813298329)
|
| 3332 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3333 |
+
| > step_time: 0.2685 (0.27989031957543414)
|
| 3334 |
+
| > loader_time: 0.0087 (0.008736454922219982)
|
| 3335 |
+
|
| 3336 |
+
|
| 3337 |
+
[1m --> TIME: 2025-12-25 22:54:49 -- STEP: 71/2207 -- GLOBAL_STEP: 38900[0m
|
| 3338 |
+
| > loss_text_ce: 0.027880392968654633 (0.029597033156265676)
|
| 3339 |
+
| > loss_mel_ce: 3.79910945892334 (3.5837998692418487)
|
| 3340 |
+
| > loss: 0.05979671701788902 (0.05645932671679577)
|
| 3341 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3342 |
+
| > step_time: 0.2663 (0.2778344355838401)
|
| 3343 |
+
| > loader_time: 0.0083 (0.008639849407572142)
|
| 3344 |
+
|
| 3345 |
+
|
| 3346 |
+
[1m --> TIME: 2025-12-25 22:54:58 -- STEP: 96/2207 -- GLOBAL_STEP: 38925[0m
|
| 3347 |
+
| > loss_text_ce: 0.03157971054315567 (0.029636889308070142)
|
| 3348 |
+
| > loss_mel_ce: 3.9575111865997314 (3.5846227382620177)
|
| 3349 |
+
| > loss: 0.06232954561710358 (0.05647280680326124)
|
| 3350 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3351 |
+
| > step_time: 0.2881 (0.2783839354912441)
|
| 3352 |
+
| > loader_time: 0.0087 (0.008627295494079588)
|
| 3353 |
+
|
| 3354 |
+
|
| 3355 |
+
[1m --> TIME: 2025-12-25 22:55:08 -- STEP: 121/2207 -- GLOBAL_STEP: 38950[0m
|
| 3356 |
+
| > loss_text_ce: 0.03042028285562992 (0.0296441166515439)
|
| 3357 |
+
| > loss_mel_ce: 3.6249642372131348 (3.585047453888192)
|
| 3358 |
+
| > loss: 0.05711538344621658 (0.056479555854866324)
|
| 3359 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3360 |
+
| > step_time: 0.2666 (0.2787145189017306)
|
| 3361 |
+
| > loader_time: 0.0115 (0.00864527245198399)
|
| 3362 |
+
|
| 3363 |
+
|
| 3364 |
+
[1m --> TIME: 2025-12-25 22:55:18 -- STEP: 146/2207 -- GLOBAL_STEP: 38975[0m
|
| 3365 |
+
| > loss_text_ce: 0.028556648641824722 (0.029586244846553835)
|
| 3366 |
+
| > loss_mel_ce: 3.518200397491455 (3.5848072946888125)
|
| 3367 |
+
| > loss: 0.05541807785630226 (0.056474899107666864)
|
| 3368 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3369 |
+
| > step_time: 0.2919 (0.27909766158012517)
|
| 3370 |
+
| > loader_time: 0.0091 (0.008660897816697205)
|
| 3371 |
+
|
| 3372 |
+
|
| 3373 |
+
[1m --> TIME: 2025-12-25 22:55:27 -- STEP: 171/2207 -- GLOBAL_STEP: 39000[0m
|
| 3374 |
+
| > loss_text_ce: 0.030042223632335663 (0.02964304372319701)
|
| 3375 |
+
| > loss_mel_ce: 4.06670618057251 (3.596818164078116)
|
| 3376 |
+
| > loss: 0.06401169300079346 (0.05666345636747037)
|
| 3377 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3378 |
+
| > step_time: 0.2868 (0.27861488911143545)
|
| 3379 |
+
| > loader_time: 0.0081 (0.008628097891110425)
|
| 3380 |
+
|
| 3381 |
+
|
| 3382 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_39000.pth
|
| 3383 |
+
|
| 3384 |
+
[1m --> TIME: 2025-12-25 22:56:01 -- STEP: 196/2207 -- GLOBAL_STEP: 39025[0m
|
| 3385 |
+
| > loss_text_ce: 0.03287003934383392 (0.029618238403024723)
|
| 3386 |
+
| > loss_mel_ce: 3.674941301345825 (3.596850484001393)
|
| 3387 |
+
| > loss: 0.05793455243110657 (0.05666357379558743)
|
| 3388 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3389 |
+
| > step_time: 0.2686 (0.2784751677999693)
|
| 3390 |
+
| > loader_time: 0.0089 (0.008653218648871596)
|
| 3391 |
+
|
| 3392 |
+
|
| 3393 |
+
[1m --> TIME: 2025-12-25 22:56:11 -- STEP: 221/2207 -- GLOBAL_STEP: 39050[0m
|
| 3394 |
+
| > loss_text_ce: 0.02911822497844696 (0.029611257996359563)
|
| 3395 |
+
| > loss_mel_ce: 3.636963367462158 (3.5929136071269867)
|
| 3396 |
+
| > loss: 0.057282526046037674 (0.05660195104204691)
|
| 3397 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3398 |
+
| > step_time: 0.266 (0.2779540767497066)
|
| 3399 |
+
| > loader_time: 0.0082 (0.008660237713636856)
|
| 3400 |
+
|
| 3401 |
+
|
| 3402 |
+
[1m --> TIME: 2025-12-25 22:56:20 -- STEP: 246/2207 -- GLOBAL_STEP: 39075[0m
|
| 3403 |
+
| > loss_text_ce: 0.031447988003492355 (0.029672772532737837)
|
| 3404 |
+
| > loss_mel_ce: 4.269295692443848 (3.593854075524865)
|
| 3405 |
+
| > loss: 0.06719911843538284 (0.056617607021840606)
|
| 3406 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3407 |
+
| > step_time: 0.2654 (0.27758471461815565)
|
| 3408 |
+
| > loader_time: 0.0084 (0.008639785332408377)
|
| 3409 |
+
|
| 3410 |
+
|
| 3411 |
+
[1m --> TIME: 2025-12-25 22:56:30 -- STEP: 271/2207 -- GLOBAL_STEP: 39100[0m
|
| 3412 |
+
| > loss_text_ce: 0.02916189655661583 (0.0296619271660203)
|
| 3413 |
+
| > loss_mel_ce: 3.429058313369751 (3.600270057516344)
|
| 3414 |
+
| > loss: 0.05403469130396843 (0.05671768727612671)
|
| 3415 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3416 |
+
| > step_time: 0.2871 (0.2776994520447791)
|
| 3417 |
+
| > loader_time: 0.0086 (0.008647449781973864)
|
| 3418 |
+
|
| 3419 |
+
|
| 3420 |
+
[1m --> TIME: 2025-12-25 22:56:39 -- STEP: 296/2207 -- GLOBAL_STEP: 39125[0m
|
| 3421 |
+
| > loss_text_ce: 0.029262222349643707 (0.029640842984254297)
|
| 3422 |
+
| > loss_mel_ce: 3.815509080886841 (3.6035642704448185)
|
| 3423 |
+
| > loss: 0.06007455289363861 (0.05676882992171354)
|
| 3424 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3425 |
+
| > step_time: 0.2907 (0.27762382900392685)
|
| 3426 |
+
| > loader_time: 0.0084 (0.008627687757079667)
|
| 3427 |
+
|
| 3428 |
+
|
| 3429 |
+
[1m --> TIME: 2025-12-25 22:56:49 -- STEP: 321/2207 -- GLOBAL_STEP: 39150[0m
|
| 3430 |
+
| > loss_text_ce: 0.02877514809370041 (0.029627905183724153)
|
| 3431 |
+
| > loss_mel_ce: 3.4497289657592773 (3.6070506802971862)
|
| 3432 |
+
| > loss: 0.054351627826690674 (0.05682310293721632)
|
| 3433 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3434 |
+
| > step_time: 0.2863 (0.277438243230184)
|
| 3435 |
+
| > loader_time: 0.0086 (0.008610531548473322)
|
| 3436 |
+
|
| 3437 |
+
|
| 3438 |
+
[1m --> TIME: 2025-12-25 22:56:59 -- STEP: 346/2207 -- GLOBAL_STEP: 39175[0m
|
| 3439 |
+
| > loss_text_ce: 0.02888738177716732 (0.029668747565865173)
|
| 3440 |
+
| > loss_mel_ce: 3.452221393585205 (3.6055006898207473)
|
| 3441 |
+
| > loss: 0.05439232289791107 (0.05679952246648382)
|
| 3442 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3443 |
+
| > step_time: 0.2859 (0.27808844629739765)
|
| 3444 |
+
| > loader_time: 0.0083 (0.008607198737260234)
|
| 3445 |
+
|
| 3446 |
+
|
| 3447 |
+
[1m --> TIME: 2025-12-25 22:57:09 -- STEP: 371/2207 -- GLOBAL_STEP: 39200[0m
|
| 3448 |
+
| > loss_text_ce: 0.029622366651892662 (0.029695368494908763)
|
| 3449 |
+
| > loss_mel_ce: 3.641005277633667 (3.608730543655848)
|
| 3450 |
+
| > loss: 0.05735355615615845 (0.05685040487272719)
|
| 3451 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3452 |
+
| > step_time: 0.2868 (0.2785003827909899)
|
| 3453 |
+
| > loader_time: 0.0081 (0.008591977091169748)
|
| 3454 |
+
|
| 3455 |
+
|
| 3456 |
+
[1m --> TIME: 2025-12-25 22:57:18 -- STEP: 396/2207 -- GLOBAL_STEP: 39225[0m
|
| 3457 |
+
| > loss_text_ce: 0.030680537223815918 (0.02972576768617287)
|
| 3458 |
+
| > loss_mel_ce: 3.5531909465789795 (3.6070802554939734)
|
| 3459 |
+
| > loss: 0.05599799007177353 (0.05682509407551601)
|
| 3460 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3461 |
+
| > step_time: 0.2871 (0.27845537903332956)
|
| 3462 |
+
| > loader_time: 0.0093 (0.008621183910755203)
|
| 3463 |
+
|
| 3464 |
+
|
| 3465 |
+
[1m --> TIME: 2025-12-25 22:57:28 -- STEP: 421/2207 -- GLOBAL_STEP: 39250[0m
|
| 3466 |
+
| > loss_text_ce: 0.030776092782616615 (0.02973807842477342)
|
| 3467 |
+
| > loss_mel_ce: 3.840707302093506 (3.6107438641319365)
|
| 3468 |
+
| > loss: 0.06049192696809769 (0.056882530326939525)
|
| 3469 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3470 |
+
| > step_time: 0.2547 (0.278183373023099)
|
| 3471 |
+
| > loader_time: 0.0079 (0.008620404291039701)
|
| 3472 |
+
|
| 3473 |
+
|
| 3474 |
+
[1m --> TIME: 2025-12-25 22:57:37 -- STEP: 446/2207 -- GLOBAL_STEP: 39275[0m
|
| 3475 |
+
| > loss_text_ce: 0.0301694106310606 (0.029761504331779053)
|
| 3476 |
+
| > loss_mel_ce: 3.500668525695801 (3.6123338857573777)
|
| 3477 |
+
| > loss: 0.05516934394836426 (0.05690774047588553)
|
| 3478 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3479 |
+
| > step_time: 0.2669 (0.27833213987906424)
|
| 3480 |
+
| > loader_time: 0.0083 (0.008610557547599217)
|
| 3481 |
+
|
| 3482 |
+
|
| 3483 |
+
[1m --> TIME: 2025-12-25 22:57:47 -- STEP: 471/2207 -- GLOBAL_STEP: 39300[0m
|
| 3484 |
+
| > loss_text_ce: 0.026033615693449974 (0.029765229846198595)
|
| 3485 |
+
| > loss_mel_ce: 3.383991003036499 (3.6140813483300986)
|
| 3486 |
+
| > loss: 0.05328163504600525 (0.05693510279945253)
|
| 3487 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3488 |
+
| > step_time: 0.2868 (0.2784255613946613)
|
| 3489 |
+
| > loader_time: 0.0083 (0.008603990963816397)
|
| 3490 |
+
|
| 3491 |
+
|
| 3492 |
+
[1m --> TIME: 2025-12-25 22:57:56 -- STEP: 496/2207 -- GLOBAL_STEP: 39325[0m
|
| 3493 |
+
| > loss_text_ce: 0.027476051822304726 (0.029741447332555487)
|
| 3494 |
+
| > loss_mel_ce: 3.541102170944214 (3.6134187833916758)
|
| 3495 |
+
| > loss: 0.05575903505086899 (0.05692437862707001)
|
| 3496 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3497 |
+
| > step_time: 0.2905 (0.2784613966941835)
|
| 3498 |
+
| > loader_time: 0.0082 (0.008596906258213912)
|
| 3499 |
+
|
| 3500 |
+
|
| 3501 |
+
[1m --> TIME: 2025-12-25 22:58:06 -- STEP: 521/2207 -- GLOBAL_STEP: 39350[0m
|
| 3502 |
+
| > loss_text_ce: 0.031287211924791336 (0.02973850787887189)
|
| 3503 |
+
| > loss_mel_ce: 3.854926586151123 (3.61423890650158)
|
| 3504 |
+
| > loss: 0.06072209030389786 (0.056937147117793664)
|
| 3505 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3506 |
+
| > step_time: 0.2908 (0.2785604297542757)
|
| 3507 |
+
| > loader_time: 0.0084 (0.008592438560529778)
|
| 3508 |
+
|
| 3509 |
+
|
| 3510 |
+
[1m --> TIME: 2025-12-25 22:58:16 -- STEP: 546/2207 -- GLOBAL_STEP: 39375[0m
|
| 3511 |
+
| > loss_text_ce: 0.026341162621974945 (0.02974348923962413)
|
| 3512 |
+
| > loss_mel_ce: 3.6721277236938477 (3.6165085406530473)
|
| 3513 |
+
| > loss: 0.05778857693076134 (0.05697268796163598)
|
| 3514 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3515 |
+
| > step_time: 0.2964 (0.2787704288741173)
|
| 3516 |
+
| > loader_time: 0.0084 (0.0085865678368034)
|
| 3517 |
+
|
| 3518 |
+
|
| 3519 |
+
[1m --> TIME: 2025-12-25 22:58:25 -- STEP: 571/2207 -- GLOBAL_STEP: 39400[0m
|
| 3520 |
+
| > loss_text_ce: 0.034135058522224426 (0.029756738372055745)
|
| 3521 |
+
| > loss_mel_ce: 3.6609606742858887 (3.6157480868691954)
|
| 3522 |
+
| > loss: 0.0577358715236187 (0.05696101288348695)
|
| 3523 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3524 |
+
| > step_time: 0.266 (0.27871264467723744)
|
| 3525 |
+
| > loader_time: 0.0086 (0.008592329593130676)
|
| 3526 |
+
|
| 3527 |
+
|
| 3528 |
+
[1m --> TIME: 2025-12-25 22:58:35 -- STEP: 596/2207 -- GLOBAL_STEP: 39425[0m
|
| 3529 |
+
| > loss_text_ce: 0.030424287542700768 (0.029761506844586973)
|
| 3530 |
+
| > loss_mel_ce: 4.100286960601807 (3.6176264838084284)
|
| 3531 |
+
| > loss: 0.06454236060380936 (0.0569904373283654)
|
| 3532 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3533 |
+
| > step_time: 0.2659 (0.2784905201636706)
|
| 3534 |
+
| > loader_time: 0.0095 (0.008591657516940333)
|
| 3535 |
+
|
| 3536 |
+
|
| 3537 |
+
[1m --> TIME: 2025-12-25 22:58:44 -- STEP: 621/2207 -- GLOBAL_STEP: 39450[0m
|
| 3538 |
+
| > loss_text_ce: 0.02930794656276703 (0.02975467073576654)
|
| 3539 |
+
| > loss_mel_ce: 3.5583994388580322 (3.6180391826107505)
|
| 3540 |
+
| > loss: 0.05605792626738548 (0.05699677894344842)
|
| 3541 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3542 |
+
| > step_time: 0.2906 (0.2784443750089685)
|
| 3543 |
+
| > loader_time: 0.0087 (0.008593411837223078)
|
| 3544 |
+
|
| 3545 |
+
|
| 3546 |
+
[1m --> TIME: 2025-12-25 22:58:54 -- STEP: 646/2207 -- GLOBAL_STEP: 39475[0m
|
| 3547 |
+
| > loss_text_ce: 0.03069169633090496 (0.029768290600794202)
|
| 3548 |
+
| > loss_mel_ce: 3.521172046661377 (3.6189163698131463)
|
| 3549 |
+
| > loss: 0.05549786984920502 (0.057010697806398176)
|
| 3550 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3551 |
+
| > step_time: 0.2679 (0.2783926928375529)
|
| 3552 |
+
| > loader_time: 0.0083 (0.008589169189287791)
|
| 3553 |
+
|
| 3554 |
+
|
| 3555 |
+
[1m --> TIME: 2025-12-25 22:59:04 -- STEP: 671/2207 -- GLOBAL_STEP: 39500[0m
|
| 3556 |
+
| > loss_text_ce: 0.030341438949108124 (0.02977386959474506)
|
| 3557 |
+
| > loss_mel_ce: 3.249605417251587 (3.6181997158488297)
|
| 3558 |
+
| > loss: 0.05124916881322861 (0.05699958726097502)
|
| 3559 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3560 |
+
| > step_time: 0.287 (0.2783844772998339)
|
| 3561 |
+
| > loader_time: 0.0086 (0.00859433314839348)
|
| 3562 |
+
|
| 3563 |
+
|
| 3564 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_39500.pth
|
| 3565 |
+
|
| 3566 |
+
[1m --> TIME: 2025-12-25 22:59:38 -- STEP: 696/2207 -- GLOBAL_STEP: 39525[0m
|
| 3567 |
+
| > loss_text_ce: 0.02982802875339985 (0.029777978474659652)
|
| 3568 |
+
| > loss_mel_ce: 3.459867000579834 (3.6162394134477642)
|
| 3569 |
+
| > loss: 0.05452648550271988 (0.05696902174138644)
|
| 3570 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3571 |
+
| > step_time: 0.266 (0.2783328764054967)
|
| 3572 |
+
| > loader_time: 0.0083 (0.008594774651801456)
|
| 3573 |
+
|
| 3574 |
+
|
| 3575 |
+
[1m --> TIME: 2025-12-25 22:59:47 -- STEP: 721/2207 -- GLOBAL_STEP: 39550[0m
|
| 3576 |
+
| > loss_text_ce: 0.02891617640852928 (0.029764441021520455)
|
| 3577 |
+
| > loss_mel_ce: 3.89870023727417 (3.6153623475777135)
|
| 3578 |
+
| > loss: 0.06136900559067726 (0.05695510605523224)
|
| 3579 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3580 |
+
| > step_time: 0.2681 (0.278415539724321)
|
| 3581 |
+
| > loader_time: 0.0082 (0.008589160591157241)
|
| 3582 |
+
|
| 3583 |
+
|
| 3584 |
+
[1m --> TIME: 2025-12-25 22:59:57 -- STEP: 746/2207 -- GLOBAL_STEP: 39575[0m
|
| 3585 |
+
| > loss_text_ce: 0.030313782393932343 (0.02977387412094159)
|
| 3586 |
+
| > loss_mel_ce: 3.4513895511627197 (3.6148199955196234)
|
| 3587 |
+
| > loss: 0.054401613771915436 (0.056946779204277806)
|
| 3588 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3589 |
+
| > step_time: 0.2657 (0.2783530273002853)
|
| 3590 |
+
| > loader_time: 0.0081 (0.008585050022953646)
|
| 3591 |
+
|
| 3592 |
+
|
| 3593 |
+
[1m --> TIME: 2025-12-25 23:00:07 -- STEP: 771/2207 -- GLOBAL_STEP: 39600[0m
|
| 3594 |
+
| > loss_text_ce: 0.030108636245131493 (0.02979994634101131)
|
| 3595 |
+
| > loss_mel_ce: 3.5901265144348145 (3.614422431097266)
|
| 3596 |
+
| > loss: 0.05656617507338524 (0.05694097464109362)
|
| 3597 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3598 |
+
| > step_time: 0.2685 (0.27854412353456276)
|
| 3599 |
+
| > loader_time: 0.0082 (0.008581166137517205)
|
| 3600 |
+
|
| 3601 |
+
|
| 3602 |
+
[1m --> TIME: 2025-12-25 23:00:16 -- STEP: 796/2207 -- GLOBAL_STEP: 39625[0m
|
| 3603 |
+
| > loss_text_ce: 0.028148362413048744 (0.029804366507366224)
|
| 3604 |
+
| > loss_mel_ce: 3.631502866744995 (3.614537371163393)
|
| 3605 |
+
| > loss: 0.05718205124139786 (0.056942839643464)
|
| 3606 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3607 |
+
| > step_time: 0.2903 (0.27852271220192865)
|
| 3608 |
+
| > loader_time: 0.0086 (0.008583893728016616)
|
| 3609 |
+
|
| 3610 |
+
|
| 3611 |
+
[1m --> TIME: 2025-12-25 23:00:26 -- STEP: 821/2207 -- GLOBAL_STEP: 39650[0m
|
| 3612 |
+
| > loss_text_ce: 0.030395232141017914 (0.02979260782431017)
|
| 3613 |
+
| > loss_mel_ce: 3.883284091949463 (3.6156775983329528)
|
| 3614 |
+
| > loss: 0.06115124002099037 (0.05696047196362107)
|
| 3615 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3616 |
+
| > step_time: 0.2663 (0.2784293313554959)
|
| 3617 |
+
| > loader_time: 0.0083 (0.008585716716845349)
|
| 3618 |
+
|
| 3619 |
+
|
| 3620 |
+
[1m --> TIME: 2025-12-25 23:00:35 -- STEP: 846/2207 -- GLOBAL_STEP: 39675[0m
|
| 3621 |
+
| > loss_text_ce: 0.031339604407548904 (0.02979312977592855)
|
| 3622 |
+
| > loss_mel_ce: 3.650805950164795 (3.6148191882081635)
|
| 3623 |
+
| > loss: 0.05753352493047714 (0.05694706745480954)
|
| 3624 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3625 |
+
| > step_time: 0.2622 (0.27842727648732823)
|
| 3626 |
+
| > loader_time: 0.0083 (0.00858143176311015)
|
| 3627 |
+
|
| 3628 |
+
|
| 3629 |
+
[1m --> TIME: 2025-12-25 23:00:45 -- STEP: 871/2207 -- GLOBAL_STEP: 39700[0m
|
| 3630 |
+
| > loss_text_ce: 0.03081549145281315 (0.02978994881030681)
|
| 3631 |
+
| > loss_mel_ce: 3.490368604660034 (3.614108407812196)
|
| 3632 |
+
| > loss: 0.055018503218889236 (0.05693591180782503)
|
| 3633 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3634 |
+
| > step_time: 0.2873 (0.2784750234382711)
|
| 3635 |
+
| > loader_time: 0.0081 (0.008579106883805606)
|
| 3636 |
+
|
| 3637 |
+
|
| 3638 |
+
[1m --> TIME: 2025-12-25 23:00:54 -- STEP: 896/2207 -- GLOBAL_STEP: 39725[0m
|
| 3639 |
+
| > loss_text_ce: 0.027041181921958923 (0.02977882997531976)
|
| 3640 |
+
| > loss_mel_ce: 3.747934103012085 (3.61121851711401)
|
| 3641 |
+
| > loss: 0.05898398905992508 (0.056890583530600565)
|
| 3642 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3643 |
+
| > step_time: 0.2588 (0.27841260736542095)
|
| 3644 |
+
| > loader_time: 0.0084 (0.008576560233320511)
|
| 3645 |
+
|
| 3646 |
+
|
| 3647 |
+
[1m --> TIME: 2025-12-25 23:01:04 -- STEP: 921/2207 -- GLOBAL_STEP: 39750[0m
|
| 3648 |
+
| > loss_text_ce: 0.029584204778075218 (0.02976418084118123)
|
| 3649 |
+
| > loss_mel_ce: 3.8310155868530273 (3.611129350693296)
|
| 3650 |
+
| > loss: 0.0603218711912632 (0.05688896142229453)
|
| 3651 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3652 |
+
| > step_time: 0.2697 (0.27894134759644074)
|
| 3653 |
+
| > loader_time: 0.0081 (0.008570200457246767)
|
| 3654 |
+
|
| 3655 |
+
|
| 3656 |
+
[1m --> TIME: 2025-12-25 23:01:14 -- STEP: 946/2207 -- GLOBAL_STEP: 39775[0m
|
| 3657 |
+
| > loss_text_ce: 0.02822069078683853 (0.029761013336652934)
|
| 3658 |
+
| > loss_mel_ce: 3.722810745239258 (3.611200787552316)
|
| 3659 |
+
| > loss: 0.058609865605831146 (0.05689002812129245)
|
| 3660 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3661 |
+
| > step_time: 0.2902 (0.2789813814405154)
|
| 3662 |
+
| > loader_time: 0.0084 (0.00856654134915697)
|
| 3663 |
+
|
| 3664 |
+
|
| 3665 |
+
[1m --> TIME: 2025-12-25 23:01:24 -- STEP: 971/2207 -- GLOBAL_STEP: 39800[0m
|
| 3666 |
+
| > loss_text_ce: 0.02889234758913517 (0.02975867978507225)
|
| 3667 |
+
| > loss_mel_ce: 3.5812318325042725 (3.6113274942097537)
|
| 3668 |
+
| > loss: 0.05640818923711777 (0.056891971456709885)
|
| 3669 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3670 |
+
| > step_time: 0.267 (0.2790188251636791)
|
| 3671 |
+
| > loader_time: 0.0084 (0.008567510500749774)
|
| 3672 |
+
|
| 3673 |
+
|
| 3674 |
+
[1m --> TIME: 2025-12-25 23:01:33 -- STEP: 996/2207 -- GLOBAL_STEP: 39825[0m
|
| 3675 |
+
| > loss_text_ce: 0.029749302193522453 (0.02974164036727215)
|
| 3676 |
+
| > loss_mel_ce: 3.618267059326172 (3.6113973291044736)
|
| 3677 |
+
| > loss: 0.057000257074832916 (0.056892796382158384)
|
| 3678 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3679 |
+
| > step_time: 0.2894 (0.27901260058085114)
|
| 3680 |
+
| > loader_time: 0.0089 (0.00856977413935834)
|
| 3681 |
+
|
| 3682 |
+
|
| 3683 |
+
[1m --> TIME: 2025-12-25 23:01:43 -- STEP: 1021/2207 -- GLOBAL_STEP: 39850[0m
|
| 3684 |
+
| > loss_text_ce: 0.03340991586446762 (0.02974465981045561)
|
| 3685 |
+
| > loss_mel_ce: 3.4717774391174316 (3.61212956146442)
|
| 3686 |
+
| > loss: 0.054768551141023636 (0.05690428469399396)
|
| 3687 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3688 |
+
| > step_time: 0.2864 (0.27886500134407355)
|
| 3689 |
+
| > loader_time: 0.0082 (0.008562140553518331)
|
| 3690 |
+
|
| 3691 |
+
|
| 3692 |
+
[1m --> TIME: 2025-12-25 23:01:52 -- STEP: 1046/2207 -- GLOBAL_STEP: 39875[0m
|
| 3693 |
+
| > loss_text_ce: 0.029041022062301636 (0.029746832057164118)
|
| 3694 |
+
| > loss_mel_ce: 3.907212734222412 (3.6114165358971917)
|
| 3695 |
+
| > loss: 0.06150396540760994 (0.056893177615410954)
|
| 3696 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3697 |
+
| > step_time: 0.2663 (0.2787794788754464)
|
| 3698 |
+
| > loader_time: 0.0079 (0.008556656354244083)
|
| 3699 |
+
|
| 3700 |
+
|
| 3701 |
+
[1m --> TIME: 2025-12-25 23:02:02 -- STEP: 1071/2207 -- GLOBAL_STEP: 39900[0m
|
| 3702 |
+
| > loss_text_ce: 0.029124662280082703 (0.029759783663001717)
|
| 3703 |
+
| > loss_mel_ce: 3.719606399536133 (3.6103497381860366)
|
| 3704 |
+
| > loss: 0.058573924005031586 (0.0568767112691084)
|
| 3705 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3706 |
+
| > step_time: 0.2899 (0.2789954477999389)
|
| 3707 |
+
| > loader_time: 0.0081 (0.008551198672624978)
|
| 3708 |
+
|
| 3709 |
+
|
| 3710 |
+
[1m --> TIME: 2025-12-25 23:02:12 -- STEP: 1096/2207 -- GLOBAL_STEP: 39925[0m
|
| 3711 |
+
| > loss_text_ce: 0.029542306438088417 (0.029760921610192986)
|
| 3712 |
+
| > loss_mel_ce: 3.8775267601013184 (3.611910603342266)
|
| 3713 |
+
| > loss: 0.06104795262217522 (0.0569011175658309)
|
| 3714 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3715 |
+
| > step_time: 0.2656 (0.2788995543535606)
|
| 3716 |
+
| > loader_time: 0.0082 (0.00854634564288341)
|
| 3717 |
+
|
| 3718 |
+
|
| 3719 |
+
[1m --> TIME: 2025-12-25 23:02:21 -- STEP: 1121/2207 -- GLOBAL_STEP: 39950[0m
|
| 3720 |
+
| > loss_text_ce: 0.03094012476503849 (0.02975936684288902)
|
| 3721 |
+
| > loss_mel_ce: 3.4471077919006348 (3.612158180027536)
|
| 3722 |
+
| > loss: 0.05434449762105942 (0.05690496165993274)
|
| 3723 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3724 |
+
| > step_time: 0.2908 (0.2789293962541589)
|
| 3725 |
+
| > loader_time: 0.0084 (0.00853881550941161)
|
| 3726 |
+
|
| 3727 |
+
|
| 3728 |
+
[1m --> TIME: 2025-12-25 23:02:31 -- STEP: 1146/2207 -- GLOBAL_STEP: 39975[0m
|
| 3729 |
+
| > loss_text_ce: 0.030415263026952744 (0.02975456950695207)
|
| 3730 |
+
| > loss_mel_ce: 3.6185507774353027 (3.6124176415265334)
|
| 3731 |
+
| > loss: 0.05701509490609169 (0.056908940782617705)
|
| 3732 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3733 |
+
| > step_time: 0.2663 (0.27887936058676865)
|
| 3734 |
+
| > loader_time: 0.0083 (0.008533920917211406)
|
| 3735 |
+
|
| 3736 |
+
|
| 3737 |
+
[1m --> TIME: 2025-12-25 23:02:40 -- STEP: 1171/2207 -- GLOBAL_STEP: 40000[0m
|
| 3738 |
+
| > loss_text_ce: 0.02989851124584675 (0.02975791666169995)
|
| 3739 |
+
| > loss_mel_ce: 3.5182437896728516 (3.611958290958486)
|
| 3740 |
+
| > loss: 0.055439721792936325 (0.05690181572515162)
|
| 3741 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3742 |
+
| > step_time: 0.2864 (0.2789448537346033)
|
| 3743 |
+
| > loader_time: 0.0085 (0.008532983639828677)
|
| 3744 |
+
|
| 3745 |
+
|
| 3746 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_40000.pth
|
| 3747 |
+
|
| 3748 |
+
[1m --> TIME: 2025-12-25 23:03:15 -- STEP: 1196/2207 -- GLOBAL_STEP: 40025[0m
|
| 3749 |
+
| > loss_text_ce: 0.03148094192147255 (0.029748822428460504)
|
| 3750 |
+
| > loss_mel_ce: 3.4673736095428467 (3.611786084828967)
|
| 3751 |
+
| > loss: 0.05466960370540619 (0.056898982902633684)
|
| 3752 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3753 |
+
| > step_time: 0.2658 (0.2789259513086296)
|
| 3754 |
+
| > loader_time: 0.0086 (0.008535032686979875)
|
| 3755 |
+
|
| 3756 |
+
|
| 3757 |
+
[1m --> TIME: 2025-12-25 23:03:24 -- STEP: 1221/2207 -- GLOBAL_STEP: 40050[0m
|
| 3758 |
+
| > loss_text_ce: 0.02971072308719158 (0.029745281688460162)
|
| 3759 |
+
| > loss_mel_ce: 3.9994726181030273 (3.6130270969662677)
|
| 3760 |
+
| > loss: 0.06295599043369293 (0.05691831839026044)
|
| 3761 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3762 |
+
| > step_time: 0.286 (0.2789905018536988)
|
| 3763 |
+
| > loader_time: 0.0082 (0.00852910443273946)
|
| 3764 |
+
|
| 3765 |
+
|
| 3766 |
+
[1m --> TIME: 2025-12-25 23:03:34 -- STEP: 1246/2207 -- GLOBAL_STEP: 40075[0m
|
| 3767 |
+
| > loss_text_ce: 0.0299287810921669 (0.029749659686850123)
|
| 3768 |
+
| > loss_mel_ce: 3.4872939586639404 (3.6126014296545454)
|
| 3769 |
+
| > loss: 0.05495660379528999 (0.05691173574909637)
|
| 3770 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3771 |
+
| > step_time: 0.297 (0.27896303541204864)
|
| 3772 |
+
| > loader_time: 0.0078 (0.008520916032561521)
|
| 3773 |
+
|
| 3774 |
+
|
| 3775 |
+
[1m --> TIME: 2025-12-25 23:03:44 -- STEP: 1271/2207 -- GLOBAL_STEP: 40100[0m
|
| 3776 |
+
| > loss_text_ce: 0.029985103756189346 (0.0297463247750356)
|
| 3777 |
+
| > loss_mel_ce: 3.954482316970825 (3.61342547391926)
|
| 3778 |
+
| > loss: 0.0622573047876358 (0.056924559337172685)
|
| 3779 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3780 |
+
| > step_time: 0.2682 (0.27898827319441555)
|
| 3781 |
+
| > loader_time: 0.0081 (0.008520963715343938)
|
| 3782 |
+
|
| 3783 |
+
|
| 3784 |
+
[1m --> TIME: 2025-12-25 23:03:53 -- STEP: 1296/2207 -- GLOBAL_STEP: 40125[0m
|
| 3785 |
+
| > loss_text_ce: 0.029612408950924873 (0.02974631753175255)
|
| 3786 |
+
| > loss_mel_ce: 3.622408866882324 (3.6139920856113785)
|
| 3787 |
+
| > loss: 0.05706283077597618 (0.05693341253532302)
|
| 3788 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3789 |
+
| > step_time: 0.2686 (0.278933043649167)
|
| 3790 |
+
| > loader_time: 0.0084 (0.008521564396811116)
|
| 3791 |
+
|
| 3792 |
+
|
| 3793 |
+
[1m --> TIME: 2025-12-25 23:04:03 -- STEP: 1321/2207 -- GLOBAL_STEP: 40150[0m
|
| 3794 |
+
| > loss_text_ce: 0.03182736784219742 (0.029744594642028135)
|
| 3795 |
+
| > loss_mel_ce: 3.698615789413452 (3.614078953805789)
|
| 3796 |
+
| > loss: 0.058288175612688065 (0.056934742929581686)
|
| 3797 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3798 |
+
| > step_time: 0.2659 (0.2789640589070806)
|
| 3799 |
+
| > loader_time: 0.0083 (0.008519134405974996)
|
| 3800 |
+
|
| 3801 |
+
|
| 3802 |
+
[1m --> TIME: 2025-12-25 23:04:12 -- STEP: 1346/2207 -- GLOBAL_STEP: 40175[0m
|
| 3803 |
+
| > loss_text_ce: 0.03140600398182869 (0.02974822322933493)
|
| 3804 |
+
| > loss_mel_ce: 3.4985225200653076 (3.614298246236294)
|
| 3805 |
+
| > loss: 0.055155131965875626 (0.05693822606213466)
|
| 3806 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3807 |
+
| > step_time: 0.2874 (0.27896740301934236)
|
| 3808 |
+
| > loader_time: 0.0076 (0.00851522051738949)
|
| 3809 |
+
|
| 3810 |
+
|
| 3811 |
+
[1m --> TIME: 2025-12-25 23:04:22 -- STEP: 1371/2207 -- GLOBAL_STEP: 40200[0m
|
| 3812 |
+
| > loss_text_ce: 0.03141113370656967 (0.029744146875412162)
|
| 3813 |
+
| > loss_mel_ce: 3.4364068508148193 (3.613910169517968)
|
| 3814 |
+
| > loss: 0.05418465659022331 (0.05693209867443504)
|
| 3815 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3816 |
+
| > step_time: 0.2662 (0.2790220215057995)
|
| 3817 |
+
| > loader_time: 0.0081 (0.008518126826491524)
|
| 3818 |
+
|
| 3819 |
+
|
| 3820 |
+
[1m --> TIME: 2025-12-25 23:04:31 -- STEP: 1396/2207 -- GLOBAL_STEP: 40225[0m
|
| 3821 |
+
| > loss_text_ce: 0.030799586325883865 (0.029738139511915965)
|
| 3822 |
+
| > loss_mel_ce: 3.546100378036499 (3.6135670677638667)
|
| 3823 |
+
| > loss: 0.0558890625834465 (0.056926643849486584)
|
| 3824 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3825 |
+
| > step_time: 0.2902 (0.27897558837360487)
|
| 3826 |
+
| > loader_time: 0.0086 (0.008515173350500865)
|
| 3827 |
+
|
| 3828 |
+
|
| 3829 |
+
[1m --> TIME: 2025-12-25 23:04:41 -- STEP: 1421/2207 -- GLOBAL_STEP: 40250[0m
|
| 3830 |
+
| > loss_text_ce: 0.028440283611416817 (0.029737424797507593)
|
| 3831 |
+
| > loss_mel_ce: 3.4774391651153564 (3.6128199160895997)
|
| 3832 |
+
| > loss: 0.05477936565876007 (0.0569149584369832)
|
| 3833 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3834 |
+
| > step_time: 0.2673 (0.27897474918459425)
|
| 3835 |
+
| > loader_time: 0.0082 (0.008511631024043223)
|
| 3836 |
+
|
| 3837 |
+
|
| 3838 |
+
[1m --> TIME: 2025-12-25 23:04:50 -- STEP: 1446/2207 -- GLOBAL_STEP: 40275[0m
|
| 3839 |
+
| > loss_text_ce: 0.02816746011376381 (0.02974252270490838)
|
| 3840 |
+
| > loss_mel_ce: 3.5898444652557373 (3.6136434417872345)
|
| 3841 |
+
| > loss: 0.05653143674135208 (0.05692790568462296)
|
| 3842 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3843 |
+
| > step_time: 0.2669 (0.278870635837275)
|
| 3844 |
+
| > loader_time: 0.0084 (0.008507927588572318)
|
| 3845 |
+
|
| 3846 |
+
|
| 3847 |
+
[1m --> TIME: 2025-12-25 23:05:00 -- STEP: 1471/2207 -- GLOBAL_STEP: 40300[0m
|
| 3848 |
+
| > loss_text_ce: 0.028746966272592545 (0.029744213502689496)
|
| 3849 |
+
| > loss_mel_ce: 3.635441541671753 (3.6134743189665834)
|
| 3850 |
+
| > loss: 0.057252947241067886 (0.0569252895607906)
|
| 3851 |
+
| > grad_norm: 0 (0.0)
|
| 3852 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3853 |
+
| > step_time: 0.2863 (0.2789106740180038)
|
| 3854 |
+
| > loader_time: 0.0084 (0.008506205680012141)
|
| 3855 |
+
|
| 3856 |
+
|
| 3857 |
+
[1m --> TIME: 2025-12-25 23:05:10 -- STEP: 1496/2207 -- GLOBAL_STEP: 40325[0m
|
| 3858 |
+
| > loss_text_ce: 0.02912270464003086 (0.029749243337172954)
|
| 3859 |
+
| > loss_mel_ce: 3.558004140853882 (3.6136459988706253)
|
| 3860 |
+
| > loss: 0.05604885518550873 (0.05692805065698721)
|
| 3861 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3862 |
+
| > step_time: 0.266 (0.2788560221539457)
|
| 3863 |
+
| > loader_time: 0.0084 (0.008501622606726262)
|
| 3864 |
+
|
| 3865 |
+
|
| 3866 |
+
[1m --> TIME: 2025-12-25 23:05:19 -- STEP: 1521/2207 -- GLOBAL_STEP: 40350[0m
|
| 3867 |
+
| > loss_text_ce: 0.030642544850707054 (0.029751243849850107)
|
| 3868 |
+
| > loss_mel_ce: 3.9657137393951416 (3.6128930175563054)
|
| 3869 |
+
| > loss: 0.06244306638836861 (0.05691631658052942)
|
| 3870 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3871 |
+
| > step_time: 0.2648 (0.2787987726601541)
|
| 3872 |
+
| > loader_time: 0.0081 (0.008497699791471244)
|
| 3873 |
+
|
| 3874 |
+
|
| 3875 |
+
[1m --> TIME: 2025-12-25 23:05:29 -- STEP: 1546/2207 -- GLOBAL_STEP: 40375[0m
|
| 3876 |
+
| > loss_text_ce: 0.030887937173247337 (0.029752243659609656)
|
| 3877 |
+
| > loss_mel_ce: 3.792837142944336 (3.6131707175428778)
|
| 3878 |
+
| > loss: 0.059745702892541885 (0.05692067126204941)
|
| 3879 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3880 |
+
| > step_time: 0.2898 (0.27879118657204616)
|
| 3881 |
+
| > loader_time: 0.008 (0.008493138161544506)
|
| 3882 |
+
|
| 3883 |
+
|
| 3884 |
+
[1m --> TIME: 2025-12-25 23:05:39 -- STEP: 1571/2207 -- GLOBAL_STEP: 40400[0m
|
| 3885 |
+
| > loss_text_ce: 0.02888145111501217 (0.029755057631870276)
|
| 3886 |
+
| > loss_mel_ce: 3.4417219161987305 (3.6131780443488983)
|
| 3887 |
+
| > loss: 0.05422817915678024 (0.0569208297193202)
|
| 3888 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3889 |
+
| > step_time: 0.2656 (0.27877123806599485)
|
| 3890 |
+
| > loader_time: 0.0079 (0.008496530643319872)
|
| 3891 |
+
|
| 3892 |
+
|
| 3893 |
+
[1m --> TIME: 2025-12-25 23:05:50 -- STEP: 1596/2207 -- GLOBAL_STEP: 40425[0m
|
| 3894 |
+
| > loss_text_ce: 0.027877559885382652 (0.029753124263174925)
|
| 3895 |
+
| > loss_mel_ce: 3.4630205631256104 (3.6134046668695627)
|
| 3896 |
+
| > loss: 0.05454528331756592 (0.05692434048555547)
|
| 3897 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3898 |
+
| > step_time: 0.2866 (0.278835178467265)
|
| 3899 |
+
| > loader_time: 0.0077 (0.008500422600815469)
|
| 3900 |
+
|
| 3901 |
+
|
| 3902 |
+
[1m --> TIME: 2025-12-25 23:06:00 -- STEP: 1621/2207 -- GLOBAL_STEP: 40450[0m
|
| 3903 |
+
| > loss_text_ce: 0.030684292316436768 (0.02975047069023157)
|
| 3904 |
+
| > loss_mel_ce: 3.486475944519043 (3.6124916863544425)
|
| 3905 |
+
| > loss: 0.054955627769231796 (0.056910033701453054)
|
| 3906 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3907 |
+
| > step_time: 0.2906 (0.27882223905861014)
|
| 3908 |
+
| > loader_time: 0.0081 (0.008501495276609369)
|
| 3909 |
+
|
| 3910 |
+
|
| 3911 |
+
[1m --> TIME: 2025-12-25 23:06:10 -- STEP: 1646/2207 -- GLOBAL_STEP: 40475[0m
|
| 3912 |
+
| > loss_text_ce: 0.02948589250445366 (0.029750664910824374)
|
| 3913 |
+
| > loss_mel_ce: 3.6549928188323975 (3.6118761279313563)
|
| 3914 |
+
| > loss: 0.05756998062133789 (0.05690041863623176)
|
| 3915 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3916 |
+
| > step_time: 0.2911 (0.27883886030783456)
|
| 3917 |
+
| > loader_time: 0.0074 (0.008501709159350482)
|
| 3918 |
+
|
| 3919 |
+
|
| 3920 |
+
[1m --> TIME: 2025-12-25 23:06:19 -- STEP: 1671/2207 -- GLOBAL_STEP: 40500[0m
|
| 3921 |
+
| > loss_text_ce: 0.03008822537958622 (0.029742146613113113)
|
| 3922 |
+
| > loss_mel_ce: 3.4990108013153076 (3.6117115420256454)
|
| 3923 |
+
| > loss: 0.05514217168092728 (0.05689771388245512)
|
| 3924 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3925 |
+
| > step_time: 0.3025 (0.2788830218666114)
|
| 3926 |
+
| > loader_time: 0.0092 (0.008500318481564023)
|
| 3927 |
+
|
| 3928 |
+
|
| 3929 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_40500.pth
|
| 3930 |
+
|
| 3931 |
+
[1m --> TIME: 2025-12-25 23:07:02 -- STEP: 1696/2207 -- GLOBAL_STEP: 40525[0m
|
| 3932 |
+
| > loss_text_ce: 0.029432693496346474 (0.02974191771705969)
|
| 3933 |
+
| > loss_mel_ce: 3.454921007156372 (3.6116017740008965)
|
| 3934 |
+
| > loss: 0.05444302782416344 (0.05689599518093086)
|
| 3935 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3936 |
+
| > step_time: 0.2667 (0.27885347028385876)
|
| 3937 |
+
| > loader_time: 0.0079 (0.008500409294974132)
|
| 3938 |
+
|
| 3939 |
+
|
| 3940 |
+
[1m --> TIME: 2025-12-25 23:07:12 -- STEP: 1721/2207 -- GLOBAL_STEP: 40550[0m
|
| 3941 |
+
| > loss_text_ce: 0.030015984550118446 (0.0297531664191828)
|
| 3942 |
+
| > loss_mel_ce: 3.733276844024658 (3.6113192194218833)
|
| 3943 |
+
| > loss: 0.058801449835300446 (0.056891756022932345)
|
| 3944 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3945 |
+
| > step_time: 0.2898 (0.2788486998833448)
|
| 3946 |
+
| > loader_time: 0.0078 (0.008504240544710932)
|
| 3947 |
+
|
| 3948 |
+
|
| 3949 |
+
[1m --> TIME: 2025-12-25 23:07:22 -- STEP: 1746/2207 -- GLOBAL_STEP: 40575[0m
|
| 3950 |
+
| > loss_text_ce: 0.02842448651790619 (0.029748099739796655)
|
| 3951 |
+
| > loss_mel_ce: 3.4693422317504883 (3.610929530237685)
|
| 3952 |
+
| > loss: 0.05465260520577431 (0.05688558796683126)
|
| 3953 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3954 |
+
| > step_time: 0.2867 (0.2788177516998427)
|
| 3955 |
+
| > loader_time: 0.0075 (0.008512936506894043)
|
| 3956 |
+
|
| 3957 |
+
|
| 3958 |
+
[1m --> TIME: 2025-12-25 23:07:32 -- STEP: 1771/2207 -- GLOBAL_STEP: 40600[0m
|
| 3959 |
+
| > loss_text_ce: 0.030122844502329826 (0.029744206735772778)
|
| 3960 |
+
| > loss_mel_ce: 3.4766221046447754 (3.6107503892606694)
|
| 3961 |
+
| > loss: 0.05479288846254349 (0.05688272806158985)
|
| 3962 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3963 |
+
| > step_time: 0.2904 (0.27884539138927444)
|
| 3964 |
+
| > loader_time: 0.0085 (0.008512757445108682)
|
| 3965 |
+
|
| 3966 |
+
|
| 3967 |
+
[1m --> TIME: 2025-12-25 23:07:43 -- STEP: 1796/2207 -- GLOBAL_STEP: 40625[0m
|
| 3968 |
+
| > loss_text_ce: 0.03249812498688698 (0.029744323826330674)
|
| 3969 |
+
| > loss_mel_ce: 3.453523874282837 (3.610680605494365)
|
| 3970 |
+
| > loss: 0.054469093680381775 (0.05688163952358514)
|
| 3971 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3972 |
+
| > step_time: 0.2869 (0.27883126940121844)
|
| 3973 |
+
| > loader_time: 0.0083 (0.008518035533964505)
|
| 3974 |
+
|
| 3975 |
+
|
| 3976 |
+
[1m --> TIME: 2025-12-25 23:07:54 -- STEP: 1821/2207 -- GLOBAL_STEP: 40650[0m
|
| 3977 |
+
| > loss_text_ce: 0.029228653758764267 (0.02974280074058752)
|
| 3978 |
+
| > loss_mel_ce: 3.4582104682922363 (3.61082390728133)
|
| 3979 |
+
| > loss: 0.05449123680591583 (0.056883854815048006)
|
| 3980 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3981 |
+
| > step_time: 0.2874 (0.27876564685228644)
|
| 3982 |
+
| > loader_time: 0.0073 (0.00922185140020046)
|
| 3983 |
+
|
| 3984 |
+
|
| 3985 |
+
[1m --> TIME: 2025-12-25 23:08:03 -- STEP: 1846/2207 -- GLOBAL_STEP: 40675[0m
|
| 3986 |
+
| > loss_text_ce: 0.028234237805008888 (0.029742555762414104)
|
| 3987 |
+
| > loss_mel_ce: 3.7793359756469727 (3.6107135387309066)
|
| 3988 |
+
| > loss: 0.0594932846724987 (0.056882126478807235)
|
| 3989 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3990 |
+
| > step_time: 0.2976 (0.27880408433250814)
|
| 3991 |
+
| > loader_time: 0.0091 (0.009208142176222064)
|
| 3992 |
+
|
| 3993 |
+
|
| 3994 |
+
[1m --> TIME: 2025-12-25 23:08:17 -- STEP: 1871/2207 -- GLOBAL_STEP: 40700[0m
|
| 3995 |
+
| > loss_text_ce: 0.0300439465790987 (0.029740850960329086)
|
| 3996 |
+
| > loss_mel_ce: 3.585555076599121 (3.6110475188203828)
|
| 3997 |
+
| > loss: 0.05649373307824135 (0.05688731828311239)
|
| 3998 |
+
| > current_lr: 1.9999968575196125e-05
|
| 3999 |
+
| > step_time: 0.2669 (0.2789339188392754)
|
| 4000 |
+
| > loader_time: 0.0072 (0.011236500192109063)
|
| 4001 |
+
|
| 4002 |
+
|
| 4003 |
+
[1m --> TIME: 2025-12-25 23:08:32 -- STEP: 1896/2207 -- GLOBAL_STEP: 40725[0m
|
| 4004 |
+
| > loss_text_ce: 0.028897859156131744 (0.029737136566437748)
|
| 4005 |
+
| > loss_mel_ce: 3.449845790863037 (3.611472120013418)
|
| 4006 |
+
| > loss: 0.05435536801815033 (0.05689389463356903)
|
| 4007 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4008 |
+
| > step_time: 0.2964 (0.27894477426754233)
|
| 4009 |
+
| > loader_time: 0.0073 (0.011195766649165748)
|
| 4010 |
+
|
| 4011 |
+
|
| 4012 |
+
[1m --> TIME: 2025-12-25 23:08:42 -- STEP: 1921/2207 -- GLOBAL_STEP: 40750[0m
|
| 4013 |
+
| > loss_text_ce: 0.031365878880023956 (0.029740173518967964)
|
| 4014 |
+
| > loss_mel_ce: 3.4094204902648926 (3.6111755968071035)
|
| 4015 |
+
| > loss: 0.053762286901474 (0.056889308912253876)
|
| 4016 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4017 |
+
| > step_time: 0.2687 (0.27886572472444704)
|
| 4018 |
+
| > loader_time: 0.0096 (0.011161029866311414)
|
| 4019 |
+
|
| 4020 |
+
|
| 4021 |
+
[1m --> TIME: 2025-12-25 23:08:51 -- STEP: 1946/2207 -- GLOBAL_STEP: 40775[0m
|
| 4022 |
+
| > loss_text_ce: 0.030574671924114227 (0.02974298447351386)
|
| 4023 |
+
| > loss_mel_ce: 3.5929527282714844 (3.610944538420298)
|
| 4024 |
+
| > loss: 0.0566176138818264 (0.05688574254313695)
|
| 4025 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4026 |
+
| > step_time: 0.2871 (0.27889007299311447)
|
| 4027 |
+
| > loader_time: 0.0083 (0.011138449469916266)
|
| 4028 |
+
|
| 4029 |
+
|
| 4030 |
+
[1m --> TIME: 2025-12-25 23:09:01 -- STEP: 1971/2207 -- GLOBAL_STEP: 40800[0m
|
| 4031 |
+
| > loss_text_ce: 0.030174532905220985 (0.02974796426997065)
|
| 4032 |
+
| > loss_mel_ce: 3.7209079265594482 (3.6107909564497747)
|
| 4033 |
+
| > loss: 0.05861066281795502 (0.05688342062887776)
|
| 4034 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4035 |
+
| > step_time: 0.2665 (0.27888417885062744)
|
| 4036 |
+
| > loader_time: 0.0074 (0.011107026230315167)
|
| 4037 |
+
|
| 4038 |
+
|
| 4039 |
+
[1m --> TIME: 2025-12-25 23:09:12 -- STEP: 1996/2207 -- GLOBAL_STEP: 40825[0m
|
| 4040 |
+
| > loss_text_ce: 0.02962142415344715 (0.029748120858429192)
|
| 4041 |
+
| > loss_mel_ce: 3.518599033355713 (3.6110308429521165)
|
| 4042 |
+
| > loss: 0.05544094368815422 (0.05688717130399896)
|
| 4043 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4044 |
+
| > step_time: 0.2625 (0.27884237823600966)
|
| 4045 |
+
| > loader_time: 0.0074 (0.0110735993585988)
|
| 4046 |
+
|
| 4047 |
+
|
| 4048 |
+
[1m --> TIME: 2025-12-25 23:09:22 -- STEP: 2021/2207 -- GLOBAL_STEP: 40850[0m
|
| 4049 |
+
| > loss_text_ce: 0.027448350563645363 (0.02974253965524209)
|
| 4050 |
+
| > loss_mel_ce: 3.5648715496063232 (3.610963821057221)
|
| 4051 |
+
| > loss: 0.056129999458789825 (0.05688603687985351)
|
| 4052 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4053 |
+
| > step_time: 0.2903 (0.27886422939196687)
|
| 4054 |
+
| > loader_time: 0.0099 (0.011035521746507567)
|
| 4055 |
+
|
| 4056 |
+
|
| 4057 |
+
[1m --> TIME: 2025-12-25 23:09:31 -- STEP: 2046/2207 -- GLOBAL_STEP: 40875[0m
|
| 4058 |
+
| > loss_text_ce: 0.029925277456641197 (0.02974282570623903)
|
| 4059 |
+
| > loss_mel_ce: 3.4512526988983154 (3.6096017844283104)
|
| 4060 |
+
| > loss: 0.054393406957387924 (0.056864759528814625)
|
| 4061 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4062 |
+
| > step_time: 0.2924 (0.27883105182461226)
|
| 4063 |
+
| > loader_time: 0.0073 (0.011007315019475177)
|
| 4064 |
+
|
| 4065 |
+
|
| 4066 |
+
[1m --> TIME: 2025-12-25 23:09:41 -- STEP: 2071/2207 -- GLOBAL_STEP: 40900[0m
|
| 4067 |
+
| > loss_text_ce: 0.028552064672112465 (0.029741530611082184)
|
| 4068 |
+
| > loss_mel_ce: 3.4174840450286865 (3.6095752421351)
|
| 4069 |
+
| > loss: 0.05384431406855583 (0.05686432456679529)
|
| 4070 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4071 |
+
| > step_time: 0.2688 (0.27878586455049953)
|
| 4072 |
+
| > loader_time: 0.0086 (0.01097536420891218)
|
| 4073 |
+
|
| 4074 |
+
|
| 4075 |
+
[1m --> TIME: 2025-12-25 23:09:54 -- STEP: 2096/2207 -- GLOBAL_STEP: 40925[0m
|
| 4076 |
+
| > loss_text_ce: 0.029125656932592392 (0.02974044233880482)
|
| 4077 |
+
| > loss_mel_ce: 3.63470196723938 (3.6096817700917483)
|
| 4078 |
+
| > loss: 0.057247307151556015 (0.0568659720579802)
|
| 4079 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4080 |
+
| > step_time: 0.291 (0.27874305901181573)
|
| 4081 |
+
| > loader_time: 0.0101 (0.010946607202973996)
|
| 4082 |
+
|
| 4083 |
+
|
| 4084 |
+
[1m --> TIME: 2025-12-25 23:10:04 -- STEP: 2121/2207 -- GLOBAL_STEP: 40950[0m
|
| 4085 |
+
| > loss_text_ce: 0.028683358803391457 (0.029741003345355625)
|
| 4086 |
+
| > loss_mel_ce: 3.5828611850738525 (3.6098853766159427)
|
| 4087 |
+
| > loss: 0.056430384516716 (0.056869162175451314)
|
| 4088 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4089 |
+
| > step_time: 0.2687 (0.27869246793996005)
|
| 4090 |
+
| > loader_time: 0.0073 (0.010915133594511153)
|
| 4091 |
+
|
| 4092 |
+
|
| 4093 |
+
[1m --> TIME: 2025-12-25 23:10:14 -- STEP: 2146/2207 -- GLOBAL_STEP: 40975[0m
|
| 4094 |
+
| > loss_text_ce: 0.028628891333937645 (0.029737809564456524)
|
| 4095 |
+
| > loss_mel_ce: 3.340278387069702 (3.6100781128324306)
|
| 4096 |
+
| > loss: 0.052639175206422806 (0.05687212377599154)
|
| 4097 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4098 |
+
| > step_time: 0.291 (0.27872393937835227)
|
| 4099 |
+
| > loader_time: 0.0076 (0.010890124117450424)
|
| 4100 |
+
|
| 4101 |
+
|
| 4102 |
+
[1m --> TIME: 2025-12-25 23:10:24 -- STEP: 2171/2207 -- GLOBAL_STEP: 41000[0m
|
| 4103 |
+
| > loss_text_ce: 0.029838191345334053 (0.02973214871426405)
|
| 4104 |
+
| > loss_mel_ce: 3.6607308387756348 (3.6093667838387855)
|
| 4105 |
+
| > loss: 0.0576651394367218 (0.056860920807105586)
|
| 4106 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4107 |
+
| > step_time: 0.2879 (0.27874778339468526)
|
| 4108 |
+
| > loader_time: 0.0086 (0.010866279373625803)
|
| 4109 |
+
|
| 4110 |
+
|
| 4111 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_41000.pth
|
| 4112 |
+
|
| 4113 |
+
[1m --> TIME: 2025-12-25 23:11:00 -- STEP: 2196/2207 -- GLOBAL_STEP: 41025[0m
|
| 4114 |
+
| > loss_text_ce: 0.026629755273461342 (0.029733386280893847)
|
| 4115 |
+
| > loss_mel_ce: 3.287623643875122 (3.6094359905341933)
|
| 4116 |
+
| > loss: 0.05178520828485489 (0.05686202149783922)
|
| 4117 |
+
| > current_lr: 1.9999968575196125e-05
|
| 4118 |
+
| > step_time: 0.2954 (0.27874378873567934)
|
| 4119 |
+
| > loader_time: 0.0078 (0.010839057967528192)
|
| 4120 |
+
|
| 4121 |
+
|
| 4122 |
+
[1m > EVALUATION [0m
|
| 4123 |
+
|
| 4124 |
+
|
| 4125 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 4126 |
+
| > avg_loader_time:[91m 0.08280639390687683 [0m(+0.0025067152203740173)
|
| 4127 |
+
| > avg_loss_text_ce:[92m 0.02873322959183841 [0m(-0.00011923722922801971)
|
| 4128 |
+
| > avg_loss_mel_ce:[92m 3.719441417101267 [0m(-0.011220292465107029)
|
| 4129 |
+
| > avg_loss:[92m 3.7481746480271623 [0m(-0.011339513031212167)
|
| 4130 |
+
|
| 4131 |
+
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_41036.pth
|