Checkpoint at step 38829
Browse files- best_model.pth +1 -1
- best_model_38829.pth +3 -0
- checkpoint_38000.pth +3 -0
- checkpoint_38500.pth +3 -0
- events.out.tfevents.1766698157.0f9434978106.1562.0 +2 -2
- trainer_0_log.txt +826 -0
best_model.pth
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 5607926805
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b7732e2bcf3a01ddfa445c849b926edb16f5ca00b3c77af393acabaf9904100e
|
| 3 |
size 5607926805
|
best_model_38829.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b7732e2bcf3a01ddfa445c849b926edb16f5ca00b3c77af393acabaf9904100e
|
| 3 |
+
size 5607926805
|
checkpoint_38000.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e7b42a682e9e3c92aeced6a2f779ba5260ed3806d1498046f8cf92b00ae0aa1f
|
| 3 |
+
size 5607926741
|
checkpoint_38500.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:af97be5e200db5345cfd7f5d004fc2b4e68af7c8c2595bc792899385b24da36f
|
| 3 |
+
size 5607926741
|
events.out.tfevents.1766698157.0f9434978106.1562.0
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b9b4be0d547695e0bef2de31954d1fe9f646504118d0faa3a60e0af5a85bef4a
|
| 3 |
+
size 43448
|
trainer_0_log.txt
CHANGED
|
@@ -2484,3 +2484,829 @@
|
|
| 2484 |
| > avg_loss:[92m 3.764977015353538 [0m(-0.007839202880859375)
|
| 2485 |
|
| 2486 |
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_36622.pth
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2484 |
| > avg_loss:[92m 3.764977015353538 [0m(-0.007839202880859375)
|
| 2485 |
|
| 2486 |
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_36622.pth
|
| 2487 |
+
|
| 2488 |
+
[4m[1m > EPOCH: 4/12[0m
|
| 2489 |
+
--> /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000
|
| 2490 |
+
|
| 2491 |
+
[1m > TRAINING (2025-12-25 22:33:50) [0m
|
| 2492 |
+
|
| 2493 |
+
[1m --> TIME: 2025-12-25 22:33:55 -- STEP: 3/2207 -- GLOBAL_STEP: 36625[0m
|
| 2494 |
+
| > loss_text_ce: 0.03105994127690792 (0.02996816858649254)
|
| 2495 |
+
| > loss_mel_ce: 3.4819118976593018 (3.44530455271403)
|
| 2496 |
+
| > loss: 0.054890185594558716 (0.054301136483748756)
|
| 2497 |
+
| > current_lr: 1.999998232354375e-05
|
| 2498 |
+
| > step_time: 0.2901 (0.28107166290283203)
|
| 2499 |
+
| > loader_time: 0.0071 (0.0072409311930338545)
|
| 2500 |
+
|
| 2501 |
+
|
| 2502 |
+
[1m --> TIME: 2025-12-25 22:34:04 -- STEP: 28/2207 -- GLOBAL_STEP: 36650[0m
|
| 2503 |
+
| > loss_text_ce: 0.029771940782666206 (0.03024066352684583)
|
| 2504 |
+
| > loss_mel_ce: 3.6021928787231445 (3.6194701875959123)
|
| 2505 |
+
| > loss: 0.05674945190548897 (0.05702673178166151)
|
| 2506 |
+
| > current_lr: 1.999998232354375e-05
|
| 2507 |
+
| > step_time: 0.2914 (0.2849241665431431)
|
| 2508 |
+
| > loader_time: 0.0092 (0.008658255849565779)
|
| 2509 |
+
|
| 2510 |
+
|
| 2511 |
+
[1m --> TIME: 2025-12-25 22:34:14 -- STEP: 53/2207 -- GLOBAL_STEP: 36675[0m
|
| 2512 |
+
| > loss_text_ce: 0.02943609096109867 (0.030146581841246137)
|
| 2513 |
+
| > loss_mel_ce: 3.5718448162078857 (3.6544206907164374)
|
| 2514 |
+
| > loss: 0.056270014494657516 (0.05757136354750057)
|
| 2515 |
+
| > current_lr: 1.999998232354375e-05
|
| 2516 |
+
| > step_time: 0.2877 (0.28131336985893973)
|
| 2517 |
+
| > loader_time: 0.0094 (0.008648705932329286)
|
| 2518 |
+
|
| 2519 |
+
|
| 2520 |
+
[1m --> TIME: 2025-12-25 22:34:24 -- STEP: 78/2207 -- GLOBAL_STEP: 36700[0m
|
| 2521 |
+
| > loss_text_ce: 0.028913235291838646 (0.030251895937208947)
|
| 2522 |
+
| > loss_mel_ce: 4.0025954246521 (3.668589683679434)
|
| 2523 |
+
| > loss: 0.06299231946468353 (0.05779439960725796)
|
| 2524 |
+
| > current_lr: 1.999998232354375e-05
|
| 2525 |
+
| > step_time: 0.2624 (0.2802002215996768)
|
| 2526 |
+
| > loader_time: 0.0088 (0.008577154232905462)
|
| 2527 |
+
|
| 2528 |
+
|
| 2529 |
+
[1m --> TIME: 2025-12-25 22:34:33 -- STEP: 103/2207 -- GLOBAL_STEP: 36725[0m
|
| 2530 |
+
| > loss_text_ce: 0.028429260477423668 (0.03008302192664841)
|
| 2531 |
+
| > loss_mel_ce: 3.5342626571655273 (3.6610913137787753)
|
| 2532 |
+
| > loss: 0.05566706135869026 (0.05767459892532201)
|
| 2533 |
+
| > current_lr: 1.999998232354375e-05
|
| 2534 |
+
| > step_time: 0.2684 (0.2795189422311137)
|
| 2535 |
+
| > loader_time: 0.0083 (0.008622530594612793)
|
| 2536 |
+
|
| 2537 |
+
|
| 2538 |
+
[1m --> TIME: 2025-12-25 22:34:43 -- STEP: 128/2207 -- GLOBAL_STEP: 36750[0m
|
| 2539 |
+
| > loss_text_ce: 0.03029327280819416 (0.030092049841186963)
|
| 2540 |
+
| > loss_mel_ce: 3.853541851043701 (3.6669089552015066)
|
| 2541 |
+
| > loss: 0.0606849230825901 (0.05776564060943201)
|
| 2542 |
+
| > current_lr: 1.999998232354375e-05
|
| 2543 |
+
| > step_time: 0.2612 (0.27848333865404146)
|
| 2544 |
+
| > loader_time: 0.0084 (0.008639754727482797)
|
| 2545 |
+
|
| 2546 |
+
|
| 2547 |
+
[1m --> TIME: 2025-12-25 22:34:52 -- STEP: 153/2207 -- GLOBAL_STEP: 36775[0m
|
| 2548 |
+
| > loss_text_ce: 0.03286995738744736 (0.030166215805056826)
|
| 2549 |
+
| > loss_mel_ce: 3.510187864303589 (3.664347862106523)
|
| 2550 |
+
| > loss: 0.055360279977321625 (0.0577267823996497)
|
| 2551 |
+
| > current_lr: 1.999998232354375e-05
|
| 2552 |
+
| > step_time: 0.2663 (0.27807637133629515)
|
| 2553 |
+
| > loader_time: 0.0084 (0.008611130558587369)
|
| 2554 |
+
|
| 2555 |
+
|
| 2556 |
+
[1m --> TIME: 2025-12-25 22:35:02 -- STEP: 178/2207 -- GLOBAL_STEP: 36800[0m
|
| 2557 |
+
| > loss_text_ce: 0.030874377116560936 (0.03017476186407416)
|
| 2558 |
+
| > loss_mel_ce: 3.9232735633850098 (3.6651728488086315)
|
| 2559 |
+
| > loss: 0.0617835633456707 (0.05773980637196074)
|
| 2560 |
+
| > current_lr: 1.999998232354375e-05
|
| 2561 |
+
| > step_time: 0.2868 (0.2785130605269017)
|
| 2562 |
+
| > loader_time: 0.0085 (0.008594933520542102)
|
| 2563 |
+
|
| 2564 |
+
|
| 2565 |
+
[1m --> TIME: 2025-12-25 22:35:11 -- STEP: 203/2207 -- GLOBAL_STEP: 36825[0m
|
| 2566 |
+
| > loss_text_ce: 0.030363690108060837 (0.03013063363473991)
|
| 2567 |
+
| > loss_mel_ce: 3.4945907592773438 (3.659278900165276)
|
| 2568 |
+
| > loss: 0.05507741495966911 (0.05764702393723825)
|
| 2569 |
+
| > current_lr: 1.999998232354375e-05
|
| 2570 |
+
| > step_time: 0.2617 (0.2785147681024865)
|
| 2571 |
+
| > loader_time: 0.0086 (0.00858109220495365)
|
| 2572 |
+
|
| 2573 |
+
|
| 2574 |
+
[1m --> TIME: 2025-12-25 22:35:21 -- STEP: 228/2207 -- GLOBAL_STEP: 36850[0m
|
| 2575 |
+
| > loss_text_ce: 0.030972639098763466 (0.03010655987779038)
|
| 2576 |
+
| > loss_mel_ce: 3.7197787761688232 (3.6538343460936296)
|
| 2577 |
+
| > loss: 0.05860549211502075 (0.05756157659517045)
|
| 2578 |
+
| > current_lr: 1.999998232354375e-05
|
| 2579 |
+
| > step_time: 0.2683 (0.27817169616096915)
|
| 2580 |
+
| > loader_time: 0.0087 (0.008623308257052778)
|
| 2581 |
+
|
| 2582 |
+
|
| 2583 |
+
[1m --> TIME: 2025-12-25 22:35:30 -- STEP: 253/2207 -- GLOBAL_STEP: 36875[0m
|
| 2584 |
+
| > loss_text_ce: 0.030273092910647392 (0.03008509619463338)
|
| 2585 |
+
| > loss_mel_ce: 3.9639713764190674 (3.6507256916860347)
|
| 2586 |
+
| > loss: 0.06241007149219513 (0.0575126685025428)
|
| 2587 |
+
| > current_lr: 1.999998232354375e-05
|
| 2588 |
+
| > step_time: 0.2861 (0.2783028900387732)
|
| 2589 |
+
| > loader_time: 0.009 (0.00861876284180894)
|
| 2590 |
+
|
| 2591 |
+
|
| 2592 |
+
[1m --> TIME: 2025-12-25 22:35:40 -- STEP: 278/2207 -- GLOBAL_STEP: 36900[0m
|
| 2593 |
+
| > loss_text_ce: 0.030992211773991585 (0.03001686306201297)
|
| 2594 |
+
| > loss_mel_ce: 3.7320902347564697 (3.652917193851883)
|
| 2595 |
+
| > loss: 0.0587981641292572 (0.057545844599497395)
|
| 2596 |
+
| > current_lr: 1.999998232354375e-05
|
| 2597 |
+
| > step_time: 0.2622 (0.27844631243095136)
|
| 2598 |
+
| > loader_time: 0.0084 (0.00862825259887915)
|
| 2599 |
+
|
| 2600 |
+
|
| 2601 |
+
[1m --> TIME: 2025-12-25 22:35:50 -- STEP: 303/2207 -- GLOBAL_STEP: 36925[0m
|
| 2602 |
+
| > loss_text_ce: 0.03255581855773926 (0.030041390728734112)
|
| 2603 |
+
| > loss_mel_ce: 3.900932788848877 (3.6536370968267864)
|
| 2604 |
+
| > loss: 0.06146075949072838 (0.057557476321757825)
|
| 2605 |
+
| > current_lr: 1.999998232354375e-05
|
| 2606 |
+
| > step_time: 0.2623 (0.2784324746714174)
|
| 2607 |
+
| > loader_time: 0.0094 (0.008622584956707344)
|
| 2608 |
+
|
| 2609 |
+
|
| 2610 |
+
[1m --> TIME: 2025-12-25 22:35:59 -- STEP: 328/2207 -- GLOBAL_STEP: 36950[0m
|
| 2611 |
+
| > loss_text_ce: 0.030865592882037163 (0.03004381339997053)
|
| 2612 |
+
| > loss_mel_ce: 3.6935110092163086 (3.6496512431923938)
|
| 2613 |
+
| > loss: 0.0581933856010437 (0.05749523520424235)
|
| 2614 |
+
| > current_lr: 1.999998232354375e-05
|
| 2615 |
+
| > step_time: 0.2652 (0.27809023566362334)
|
| 2616 |
+
| > loader_time: 0.0081 (0.008602350223355183)
|
| 2617 |
+
|
| 2618 |
+
|
| 2619 |
+
[1m --> TIME: 2025-12-25 22:36:09 -- STEP: 353/2207 -- GLOBAL_STEP: 36975[0m
|
| 2620 |
+
| > loss_text_ce: 0.02890920825302601 (0.030006446667205512)
|
| 2621 |
+
| > loss_mel_ce: 3.693331718444824 (3.6539445402939665)
|
| 2622 |
+
| > loss: 0.05816001445055008 (0.05756173409500473)
|
| 2623 |
+
| > current_lr: 1.999998232354375e-05
|
| 2624 |
+
| > step_time: 0.2864 (0.278261472415654)
|
| 2625 |
+
| > loader_time: 0.0084 (0.008600345752056876)
|
| 2626 |
+
|
| 2627 |
+
|
| 2628 |
+
[1m --> TIME: 2025-12-25 22:36:18 -- STEP: 378/2207 -- GLOBAL_STEP: 37000[0m
|
| 2629 |
+
| > loss_text_ce: 0.029319290071725845 (0.030011727331728532)
|
| 2630 |
+
| > loss_mel_ce: 3.561335802078247 (3.6513603522033287)
|
| 2631 |
+
| > loss: 0.056103985756635666 (0.057521438659735455)
|
| 2632 |
+
| > current_lr: 1.999998232354375e-05
|
| 2633 |
+
| > step_time: 0.2659 (0.2782370728790446)
|
| 2634 |
+
| > loader_time: 0.0082 (0.008602334078026835)
|
| 2635 |
+
|
| 2636 |
+
|
| 2637 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_37000.pth
|
| 2638 |
+
|
| 2639 |
+
[1m --> TIME: 2025-12-25 22:36:53 -- STEP: 403/2207 -- GLOBAL_STEP: 37025[0m
|
| 2640 |
+
| > loss_text_ce: 0.02902626246213913 (0.030000154802543352)
|
| 2641 |
+
| > loss_mel_ce: 3.6377151012420654 (3.652060526476309)
|
| 2642 |
+
| > loss: 0.05729283392429352 (0.057532198070814236)
|
| 2643 |
+
| > current_lr: 1.999998232354375e-05
|
| 2644 |
+
| > step_time: 0.3046 (0.2781177886366549)
|
| 2645 |
+
| > loader_time: 0.0086 (0.008621495651548024)
|
| 2646 |
+
|
| 2647 |
+
|
| 2648 |
+
[1m --> TIME: 2025-12-25 22:37:02 -- STEP: 428/2207 -- GLOBAL_STEP: 37050[0m
|
| 2649 |
+
| > loss_text_ce: 0.031081179156899452 (0.03000510779914455)
|
| 2650 |
+
| > loss_mel_ce: 3.479534149169922 (3.653535674665576)
|
| 2651 |
+
| > loss: 0.05485336482524872 (0.05755532465909965)
|
| 2652 |
+
| > current_lr: 1.999998232354375e-05
|
| 2653 |
+
| > step_time: 0.2693 (0.27818728217454713)
|
| 2654 |
+
| > loader_time: 0.0082 (0.008625869439027021)
|
| 2655 |
+
|
| 2656 |
+
|
| 2657 |
+
[1m --> TIME: 2025-12-25 22:37:12 -- STEP: 453/2207 -- GLOBAL_STEP: 37075[0m
|
| 2658 |
+
| > loss_text_ce: 0.030318038538098335 (0.02997327927771388)
|
| 2659 |
+
| > loss_mel_ce: 3.654078483581543 (3.6520691849538034)
|
| 2660 |
+
| > loss: 0.05756869539618492 (0.0575319134340381)
|
| 2661 |
+
| > current_lr: 1.999998232354375e-05
|
| 2662 |
+
| > step_time: 0.2688 (0.27811358832891936)
|
| 2663 |
+
| > loader_time: 0.0099 (0.0086262257683356)
|
| 2664 |
+
|
| 2665 |
+
|
| 2666 |
+
[1m --> TIME: 2025-12-25 22:37:21 -- STEP: 478/2207 -- GLOBAL_STEP: 37100[0m
|
| 2667 |
+
| > loss_text_ce: 0.028813056647777557 (0.029980088237314545)
|
| 2668 |
+
| > loss_mel_ce: 3.8569045066833496 (3.649504156292233)
|
| 2669 |
+
| > loss: 0.06071433797478676 (0.057491941252824655)
|
| 2670 |
+
| > current_lr: 1.999998232354375e-05
|
| 2671 |
+
| > step_time: 0.2905 (0.2779884847138217)
|
| 2672 |
+
| > loader_time: 0.0092 (0.008628546942228043)
|
| 2673 |
+
|
| 2674 |
+
|
| 2675 |
+
[1m --> TIME: 2025-12-25 22:37:31 -- STEP: 503/2207 -- GLOBAL_STEP: 37125[0m
|
| 2676 |
+
| > loss_text_ce: 0.028841480612754822 (0.02996080491699234)
|
| 2677 |
+
| > loss_mel_ce: 3.694622278213501 (3.6457357250198457)
|
| 2678 |
+
| > loss: 0.05817912146449089 (0.05743275820527589)
|
| 2679 |
+
| > current_lr: 1.999998232354375e-05
|
| 2680 |
+
| > step_time: 0.3066 (0.27823117902691274)
|
| 2681 |
+
| > loader_time: 0.0094 (0.008629650529288857)
|
| 2682 |
+
|
| 2683 |
+
|
| 2684 |
+
[1m --> TIME: 2025-12-25 22:37:41 -- STEP: 528/2207 -- GLOBAL_STEP: 37150[0m
|
| 2685 |
+
| > loss_text_ce: 0.0324242040514946 (0.029954439431259576)
|
| 2686 |
+
| > loss_mel_ce: 3.599579095840454 (3.643170359459791)
|
| 2687 |
+
| > loss: 0.05675005167722702 (0.05739257489613286)
|
| 2688 |
+
| > current_lr: 1.999998232354375e-05
|
| 2689 |
+
| > step_time: 0.2677 (0.2782137624242089)
|
| 2690 |
+
| > loader_time: 0.0081 (0.008619090824416179)
|
| 2691 |
+
|
| 2692 |
+
|
| 2693 |
+
[1m --> TIME: 2025-12-25 22:37:50 -- STEP: 553/2207 -- GLOBAL_STEP: 37175[0m
|
| 2694 |
+
| > loss_text_ce: 0.03125845268368721 (0.029944292011975595)
|
| 2695 |
+
| > loss_mel_ce: 3.5502774715423584 (3.641215753813981)
|
| 2696 |
+
| > loss: 0.055961497128009796 (0.05736187563208732)
|
| 2697 |
+
| > current_lr: 1.999998232354375e-05
|
| 2698 |
+
| > step_time: 0.2691 (0.2781586776374692)
|
| 2699 |
+
| > loader_time: 0.0084 (0.008614579764671384)
|
| 2700 |
+
|
| 2701 |
+
|
| 2702 |
+
[1m --> TIME: 2025-12-25 22:38:00 -- STEP: 578/2207 -- GLOBAL_STEP: 37200[0m
|
| 2703 |
+
| > loss_text_ce: 0.029963556677103043 (0.029942966449786636)
|
| 2704 |
+
| > loss_mel_ce: 3.442197322845459 (3.6411196690529706)
|
| 2705 |
+
| > loss: 0.054252512753009796 (0.057360353592821885)
|
| 2706 |
+
| > current_lr: 1.999998232354375e-05
|
| 2707 |
+
| > step_time: 0.2866 (0.27815267165226915)
|
| 2708 |
+
| > loader_time: 0.0081 (0.008625608827003545)
|
| 2709 |
+
|
| 2710 |
+
|
| 2711 |
+
[1m --> TIME: 2025-12-25 22:38:10 -- STEP: 603/2207 -- GLOBAL_STEP: 37225[0m
|
| 2712 |
+
| > loss_text_ce: 0.0302469152957201 (0.029954587271557517)
|
| 2713 |
+
| > loss_mel_ce: 3.442945957183838 (3.639026295881763)
|
| 2714 |
+
| > loss: 0.05426863953471184 (0.05732782621857143)
|
| 2715 |
+
| > current_lr: 1.999998232354375e-05
|
| 2716 |
+
| > step_time: 0.2623 (0.2782686061131618)
|
| 2717 |
+
| > loader_time: 0.0084 (0.00862701852523271)
|
| 2718 |
+
|
| 2719 |
+
|
| 2720 |
+
[1m --> TIME: 2025-12-25 22:38:19 -- STEP: 628/2207 -- GLOBAL_STEP: 37250[0m
|
| 2721 |
+
| > loss_text_ce: 0.03177732229232788 (0.029956031943653608)
|
| 2722 |
+
| > loss_mel_ce: 3.577380418777466 (3.6397846052601093)
|
| 2723 |
+
| > loss: 0.0563930906355381 (0.05733969737034125)
|
| 2724 |
+
| > current_lr: 1.999998232354375e-05
|
| 2725 |
+
| > step_time: 0.2866 (0.2782954900128066)
|
| 2726 |
+
| > loader_time: 0.0084 (0.008625051018538736)
|
| 2727 |
+
|
| 2728 |
+
|
| 2729 |
+
[1m --> TIME: 2025-12-25 22:38:29 -- STEP: 653/2207 -- GLOBAL_STEP: 37275[0m
|
| 2730 |
+
| > loss_text_ce: 0.029098352417349815 (0.029975488319833283)
|
| 2731 |
+
| > loss_mel_ce: 3.4419867992401123 (3.639289094697098)
|
| 2732 |
+
| > loss: 0.054235704243183136 (0.05733225902522505)
|
| 2733 |
+
| > current_lr: 1.999998232354375e-05
|
| 2734 |
+
| > step_time: 0.269 (0.2780567624850075)
|
| 2735 |
+
| > loader_time: 0.0092 (0.008621339593510524)
|
| 2736 |
+
|
| 2737 |
+
|
| 2738 |
+
[1m --> TIME: 2025-12-25 22:38:38 -- STEP: 678/2207 -- GLOBAL_STEP: 37300[0m
|
| 2739 |
+
| > loss_text_ce: 0.031218064948916435 (0.029965481970865817)
|
| 2740 |
+
| > loss_mel_ce: 4.1310834884643555 (3.640433362099978)
|
| 2741 |
+
| > loss: 0.06503596156835556 (0.0573499818609967)
|
| 2742 |
+
| > current_lr: 1.999998232354375e-05
|
| 2743 |
+
| > step_time: 0.2497 (0.27805528493054144)
|
| 2744 |
+
| > loader_time: 0.0102 (0.008618066444509494)
|
| 2745 |
+
|
| 2746 |
+
|
| 2747 |
+
[1m --> TIME: 2025-12-25 22:38:48 -- STEP: 703/2207 -- GLOBAL_STEP: 37325[0m
|
| 2748 |
+
| > loss_text_ce: 0.02868487685918808 (0.029973591098914277)
|
| 2749 |
+
| > loss_mel_ce: 3.4746909141540527 (3.6421279639302426)
|
| 2750 |
+
| > loss: 0.05474024638533592 (0.05737658672744824)
|
| 2751 |
+
| > grad_norm: 0 (0.0)
|
| 2752 |
+
| > current_lr: 1.999998232354375e-05
|
| 2753 |
+
| > step_time: 0.3124 (0.27806359300572686)
|
| 2754 |
+
| > loader_time: 0.0093 (0.008615056617161618)
|
| 2755 |
+
|
| 2756 |
+
|
| 2757 |
+
[1m --> TIME: 2025-12-25 22:38:57 -- STEP: 728/2207 -- GLOBAL_STEP: 37350[0m
|
| 2758 |
+
| > loss_text_ce: 0.03152864798903465 (0.029981767566833687)
|
| 2759 |
+
| > loss_mel_ce: 3.585728883743286 (3.6425413895439327)
|
| 2760 |
+
| > loss: 0.05651964992284775 (0.05738317426601118)
|
| 2761 |
+
| > current_lr: 1.999998232354375e-05
|
| 2762 |
+
| > step_time: 0.2877 (0.2781706206746151)
|
| 2763 |
+
| > loader_time: 0.0083 (0.008615102414246435)
|
| 2764 |
+
|
| 2765 |
+
|
| 2766 |
+
[1m --> TIME: 2025-12-25 22:39:07 -- STEP: 753/2207 -- GLOBAL_STEP: 37375[0m
|
| 2767 |
+
| > loss_text_ce: 0.028128303587436676 (0.0299746298180475)
|
| 2768 |
+
| > loss_mel_ce: 3.8263421058654785 (3.6436999217764168)
|
| 2769 |
+
| > loss: 0.06022610142827034 (0.05740116481702641)
|
| 2770 |
+
| > current_lr: 1.999998232354375e-05
|
| 2771 |
+
| > step_time: 0.2657 (0.27805506723968754)
|
| 2772 |
+
| > loader_time: 0.0086 (0.008607410656345155)
|
| 2773 |
+
|
| 2774 |
+
|
| 2775 |
+
[1m --> TIME: 2025-12-25 22:39:16 -- STEP: 778/2207 -- GLOBAL_STEP: 37400[0m
|
| 2776 |
+
| > loss_text_ce: 0.026816098019480705 (0.029975097966416328)
|
| 2777 |
+
| > loss_mel_ce: 3.6117193698883057 (3.6442726237608425)
|
| 2778 |
+
| > loss: 0.05685211718082428 (0.057410120595351895)
|
| 2779 |
+
| > current_lr: 1.999998232354375e-05
|
| 2780 |
+
| > step_time: 0.2866 (0.2779938978523702)
|
| 2781 |
+
| > loader_time: 0.0081 (0.008602352865557434)
|
| 2782 |
+
|
| 2783 |
+
|
| 2784 |
+
[1m --> TIME: 2025-12-25 22:39:26 -- STEP: 803/2207 -- GLOBAL_STEP: 37425[0m
|
| 2785 |
+
| > loss_text_ce: 0.03060527890920639 (0.029983045994901418)
|
| 2786 |
+
| > loss_mel_ce: 3.741201162338257 (3.6455449229605033)
|
| 2787 |
+
| > loss: 0.05893447622656822 (0.05743012446469417)
|
| 2788 |
+
| > current_lr: 1.999998232354375e-05
|
| 2789 |
+
| > step_time: 0.2864 (0.2780827617882788)
|
| 2790 |
+
| > loader_time: 0.0083 (0.008599410169893601)
|
| 2791 |
+
|
| 2792 |
+
|
| 2793 |
+
[1m --> TIME: 2025-12-25 22:39:35 -- STEP: 828/2207 -- GLOBAL_STEP: 37450[0m
|
| 2794 |
+
| > loss_text_ce: 0.029512014240026474 (0.029970357597216172)
|
| 2795 |
+
| > loss_mel_ce: 3.6994731426239014 (3.6449834413574527)
|
| 2796 |
+
| > loss: 0.05826539173722267 (0.0574211530558369)
|
| 2797 |
+
| > current_lr: 1.999998232354375e-05
|
| 2798 |
+
| > step_time: 0.2874 (0.2780538054480067)
|
| 2799 |
+
| > loader_time: 0.0089 (0.008598778270868867)
|
| 2800 |
+
|
| 2801 |
+
|
| 2802 |
+
[1m --> TIME: 2025-12-25 22:39:45 -- STEP: 853/2207 -- GLOBAL_STEP: 37475[0m
|
| 2803 |
+
| > loss_text_ce: 0.02995569072663784 (0.029969920863911665)
|
| 2804 |
+
| > loss_mel_ce: 3.487165927886963 (3.6442594813294327)
|
| 2805 |
+
| > loss: 0.054955024272203445 (0.05740983434816318)
|
| 2806 |
+
| > current_lr: 1.999998232354375e-05
|
| 2807 |
+
| > step_time: 0.2656 (0.2777623308501515)
|
| 2808 |
+
| > loader_time: 0.0087 (0.008607358753611308)
|
| 2809 |
+
|
| 2810 |
+
|
| 2811 |
+
[1m --> TIME: 2025-12-25 22:39:54 -- STEP: 878/2207 -- GLOBAL_STEP: 37500[0m
|
| 2812 |
+
| > loss_text_ce: 0.030305203050374985 (0.029962425190183186)
|
| 2813 |
+
| > loss_mel_ce: 3.3244807720184326 (3.6446188490048637)
|
| 2814 |
+
| > loss: 0.052418529987335205 (0.05741533235665877)
|
| 2815 |
+
| > current_lr: 1.999998232354375e-05
|
| 2816 |
+
| > step_time: 0.266 (0.27772207194960447)
|
| 2817 |
+
| > loader_time: 0.0087 (0.008601510171738193)
|
| 2818 |
+
|
| 2819 |
+
|
| 2820 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_37500.pth
|
| 2821 |
+
|
| 2822 |
+
[1m --> TIME: 2025-12-25 22:40:28 -- STEP: 903/2207 -- GLOBAL_STEP: 37525[0m
|
| 2823 |
+
| > loss_text_ce: 0.029249630868434906 (0.02996275690719403)
|
| 2824 |
+
| > loss_mel_ce: 3.9340837001800537 (3.6443699478178995)
|
| 2825 |
+
| > loss: 0.061927083879709244 (0.05741144846691642)
|
| 2826 |
+
| > current_lr: 1.999998232354375e-05
|
| 2827 |
+
| > step_time: 0.2698 (0.27767021737885483)
|
| 2828 |
+
| > loader_time: 0.0087 (0.008601221399317812)
|
| 2829 |
+
|
| 2830 |
+
|
| 2831 |
+
[1m --> TIME: 2025-12-25 22:40:38 -- STEP: 928/2207 -- GLOBAL_STEP: 37550[0m
|
| 2832 |
+
| > loss_text_ce: 0.031362101435661316 (0.029974632320427817)
|
| 2833 |
+
| > loss_mel_ce: 3.7293150424957275 (3.6447094666032966)
|
| 2834 |
+
| > loss: 0.05876057967543602 (0.05741693899998891)
|
| 2835 |
+
| > current_lr: 1.999998232354375e-05
|
| 2836 |
+
| > step_time: 0.2664 (0.2779292831646983)
|
| 2837 |
+
| > loader_time: 0.0081 (0.008605776675816239)
|
| 2838 |
+
|
| 2839 |
+
|
| 2840 |
+
[1m --> TIME: 2025-12-25 22:40:48 -- STEP: 953/2207 -- GLOBAL_STEP: 37575[0m
|
| 2841 |
+
| > loss_text_ce: 0.028790520504117012 (0.029958822179081433)
|
| 2842 |
+
| > loss_mel_ce: 3.503146171569824 (3.6438614136274574)
|
| 2843 |
+
| > loss: 0.05518651008605957 (0.05740344113155277)
|
| 2844 |
+
| > current_lr: 1.999998232354375e-05
|
| 2845 |
+
| > step_time: 0.2899 (0.2779601148143017)
|
| 2846 |
+
| > loader_time: 0.0084 (0.008605077659470835)
|
| 2847 |
+
|
| 2848 |
+
|
| 2849 |
+
[1m --> TIME: 2025-12-25 22:40:57 -- STEP: 978/2207 -- GLOBAL_STEP: 37600[0m
|
| 2850 |
+
| > loss_text_ce: 0.028217358514666557 (0.029947899000105926)
|
| 2851 |
+
| > loss_mel_ce: 3.5288774967193604 (3.6439758544075476)
|
| 2852 |
+
| > loss: 0.05557960644364357 (0.057405058599261906)
|
| 2853 |
+
| > current_lr: 1.999998232354375e-05
|
| 2854 |
+
| > step_time: 0.2661 (0.2779681529004147)
|
| 2855 |
+
| > loader_time: 0.0086 (0.008604847824159085)
|
| 2856 |
+
|
| 2857 |
+
|
| 2858 |
+
[1m --> TIME: 2025-12-25 22:41:07 -- STEP: 1003/2207 -- GLOBAL_STEP: 37625[0m
|
| 2859 |
+
| > loss_text_ce: 0.0324404202401638 (0.029962233818346933)
|
| 2860 |
+
| > loss_mel_ce: 3.6750693321228027 (3.644574325320016)
|
| 2861 |
+
| > loss: 0.05792983993887901 (0.05741463369083547)
|
| 2862 |
+
| > current_lr: 1.999998232354375e-05
|
| 2863 |
+
| > step_time: 0.2673 (0.27785040778391124)
|
| 2864 |
+
| > loader_time: 0.01 (0.008609181505851706)
|
| 2865 |
+
|
| 2866 |
+
|
| 2867 |
+
[1m --> TIME: 2025-12-25 22:41:16 -- STEP: 1028/2207 -- GLOBAL_STEP: 37650[0m
|
| 2868 |
+
| > loss_text_ce: 0.029688077047467232 (0.029961557549710752)
|
| 2869 |
+
| > loss_mel_ce: 3.7994959354400635 (3.644482085213124)
|
| 2870 |
+
| > loss: 0.05983100086450577 (0.05741318187346602)
|
| 2871 |
+
| > current_lr: 1.999998232354375e-05
|
| 2872 |
+
| > step_time: 0.2691 (0.27780350590494335)
|
| 2873 |
+
| > loader_time: 0.0107 (0.008614934603991674)
|
| 2874 |
+
|
| 2875 |
+
|
| 2876 |
+
[1m --> TIME: 2025-12-25 22:41:26 -- STEP: 1053/2207 -- GLOBAL_STEP: 37675[0m
|
| 2877 |
+
| > loss_text_ce: 0.03160705789923668 (0.029969700220317808)
|
| 2878 |
+
| > loss_mel_ce: 3.428128957748413 (3.6446200422411983)
|
| 2879 |
+
| > loss: 0.05405837669968605 (0.05741546468956065)
|
| 2880 |
+
| > current_lr: 1.999998232354375e-05
|
| 2881 |
+
| > step_time: 0.266 (0.2777721417708725)
|
| 2882 |
+
| > loader_time: 0.0093 (0.008618845219625705)
|
| 2883 |
+
|
| 2884 |
+
|
| 2885 |
+
[1m --> TIME: 2025-12-25 22:41:35 -- STEP: 1078/2207 -- GLOBAL_STEP: 37700[0m
|
| 2886 |
+
| > loss_text_ce: 0.03297491744160652 (0.02996745951998986)
|
| 2887 |
+
| > loss_mel_ce: 3.8216848373413086 (3.6452721974402955)
|
| 2888 |
+
| > loss: 0.06022905930876732 (0.05742561961273647)
|
| 2889 |
+
| > current_lr: 1.999998232354375e-05
|
| 2890 |
+
| > step_time: 0.2958 (0.2777335730023638)
|
| 2891 |
+
| > loader_time: 0.0085 (0.008618534827718927)
|
| 2892 |
+
|
| 2893 |
+
|
| 2894 |
+
[1m --> TIME: 2025-12-25 22:41:45 -- STEP: 1103/2207 -- GLOBAL_STEP: 37725[0m
|
| 2895 |
+
| > loss_text_ce: 0.031031278893351555 (0.02996064620371148)
|
| 2896 |
+
| > loss_mel_ce: 3.7310454845428467 (3.644437807209366)
|
| 2897 |
+
| > loss: 0.05878245085477829 (0.05741247580009271)
|
| 2898 |
+
| > current_lr: 1.999998232354375e-05
|
| 2899 |
+
| > step_time: 0.2901 (0.27779128285612914)
|
| 2900 |
+
| > loader_time: 0.0088 (0.008617082294937921)
|
| 2901 |
+
|
| 2902 |
+
|
| 2903 |
+
[1m --> TIME: 2025-12-25 22:41:54 -- STEP: 1128/2207 -- GLOBAL_STEP: 37750[0m
|
| 2904 |
+
| > loss_text_ce: 0.029448486864566803 (0.029960699085516077)
|
| 2905 |
+
| > loss_mel_ce: 3.6037747859954834 (3.6446615334943693)
|
| 2906 |
+
| > loss: 0.05676911398768425 (0.057415972351499485)
|
| 2907 |
+
| > current_lr: 1.999998232354375e-05
|
| 2908 |
+
| > step_time: 0.2913 (0.27784293509544195)
|
| 2909 |
+
| > loader_time: 0.009 (0.008619362580860762)
|
| 2910 |
+
|
| 2911 |
+
|
| 2912 |
+
[1m --> TIME: 2025-12-25 22:42:04 -- STEP: 1153/2207 -- GLOBAL_STEP: 37775[0m
|
| 2913 |
+
| > loss_text_ce: 0.031543221324682236 (0.029968640774307515)
|
| 2914 |
+
| > loss_mel_ce: 3.5673203468322754 (3.643751137378625)
|
| 2915 |
+
| > loss: 0.05623224377632141 (0.05740187150738804)
|
| 2916 |
+
| > current_lr: 1.999998232354375e-05
|
| 2917 |
+
| > step_time: 0.2535 (0.277768083375532)
|
| 2918 |
+
| > loader_time: 0.0087 (0.008621754898574183)
|
| 2919 |
+
|
| 2920 |
+
|
| 2921 |
+
[1m --> TIME: 2025-12-25 22:42:13 -- STEP: 1178/2207 -- GLOBAL_STEP: 37800[0m
|
| 2922 |
+
| > loss_text_ce: 0.029509419575333595 (0.029962012031967855)
|
| 2923 |
+
| > loss_mel_ce: 3.6230149269104004 (3.6454181588161596)
|
| 2924 |
+
| > loss: 0.057070691138505936 (0.057427815141089904)
|
| 2925 |
+
| > current_lr: 1.999998232354375e-05
|
| 2926 |
+
| > step_time: 0.3203 (0.2778549844016446)
|
| 2927 |
+
| > loader_time: 0.0082 (0.008618397097433774)
|
| 2928 |
+
|
| 2929 |
+
|
| 2930 |
+
[1m --> TIME: 2025-12-25 22:42:23 -- STEP: 1203/2207 -- GLOBAL_STEP: 37825[0m
|
| 2931 |
+
| > loss_text_ce: 0.029262254014611244 (0.02996188421684905)
|
| 2932 |
+
| > loss_mel_ce: 3.571575164794922 (3.6449717343695855)
|
| 2933 |
+
| > loss: 0.05626308545470238 (0.0574208377602689)
|
| 2934 |
+
| > current_lr: 1.999998232354375e-05
|
| 2935 |
+
| > step_time: 0.267 (0.2778606819095754)
|
| 2936 |
+
| > loader_time: 0.0083 (0.008618712722511962)
|
| 2937 |
+
|
| 2938 |
+
|
| 2939 |
+
[1m --> TIME: 2025-12-25 22:42:33 -- STEP: 1228/2207 -- GLOBAL_STEP: 37850[0m
|
| 2940 |
+
| > loss_text_ce: 0.029146136716008186 (0.029966183125074795)
|
| 2941 |
+
| > loss_mel_ce: 3.8141956329345703 (3.644844881098124)
|
| 2942 |
+
| > loss: 0.06005221605300903 (0.05741892284819863)
|
| 2943 |
+
| > current_lr: 1.999998232354375e-05
|
| 2944 |
+
| > step_time: 0.2862 (0.27783506953366793)
|
| 2945 |
+
| > loader_time: 0.0082 (0.008616887591172514)
|
| 2946 |
+
|
| 2947 |
+
|
| 2948 |
+
[1m --> TIME: 2025-12-25 22:42:42 -- STEP: 1253/2207 -- GLOBAL_STEP: 37875[0m
|
| 2949 |
+
| > loss_text_ce: 0.027041716501116753 (0.029961506726855006)
|
| 2950 |
+
| > loss_mel_ce: 3.4550020694732666 (3.6436915842895035)
|
| 2951 |
+
| > loss: 0.05440693348646164 (0.05740082951623825)
|
| 2952 |
+
| > current_lr: 1.999998232354375e-05
|
| 2953 |
+
| > step_time: 0.2869 (0.27790803939746267)
|
| 2954 |
+
| > loader_time: 0.0086 (0.008618124751024797)
|
| 2955 |
+
|
| 2956 |
+
|
| 2957 |
+
[1m --> TIME: 2025-12-25 22:42:52 -- STEP: 1278/2207 -- GLOBAL_STEP: 37900[0m
|
| 2958 |
+
| > loss_text_ce: 0.02990317903459072 (0.029951240548190657)
|
| 2959 |
+
| > loss_mel_ce: 3.859337568283081 (3.6439061856978765)
|
| 2960 |
+
| > loss: 0.06076938658952713 (0.05740402224991056)
|
| 2961 |
+
| > current_lr: 1.999998232354375e-05
|
| 2962 |
+
| > step_time: 0.2663 (0.2779029520091503)
|
| 2963 |
+
| > loader_time: 0.0078 (0.008616868132530072)
|
| 2964 |
+
|
| 2965 |
+
|
| 2966 |
+
[1m --> TIME: 2025-12-25 22:43:02 -- STEP: 1303/2207 -- GLOBAL_STEP: 37925[0m
|
| 2967 |
+
| > loss_text_ce: 0.02820732444524765 (0.029937458584115128)
|
| 2968 |
+
| > loss_mel_ce: 3.499760866165161 (3.6441128797377442)
|
| 2969 |
+
| > loss: 0.05512450262904167 (0.057407036498017434)
|
| 2970 |
+
| > current_lr: 1.999998232354375e-05
|
| 2971 |
+
| > step_time: 0.2679 (0.27799866881996327)
|
| 2972 |
+
| > loader_time: 0.009 (0.008615810872954047)
|
| 2973 |
+
|
| 2974 |
+
|
| 2975 |
+
[1m --> TIME: 2025-12-25 22:43:11 -- STEP: 1328/2207 -- GLOBAL_STEP: 37950[0m
|
| 2976 |
+
| > loss_text_ce: 0.030363403260707855 (0.02993688016641795)
|
| 2977 |
+
| > loss_mel_ce: 3.7565155029296875 (3.642705530825869)
|
| 2978 |
+
| > loss: 0.05916998162865639 (0.057385037633511854)
|
| 2979 |
+
| > current_lr: 1.999998232354375e-05
|
| 2980 |
+
| > step_time: 0.2633 (0.27797997536429425)
|
| 2981 |
+
| > loader_time: 0.0085 (0.008615866303443917)
|
| 2982 |
+
|
| 2983 |
+
|
| 2984 |
+
[1m --> TIME: 2025-12-25 22:43:21 -- STEP: 1353/2207 -- GLOBAL_STEP: 37975[0m
|
| 2985 |
+
| > loss_text_ce: 0.028071854263544083 (0.029932902425153115)
|
| 2986 |
+
| > loss_mel_ce: 3.5915145874023438 (3.6415943819244805)
|
| 2987 |
+
| > loss: 0.056556038558483124 (0.05736761378552503)
|
| 2988 |
+
| > current_lr: 1.999998232354375e-05
|
| 2989 |
+
| > step_time: 0.2906 (0.27801203903936944)
|
| 2990 |
+
| > loader_time: 0.0084 (0.008614424326996056)
|
| 2991 |
+
|
| 2992 |
+
|
| 2993 |
+
[1m --> TIME: 2025-12-25 22:43:30 -- STEP: 1378/2207 -- GLOBAL_STEP: 38000[0m
|
| 2994 |
+
| > loss_text_ce: 0.030603187158703804 (0.029933353023622482)
|
| 2995 |
+
| > loss_mel_ce: 3.421797752380371 (3.6417505779183323)
|
| 2996 |
+
| > loss: 0.05394376441836357 (0.05737006139275125)
|
| 2997 |
+
| > current_lr: 1.999998232354375e-05
|
| 2998 |
+
| > step_time: 0.2655 (0.2780031881069413)
|
| 2999 |
+
| > loader_time: 0.0082 (0.008609537462364607)
|
| 3000 |
+
|
| 3001 |
+
|
| 3002 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_38000.pth
|
| 3003 |
+
|
| 3004 |
+
[1m --> TIME: 2025-12-25 22:44:05 -- STEP: 1403/2207 -- GLOBAL_STEP: 38025[0m
|
| 3005 |
+
| > loss_text_ce: 0.029409460723400116 (0.029932489981098168)
|
| 3006 |
+
| > loss_mel_ce: 3.524285078048706 (3.6420105356704497)
|
| 3007 |
+
| > loss: 0.055526476353406906 (0.05737410975369231)
|
| 3008 |
+
| > current_lr: 1.999998232354375e-05
|
| 3009 |
+
| > step_time: 0.3004 (0.27802305687179113)
|
| 3010 |
+
| > loader_time: 0.0078 (0.008608610393825974)
|
| 3011 |
+
|
| 3012 |
+
|
| 3013 |
+
[1m --> TIME: 2025-12-25 22:44:15 -- STEP: 1428/2207 -- GLOBAL_STEP: 38050[0m
|
| 3014 |
+
| > loss_text_ce: 0.031176967546343803 (0.02992752044271229)
|
| 3015 |
+
| > loss_mel_ce: 3.6474177837371826 (3.641771961660947)
|
| 3016 |
+
| > loss: 0.05747804418206215 (0.05737030438828368)
|
| 3017 |
+
| > current_lr: 1.999998232354375e-05
|
| 3018 |
+
| > step_time: 0.2873 (0.27815087703095803)
|
| 3019 |
+
| > loader_time: 0.0085 (0.008607626629142875)
|
| 3020 |
+
|
| 3021 |
+
|
| 3022 |
+
[1m --> TIME: 2025-12-25 22:44:24 -- STEP: 1453/2207 -- GLOBAL_STEP: 38075[0m
|
| 3023 |
+
| > loss_text_ce: 0.031893592327833176 (0.029921248606460633)
|
| 3024 |
+
| > loss_mel_ce: 3.9198994636535645 (3.6408126925240705)
|
| 3025 |
+
| > loss: 0.06174676492810249 (0.05735521780997412)
|
| 3026 |
+
| > current_lr: 1.999998232354375e-05
|
| 3027 |
+
| > step_time: 0.2906 (0.2781576319390631)
|
| 3028 |
+
| > loader_time: 0.008 (0.008612065502305899)
|
| 3029 |
+
|
| 3030 |
+
|
| 3031 |
+
[1m --> TIME: 2025-12-25 22:44:34 -- STEP: 1478/2207 -- GLOBAL_STEP: 38100[0m
|
| 3032 |
+
| > loss_text_ce: 0.02910993993282318 (0.02992083561255096)
|
| 3033 |
+
| > loss_mel_ce: 3.4966020584106445 (3.640296582117456)
|
| 3034 |
+
| > loss: 0.05508925020694733 (0.05734714712690517)
|
| 3035 |
+
| > current_lr: 1.999998232354375e-05
|
| 3036 |
+
| > step_time: 0.2668 (0.2782023540207757)
|
| 3037 |
+
| > loader_time: 0.0086 (0.008616878957322553)
|
| 3038 |
+
|
| 3039 |
+
|
| 3040 |
+
[1m --> TIME: 2025-12-25 22:44:43 -- STEP: 1503/2207 -- GLOBAL_STEP: 38125[0m
|
| 3041 |
+
| > loss_text_ce: 0.029753051698207855 (0.0299190978754463)
|
| 3042 |
+
| > loss_mel_ce: 3.6692380905151367 (3.6395860798582595)
|
| 3043 |
+
| > loss: 0.05779673531651497 (0.05733601837923982)
|
| 3044 |
+
| > current_lr: 1.999998232354375e-05
|
| 3045 |
+
| > step_time: 0.2904 (0.2781783658825234)
|
| 3046 |
+
| > loader_time: 0.0094 (0.008628880430362421)
|
| 3047 |
+
|
| 3048 |
+
|
| 3049 |
+
[1m --> TIME: 2025-12-25 22:44:53 -- STEP: 1528/2207 -- GLOBAL_STEP: 38150[0m
|
| 3050 |
+
| > loss_text_ce: 0.03184933215379715 (0.02992773857894134)
|
| 3051 |
+
| > loss_mel_ce: 3.628498077392578 (3.638540097244124)
|
| 3052 |
+
| > loss: 0.057192929089069366 (0.05731980991184087)
|
| 3053 |
+
| > current_lr: 1.999998232354375e-05
|
| 3054 |
+
| > step_time: 0.2901 (0.2783152515663527)
|
| 3055 |
+
| > loader_time: 0.0087 (0.008629494312546017)
|
| 3056 |
+
|
| 3057 |
+
|
| 3058 |
+
[1m --> TIME: 2025-12-25 22:45:03 -- STEP: 1553/2207 -- GLOBAL_STEP: 38175[0m
|
| 3059 |
+
| > loss_text_ce: 0.030513882637023926 (0.029926046663121646)
|
| 3060 |
+
| > loss_mel_ce: 3.6318204402923584 (3.637682948969138)
|
| 3061 |
+
| > loss: 0.05722397565841675 (0.05730639052675067)
|
| 3062 |
+
| > current_lr: 1.999998232354375e-05
|
| 3063 |
+
| > step_time: 0.29 (0.2783473657164664)
|
| 3064 |
+
| > loader_time: 0.0083 (0.008629536060694335)
|
| 3065 |
+
|
| 3066 |
+
|
| 3067 |
+
[1m --> TIME: 2025-12-25 22:45:13 -- STEP: 1578/2207 -- GLOBAL_STEP: 38200[0m
|
| 3068 |
+
| > loss_text_ce: 0.029088621959090233 (0.029927844978415694)
|
| 3069 |
+
| > loss_mel_ce: 4.183952331542969 (3.6371605727155862)
|
| 3070 |
+
| > loss: 0.06582876294851303 (0.057298256495096386)
|
| 3071 |
+
| > current_lr: 1.999998232354375e-05
|
| 3072 |
+
| > step_time: 0.2683 (0.27839019090050543)
|
| 3073 |
+
| > loader_time: 0.0083 (0.008629074568077643)
|
| 3074 |
+
|
| 3075 |
+
|
| 3076 |
+
[1m --> TIME: 2025-12-25 22:45:22 -- STEP: 1603/2207 -- GLOBAL_STEP: 38225[0m
|
| 3077 |
+
| > loss_text_ce: 0.031632233411073685 (0.029920333266695142)
|
| 3078 |
+
| > loss_mel_ce: 3.376102924346924 (3.63638518188272)
|
| 3079 |
+
| > loss: 0.05324586108326912 (0.05728602363813392)
|
| 3080 |
+
| > current_lr: 1.999998232354375e-05
|
| 3081 |
+
| > step_time: 0.2865 (0.278387498052434)
|
| 3082 |
+
| > loader_time: 0.0084 (0.00862577626352674)
|
| 3083 |
+
|
| 3084 |
+
|
| 3085 |
+
[1m --> TIME: 2025-12-25 22:45:32 -- STEP: 1628/2207 -- GLOBAL_STEP: 38250[0m
|
| 3086 |
+
| > loss_text_ce: 0.030939539894461632 (0.02991920680892299)
|
| 3087 |
+
| > loss_mel_ce: 3.553030014038086 (3.6359587989802455)
|
| 3088 |
+
| > loss: 0.055999524891376495 (0.057279343801465904)
|
| 3089 |
+
| > current_lr: 1.999998232354375e-05
|
| 3090 |
+
| > step_time: 0.266 (0.27839034297835313)
|
| 3091 |
+
| > loader_time: 0.0084 (0.008626182922860041)
|
| 3092 |
+
|
| 3093 |
+
|
| 3094 |
+
[1m --> TIME: 2025-12-25 22:45:41 -- STEP: 1653/2207 -- GLOBAL_STEP: 38275[0m
|
| 3095 |
+
| > loss_text_ce: 0.03499649092555046 (0.029920489666448242)
|
| 3096 |
+
| > loss_mel_ce: 3.760573387145996 (3.6355771054950408)
|
| 3097 |
+
| > loss: 0.05930577963590622 (0.057273399880346354)
|
| 3098 |
+
| > current_lr: 1.999998232354375e-05
|
| 3099 |
+
| > step_time: 0.4071 (0.2784371590946055)
|
| 3100 |
+
| > loader_time: 0.0096 (0.008629374985830473)
|
| 3101 |
+
|
| 3102 |
+
|
| 3103 |
+
[1m --> TIME: 2025-12-25 22:45:51 -- STEP: 1678/2207 -- GLOBAL_STEP: 38300[0m
|
| 3104 |
+
| > loss_text_ce: 0.028202196583151817 (0.029923600529867787)
|
| 3105 |
+
| > loss_mel_ce: 3.4287939071655273 (3.6357076618469666)
|
| 3106 |
+
| > loss: 0.054015565663576126 (0.05727548843096493)
|
| 3107 |
+
| > current_lr: 1.999998232354375e-05
|
| 3108 |
+
| > step_time: 0.2871 (0.27842674010984947)
|
| 3109 |
+
| > loader_time: 0.0088 (0.008628455049517054)
|
| 3110 |
+
|
| 3111 |
+
|
| 3112 |
+
[1m --> TIME: 2025-12-25 22:46:01 -- STEP: 1703/2207 -- GLOBAL_STEP: 38325[0m
|
| 3113 |
+
| > loss_text_ce: 0.031443193554878235 (0.02991979562306117)
|
| 3114 |
+
| > loss_mel_ce: 3.6756880283355713 (3.6351234877872534)
|
| 3115 |
+
| > loss: 0.057923924177885056 (0.05726630125616973)
|
| 3116 |
+
| > current_lr: 1.999998232354375e-05
|
| 3117 |
+
| > step_time: 0.2963 (0.2784217677953588)
|
| 3118 |
+
| > loader_time: 0.0086 (0.008628461897128209)
|
| 3119 |
+
|
| 3120 |
+
|
| 3121 |
+
[1m --> TIME: 2025-12-25 22:46:10 -- STEP: 1728/2207 -- GLOBAL_STEP: 38350[0m
|
| 3122 |
+
| > loss_text_ce: 0.02784154564142227 (0.02991638615774853)
|
| 3123 |
+
| > loss_mel_ce: 3.4615559577941895 (3.634835919710222)
|
| 3124 |
+
| > loss: 0.0545218363404274 (0.057261754732785955)
|
| 3125 |
+
| > current_lr: 1.999998232354375e-05
|
| 3126 |
+
| > step_time: 0.2612 (0.27838417473766514)
|
| 3127 |
+
| > loader_time: 0.0085 (0.008628837626289444)
|
| 3128 |
+
|
| 3129 |
+
|
| 3130 |
+
[1m --> TIME: 2025-12-25 22:46:20 -- STEP: 1753/2207 -- GLOBAL_STEP: 38375[0m
|
| 3131 |
+
| > loss_text_ce: 0.02997463010251522 (0.029917057400651134)
|
| 3132 |
+
| > loss_mel_ce: 3.3990094661712646 (3.634106567908341)
|
| 3133 |
+
| > loss: 0.05357787758111954 (0.05725036910217079)
|
| 3134 |
+
| > current_lr: 1.999998232354375e-05
|
| 3135 |
+
| > step_time: 0.2665 (0.27843388447541173)
|
| 3136 |
+
| > loader_time: 0.0082 (0.00862957515923283)
|
| 3137 |
+
|
| 3138 |
+
|
| 3139 |
+
[1m --> TIME: 2025-12-25 22:46:29 -- STEP: 1778/2207 -- GLOBAL_STEP: 38400[0m
|
| 3140 |
+
| > loss_text_ce: 0.0297872181981802 (0.029912889170323167)
|
| 3141 |
+
| > loss_mel_ce: 3.3810229301452637 (3.633499904746281)
|
| 3142 |
+
| > loss: 0.05329390987753868 (0.05724082485729717)
|
| 3143 |
+
| > current_lr: 1.999998232354375e-05
|
| 3144 |
+
| > step_time: 0.2665 (0.27841525726833255)
|
| 3145 |
+
| > loader_time: 0.0083 (0.008633178050123815)
|
| 3146 |
+
|
| 3147 |
+
|
| 3148 |
+
[1m --> TIME: 2025-12-25 22:46:39 -- STEP: 1803/2207 -- GLOBAL_STEP: 38425[0m
|
| 3149 |
+
| > loss_text_ce: 0.029177898541092873 (0.029909475220453056)
|
| 3150 |
+
| > loss_mel_ce: 3.5290417671203613 (3.633219836092763)
|
| 3151 |
+
| > loss: 0.05559718236327171 (0.057236395437421234)
|
| 3152 |
+
| > current_lr: 1.999998232354375e-05
|
| 3153 |
+
| > step_time: 0.2659 (0.2783339787904776)
|
| 3154 |
+
| > loader_time: 0.0081 (0.008635927952466527)
|
| 3155 |
+
|
| 3156 |
+
|
| 3157 |
+
[1m --> TIME: 2025-12-25 22:46:48 -- STEP: 1828/2207 -- GLOBAL_STEP: 38450[0m
|
| 3158 |
+
| > loss_text_ce: 0.030047830194234848 (0.029908699380881937)
|
| 3159 |
+
| > loss_mel_ce: 3.7095959186553955 (3.6330161604630877)
|
| 3160 |
+
| > loss: 0.05843193456530571 (0.05723320088538808)
|
| 3161 |
+
| > current_lr: 1.999998232354375e-05
|
| 3162 |
+
| > step_time: 0.2661 (0.27836776536194435)
|
| 3163 |
+
| > loader_time: 0.0087 (0.008636866408834324)
|
| 3164 |
+
|
| 3165 |
+
|
| 3166 |
+
[1m --> TIME: 2025-12-25 22:46:58 -- STEP: 1853/2207 -- GLOBAL_STEP: 38475[0m
|
| 3167 |
+
| > loss_text_ce: 0.032452292740345 (0.02990591950228069)
|
| 3168 |
+
| > loss_mel_ce: 3.6003735065460205 (3.6322219412968866)
|
| 3169 |
+
| > loss: 0.056762903928756714 (0.057220747777805156)
|
| 3170 |
+
| > current_lr: 1.999998232354375e-05
|
| 3171 |
+
| > step_time: 0.2693 (0.2783444079591078)
|
| 3172 |
+
| > loader_time: 0.0097 (0.00863710443071592)
|
| 3173 |
+
|
| 3174 |
+
|
| 3175 |
+
[1m --> TIME: 2025-12-25 22:47:07 -- STEP: 1878/2207 -- GLOBAL_STEP: 38500[0m
|
| 3176 |
+
| > loss_text_ce: 0.03078889101743698 (0.029902709443697405)
|
| 3177 |
+
| > loss_mel_ce: 3.5984854698181152 (3.6323180357361258)
|
| 3178 |
+
| > loss: 0.056707412004470825 (0.0572221990996085)
|
| 3179 |
+
| > current_lr: 1.999998232354375e-05
|
| 3180 |
+
| > step_time: 0.2869 (0.27833991944472497)
|
| 3181 |
+
| > loader_time: 0.0088 (0.008636435126852061)
|
| 3182 |
+
|
| 3183 |
+
|
| 3184 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_38500.pth
|
| 3185 |
+
|
| 3186 |
+
[1m --> TIME: 2025-12-25 22:47:42 -- STEP: 1903/2207 -- GLOBAL_STEP: 38525[0m
|
| 3187 |
+
| > loss_text_ce: 0.031191864982247353 (0.029901884949393668)
|
| 3188 |
+
| > loss_mel_ce: 3.4433155059814453 (3.6309039218138857)
|
| 3189 |
+
| > loss: 0.05428917706012726 (0.05720009068071184)
|
| 3190 |
+
| > current_lr: 1.999998232354375e-05
|
| 3191 |
+
| > step_time: 0.2666 (0.27832018834192745)
|
| 3192 |
+
| > loader_time: 0.0104 (0.008653416986911482)
|
| 3193 |
+
|
| 3194 |
+
|
| 3195 |
+
[1m --> TIME: 2025-12-25 22:47:51 -- STEP: 1928/2207 -- GLOBAL_STEP: 38550[0m
|
| 3196 |
+
| > loss_text_ce: 0.029839415103197098 (0.029897324268462686)
|
| 3197 |
+
| > loss_mel_ce: 3.531874656677246 (3.6298724151993205)
|
| 3198 |
+
| > loss: 0.05565178394317627 (0.057183902132947044)
|
| 3199 |
+
| > current_lr: 1.999998232354375e-05
|
| 3200 |
+
| > step_time: 0.2868 (0.2783145719049382)
|
| 3201 |
+
| > loader_time: 0.0083 (0.008661776533759987)
|
| 3202 |
+
|
| 3203 |
+
|
| 3204 |
+
[1m --> TIME: 2025-12-25 22:48:01 -- STEP: 1953/2207 -- GLOBAL_STEP: 38575[0m
|
| 3205 |
+
| > loss_text_ce: 0.03108958899974823 (0.02989386691708505)
|
| 3206 |
+
| > loss_mel_ce: 3.477349042892456 (3.6292283858510705)
|
| 3207 |
+
| > loss: 0.05481935292482376 (0.05717378514649559)
|
| 3208 |
+
| > current_lr: 1.999998232354375e-05
|
| 3209 |
+
| > step_time: 0.2866 (0.2783358122712822)
|
| 3210 |
+
| > loader_time: 0.0091 (0.008665205452055192)
|
| 3211 |
+
|
| 3212 |
+
|
| 3213 |
+
[1m --> TIME: 2025-12-25 22:48:11 -- STEP: 1978/2207 -- GLOBAL_STEP: 38600[0m
|
| 3214 |
+
| > loss_text_ce: 0.028922077268362045 (0.029890917737839254)
|
| 3215 |
+
| > loss_mel_ce: 3.5860700607299805 (3.628273621246717)
|
| 3216 |
+
| > loss: 0.05648425221443176 (0.057158820867854856)
|
| 3217 |
+
| > current_lr: 1.999998232354375e-05
|
| 3218 |
+
| > step_time: 0.2892 (0.27834809706353314)
|
| 3219 |
+
| > loader_time: 0.0078 (0.008665535760962455)
|
| 3220 |
+
|
| 3221 |
+
|
| 3222 |
+
[1m --> TIME: 2025-12-25 22:48:20 -- STEP: 2003/2207 -- GLOBAL_STEP: 38625[0m
|
| 3223 |
+
| > loss_text_ce: 0.02855519764125347 (0.029891458752412114)
|
| 3224 |
+
| > loss_mel_ce: 3.67592191696167 (3.629334512021146)
|
| 3225 |
+
| > loss: 0.05788245424628258 (0.05717540573453344)
|
| 3226 |
+
| > current_lr: 1.999998232354375e-05
|
| 3227 |
+
| > step_time: 0.2966 (0.2783079315172215)
|
| 3228 |
+
| > loader_time: 0.0084 (0.008666753530859422)
|
| 3229 |
+
|
| 3230 |
+
|
| 3231 |
+
[1m --> TIME: 2025-12-25 22:48:30 -- STEP: 2028/2207 -- GLOBAL_STEP: 38650[0m
|
| 3232 |
+
| > loss_text_ce: 0.02967739664018154 (0.029890761107272474)
|
| 3233 |
+
| > loss_mel_ce: 3.6050052642822266 (3.6285824786276515)
|
| 3234 |
+
| > loss: 0.056791916489601135 (0.05716364431567857)
|
| 3235 |
+
| > current_lr: 1.999998232354375e-05
|
| 3236 |
+
| > step_time: 0.2669 (0.2782758903456392)
|
| 3237 |
+
| > loader_time: 0.0079 (0.008665225209569095)
|
| 3238 |
+
|
| 3239 |
+
|
| 3240 |
+
[1m --> TIME: 2025-12-25 22:48:39 -- STEP: 2053/2207 -- GLOBAL_STEP: 38675[0m
|
| 3241 |
+
| > loss_text_ce: 0.03009367547929287 (0.029886923594281432)
|
| 3242 |
+
| > loss_mel_ce: 3.9077799320220947 (3.6282956396958572)
|
| 3243 |
+
| > loss: 0.06152927502989769 (0.05715910249507201)
|
| 3244 |
+
| > current_lr: 1.999998232354375e-05
|
| 3245 |
+
| > step_time: 0.2904 (0.27830021984102193)
|
| 3246 |
+
| > loader_time: 0.0086 (0.008662565127269163)
|
| 3247 |
+
|
| 3248 |
+
|
| 3249 |
+
[1m --> TIME: 2025-12-25 22:48:49 -- STEP: 2078/2207 -- GLOBAL_STEP: 38700[0m
|
| 3250 |
+
| > loss_text_ce: 0.03063501976430416 (0.029886219531290514)
|
| 3251 |
+
| > loss_mel_ce: 3.650606155395508 (3.6281097347169577)
|
| 3252 |
+
| > loss: 0.057519394904375076 (0.05715618673012851)
|
| 3253 |
+
| > current_lr: 1.999998232354375e-05
|
| 3254 |
+
| > step_time: 0.2663 (0.27834473411203914)
|
| 3255 |
+
| > loader_time: 0.0088 (0.008660224909043529)
|
| 3256 |
+
|
| 3257 |
+
|
| 3258 |
+
[1m --> TIME: 2025-12-25 22:48:59 -- STEP: 2103/2207 -- GLOBAL_STEP: 38725[0m
|
| 3259 |
+
| > loss_text_ce: 0.03134392946958542 (0.029887454237928858)
|
| 3260 |
+
| > loss_mel_ce: 3.681150436401367 (3.6269207645811705)
|
| 3261 |
+
| > loss: 0.058007724583148956 (0.05713762836800946)
|
| 3262 |
+
| > current_lr: 1.999998232354375e-05
|
| 3263 |
+
| > step_time: 0.2687 (0.2783359859990778)
|
| 3264 |
+
| > loader_time: 0.0089 (0.0086609579186977)
|
| 3265 |
+
|
| 3266 |
+
|
| 3267 |
+
[1m --> TIME: 2025-12-25 22:49:08 -- STEP: 2128/2207 -- GLOBAL_STEP: 38750[0m
|
| 3268 |
+
| > loss_text_ce: 0.029510390013456345 (0.029885379081115798)
|
| 3269 |
+
| > loss_mel_ce: 3.355654716491699 (3.626512399853621)
|
| 3270 |
+
| > loss: 0.05289320647716522 (0.05713121524724858)
|
| 3271 |
+
| > current_lr: 1.999998232354375e-05
|
| 3272 |
+
| > step_time: 0.265 (0.27830955027637605)
|
| 3273 |
+
| > loader_time: 0.0081 (0.008658326434013546)
|
| 3274 |
+
|
| 3275 |
+
|
| 3276 |
+
[1m --> TIME: 2025-12-25 22:49:18 -- STEP: 2153/2207 -- GLOBAL_STEP: 38775[0m
|
| 3277 |
+
| > loss_text_ce: 0.030590606853365898 (0.029887776393607724)
|
| 3278 |
+
| > loss_mel_ce: 3.327104091644287 (3.6258785172058046)
|
| 3279 |
+
| > loss: 0.05246397852897644 (0.05712134828656927)
|
| 3280 |
+
| > current_lr: 1.999998232354375e-05
|
| 3281 |
+
| > step_time: 0.2869 (0.27829763637937815)
|
| 3282 |
+
| > loader_time: 0.008 (0.008654784558265648)
|
| 3283 |
+
|
| 3284 |
+
|
| 3285 |
+
[1m --> TIME: 2025-12-25 22:49:27 -- STEP: 2178/2207 -- GLOBAL_STEP: 38800[0m
|
| 3286 |
+
| > loss_text_ce: 0.02868533693253994 (0.029882631433103147)
|
| 3287 |
+
| > loss_mel_ce: 3.6160542964935303 (3.62483569183297)
|
| 3288 |
+
| > loss: 0.056949056684970856 (0.057104973746780086)
|
| 3289 |
+
| > current_lr: 1.999998232354375e-05
|
| 3290 |
+
| > step_time: 0.3033 (0.2782976534303114)
|
| 3291 |
+
| > loader_time: 0.008 (0.00865215057621756)
|
| 3292 |
+
|
| 3293 |
+
|
| 3294 |
+
[1m --> TIME: 2025-12-25 22:49:37 -- STEP: 2203/2207 -- GLOBAL_STEP: 38825[0m
|
| 3295 |
+
| > loss_text_ce: 0.02815687283873558 (0.029882760663507098)
|
| 3296 |
+
| > loss_mel_ce: 3.498441219329834 (3.6245897400665976)
|
| 3297 |
+
| > loss: 0.05510309338569641 (0.05710113277008627)
|
| 3298 |
+
| > current_lr: 1.999998232354375e-05
|
| 3299 |
+
| > step_time: 0.2669 (0.2784035848694605)
|
| 3300 |
+
| > loader_time: 0.0087 (0.008649310143601072)
|
| 3301 |
+
|
| 3302 |
+
|
| 3303 |
+
[1m > EVALUATION [0m
|
| 3304 |
+
|
| 3305 |
+
|
| 3306 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 3307 |
+
| > avg_loader_time:[92m 0.08029967868650281 [0m(-0.0012138824205140814)
|
| 3308 |
+
| > avg_loss_text_ce:[92m 0.02885246682106643 [0m(-0.0001361244861539969)
|
| 3309 |
+
| > avg_loss_mel_ce:[92m 3.730661709566374 [0m(-0.0053267140646240385)
|
| 3310 |
+
| > avg_loss:[92m 3.7595141610583744 [0m(-0.005462854295163577)
|
| 3311 |
+
|
| 3312 |
+
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_38829.pth
|