Checkpoint at step 34415
Browse files- best_model.pth +2 -2
- best_model_34415.pth +3 -0
- checkpoint_33500.pth +3 -0
- checkpoint_34000.pth +3 -0
- events.out.tfevents.1766698157.0f9434978106.1562.0 +2 -2
- trainer_0_log.txt +817 -0
best_model.pth
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1d78030cb63a7760cb151c079dedf1d3cb1438c118e2a50dfe2f41c7975b2611
|
| 3 |
+
size 5607926805
|
best_model_34415.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1d78030cb63a7760cb151c079dedf1d3cb1438c118e2a50dfe2f41c7975b2611
|
| 3 |
+
size 5607926805
|
checkpoint_33500.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:af563ef1ae72154c60bd42b37e818692670eaecf769c9e4221462fd9363d5ca2
|
| 3 |
+
size 5607926741
|
checkpoint_34000.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:86c8832760a1728b3b6fe6318f082167aa64224d515dca0d1c4561595fc1694a
|
| 3 |
+
size 5607926741
|
events.out.tfevents.1766698157.0f9434978106.1562.0
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3828e86ca6e7b3a3212a3bbd5e9e1047bead839a9d2447442835d7aaac918eeb
|
| 3 |
+
size 24948
|
trainer_0_log.txt
CHANGED
|
@@ -848,3 +848,820 @@
|
|
| 848 |
| > avg_loss_mel_ce:[91m 3.761217610256092 [0m(+0.0002901312467216144)
|
| 849 |
| > avg_loss:[91m 3.7905127615542025 [0m(+0.00014067340541545903)
|
| 850 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 848 |
| > avg_loss_mel_ce:[91m 3.761217610256092 [0m(+0.0002901312467216144)
|
| 849 |
| > avg_loss:[91m 3.7905127615542025 [0m(+0.00014067340541545903)
|
| 850 |
|
| 851 |
+
|
| 852 |
+
[4m[1m > EPOCH: 2/12[0m
|
| 853 |
+
--> /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000
|
| 854 |
+
|
| 855 |
+
[1m > TRAINING (2025-12-25 21:52:25) [0m
|
| 856 |
+
|
| 857 |
+
[1m --> TIME: 2025-12-25 21:52:35 -- STEP: 17/2207 -- GLOBAL_STEP: 32225[0m
|
| 858 |
+
| > loss_text_ce: 0.030644629150629044 (0.03074425502735026)
|
| 859 |
+
| > loss_mel_ce: 3.6692090034484863 (3.6377469231100643)
|
| 860 |
+
| > loss: 0.057810213416814804 (0.0573201744871981)
|
| 861 |
+
| > current_lr: 1.999999803594879e-05
|
| 862 |
+
| > step_time: 0.2682 (0.27825629009920005)
|
| 863 |
+
| > loader_time: 0.0088 (0.009260149563060087)
|
| 864 |
+
|
| 865 |
+
|
| 866 |
+
[1m --> TIME: 2025-12-25 21:52:44 -- STEP: 42/2207 -- GLOBAL_STEP: 32250[0m
|
| 867 |
+
| > loss_text_ce: 0.03318643197417259 (0.03060376573176611)
|
| 868 |
+
| > loss_mel_ce: 3.6696979999542236 (3.675323628243946)
|
| 869 |
+
| > loss: 0.05785756930708885 (0.057905115453260284)
|
| 870 |
+
| > current_lr: 1.999999803594879e-05
|
| 871 |
+
| > step_time: 0.2908 (0.278734218506586)
|
| 872 |
+
| > loader_time: 0.0082 (0.00873149008977981)
|
| 873 |
+
|
| 874 |
+
|
| 875 |
+
[1m --> TIME: 2025-12-25 21:52:54 -- STEP: 67/2207 -- GLOBAL_STEP: 32275[0m
|
| 876 |
+
| > loss_text_ce: 0.03000219352543354 (0.03057253863940488)
|
| 877 |
+
| > loss_mel_ce: 3.9564764499664307 (3.6588066549443488)
|
| 878 |
+
| > loss: 0.06228872761130333 (0.05764654990452439)
|
| 879 |
+
| > current_lr: 1.999999803594879e-05
|
| 880 |
+
| > step_time: 0.2669 (0.27801290554786806)
|
| 881 |
+
| > loader_time: 0.008 (0.008507515067484844)
|
| 882 |
+
|
| 883 |
+
|
| 884 |
+
[1m --> TIME: 2025-12-25 21:53:03 -- STEP: 92/2207 -- GLOBAL_STEP: 32300[0m
|
| 885 |
+
| > loss_text_ce: 0.031143223866820335 (0.03044124176401807)
|
| 886 |
+
| > loss_mel_ce: 3.664954900741577 (3.662341426248136)
|
| 887 |
+
| > loss: 0.05775153264403343 (0.05769972920255816)
|
| 888 |
+
| > current_lr: 1.999999803594879e-05
|
| 889 |
+
| > step_time: 0.2877 (0.2775989589483842)
|
| 890 |
+
| > loader_time: 0.0077 (0.008520113385241964)
|
| 891 |
+
|
| 892 |
+
|
| 893 |
+
[1m --> TIME: 2025-12-25 21:53:13 -- STEP: 117/2207 -- GLOBAL_STEP: 32325[0m
|
| 894 |
+
| > loss_text_ce: 0.028810717165470123 (0.030331756808182113)
|
| 895 |
+
| > loss_mel_ce: 3.445390462875366 (3.6500292965489574)
|
| 896 |
+
| > loss: 0.05428439378738403 (0.05750564138731386)
|
| 897 |
+
| > current_lr: 1.999999803594879e-05
|
| 898 |
+
| > step_time: 0.2673 (0.2776821291344798)
|
| 899 |
+
| > loader_time: 0.0091 (0.00850140335213425)
|
| 900 |
+
|
| 901 |
+
|
| 902 |
+
[1m --> TIME: 2025-12-25 21:53:23 -- STEP: 142/2207 -- GLOBAL_STEP: 32350[0m
|
| 903 |
+
| > loss_text_ce: 0.03158501163125038 (0.03032019944258139)
|
| 904 |
+
| > loss_mel_ce: 3.631765842437744 (3.6519497179649245)
|
| 905 |
+
| > loss: 0.05723985657095909 (0.057535467328320086)
|
| 906 |
+
| > current_lr: 1.999999803594879e-05
|
| 907 |
+
| > step_time: 0.2685 (0.2780482198151065)
|
| 908 |
+
| > loader_time: 0.0075 (0.008498346301871287)
|
| 909 |
+
|
| 910 |
+
|
| 911 |
+
[1m --> TIME: 2025-12-25 21:53:32 -- STEP: 167/2207 -- GLOBAL_STEP: 32375[0m
|
| 912 |
+
| > loss_text_ce: 0.03196166083216667 (0.030338440960395837)
|
| 913 |
+
| > loss_mel_ce: 3.5371744632720947 (3.6635501927244447)
|
| 914 |
+
| > loss: 0.05576775223016739 (0.057717009843466525)
|
| 915 |
+
| > current_lr: 1.999999803594879e-05
|
| 916 |
+
| > step_time: 0.266 (0.27810066902708885)
|
| 917 |
+
| > loader_time: 0.0092 (0.008604492256027493)
|
| 918 |
+
|
| 919 |
+
|
| 920 |
+
[1m --> TIME: 2025-12-25 21:53:42 -- STEP: 192/2207 -- GLOBAL_STEP: 32400[0m
|
| 921 |
+
| > loss_text_ce: 0.028228849172592163 (0.030311426089610904)
|
| 922 |
+
| > loss_mel_ce: 3.9837417602539062 (3.6649645753204823)
|
| 923 |
+
| > loss: 0.06268703937530518 (0.057738687455033265)
|
| 924 |
+
| > current_lr: 1.999999803594879e-05
|
| 925 |
+
| > step_time: 0.2596 (0.27817040557662664)
|
| 926 |
+
| > loader_time: 0.0082 (0.008639775216579439)
|
| 927 |
+
|
| 928 |
+
|
| 929 |
+
[1m --> TIME: 2025-12-25 21:53:51 -- STEP: 217/2207 -- GLOBAL_STEP: 32425[0m
|
| 930 |
+
| > loss_text_ce: 0.027301324531435966 (0.030269390555776758)
|
| 931 |
+
| > loss_mel_ce: 3.582484245300293 (3.666859741035145)
|
| 932 |
+
| > loss: 0.05640289932489395 (0.057767642584676565)
|
| 933 |
+
| > current_lr: 1.999999803594879e-05
|
| 934 |
+
| > step_time: 0.2664 (0.2779466853163762)
|
| 935 |
+
| > loader_time: 0.0087 (0.008641325383691738)
|
| 936 |
+
|
| 937 |
+
|
| 938 |
+
[1m --> TIME: 2025-12-25 21:54:01 -- STEP: 242/2207 -- GLOBAL_STEP: 32450[0m
|
| 939 |
+
| > loss_text_ce: 0.028960829600691795 (0.030276150334903523)
|
| 940 |
+
| > loss_mel_ce: 3.4629714488983154 (3.673465207588574)
|
| 941 |
+
| > loss: 0.054561443626880646 (0.05787095861617199)
|
| 942 |
+
| > current_lr: 1.999999803594879e-05
|
| 943 |
+
| > step_time: 0.2973 (0.27818893302570724)
|
| 944 |
+
| > loader_time: 0.0091 (0.008639584888111458)
|
| 945 |
+
|
| 946 |
+
|
| 947 |
+
[1m --> TIME: 2025-12-25 21:54:10 -- STEP: 267/2207 -- GLOBAL_STEP: 32475[0m
|
| 948 |
+
| > loss_text_ce: 0.032017726451158524 (0.030240296947766334)
|
| 949 |
+
| > loss_mel_ce: 3.511160135269165 (3.6729966469025346)
|
| 950 |
+
| > loss: 0.05536215379834175 (0.05786307721158092)
|
| 951 |
+
| > current_lr: 1.999999803594879e-05
|
| 952 |
+
| > step_time: 0.267 (0.2780476500479026)
|
| 953 |
+
| > loader_time: 0.0082 (0.008624990334671529)
|
| 954 |
+
|
| 955 |
+
|
| 956 |
+
[1m --> TIME: 2025-12-25 21:54:20 -- STEP: 292/2207 -- GLOBAL_STEP: 32500[0m
|
| 957 |
+
| > loss_text_ce: 0.028349461033940315 (0.03026523592894616)
|
| 958 |
+
| > loss_mel_ce: 3.4906275272369385 (3.675368253498861)
|
| 959 |
+
| > loss: 0.054984014481306076 (0.05790052323104584)
|
| 960 |
+
| > current_lr: 1.999999803594879e-05
|
| 961 |
+
| > step_time: 0.2916 (0.2778957412667473)
|
| 962 |
+
| > loader_time: 0.009 (0.008612889949589553)
|
| 963 |
+
|
| 964 |
+
|
| 965 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_32500.pth
|
| 966 |
+
|
| 967 |
+
[1m --> TIME: 2025-12-25 21:54:55 -- STEP: 317/2207 -- GLOBAL_STEP: 32525[0m
|
| 968 |
+
| > loss_text_ce: 0.03247431665658951 (0.030264701059658068)
|
| 969 |
+
| > loss_mel_ce: 3.3276169300079346 (3.668460987343773)
|
| 970 |
+
| > loss: 0.05250142514705658 (0.0577925888686338)
|
| 971 |
+
| > current_lr: 1.999999803594879e-05
|
| 972 |
+
| > step_time: 0.2879 (0.2783055794351863)
|
| 973 |
+
| > loader_time: 0.0083 (0.008673616764297237)
|
| 974 |
+
|
| 975 |
+
|
| 976 |
+
[1m --> TIME: 2025-12-25 21:55:05 -- STEP: 342/2207 -- GLOBAL_STEP: 32550[0m
|
| 977 |
+
| > loss_text_ce: 0.03027077578008175 (0.030276037883340268)
|
| 978 |
+
| > loss_mel_ce: 3.55334734916687 (3.6658302308523165)
|
| 979 |
+
| > loss: 0.05599403381347656 (0.057751660425847735)
|
| 980 |
+
| > current_lr: 1.999999803594879e-05
|
| 981 |
+
| > step_time: 0.2901 (0.2782544044026159)
|
| 982 |
+
| > loader_time: 0.0105 (0.00867633443129689)
|
| 983 |
+
|
| 984 |
+
|
| 985 |
+
[1m --> TIME: 2025-12-25 21:55:14 -- STEP: 367/2207 -- GLOBAL_STEP: 32575[0m
|
| 986 |
+
| > loss_text_ce: 0.031195439398288727 (0.030276423736351064)
|
| 987 |
+
| > loss_mel_ce: 3.5194814205169678 (3.664633897734598)
|
| 988 |
+
| > loss: 0.055479325354099274 (0.05773297375529923)
|
| 989 |
+
| > current_lr: 1.999999803594879e-05
|
| 990 |
+
| > step_time: 0.2904 (0.2782727587125608)
|
| 991 |
+
| > loader_time: 0.0084 (0.008660386628610881)
|
| 992 |
+
|
| 993 |
+
|
| 994 |
+
[1m --> TIME: 2025-12-25 21:55:24 -- STEP: 392/2207 -- GLOBAL_STEP: 32600[0m
|
| 995 |
+
| > loss_text_ce: 0.03301140293478966 (0.030292556302774012)
|
| 996 |
+
| > loss_mel_ce: 3.7577428817749023 (3.6674111789586594)
|
| 997 |
+
| > loss: 0.059230536222457886 (0.057776620839627424)
|
| 998 |
+
| > current_lr: 1.999999803594879e-05
|
| 999 |
+
| > step_time: 0.2628 (0.27805731491166724)
|
| 1000 |
+
| > loader_time: 0.0084 (0.008661697105485564)
|
| 1001 |
+
|
| 1002 |
+
|
| 1003 |
+
[1m --> TIME: 2025-12-25 21:55:34 -- STEP: 417/2207 -- GLOBAL_STEP: 32625[0m
|
| 1004 |
+
| > loss_text_ce: 0.028912879526615143 (0.03028874746043619)
|
| 1005 |
+
| > loss_mel_ce: 3.5075995922088623 (3.669082329999343)
|
| 1006 |
+
| > loss: 0.05525800585746765 (0.05780267307190872)
|
| 1007 |
+
| > current_lr: 1.999999803594879e-05
|
| 1008 |
+
| > step_time: 0.2705 (0.27841527799336446)
|
| 1009 |
+
| > loader_time: 0.0088 (0.008685234353410823)
|
| 1010 |
+
|
| 1011 |
+
|
| 1012 |
+
[1m --> TIME: 2025-12-25 21:55:43 -- STEP: 442/2207 -- GLOBAL_STEP: 32650[0m
|
| 1013 |
+
| > loss_text_ce: 0.030323388054966927 (0.03029301406908359)
|
| 1014 |
+
| > loss_mel_ce: 3.5136663913726807 (3.671982834242048)
|
| 1015 |
+
| > loss: 0.05537484213709831 (0.057848060115902135)
|
| 1016 |
+
| > current_lr: 1.999999803594879e-05
|
| 1017 |
+
| > step_time: 0.2568 (0.2782933992498061)
|
| 1018 |
+
| > loader_time: 0.012 (0.008703894744631388)
|
| 1019 |
+
|
| 1020 |
+
|
| 1021 |
+
[1m --> TIME: 2025-12-25 21:55:53 -- STEP: 467/2207 -- GLOBAL_STEP: 32675[0m
|
| 1022 |
+
| > loss_text_ce: 0.026899555698037148 (0.03028931097791588)
|
| 1023 |
+
| > loss_mel_ce: 4.234764575958252 (3.674705959234299)
|
| 1024 |
+
| > loss: 0.06658849865198135 (0.057890551080987084)
|
| 1025 |
+
| > current_lr: 1.999999803594879e-05
|
| 1026 |
+
| > step_time: 0.2914 (0.2780496456383128)
|
| 1027 |
+
| > loader_time: 0.0088 (0.008718427294582063)
|
| 1028 |
+
|
| 1029 |
+
|
| 1030 |
+
[1m --> TIME: 2025-12-25 21:56:02 -- STEP: 492/2207 -- GLOBAL_STEP: 32700[0m
|
| 1031 |
+
| > loss_text_ce: 0.03227531537413597 (0.030302050720324845)
|
| 1032 |
+
| > loss_mel_ce: 3.781071424484253 (3.675778325980272)
|
| 1033 |
+
| > loss: 0.05958354100584984 (0.05790750588464543)
|
| 1034 |
+
| > current_lr: 1.999999803594879e-05
|
| 1035 |
+
| > step_time: 0.2882 (0.27804501948317867)
|
| 1036 |
+
| > loader_time: 0.0089 (0.008712711372995762)
|
| 1037 |
+
|
| 1038 |
+
|
| 1039 |
+
[1m --> TIME: 2025-12-25 21:56:12 -- STEP: 517/2207 -- GLOBAL_STEP: 32725[0m
|
| 1040 |
+
| > loss_text_ce: 0.0300380177795887 (0.030319991836781888)
|
| 1041 |
+
| > loss_mel_ce: 3.7702386379241943 (3.675113140268529)
|
| 1042 |
+
| > loss: 0.059379324316978455 (0.05789739268199618)
|
| 1043 |
+
| > current_lr: 1.999999803594879e-05
|
| 1044 |
+
| > step_time: 0.2557 (0.2779747743440551)
|
| 1045 |
+
| > loader_time: 0.0085 (0.008707183472653651)
|
| 1046 |
+
|
| 1047 |
+
|
| 1048 |
+
[1m --> TIME: 2025-12-25 21:56:21 -- STEP: 542/2207 -- GLOBAL_STEP: 32750[0m
|
| 1049 |
+
| > loss_text_ce: 0.030097596347332 (0.030324528368389253)
|
| 1050 |
+
| > loss_mel_ce: 3.691117286682129 (3.674628809809245)
|
| 1051 |
+
| > loss: 0.0581439808011055 (0.05788989590542783)
|
| 1052 |
+
| > current_lr: 1.999999803594879e-05
|
| 1053 |
+
| > step_time: 0.2976 (0.27829050621863227)
|
| 1054 |
+
| > loader_time: 0.0085 (0.008698109771052847)
|
| 1055 |
+
|
| 1056 |
+
|
| 1057 |
+
[1m --> TIME: 2025-12-25 21:56:31 -- STEP: 567/2207 -- GLOBAL_STEP: 32775[0m
|
| 1058 |
+
| > loss_text_ce: 0.03361915424466133 (0.030339587384562224)
|
| 1059 |
+
| > loss_mel_ce: 3.7000818252563477 (3.6757885244998563)
|
| 1060 |
+
| > loss: 0.05833907797932625 (0.0579082517436257)
|
| 1061 |
+
| > current_lr: 1.999999803594879e-05
|
| 1062 |
+
| > step_time: 0.2867 (0.27828967760479634)
|
| 1063 |
+
| > loader_time: 0.0085 (0.008695398899192745)
|
| 1064 |
+
|
| 1065 |
+
|
| 1066 |
+
[1m --> TIME: 2025-12-25 21:56:40 -- STEP: 592/2207 -- GLOBAL_STEP: 32800[0m
|
| 1067 |
+
| > loss_text_ce: 0.028320519253611565 (0.03032306390751556)
|
| 1068 |
+
| > loss_mel_ce: 3.6912243366241455 (3.6763477075744317)
|
| 1069 |
+
| > loss: 0.05811788886785507 (0.05791673079963672)
|
| 1070 |
+
| > current_lr: 1.999999803594879e-05
|
| 1071 |
+
| > step_time: 0.2894 (0.27828324324375864)
|
| 1072 |
+
| > loader_time: 0.0083 (0.008699575388753737)
|
| 1073 |
+
|
| 1074 |
+
|
| 1075 |
+
[1m --> TIME: 2025-12-25 21:56:50 -- STEP: 617/2207 -- GLOBAL_STEP: 32825[0m
|
| 1076 |
+
| > loss_text_ce: 0.03405119478702545 (0.030333355691048468)
|
| 1077 |
+
| > loss_mel_ce: 3.8293676376342773 (3.675671305803271)
|
| 1078 |
+
| > loss: 0.06036591902375221 (0.057906322833301556)
|
| 1079 |
+
| > current_lr: 1.999999803594879e-05
|
| 1080 |
+
| > step_time: 0.2656 (0.2782943175329767)
|
| 1081 |
+
| > loader_time: 0.0087 (0.00870424074330631)
|
| 1082 |
+
|
| 1083 |
+
|
| 1084 |
+
[1m --> TIME: 2025-12-25 21:56:59 -- STEP: 642/2207 -- GLOBAL_STEP: 32850[0m
|
| 1085 |
+
| > loss_text_ce: 0.028031745925545692 (0.030321214452525166)
|
| 1086 |
+
| > loss_mel_ce: 3.541181802749634 (3.6752061461362513)
|
| 1087 |
+
| > loss: 0.05576896294951439 (0.05789886501463959)
|
| 1088 |
+
| > current_lr: 1.999999803594879e-05
|
| 1089 |
+
| > step_time: 0.2875 (0.2780870615136213)
|
| 1090 |
+
| > loader_time: 0.0078 (0.008692007569880505)
|
| 1091 |
+
|
| 1092 |
+
|
| 1093 |
+
[1m --> TIME: 2025-12-25 21:57:09 -- STEP: 667/2207 -- GLOBAL_STEP: 32875[0m
|
| 1094 |
+
| > loss_text_ce: 0.031378358602523804 (0.030300467502677637)
|
| 1095 |
+
| > loss_mel_ce: 3.7730464935302734 (3.6738344028793173)
|
| 1096 |
+
| > loss: 0.059444136917591095 (0.057877107348845984)
|
| 1097 |
+
| > current_lr: 1.999999803594879e-05
|
| 1098 |
+
| > step_time: 0.2664 (0.27820306560625)
|
| 1099 |
+
| > loader_time: 0.0086 (0.008698579968362378)
|
| 1100 |
+
|
| 1101 |
+
|
| 1102 |
+
[1m --> TIME: 2025-12-25 21:57:19 -- STEP: 692/2207 -- GLOBAL_STEP: 32900[0m
|
| 1103 |
+
| > loss_text_ce: 0.029866935685276985 (0.030291089666953493)
|
| 1104 |
+
| > loss_mel_ce: 3.6282331943511963 (3.6739197332045936)
|
| 1105 |
+
| > loss: 0.057157814502716064 (0.057878294087558804)
|
| 1106 |
+
| > current_lr: 1.999999803594879e-05
|
| 1107 |
+
| > step_time: 0.2913 (0.278215139932026)
|
| 1108 |
+
| > loader_time: 0.0096 (0.008691807013715623)
|
| 1109 |
+
|
| 1110 |
+
|
| 1111 |
+
[1m --> TIME: 2025-12-25 21:57:29 -- STEP: 717/2207 -- GLOBAL_STEP: 32925[0m
|
| 1112 |
+
| > loss_text_ce: 0.02960648387670517 (0.030291712864963245)
|
| 1113 |
+
| > loss_mel_ce: 3.7562386989593506 (3.67462095688909)
|
| 1114 |
+
| > loss: 0.059153832495212555 (0.05788926044517979)
|
| 1115 |
+
| > current_lr: 1.999999803594879e-05
|
| 1116 |
+
| > step_time: 0.291 (0.27857414615370385)
|
| 1117 |
+
| > loader_time: 0.008 (0.008683921236705913)
|
| 1118 |
+
|
| 1119 |
+
|
| 1120 |
+
[1m --> TIME: 2025-12-25 21:57:38 -- STEP: 742/2207 -- GLOBAL_STEP: 32950[0m
|
| 1121 |
+
| > loss_text_ce: 0.0325959138572216 (0.030291151385904967)
|
| 1122 |
+
| > loss_mel_ce: 3.4901907444000244 (3.6749090608560815)
|
| 1123 |
+
| > loss: 0.0550435408949852 (0.05789375328772633)
|
| 1124 |
+
| > current_lr: 1.999999803594879e-05
|
| 1125 |
+
| > step_time: 0.2663 (0.2787341795199001)
|
| 1126 |
+
| > loader_time: 0.0083 (0.00867371096443937)
|
| 1127 |
+
|
| 1128 |
+
|
| 1129 |
+
[1m --> TIME: 2025-12-25 21:57:48 -- STEP: 767/2207 -- GLOBAL_STEP: 32975[0m
|
| 1130 |
+
| > loss_text_ce: 0.030906755477190018 (0.030276066882261397)
|
| 1131 |
+
| > loss_mel_ce: 3.6728551387786865 (3.673777471154423)
|
| 1132 |
+
| > loss: 0.05787127837538719 (0.057875836508486996)
|
| 1133 |
+
| > grad_norm: 0 (0.0)
|
| 1134 |
+
| > current_lr: 1.999999803594879e-05
|
| 1135 |
+
| > step_time: 0.3103 (0.278835452675353)
|
| 1136 |
+
| > loader_time: 0.0083 (0.008658741723449786)
|
| 1137 |
+
|
| 1138 |
+
|
| 1139 |
+
[1m --> TIME: 2025-12-25 21:57:57 -- STEP: 792/2207 -- GLOBAL_STEP: 33000[0m
|
| 1140 |
+
| > loss_text_ce: 0.02748909592628479 (0.03027678717567463)
|
| 1141 |
+
| > loss_mel_ce: 3.8186678886413574 (3.674680231496541)
|
| 1142 |
+
| > loss: 0.06009620428085327 (0.0578899534014665)
|
| 1143 |
+
| > current_lr: 1.999999803594879e-05
|
| 1144 |
+
| > step_time: 0.2686 (0.2786936564276915)
|
| 1145 |
+
| > loader_time: 0.0082 (0.008654127217302424)
|
| 1146 |
+
|
| 1147 |
+
|
| 1148 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_33000.pth
|
| 1149 |
+
|
| 1150 |
+
[1m --> TIME: 2025-12-25 21:58:33 -- STEP: 817/2207 -- GLOBAL_STEP: 33025[0m
|
| 1151 |
+
| > loss_text_ce: 0.030527416616678238 (0.03026729809391601)
|
| 1152 |
+
| > loss_mel_ce: 3.5704305171966553 (3.6735081704768877)
|
| 1153 |
+
| > loss: 0.0562649667263031 (0.05787149168359459)
|
| 1154 |
+
| > current_lr: 1.999999803594879e-05
|
| 1155 |
+
| > step_time: 0.2881 (0.27876620105897465)
|
| 1156 |
+
| > loader_time: 0.0088 (0.008675259796758907)
|
| 1157 |
+
|
| 1158 |
+
|
| 1159 |
+
[1m --> TIME: 2025-12-25 21:58:43 -- STEP: 842/2207 -- GLOBAL_STEP: 33050[0m
|
| 1160 |
+
| > loss_text_ce: 0.02966105192899704 (0.030273577207234864)
|
| 1161 |
+
| > loss_mel_ce: 3.6885597705841064 (3.6730167721342872)
|
| 1162 |
+
| > loss: 0.058097198605537415 (0.057863911680187)
|
| 1163 |
+
| > current_lr: 1.999999803594879e-05
|
| 1164 |
+
| > step_time: 0.2693 (0.2787655634438338)
|
| 1165 |
+
| > loader_time: 0.0079 (0.008677821544456931)
|
| 1166 |
+
|
| 1167 |
+
|
| 1168 |
+
[1m --> TIME: 2025-12-25 21:58:52 -- STEP: 867/2207 -- GLOBAL_STEP: 33075[0m
|
| 1169 |
+
| > loss_text_ce: 0.028825702145695686 (0.030258222198438205)
|
| 1170 |
+
| > loss_mel_ce: 3.6814537048339844 (3.671727597094737)
|
| 1171 |
+
| > loss: 0.057973116636276245 (0.05784352839577431)
|
| 1172 |
+
| > current_lr: 1.999999803594879e-05
|
| 1173 |
+
| > step_time: 0.2685 (0.27877341504740444)
|
| 1174 |
+
| > loader_time: 0.0081 (0.008671017804085314)
|
| 1175 |
+
|
| 1176 |
+
|
| 1177 |
+
[1m --> TIME: 2025-12-25 21:59:02 -- STEP: 892/2207 -- GLOBAL_STEP: 33100[0m
|
| 1178 |
+
| > loss_text_ce: 0.030576463788747787 (0.030256132710090266)
|
| 1179 |
+
| > loss_mel_ce: 3.570002317428589 (3.671212342020642)
|
| 1180 |
+
| > loss: 0.056259043514728546 (0.05783544487482763)
|
| 1181 |
+
| > current_lr: 1.999999803594879e-05
|
| 1182 |
+
| > step_time: 0.2668 (0.2791920788084978)
|
| 1183 |
+
| > loader_time: 0.0082 (0.008667366387063608)
|
| 1184 |
+
|
| 1185 |
+
|
| 1186 |
+
[1m --> TIME: 2025-12-25 21:59:12 -- STEP: 917/2207 -- GLOBAL_STEP: 33125[0m
|
| 1187 |
+
| > loss_text_ce: 0.029416024684906006 (0.030255095102908566)
|
| 1188 |
+
| > loss_mel_ce: 3.507622241973877 (3.6714974981097903)
|
| 1189 |
+
| > loss: 0.05526622384786606 (0.05783988422873625)
|
| 1190 |
+
| > current_lr: 1.999999803594879e-05
|
| 1191 |
+
| > step_time: 0.2863 (0.2792648502108574)
|
| 1192 |
+
| > loader_time: 0.0083 (0.008657365224873615)
|
| 1193 |
+
|
| 1194 |
+
|
| 1195 |
+
[1m --> TIME: 2025-12-25 21:59:21 -- STEP: 942/2207 -- GLOBAL_STEP: 33150[0m
|
| 1196 |
+
| > loss_text_ce: 0.03284243866801262 (0.030242854927027808)
|
| 1197 |
+
| > loss_mel_ce: 3.6587798595428467 (3.6697902282079062)
|
| 1198 |
+
| > loss: 0.057681597769260406 (0.05781301688363344)
|
| 1199 |
+
| > current_lr: 1.999999803594879e-05
|
| 1200 |
+
| > step_time: 0.2667 (0.2791557732154861)
|
| 1201 |
+
| > loader_time: 0.0133 (0.008651209484999346)
|
| 1202 |
+
|
| 1203 |
+
|
| 1204 |
+
[1m --> TIME: 2025-12-25 21:59:31 -- STEP: 967/2207 -- GLOBAL_STEP: 33175[0m
|
| 1205 |
+
| > loss_text_ce: 0.030261661857366562 (0.030241678204029152)
|
| 1206 |
+
| > loss_mel_ce: 3.6733038425445557 (3.669723479922826)
|
| 1207 |
+
| > loss: 0.05786821246147156 (0.05781195555055646)
|
| 1208 |
+
| > current_lr: 1.999999803594879e-05
|
| 1209 |
+
| > step_time: 0.2905 (0.27909705434244925)
|
| 1210 |
+
| > loader_time: 0.0091 (0.008685013870665304)
|
| 1211 |
+
|
| 1212 |
+
|
| 1213 |
+
[1m --> TIME: 2025-12-25 21:59:41 -- STEP: 992/2207 -- GLOBAL_STEP: 33200[0m
|
| 1214 |
+
| > loss_text_ce: 0.03123987279832363 (0.030240577547961184)
|
| 1215 |
+
| > loss_mel_ce: 3.674602746963501 (3.6708724700635478)
|
| 1216 |
+
| > loss: 0.05790379270911217 (0.05782989132398318)
|
| 1217 |
+
| > current_lr: 1.999999803594879e-05
|
| 1218 |
+
| > step_time: 0.2671 (0.27923738139290954)
|
| 1219 |
+
| > loader_time: 0.0091 (0.008703768974350344)
|
| 1220 |
+
|
| 1221 |
+
|
| 1222 |
+
[1m --> TIME: 2025-12-25 21:59:50 -- STEP: 1017/2207 -- GLOBAL_STEP: 33225[0m
|
| 1223 |
+
| > loss_text_ce: 0.032771430909633636 (0.03024645330111479)
|
| 1224 |
+
| > loss_mel_ce: 3.5884199142456055 (3.670531275232516)
|
| 1225 |
+
| > loss: 0.05658111348748207 (0.057824651968016144)
|
| 1226 |
+
| > current_lr: 1.999999803594879e-05
|
| 1227 |
+
| > step_time: 0.2864 (0.2793125281057291)
|
| 1228 |
+
| > loader_time: 0.0086 (0.008703793044638844)
|
| 1229 |
+
|
| 1230 |
+
|
| 1231 |
+
[1m --> TIME: 2025-12-25 22:00:00 -- STEP: 1042/2207 -- GLOBAL_STEP: 33250[0m
|
| 1232 |
+
| > loss_text_ce: 0.0303847286850214 (0.03024321235001316)
|
| 1233 |
+
| > loss_mel_ce: 3.5909793376922607 (3.6696448705933142)
|
| 1234 |
+
| > loss: 0.056583814322948456 (0.057810751260070556)
|
| 1235 |
+
| > current_lr: 1.999999803594879e-05
|
| 1236 |
+
| > step_time: 0.2667 (0.27942793360140855)
|
| 1237 |
+
| > loader_time: 0.0082 (0.008699765589781777)
|
| 1238 |
+
|
| 1239 |
+
|
| 1240 |
+
[1m --> TIME: 2025-12-25 22:00:10 -- STEP: 1067/2207 -- GLOBAL_STEP: 33275[0m
|
| 1241 |
+
| > loss_text_ce: 0.028887305408716202 (0.030242266870106393)
|
| 1242 |
+
| > loss_mel_ce: 3.5442392826080322 (3.6697516859192216)
|
| 1243 |
+
| > loss: 0.055830102413892746 (0.05781240547259164)
|
| 1244 |
+
| > current_lr: 1.999999803594879e-05
|
| 1245 |
+
| > step_time: 0.2965 (0.279559671711229)
|
| 1246 |
+
| > loader_time: 0.0085 (0.008694199166123623)
|
| 1247 |
+
|
| 1248 |
+
|
| 1249 |
+
[1m --> TIME: 2025-12-25 22:00:20 -- STEP: 1092/2207 -- GLOBAL_STEP: 33300[0m
|
| 1250 |
+
| > loss_text_ce: 0.030024928972125053 (0.030229838176088018)
|
| 1251 |
+
| > loss_mel_ce: 3.80499005317688 (3.6698368809598714)
|
| 1252 |
+
| > loss: 0.05992211028933525 (0.05781354245835498)
|
| 1253 |
+
| > current_lr: 1.999999803594879e-05
|
| 1254 |
+
| > step_time: 0.2903 (0.2796051604843838)
|
| 1255 |
+
| > loader_time: 0.0083 (0.008684265744555126)
|
| 1256 |
+
|
| 1257 |
+
|
| 1258 |
+
[1m --> TIME: 2025-12-25 22:00:29 -- STEP: 1117/2207 -- GLOBAL_STEP: 33325[0m
|
| 1259 |
+
| > loss_text_ce: 0.03003372810781002 (0.030236679558694735)
|
| 1260 |
+
| > loss_mel_ce: 3.4917540550231934 (3.6688258673532164)
|
| 1261 |
+
| > loss: 0.05502793565392494 (0.05779785226660319)
|
| 1262 |
+
| > current_lr: 1.999999803594879e-05
|
| 1263 |
+
| > step_time: 0.2914 (0.2795455159090839)
|
| 1264 |
+
| > loader_time: 0.0086 (0.008676094996043363)
|
| 1265 |
+
|
| 1266 |
+
|
| 1267 |
+
[1m --> TIME: 2025-12-25 22:00:38 -- STEP: 1142/2207 -- GLOBAL_STEP: 33350[0m
|
| 1268 |
+
| > loss_text_ce: 0.030257757753133774 (0.030232944582864615)
|
| 1269 |
+
| > loss_mel_ce: 3.6511855125427246 (3.669242637587513)
|
| 1270 |
+
| > loss: 0.057522550225257874 (0.05780430595127899)
|
| 1271 |
+
| > current_lr: 1.999999803594879e-05
|
| 1272 |
+
| > step_time: 0.2662 (0.27943540892960095)
|
| 1273 |
+
| > loader_time: 0.008 (0.008670533809895686)
|
| 1274 |
+
|
| 1275 |
+
|
| 1276 |
+
[1m --> TIME: 2025-12-25 22:00:48 -- STEP: 1167/2207 -- GLOBAL_STEP: 33375[0m
|
| 1277 |
+
| > loss_text_ce: 0.031116589903831482 (0.03024675576263702)
|
| 1278 |
+
| > loss_mel_ce: 3.794879674911499 (3.667864134910277)
|
| 1279 |
+
| > loss: 0.059781190007925034 (0.057782982646039927)
|
| 1280 |
+
| > current_lr: 1.999999803594879e-05
|
| 1281 |
+
| > step_time: 0.2691 (0.27940349488965227)
|
| 1282 |
+
| > loader_time: 0.008 (0.008661381893926125)
|
| 1283 |
+
|
| 1284 |
+
|
| 1285 |
+
[1m --> TIME: 2025-12-25 22:00:58 -- STEP: 1192/2207 -- GLOBAL_STEP: 33400[0m
|
| 1286 |
+
| > loss_text_ce: 0.029510894790291786 (0.03025074678335664)
|
| 1287 |
+
| > loss_mel_ce: 3.670255184173584 (3.669161499546679)
|
| 1288 |
+
| > loss: 0.05780884623527527 (0.05780331633560007)
|
| 1289 |
+
| > current_lr: 1.999999803594879e-05
|
| 1290 |
+
| > step_time: 0.2859 (0.27940681616731927)
|
| 1291 |
+
| > loader_time: 0.0088 (0.00865318091123696)
|
| 1292 |
+
|
| 1293 |
+
|
| 1294 |
+
[1m --> TIME: 2025-12-25 22:01:07 -- STEP: 1217/2207 -- GLOBAL_STEP: 33425[0m
|
| 1295 |
+
| > loss_text_ce: 0.02882526069879532 (0.030238762491779828)
|
| 1296 |
+
| > loss_mel_ce: 3.5955259799957275 (3.6690606788817064)
|
| 1297 |
+
| > loss: 0.056630488485097885 (0.05780155376677346)
|
| 1298 |
+
| > current_lr: 1.999999803594879e-05
|
| 1299 |
+
| > step_time: 0.2665 (0.2794938902435679)
|
| 1300 |
+
| > loader_time: 0.0084 (0.0086473687907762)
|
| 1301 |
+
|
| 1302 |
+
|
| 1303 |
+
[1m --> TIME: 2025-12-25 22:01:17 -- STEP: 1242/2207 -- GLOBAL_STEP: 33450[0m
|
| 1304 |
+
| > loss_text_ce: 0.03155346214771271 (0.030248059333270585)
|
| 1305 |
+
| > loss_mel_ce: 3.567056894302368 (3.67004957559988)
|
| 1306 |
+
| > loss: 0.056228287518024445 (0.05781715054022995)
|
| 1307 |
+
| > current_lr: 1.999999803594879e-05
|
| 1308 |
+
| > step_time: 0.2689 (0.2794395404929317)
|
| 1309 |
+
| > loader_time: 0.0082 (0.008643939491247402)
|
| 1310 |
+
|
| 1311 |
+
|
| 1312 |
+
[1m --> TIME: 2025-12-25 22:01:26 -- STEP: 1267/2207 -- GLOBAL_STEP: 33475[0m
|
| 1313 |
+
| > loss_text_ce: 0.0317721925675869 (0.030251537341603917)
|
| 1314 |
+
| > loss_mel_ce: 3.5550684928894043 (3.669930552909388)
|
| 1315 |
+
| > loss: 0.056044384837150574 (0.05781534515856377)
|
| 1316 |
+
| > current_lr: 1.999999803594879e-05
|
| 1317 |
+
| > step_time: 0.256 (0.27941572242772955)
|
| 1318 |
+
| > loader_time: 0.0083 (0.008635847984380201)
|
| 1319 |
+
|
| 1320 |
+
|
| 1321 |
+
[1m --> TIME: 2025-12-25 22:01:36 -- STEP: 1292/2207 -- GLOBAL_STEP: 33500[0m
|
| 1322 |
+
| > loss_text_ce: 0.028758108615875244 (0.030253066839909776)
|
| 1323 |
+
| > loss_mel_ce: 3.437675952911377 (3.6696313344657243)
|
| 1324 |
+
| > loss: 0.05416303128004074 (0.057810693763011914)
|
| 1325 |
+
| > current_lr: 1.999999803594879e-05
|
| 1326 |
+
| > step_time: 0.2909 (0.2793923804634496)
|
| 1327 |
+
| > loader_time: 0.008 (0.00863082619274364)
|
| 1328 |
+
|
| 1329 |
+
|
| 1330 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_33500.pth
|
| 1331 |
+
|
| 1332 |
+
[1m --> TIME: 2025-12-25 22:02:11 -- STEP: 1317/2207 -- GLOBAL_STEP: 33525[0m
|
| 1333 |
+
| > loss_text_ce: 0.028247222304344177 (0.030245677291520997)
|
| 1334 |
+
| > loss_mel_ce: 3.6836891174316406 (3.6697709028161474)
|
| 1335 |
+
| > loss: 0.05799900367856026 (0.057812759062966866)
|
| 1336 |
+
| > current_lr: 1.999999803594879e-05
|
| 1337 |
+
| > step_time: 0.29 (0.2794235628847832)
|
| 1338 |
+
| > loader_time: 0.0083 (0.008632874615552663)
|
| 1339 |
+
|
| 1340 |
+
|
| 1341 |
+
[1m --> TIME: 2025-12-25 22:02:20 -- STEP: 1342/2207 -- GLOBAL_STEP: 33550[0m
|
| 1342 |
+
| > loss_text_ce: 0.028736207634210587 (0.030240909392139064)
|
| 1343 |
+
| > loss_mel_ce: 3.5469231605529785 (3.6696852580623376)
|
| 1344 |
+
| > loss: 0.05586967617273331 (0.05781134636586893)
|
| 1345 |
+
| > current_lr: 1.999999803594879e-05
|
| 1346 |
+
| > step_time: 0.2901 (0.27931749269965844)
|
| 1347 |
+
| > loader_time: 0.0086 (0.008635605323154771)
|
| 1348 |
+
|
| 1349 |
+
|
| 1350 |
+
[1m --> TIME: 2025-12-25 22:02:30 -- STEP: 1367/2207 -- GLOBAL_STEP: 33575[0m
|
| 1351 |
+
| > loss_text_ce: 0.02937578409910202 (0.030231545109032015)
|
| 1352 |
+
| > loss_mel_ce: 3.4941675662994385 (3.6688636690487204)
|
| 1353 |
+
| > loss: 0.05505536496639252 (0.05779836272606904)
|
| 1354 |
+
| > current_lr: 1.999999803594879e-05
|
| 1355 |
+
| > step_time: 0.2904 (0.2793314978425835)
|
| 1356 |
+
| > loader_time: 0.0086 (0.008634327970112685)
|
| 1357 |
+
|
| 1358 |
+
|
| 1359 |
+
[1m --> TIME: 2025-12-25 22:02:39 -- STEP: 1392/2207 -- GLOBAL_STEP: 33600[0m
|
| 1360 |
+
| > loss_text_ce: 0.03153255954384804 (0.030230863494852066)
|
| 1361 |
+
| > loss_mel_ce: 3.711127281188965 (3.6684994957912935)
|
| 1362 |
+
| > loss: 0.05847905948758125 (0.05779266186679401)
|
| 1363 |
+
| > current_lr: 1.999999803594879e-05
|
| 1364 |
+
| > step_time: 0.2693 (0.2792636550020896)
|
| 1365 |
+
| > loader_time: 0.0083 (0.008628269721721777)
|
| 1366 |
+
|
| 1367 |
+
|
| 1368 |
+
[1m --> TIME: 2025-12-25 22:02:49 -- STEP: 1417/2207 -- GLOBAL_STEP: 33625[0m
|
| 1369 |
+
| > loss_text_ce: 0.03205839917063713 (0.030227988671797972)
|
| 1370 |
+
| > loss_mel_ce: 3.6176953315734863 (3.6695709817411206)
|
| 1371 |
+
| > loss: 0.05702740326523781 (0.05780935891619346)
|
| 1372 |
+
| > current_lr: 1.999999803594879e-05
|
| 1373 |
+
| > step_time: 0.2874 (0.2791883807495266)
|
| 1374 |
+
| > loader_time: 0.0083 (0.00862536686107821)
|
| 1375 |
+
|
| 1376 |
+
|
| 1377 |
+
[1m --> TIME: 2025-12-25 22:02:58 -- STEP: 1442/2207 -- GLOBAL_STEP: 33650[0m
|
| 1378 |
+
| > loss_text_ce: 0.03024265542626381 (0.030219388663272752)
|
| 1379 |
+
| > loss_mel_ce: 3.729968547821045 (3.669712749334381)
|
| 1380 |
+
| > loss: 0.058753300458192825 (0.05781143966251039)
|
| 1381 |
+
| > current_lr: 1.999999803594879e-05
|
| 1382 |
+
| > step_time: 0.2663 (0.2791618735383518)
|
| 1383 |
+
| > loader_time: 0.0086 (0.008622546797817193)
|
| 1384 |
+
|
| 1385 |
+
|
| 1386 |
+
[1m --> TIME: 2025-12-25 22:03:08 -- STEP: 1467/2207 -- GLOBAL_STEP: 33675[0m
|
| 1387 |
+
| > loss_text_ce: 0.03239262104034424 (0.030222916188405415)
|
| 1388 |
+
| > loss_mel_ce: 3.5529046058654785 (3.6694573380951994)
|
| 1389 |
+
| > loss: 0.05602026730775833 (0.05780750398042254)
|
| 1390 |
+
| > current_lr: 1.999999803594879e-05
|
| 1391 |
+
| > step_time: 0.2686 (0.27914598675593266)
|
| 1392 |
+
| > loader_time: 0.008 (0.008616991244617093)
|
| 1393 |
+
|
| 1394 |
+
|
| 1395 |
+
[1m --> TIME: 2025-12-25 22:03:17 -- STEP: 1492/2207 -- GLOBAL_STEP: 33700[0m
|
| 1396 |
+
| > loss_text_ce: 0.030658289790153503 (0.030223791083026346)
|
| 1397 |
+
| > loss_mel_ce: 3.542323589324951 (3.6694255284265926)
|
| 1398 |
+
| > loss: 0.05582784116268158 (0.05780702062330679)
|
| 1399 |
+
| > current_lr: 1.999999803594879e-05
|
| 1400 |
+
| > step_time: 0.2684 (0.27912762833025095)
|
| 1401 |
+
| > loader_time: 0.0086 (0.008616551956606301)
|
| 1402 |
+
|
| 1403 |
+
|
| 1404 |
+
[1m --> TIME: 2025-12-25 22:03:27 -- STEP: 1517/2207 -- GLOBAL_STEP: 33725[0m
|
| 1405 |
+
| > loss_text_ce: 0.028873823583126068 (0.030219121684486646)
|
| 1406 |
+
| > loss_mel_ce: 3.7265751361846924 (3.6689475296199054)
|
| 1407 |
+
| > loss: 0.05867889150977135 (0.057799478932655225)
|
| 1408 |
+
| > current_lr: 1.999999803594879e-05
|
| 1409 |
+
| > step_time: 0.287 (0.2791129054407106)
|
| 1410 |
+
| > loader_time: 0.0084 (0.008614474405327682)
|
| 1411 |
+
|
| 1412 |
+
|
| 1413 |
+
[1m --> TIME: 2025-12-25 22:03:37 -- STEP: 1542/2207 -- GLOBAL_STEP: 33750[0m
|
| 1414 |
+
| > loss_text_ce: 0.03084191493690014 (0.03021548565926487)
|
| 1415 |
+
| > loss_mel_ce: 4.272586822509766 (3.6690947370925)
|
| 1416 |
+
| > loss: 0.06724107265472412 (0.05780172223362939)
|
| 1417 |
+
| > current_lr: 1.999999803594879e-05
|
| 1418 |
+
| > step_time: 0.2877 (0.27913414272352854)
|
| 1419 |
+
| > loader_time: 0.0084 (0.008611800605684873)
|
| 1420 |
+
|
| 1421 |
+
|
| 1422 |
+
[1m --> TIME: 2025-12-25 22:03:46 -- STEP: 1567/2207 -- GLOBAL_STEP: 33775[0m
|
| 1423 |
+
| > loss_text_ce: 0.029085008427500725 (0.030216344095524057)
|
| 1424 |
+
| > loss_mel_ce: 3.675722360610962 (3.6684183485582524)
|
| 1425 |
+
| > loss: 0.05788761377334595 (0.057791167074943986)
|
| 1426 |
+
| > current_lr: 1.999999803594879e-05
|
| 1427 |
+
| > step_time: 0.2609 (0.27905297522797423)
|
| 1428 |
+
| > loader_time: 0.008 (0.00861395265010793)
|
| 1429 |
+
|
| 1430 |
+
|
| 1431 |
+
[1m --> TIME: 2025-12-25 22:03:56 -- STEP: 1592/2207 -- GLOBAL_STEP: 33800[0m
|
| 1432 |
+
| > loss_text_ce: 0.03128354623913765 (0.030217592962479323)
|
| 1433 |
+
| > loss_mel_ce: 3.6151351928710938 (3.6686241733069402)
|
| 1434 |
+
| > loss: 0.05697529390454292 (0.057794402596826215)
|
| 1435 |
+
| > current_lr: 1.999999803594879e-05
|
| 1436 |
+
| > step_time: 0.2688 (0.2791039979637567)
|
| 1437 |
+
| > loader_time: 0.0081 (0.008609625861872383)
|
| 1438 |
+
|
| 1439 |
+
|
| 1440 |
+
[1m --> TIME: 2025-12-25 22:04:05 -- STEP: 1617/2207 -- GLOBAL_STEP: 33825[0m
|
| 1441 |
+
| > loss_text_ce: 0.028969425708055496 (0.030217405420139974)
|
| 1442 |
+
| > loss_mel_ce: 3.527372121810913 (3.6690204147947525)
|
| 1443 |
+
| > loss: 0.05556783825159073 (0.057800590941289635)
|
| 1444 |
+
| > current_lr: 1.999999803594879e-05
|
| 1445 |
+
| > step_time: 0.2905 (0.2791682954326998)
|
| 1446 |
+
| > loader_time: 0.0085 (0.00860697233758005)
|
| 1447 |
+
|
| 1448 |
+
|
| 1449 |
+
[1m --> TIME: 2025-12-25 22:04:15 -- STEP: 1642/2207 -- GLOBAL_STEP: 33850[0m
|
| 1450 |
+
| > loss_text_ce: 0.030807415023446083 (0.030216442809035224)
|
| 1451 |
+
| > loss_mel_ce: 3.4754300117492676 (3.6690939972664935)
|
| 1452 |
+
| > loss: 0.05478496104478836 (0.05780172562873245)
|
| 1453 |
+
| > current_lr: 1.999999803594879e-05
|
| 1454 |
+
| > step_time: 0.2665 (0.2790928322563216)
|
| 1455 |
+
| > loader_time: 0.0082 (0.008601953719042685)
|
| 1456 |
+
|
| 1457 |
+
|
| 1458 |
+
[1m --> TIME: 2025-12-25 22:04:24 -- STEP: 1667/2207 -- GLOBAL_STEP: 33875[0m
|
| 1459 |
+
| > loss_text_ce: 0.030791722238063812 (0.03021561699999294)
|
| 1460 |
+
| > loss_mel_ce: 3.9251177310943604 (3.6690652802380965)
|
| 1461 |
+
| > loss: 0.06181108579039574 (0.05780126402700954)
|
| 1462 |
+
| > current_lr: 1.999999803594879e-05
|
| 1463 |
+
| > step_time: 0.2688 (0.2790552665891227)
|
| 1464 |
+
| > loader_time: 0.0085 (0.008597536579033673)
|
| 1465 |
+
|
| 1466 |
+
|
| 1467 |
+
[1m --> TIME: 2025-12-25 22:04:34 -- STEP: 1692/2207 -- GLOBAL_STEP: 33900[0m
|
| 1468 |
+
| > loss_text_ce: 0.030678123235702515 (0.030215419551476518)
|
| 1469 |
+
| > loss_mel_ce: 3.5805423259735107 (3.668094589214235)
|
| 1470 |
+
| > loss: 0.056425318121910095 (0.05778609389707832)
|
| 1471 |
+
| > current_lr: 1.999999803594879e-05
|
| 1472 |
+
| > step_time: 0.29 (0.279003456270159)
|
| 1473 |
+
| > loader_time: 0.0088 (0.008592242608397273)
|
| 1474 |
+
|
| 1475 |
+
|
| 1476 |
+
[1m --> TIME: 2025-12-25 22:04:43 -- STEP: 1717/2207 -- GLOBAL_STEP: 33925[0m
|
| 1477 |
+
| > loss_text_ce: 0.030814236029982567 (0.030211187655026995)
|
| 1478 |
+
| > loss_mel_ce: 3.711381435394287 (3.6675508787413986)
|
| 1479 |
+
| > loss: 0.05847180634737015 (0.05777753229119016)
|
| 1480 |
+
| > current_lr: 1.999999803594879e-05
|
| 1481 |
+
| > step_time: 0.2542 (0.2790025737164803)
|
| 1482 |
+
| > loader_time: 0.0079 (0.008590328464274764)
|
| 1483 |
+
|
| 1484 |
+
|
| 1485 |
+
[1m --> TIME: 2025-12-25 22:04:53 -- STEP: 1742/2207 -- GLOBAL_STEP: 33950[0m
|
| 1486 |
+
| > loss_text_ce: 0.028332965448498726 (0.03020891614993938)
|
| 1487 |
+
| > loss_mel_ce: 3.7369577884674072 (3.6670745178697857)
|
| 1488 |
+
| > loss: 0.058832667768001556 (0.05777005365826231)
|
| 1489 |
+
| > current_lr: 1.999999803594879e-05
|
| 1490 |
+
| > step_time: 0.2667 (0.278960460910293)
|
| 1491 |
+
| > loader_time: 0.0085 (0.008590039648500553)
|
| 1492 |
+
|
| 1493 |
+
|
| 1494 |
+
[1m --> TIME: 2025-12-25 22:05:03 -- STEP: 1767/2207 -- GLOBAL_STEP: 33975[0m
|
| 1495 |
+
| > loss_text_ce: 0.027250058948993683 (0.030207996624013293)
|
| 1496 |
+
| > loss_mel_ce: 3.6868233680725098 (3.6678104754147323)
|
| 1497 |
+
| > loss: 0.05803239718079567 (0.057781538628273635)
|
| 1498 |
+
| > current_lr: 1.999999803594879e-05
|
| 1499 |
+
| > step_time: 0.3051 (0.27899268190268356)
|
| 1500 |
+
| > loader_time: 0.008 (0.00858696976420429)
|
| 1501 |
+
|
| 1502 |
+
|
| 1503 |
+
[1m --> TIME: 2025-12-25 22:05:12 -- STEP: 1792/2207 -- GLOBAL_STEP: 34000[0m
|
| 1504 |
+
| > loss_text_ce: 0.027882235124707222 (0.030205997146757518)
|
| 1505 |
+
| > loss_mel_ce: 4.0121684074401855 (3.667537564278714)
|
| 1506 |
+
| > loss: 0.06312578916549683 (0.05777724314871841)
|
| 1507 |
+
| > current_lr: 1.999999803594879e-05
|
| 1508 |
+
| > step_time: 0.2408 (0.2789914292682488)
|
| 1509 |
+
| > loader_time: 0.0081 (0.00858304915683609)
|
| 1510 |
+
|
| 1511 |
+
|
| 1512 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_34000.pth
|
| 1513 |
+
|
| 1514 |
+
[1m --> TIME: 2025-12-25 22:05:46 -- STEP: 1817/2207 -- GLOBAL_STEP: 34025[0m
|
| 1515 |
+
| > loss_text_ce: 0.03059951215982437 (0.030205635475912262)
|
| 1516 |
+
| > loss_mel_ce: 3.632253646850586 (3.6663451021558715)
|
| 1517 |
+
| > loss: 0.0572320818901062 (0.05775860527681004)
|
| 1518 |
+
| > current_lr: 1.999999803594879e-05
|
| 1519 |
+
| > step_time: 0.2512 (0.2789730696344868)
|
| 1520 |
+
| > loader_time: 0.0086 (0.008584037874187147)
|
| 1521 |
+
|
| 1522 |
+
|
| 1523 |
+
[1m --> TIME: 2025-12-25 22:05:56 -- STEP: 1842/2207 -- GLOBAL_STEP: 34050[0m
|
| 1524 |
+
| > loss_text_ce: 0.03174057975411415 (0.030210003290167656)
|
| 1525 |
+
| > loss_mel_ce: 3.4106929302215576 (3.666705815204451)
|
| 1526 |
+
| > loss: 0.05378802493214607 (0.057764309657642544)
|
| 1527 |
+
| > current_lr: 1.999999803594879e-05
|
| 1528 |
+
| > step_time: 0.2686 (0.2789648112463765)
|
| 1529 |
+
| > loader_time: 0.0085 (0.008582787974520162)
|
| 1530 |
+
|
| 1531 |
+
|
| 1532 |
+
[1m --> TIME: 2025-12-25 22:06:06 -- STEP: 1867/2207 -- GLOBAL_STEP: 34075[0m
|
| 1533 |
+
| > loss_text_ce: 0.02972555346786976 (0.030210420446411517)
|
| 1534 |
+
| > loss_mel_ce: 3.467895030975342 (3.6663959683999066)
|
| 1535 |
+
| > loss: 0.05465032160282135 (0.057759474815294955)
|
| 1536 |
+
| > current_lr: 1.999999803594879e-05
|
| 1537 |
+
| > step_time: 0.2869 (0.2790133189439896)
|
| 1538 |
+
| > loader_time: 0.0087 (0.00858088936726549)
|
| 1539 |
+
|
| 1540 |
+
|
| 1541 |
+
[1m --> TIME: 2025-12-25 22:06:15 -- STEP: 1892/2207 -- GLOBAL_STEP: 34100[0m
|
| 1542 |
+
| > loss_text_ce: 0.030226539820432663 (0.03020907149612746)
|
| 1543 |
+
| > loss_mel_ce: 3.4116921424865723 (3.666441644977062)
|
| 1544 |
+
| > loss: 0.053779978305101395 (0.057760167433771964)
|
| 1545 |
+
| > current_lr: 1.999999803594879e-05
|
| 1546 |
+
| > step_time: 0.2666 (0.27899128954707897)
|
| 1547 |
+
| > loader_time: 0.0086 (0.008579870233072023)
|
| 1548 |
+
|
| 1549 |
+
|
| 1550 |
+
[1m --> TIME: 2025-12-25 22:06:25 -- STEP: 1917/2207 -- GLOBAL_STEP: 34125[0m
|
| 1551 |
+
| > loss_text_ce: 0.03107047639787197 (0.03020506256479084)
|
| 1552 |
+
| > loss_mel_ce: 3.5890026092529297 (3.66619321512691)
|
| 1553 |
+
| > loss: 0.056563641875982285 (0.05775622307989953)
|
| 1554 |
+
| > current_lr: 1.999999803594879e-05
|
| 1555 |
+
| > step_time: 0.266 (0.2790518238573065)
|
| 1556 |
+
| > loader_time: 0.0092 (0.008576352706972847)
|
| 1557 |
+
|
| 1558 |
+
|
| 1559 |
+
[1m --> TIME: 2025-12-25 22:06:34 -- STEP: 1942/2207 -- GLOBAL_STEP: 34150[0m
|
| 1560 |
+
| > loss_text_ce: 0.031138692051172256 (0.030203481418695437)
|
| 1561 |
+
| > loss_mel_ce: 4.088606357574463 (3.666404624923251)
|
| 1562 |
+
| > loss: 0.0643710196018219 (0.05775950165574814)
|
| 1563 |
+
| > current_lr: 1.999999803594879e-05
|
| 1564 |
+
| > step_time: 0.2618 (0.2789932387496362)
|
| 1565 |
+
| > loader_time: 0.0082 (0.008572183112037404)
|
| 1566 |
+
|
| 1567 |
+
|
| 1568 |
+
[1m --> TIME: 2025-12-25 22:06:44 -- STEP: 1967/2207 -- GLOBAL_STEP: 34175[0m
|
| 1569 |
+
| > loss_text_ce: 0.030564548447728157 (0.03020159132877127)
|
| 1570 |
+
| > loss_mel_ce: 3.5198118686676025 (3.666901490987711)
|
| 1571 |
+
| > loss: 0.05547463148832321 (0.05776723565739243)
|
| 1572 |
+
| > current_lr: 1.999999803594879e-05
|
| 1573 |
+
| > step_time: 0.2861 (0.279002703927044)
|
| 1574 |
+
| > loader_time: 0.008 (0.008569334749649417)
|
| 1575 |
+
|
| 1576 |
+
|
| 1577 |
+
[1m --> TIME: 2025-12-25 22:06:54 -- STEP: 1992/2207 -- GLOBAL_STEP: 34200[0m
|
| 1578 |
+
| > loss_text_ce: 0.030575839802622795 (0.030202181419791526)
|
| 1579 |
+
| > loss_mel_ce: 3.5924935340881348 (3.666848091475935)
|
| 1580 |
+
| > loss: 0.056610457599163055 (0.05776641050907202)
|
| 1581 |
+
| > current_lr: 1.999999803594879e-05
|
| 1582 |
+
| > step_time: 0.2897 (0.27905963259528394)
|
| 1583 |
+
| > loader_time: 0.0081 (0.008568205268507491)
|
| 1584 |
+
|
| 1585 |
+
|
| 1586 |
+
[1m --> TIME: 2025-12-25 22:07:03 -- STEP: 2017/2207 -- GLOBAL_STEP: 34225[0m
|
| 1587 |
+
| > loss_text_ce: 0.027702152729034424 (0.030196573046887044)
|
| 1588 |
+
| > loss_mel_ce: 3.5850439071655273 (3.666954867131768)
|
| 1589 |
+
| > loss: 0.05644915625452995 (0.0577679912480731)
|
| 1590 |
+
| > current_lr: 1.999999803594879e-05
|
| 1591 |
+
| > step_time: 0.2966 (0.2790709123436503)
|
| 1592 |
+
| > loader_time: 0.0083 (0.00856840096314353)
|
| 1593 |
+
|
| 1594 |
+
|
| 1595 |
+
[1m --> TIME: 2025-12-25 22:07:13 -- STEP: 2042/2207 -- GLOBAL_STEP: 34250[0m
|
| 1596 |
+
| > loss_text_ce: 0.028264036402106285 (0.03019207050106201)
|
| 1597 |
+
| > loss_mel_ce: 3.5881502628326416 (3.666963961402303)
|
| 1598 |
+
| > loss: 0.05650647357106209 (0.0577680629921619)
|
| 1599 |
+
| > current_lr: 1.999999803594879e-05
|
| 1600 |
+
| > step_time: 0.2964 (0.2790879463005248)
|
| 1601 |
+
| > loader_time: 0.0086 (0.00856652245815773)
|
| 1602 |
+
|
| 1603 |
+
|
| 1604 |
+
[1m --> TIME: 2025-12-25 22:07:23 -- STEP: 2067/2207 -- GLOBAL_STEP: 34275[0m
|
| 1605 |
+
| > loss_text_ce: 0.029042014852166176 (0.03018852376730966)
|
| 1606 |
+
| > loss_mel_ce: 3.414708375930786 (3.666221316447856)
|
| 1607 |
+
| > loss: 0.05380859971046448 (0.057756403748560586)
|
| 1608 |
+
| > current_lr: 1.999999803594879e-05
|
| 1609 |
+
| > step_time: 0.2685 (0.2791220907664375)
|
| 1610 |
+
| > loader_time: 0.0081 (0.008562659663628068)
|
| 1611 |
+
|
| 1612 |
+
|
| 1613 |
+
[1m --> TIME: 2025-12-25 22:07:32 -- STEP: 2092/2207 -- GLOBAL_STEP: 34300[0m
|
| 1614 |
+
| > loss_text_ce: 0.030513126403093338 (0.03018568095485778)
|
| 1615 |
+
| > loss_mel_ce: 3.734703779220581 (3.666197642883425)
|
| 1616 |
+
| > loss: 0.05883151292800903 (0.057755989430340465)
|
| 1617 |
+
| > current_lr: 1.999999803594879e-05
|
| 1618 |
+
| > step_time: 0.2669 (0.2791535894227981)
|
| 1619 |
+
| > loader_time: 0.0088 (0.008560543648371494)
|
| 1620 |
+
|
| 1621 |
+
|
| 1622 |
+
[1m --> TIME: 2025-12-25 22:07:42 -- STEP: 2117/2207 -- GLOBAL_STEP: 34325[0m
|
| 1623 |
+
| > loss_text_ce: 0.03083481639623642 (0.030184599548493795)
|
| 1624 |
+
| > loss_mel_ce: 3.5531423091888428 (3.666440862949356)
|
| 1625 |
+
| > loss: 0.055999644100666046 (0.05775977285067652)
|
| 1626 |
+
| > current_lr: 1.999999803594879e-05
|
| 1627 |
+
| > step_time: 0.2688 (0.2791283465130008)
|
| 1628 |
+
| > loader_time: 0.0097 (0.008558270837768483)
|
| 1629 |
+
|
| 1630 |
+
|
| 1631 |
+
[1m --> TIME: 2025-12-25 22:07:51 -- STEP: 2142/2207 -- GLOBAL_STEP: 34350[0m
|
| 1632 |
+
| > loss_text_ce: 0.028636541217565536 (0.030187789838583695)
|
| 1633 |
+
| > loss_mel_ce: 3.5387206077575684 (3.6665975250561145)
|
| 1634 |
+
| > loss: 0.055739954113960266 (0.05776227054635738)
|
| 1635 |
+
| > current_lr: 1.999999803594879e-05
|
| 1636 |
+
| > step_time: 0.2909 (0.2791033112813552)
|
| 1637 |
+
| > loader_time: 0.0083 (0.008555201477919853)
|
| 1638 |
+
|
| 1639 |
+
|
| 1640 |
+
[1m --> TIME: 2025-12-25 22:08:01 -- STEP: 2167/2207 -- GLOBAL_STEP: 34375[0m
|
| 1641 |
+
| > loss_text_ce: 0.02974676340818405 (0.030192021912296467)
|
| 1642 |
+
| > loss_mel_ce: 3.528259515762329 (3.665465648459244)
|
| 1643 |
+
| > loss: 0.05559384822845459 (0.05774465110109981)
|
| 1644 |
+
| > current_lr: 1.999999803594879e-05
|
| 1645 |
+
| > step_time: 0.2679 (0.27904570152201696)
|
| 1646 |
+
| > loader_time: 0.0079 (0.008552518922940445)
|
| 1647 |
+
|
| 1648 |
+
|
| 1649 |
+
[1m --> TIME: 2025-12-25 22:08:10 -- STEP: 2192/2207 -- GLOBAL_STEP: 34400[0m
|
| 1650 |
+
| > loss_text_ce: 0.03171277046203613 (0.03018974362325304)
|
| 1651 |
+
| > loss_mel_ce: 3.763514995574951 (3.664962656197758)
|
| 1652 |
+
| > loss: 0.05930043384432793 (0.05773675624777419)
|
| 1653 |
+
| > current_lr: 1.999999803594879e-05
|
| 1654 |
+
| > step_time: 0.2544 (0.2790777404812996)
|
| 1655 |
+
| > loader_time: 0.0082 (0.008550308372852568)
|
| 1656 |
+
|
| 1657 |
+
|
| 1658 |
+
[1m > EVALUATION [0m
|
| 1659 |
+
|
| 1660 |
+
|
| 1661 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 1662 |
+
| > avg_loader_time:[92m 0.09335947197836798 [0m(-0.019347804623681145)
|
| 1663 |
+
| > avg_loss_text_ce:[92m 0.029143485945422907 [0m(-0.0001516662714247767)
|
| 1664 |
+
| > avg_loss_mel_ce:[92m 3.743672736593195 [0m(-0.017544873662897142)
|
| 1665 |
+
| > avg_loss:[92m 3.7728162182343974 [0m(-0.01769654331980508)
|
| 1666 |
+
|
| 1667 |
+
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_34415.pth
|