Checkpoint at step 43243
Browse files- best_model.pth +1 -1
- best_model_43243.pth +3 -0
- checkpoint_42500.pth +3 -0
- checkpoint_43000.pth +3 -0
- events.out.tfevents.1766698157.0f9434978106.1562.0 +2 -2
- trainer_0_log.txt +817 -0
best_model.pth
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 5607926805
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a6b5293674a4f436164eb5ab4ad16964fff6f35ae19adeddfcb1b7df6e6e32b
|
| 3 |
size 5607926805
|
best_model_43243.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a6b5293674a4f436164eb5ab4ad16964fff6f35ae19adeddfcb1b7df6e6e32b
|
| 3 |
+
size 5607926805
|
checkpoint_42500.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:235e00ed5b85dfdabb7cd718e6b00412e334b0d5728c63d5c071322bec950e31
|
| 3 |
+
size 5607926741
|
checkpoint_43000.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:00f7e05123068e5a9daddd95c8c5696a2e4a7c44fcb2474cf64f9ec586f809bf
|
| 3 |
+
size 5607926741
|
events.out.tfevents.1766698157.0f9434978106.1562.0
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:437cde3ddf8f08e5fe046d7217a505a3798eec595682ac130863319c1e7782cc
|
| 3 |
+
size 62012
|
trainer_0_log.txt
CHANGED
|
@@ -4129,3 +4129,820 @@
|
|
| 4129 |
| > avg_loss:[92m 3.7481746480271623 [0m(-0.011339513031212167)
|
| 4130 |
|
| 4131 |
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_41036.pth
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4129 |
| > avg_loss:[92m 3.7481746480271623 [0m(-0.011339513031212167)
|
| 4130 |
|
| 4131 |
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_41036.pth
|
| 4132 |
+
|
| 4133 |
+
[4m[1m > EPOCH: 6/12[0m
|
| 4134 |
+
--> /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000
|
| 4135 |
+
|
| 4136 |
+
[1m > TRAINING (2025-12-25 23:15:44) [0m
|
| 4137 |
+
|
| 4138 |
+
[1m --> TIME: 2025-12-25 23:15:53 -- STEP: 14/2207 -- GLOBAL_STEP: 41050[0m
|
| 4139 |
+
| > loss_text_ce: 0.02931983955204487 (0.02943239640444517)
|
| 4140 |
+
| > loss_mel_ce: 3.8469035625457764 (3.5794733251844133)
|
| 4141 |
+
| > loss: 0.06056598946452141 (0.05638915167323181)
|
| 4142 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4143 |
+
| > step_time: 0.2678 (0.2743805306298392)
|
| 4144 |
+
| > loader_time: 0.0081 (0.0087794406073434)
|
| 4145 |
+
|
| 4146 |
+
|
| 4147 |
+
[1m --> TIME: 2025-12-25 23:16:02 -- STEP: 39/2207 -- GLOBAL_STEP: 41075[0m
|
| 4148 |
+
| > loss_text_ce: 0.030693653970956802 (0.02922345435199065)
|
| 4149 |
+
| > loss_mel_ce: 3.740534543991089 (3.6449042833768406)
|
| 4150 |
+
| > loss: 0.05892544239759445 (0.05740824580574647)
|
| 4151 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4152 |
+
| > step_time: 0.2659 (0.2785025926736684)
|
| 4153 |
+
| > loader_time: 0.0084 (0.008852060024554912)
|
| 4154 |
+
|
| 4155 |
+
|
| 4156 |
+
[1m --> TIME: 2025-12-25 23:16:12 -- STEP: 64/2207 -- GLOBAL_STEP: 41100[0m
|
| 4157 |
+
| > loss_text_ce: 0.031162580475211143 (0.02935216491459869)
|
| 4158 |
+
| > loss_mel_ce: 3.9580140113830566 (3.6483749002218246)
|
| 4159 |
+
| > loss: 0.062330882996320724 (0.05746448534773663)
|
| 4160 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4161 |
+
| > step_time: 0.2578 (0.28068241104483604)
|
| 4162 |
+
| > loader_time: 0.0083 (0.008796826004981995)
|
| 4163 |
+
|
| 4164 |
+
|
| 4165 |
+
[1m --> TIME: 2025-12-25 23:16:21 -- STEP: 89/2207 -- GLOBAL_STEP: 41125[0m
|
| 4166 |
+
| > loss_text_ce: 0.02822493202984333 (0.029531902811500463)
|
| 4167 |
+
| > loss_mel_ce: 3.779770851135254 (3.654785815249668)
|
| 4168 |
+
| > loss: 0.05949993431568146 (0.05756746425052707)
|
| 4169 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4170 |
+
| > step_time: 0.2691 (0.2788318060757069)
|
| 4171 |
+
| > loader_time: 0.0085 (0.008817463778377915)
|
| 4172 |
+
|
| 4173 |
+
|
| 4174 |
+
[1m --> TIME: 2025-12-25 23:16:31 -- STEP: 114/2207 -- GLOBAL_STEP: 41150[0m
|
| 4175 |
+
| > loss_text_ce: 0.02694079279899597 (0.02959705083712674)
|
| 4176 |
+
| > loss_mel_ce: 3.5190861225128174 (3.6454159146861027)
|
| 4177 |
+
| > loss: 0.05540667101740837 (0.05742207747933112)
|
| 4178 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4179 |
+
| > step_time: 0.2906 (0.2784205340502554)
|
| 4180 |
+
| > loader_time: 0.0085 (0.008711417516072585)
|
| 4181 |
+
|
| 4182 |
+
|
| 4183 |
+
[1m --> TIME: 2025-12-25 23:16:40 -- STEP: 139/2207 -- GLOBAL_STEP: 41175[0m
|
| 4184 |
+
| > loss_text_ce: 0.030321231111884117 (0.029596778343049742)
|
| 4185 |
+
| > loss_mel_ce: 3.440438747406006 (3.63434976124935)
|
| 4186 |
+
| > loss: 0.054230622947216034 (0.05724916454377792)
|
| 4187 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4188 |
+
| > step_time: 0.2514 (0.277898846770362)
|
| 4189 |
+
| > loader_time: 0.0086 (0.008672803425960397)
|
| 4190 |
+
|
| 4191 |
+
|
| 4192 |
+
[1m --> TIME: 2025-12-25 23:16:50 -- STEP: 164/2207 -- GLOBAL_STEP: 41200[0m
|
| 4193 |
+
| > loss_text_ce: 0.02963964454829693 (0.02963556687733749)
|
| 4194 |
+
| > loss_mel_ce: 3.4831862449645996 (3.6211055081065107)
|
| 4195 |
+
| > loss: 0.05488790571689606 (0.057042829215344856)
|
| 4196 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4197 |
+
| > step_time: 0.287 (0.2774369265975022)
|
| 4198 |
+
| > loader_time: 0.0088 (0.00863570992539568)
|
| 4199 |
+
|
| 4200 |
+
|
| 4201 |
+
[1m --> TIME: 2025-12-25 23:16:59 -- STEP: 189/2207 -- GLOBAL_STEP: 41225[0m
|
| 4202 |
+
| > loss_text_ce: 0.02952307090163231 (0.029772064101601405)
|
| 4203 |
+
| > loss_mel_ce: 3.33707332611084 (3.626503858616743)
|
| 4204 |
+
| > loss: 0.05260306969285011 (0.05712931128169494)
|
| 4205 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4206 |
+
| > step_time: 0.288 (0.27771097137814477)
|
| 4207 |
+
| > loader_time: 0.0085 (0.00859386958773174)
|
| 4208 |
+
|
| 4209 |
+
|
| 4210 |
+
[1m --> TIME: 2025-12-25 23:17:09 -- STEP: 214/2207 -- GLOBAL_STEP: 41250[0m
|
| 4211 |
+
| > loss_text_ce: 0.02979467809200287 (0.029797586438324405)
|
| 4212 |
+
| > loss_mel_ce: 3.5898358821868896 (3.6284380259914935)
|
| 4213 |
+
| > loss: 0.05655672773718834 (0.05715993145985581)
|
| 4214 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4215 |
+
| > step_time: 0.2658 (0.27779047845680016)
|
| 4216 |
+
| > loader_time: 0.0083 (0.00855109847594644)
|
| 4217 |
+
|
| 4218 |
+
|
| 4219 |
+
[1m --> TIME: 2025-12-25 23:17:19 -- STEP: 239/2207 -- GLOBAL_STEP: 41275[0m
|
| 4220 |
+
| > loss_text_ce: 0.030931800603866577 (0.029759130226331275)
|
| 4221 |
+
| > loss_mel_ce: 3.6647183895111084 (3.6229996272210796)
|
| 4222 |
+
| > loss: 0.05774453282356262 (0.057074355592787514)
|
| 4223 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4224 |
+
| > step_time: 0.2655 (0.2780441359994799)
|
| 4225 |
+
| > loader_time: 0.0089 (0.008533075763590667)
|
| 4226 |
+
|
| 4227 |
+
|
| 4228 |
+
[1m --> TIME: 2025-12-25 23:17:28 -- STEP: 264/2207 -- GLOBAL_STEP: 41300[0m
|
| 4229 |
+
| > loss_text_ce: 0.030077580362558365 (0.02970684285868298)
|
| 4230 |
+
| > loss_mel_ce: 3.7819011211395264 (3.623836860512242)
|
| 4231 |
+
| > loss: 0.05956216901540756 (0.057086620409267416)
|
| 4232 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4233 |
+
| > step_time: 0.2876 (0.2780812328512017)
|
| 4234 |
+
| > loader_time: 0.0086 (0.008524314923719929)
|
| 4235 |
+
|
| 4236 |
+
|
| 4237 |
+
[1m --> TIME: 2025-12-25 23:17:38 -- STEP: 289/2207 -- GLOBAL_STEP: 41325[0m
|
| 4238 |
+
| > loss_text_ce: 0.026399923488497734 (0.029675850535140318)
|
| 4239 |
+
| > loss_mel_ce: 4.021079063415527 (3.622711709626406)
|
| 4240 |
+
| > loss: 0.0632418617606163 (0.0570685556835163)
|
| 4241 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4242 |
+
| > step_time: 0.2895 (0.27836909343627064)
|
| 4243 |
+
| > loader_time: 0.0086 (0.008520363111396976)
|
| 4244 |
+
|
| 4245 |
+
|
| 4246 |
+
[1m --> TIME: 2025-12-25 23:17:47 -- STEP: 314/2207 -- GLOBAL_STEP: 41350[0m
|
| 4247 |
+
| > loss_text_ce: 0.028927525505423546 (0.029657204291385828)
|
| 4248 |
+
| > loss_mel_ce: 3.8427863121032715 (3.619545330667192)
|
| 4249 |
+
| > loss: 0.0604955293238163 (0.05701878965850089)
|
| 4250 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4251 |
+
| > step_time: 0.2677 (0.278355995560907)
|
| 4252 |
+
| > loader_time: 0.0079 (0.008499648160995195)
|
| 4253 |
+
|
| 4254 |
+
|
| 4255 |
+
[1m --> TIME: 2025-12-25 23:17:57 -- STEP: 339/2207 -- GLOBAL_STEP: 41375[0m
|
| 4256 |
+
| > loss_text_ce: 0.028595861047506332 (0.02963820909724123)
|
| 4257 |
+
| > loss_mel_ce: 3.5068726539611816 (3.6197640466830725)
|
| 4258 |
+
| > loss: 0.055241696536540985 (0.05702191027524197)
|
| 4259 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4260 |
+
| > step_time: 0.2664 (0.27811446232078324)
|
| 4261 |
+
| > loader_time: 0.008 (0.00847873997195984)
|
| 4262 |
+
|
| 4263 |
+
|
| 4264 |
+
[1m --> TIME: 2025-12-25 23:18:06 -- STEP: 364/2207 -- GLOBAL_STEP: 41400[0m
|
| 4265 |
+
| > loss_text_ce: 0.02955690771341324 (0.029612432034420117)
|
| 4266 |
+
| > loss_mel_ce: 3.468984842300415 (3.6170055604243014)
|
| 4267 |
+
| > loss: 0.05466471612453461 (0.05697840615943238)
|
| 4268 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4269 |
+
| > step_time: 0.2878 (0.27829282755380125)
|
| 4270 |
+
| > loader_time: 0.0085 (0.008466435002756647)
|
| 4271 |
+
|
| 4272 |
+
|
| 4273 |
+
[1m --> TIME: 2025-12-25 23:18:16 -- STEP: 389/2207 -- GLOBAL_STEP: 41425[0m
|
| 4274 |
+
| > loss_text_ce: 0.029290346428751945 (0.02961127540925015)
|
| 4275 |
+
| > loss_mel_ce: 3.6115591526031494 (3.616704348733309)
|
| 4276 |
+
| > loss: 0.05688827484846115 (0.056973681658276557)
|
| 4277 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4278 |
+
| > step_time: 0.2894 (0.2781374227724834)
|
| 4279 |
+
| > loader_time: 0.0085 (0.008471062373379518)
|
| 4280 |
+
|
| 4281 |
+
|
| 4282 |
+
[1m --> TIME: 2025-12-25 23:18:26 -- STEP: 414/2207 -- GLOBAL_STEP: 41450[0m
|
| 4283 |
+
| > loss_text_ce: 0.02820834331214428 (0.029588528976276302)
|
| 4284 |
+
| > loss_mel_ce: 3.448228120803833 (3.616701604663462)
|
| 4285 |
+
| > loss: 0.054319318383932114 (0.05697328334557261)
|
| 4286 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4287 |
+
| > step_time: 0.2877 (0.2781311192950187)
|
| 4288 |
+
| > loader_time: 0.0084 (0.00845796301744987)
|
| 4289 |
+
|
| 4290 |
+
|
| 4291 |
+
[1m --> TIME: 2025-12-25 23:18:35 -- STEP: 439/2207 -- GLOBAL_STEP: 41475[0m
|
| 4292 |
+
| > loss_text_ce: 0.030648235231637955 (0.029596449650297948)
|
| 4293 |
+
| > loss_mel_ce: 3.3936400413513184 (3.6180809592331733)
|
| 4294 |
+
| > loss: 0.05350450426340103 (0.056994959491396276)
|
| 4295 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4296 |
+
| > step_time: 0.2685 (0.27824544091974135)
|
| 4297 |
+
| > loader_time: 0.0084 (0.008451018626706214)
|
| 4298 |
+
|
| 4299 |
+
|
| 4300 |
+
[1m --> TIME: 2025-12-25 23:18:45 -- STEP: 464/2207 -- GLOBAL_STEP: 41500[0m
|
| 4301 |
+
| > loss_text_ce: 0.030289003625512123 (0.029578746011448574)
|
| 4302 |
+
| > loss_mel_ce: 3.512927770614624 (3.612599209978663)
|
| 4303 |
+
| > loss: 0.0553627610206604 (0.056909030536189675)
|
| 4304 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4305 |
+
| > step_time: 0.2889 (0.27817403088355874)
|
| 4306 |
+
| > loader_time: 0.0079 (0.00844478607177736)
|
| 4307 |
+
|
| 4308 |
+
|
| 4309 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_41500.pth
|
| 4310 |
+
|
| 4311 |
+
[1m --> TIME: 2025-12-25 23:19:19 -- STEP: 489/2207 -- GLOBAL_STEP: 41525[0m
|
| 4312 |
+
| > loss_text_ce: 0.028427859768271446 (0.029566011431575553)
|
| 4313 |
+
| > loss_mel_ce: 3.4678280353546143 (3.608887947897726)
|
| 4314 |
+
| > loss: 0.054628998041152954 (0.0568508430934513)
|
| 4315 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4316 |
+
| > step_time: 0.2908 (0.27810252809817065)
|
| 4317 |
+
| > loader_time: 0.0095 (0.008456921528697271)
|
| 4318 |
+
|
| 4319 |
+
|
| 4320 |
+
[1m --> TIME: 2025-12-25 23:19:29 -- STEP: 514/2207 -- GLOBAL_STEP: 41550[0m
|
| 4321 |
+
| > loss_text_ce: 0.029567204415798187 (0.02959376862743717)
|
| 4322 |
+
| > loss_mel_ce: 3.5579445362091064 (3.610846080204856)
|
| 4323 |
+
| > loss: 0.056054871529340744 (0.056881872603691506)
|
| 4324 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4325 |
+
| > step_time: 0.2966 (0.2779867338299286)
|
| 4326 |
+
| > loader_time: 0.009 (0.008459177462507328)
|
| 4327 |
+
|
| 4328 |
+
|
| 4329 |
+
[1m --> TIME: 2025-12-25 23:19:38 -- STEP: 539/2207 -- GLOBAL_STEP: 41575[0m
|
| 4330 |
+
| > loss_text_ce: 0.029899083077907562 (0.02958948267813848)
|
| 4331 |
+
| > loss_mel_ce: 3.6995041370391846 (3.6107536627321832)
|
| 4332 |
+
| > loss: 0.058271925896406174 (0.056880361619984283)
|
| 4333 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4334 |
+
| > step_time: 0.2878 (0.2778272708404483)
|
| 4335 |
+
| > loader_time: 0.008 (0.008455266757887247)
|
| 4336 |
+
|
| 4337 |
+
|
| 4338 |
+
[1m --> TIME: 2025-12-25 23:19:48 -- STEP: 564/2207 -- GLOBAL_STEP: 41600[0m
|
| 4339 |
+
| > loss_text_ce: 0.030487291514873505 (0.0296106316373809)
|
| 4340 |
+
| > loss_mel_ce: 3.475098133087158 (3.610199705928776)
|
| 4341 |
+
| > loss: 0.05477477237582207 (0.05687203649299365)
|
| 4342 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4343 |
+
| > step_time: 0.2669 (0.27777512487790246)
|
| 4344 |
+
| > loader_time: 0.0088 (0.008471915485165652)
|
| 4345 |
+
|
| 4346 |
+
|
| 4347 |
+
[1m --> TIME: 2025-12-25 23:19:58 -- STEP: 589/2207 -- GLOBAL_STEP: 41625[0m
|
| 4348 |
+
| > loss_text_ce: 0.03004053235054016 (0.029606291613144663)
|
| 4349 |
+
| > loss_mel_ce: 3.4523708820343018 (3.6090977491466227)
|
| 4350 |
+
| > loss: 0.05441267788410187 (0.056854750600553325)
|
| 4351 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4352 |
+
| > step_time: 0.2867 (0.2778470661927564)
|
| 4353 |
+
| > loader_time: 0.0089 (0.008473728225267612)
|
| 4354 |
+
|
| 4355 |
+
|
| 4356 |
+
[1m --> TIME: 2025-12-25 23:20:07 -- STEP: 614/2207 -- GLOBAL_STEP: 41650[0m
|
| 4357 |
+
| > loss_text_ce: 0.028933849185705185 (0.02958771363143424)
|
| 4358 |
+
| > loss_mel_ce: 3.54272198677063 (3.607593520456495)
|
| 4359 |
+
| > loss: 0.055807121098041534 (0.05683095674318677)
|
| 4360 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4361 |
+
| > step_time: 0.2901 (0.27800720444719634)
|
| 4362 |
+
| > loader_time: 0.0083 (0.008469419681288143)
|
| 4363 |
+
|
| 4364 |
+
|
| 4365 |
+
[1m --> TIME: 2025-12-25 23:20:17 -- STEP: 639/2207 -- GLOBAL_STEP: 41675[0m
|
| 4366 |
+
| > loss_text_ce: 0.03017072193324566 (0.029583753274640006)
|
| 4367 |
+
| > loss_mel_ce: 3.718170642852783 (3.608215835359362)
|
| 4368 |
+
| > loss: 0.0585678331553936 (0.05684061852139487)
|
| 4369 |
+
| > grad_norm: 0 (0.0)
|
| 4370 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4371 |
+
| > step_time: 0.2869 (0.27808111411678205)
|
| 4372 |
+
| > loader_time: 0.0097 (0.008479233638781345)
|
| 4373 |
+
|
| 4374 |
+
|
| 4375 |
+
[1m --> TIME: 2025-12-25 23:20:26 -- STEP: 664/2207 -- GLOBAL_STEP: 41700[0m
|
| 4376 |
+
| > loss_text_ce: 0.030205126851797104 (0.02956784174339958)
|
| 4377 |
+
| > loss_mel_ce: 3.8184919357299805 (3.607379682092782)
|
| 4378 |
+
| > loss: 0.060135889798402786 (0.05682730500260929)
|
| 4379 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4380 |
+
| > step_time: 0.2613 (0.27816425784524657)
|
| 4381 |
+
| > loader_time: 0.009 (0.008485546672200589)
|
| 4382 |
+
|
| 4383 |
+
|
| 4384 |
+
[1m --> TIME: 2025-12-25 23:20:36 -- STEP: 689/2207 -- GLOBAL_STEP: 41725[0m
|
| 4385 |
+
| > loss_text_ce: 0.027944227680563927 (0.02955748476033979)
|
| 4386 |
+
| > loss_mel_ce: 3.531876802444458 (3.6053763166048363)
|
| 4387 |
+
| > loss: 0.05562220513820648 (0.05679584060442811)
|
| 4388 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4389 |
+
| > step_time: 0.269 (0.2781369613459218)
|
| 4390 |
+
| > loader_time: 0.0088 (0.008490695316663506)
|
| 4391 |
+
|
| 4392 |
+
|
| 4393 |
+
[1m --> TIME: 2025-12-25 23:20:45 -- STEP: 714/2207 -- GLOBAL_STEP: 41750[0m
|
| 4394 |
+
| > loss_text_ce: 0.02903156913816929 (0.029563568227243524)
|
| 4395 |
+
| > loss_mel_ce: 3.48026180267334 (3.603826522827149)
|
| 4396 |
+
| > loss: 0.05483270809054375 (0.05677172011521016)
|
| 4397 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4398 |
+
| > step_time: 0.2664 (0.27805548956414233)
|
| 4399 |
+
| > loader_time: 0.009 (0.008488639396111842)
|
| 4400 |
+
|
| 4401 |
+
|
| 4402 |
+
[1m --> TIME: 2025-12-25 23:20:55 -- STEP: 739/2207 -- GLOBAL_STEP: 41775[0m
|
| 4403 |
+
| > loss_text_ce: 0.02933335304260254 (0.029572590494954375)
|
| 4404 |
+
| > loss_mel_ce: 3.8951776027679443 (3.605068329383944)
|
| 4405 |
+
| > loss: 0.061320483684539795 (0.056791264311749495)
|
| 4406 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4407 |
+
| > step_time: 0.2665 (0.27794385343507755)
|
| 4408 |
+
| > loader_time: 0.0087 (0.008489666191586295)
|
| 4409 |
+
|
| 4410 |
+
|
| 4411 |
+
[1m --> TIME: 2025-12-25 23:21:04 -- STEP: 764/2207 -- GLOBAL_STEP: 41800[0m
|
| 4412 |
+
| > loss_text_ce: 0.028919801115989685 (0.029566443547196883)
|
| 4413 |
+
| > loss_mel_ce: 3.525236129760742 (3.603446360657977)
|
| 4414 |
+
| > loss: 0.05553368479013443 (0.05676582500006984)
|
| 4415 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4416 |
+
| > step_time: 0.2654 (0.27789217576930664)
|
| 4417 |
+
| > loader_time: 0.0084 (0.008485946667756079)
|
| 4418 |
+
|
| 4419 |
+
|
| 4420 |
+
[1m --> TIME: 2025-12-25 23:21:14 -- STEP: 789/2207 -- GLOBAL_STEP: 41825[0m
|
| 4421 |
+
| > loss_text_ce: 0.031047740951180458 (0.029560877326569025)
|
| 4422 |
+
| > loss_mel_ce: 3.665689706802368 (3.6036518067310372)
|
| 4423 |
+
| > loss: 0.05776152387261391 (0.056768948129639125)
|
| 4424 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4425 |
+
| > step_time: 0.2688 (0.2779034486439442)
|
| 4426 |
+
| > loader_time: 0.0085 (0.00848975894720079)
|
| 4427 |
+
|
| 4428 |
+
|
| 4429 |
+
[1m --> TIME: 2025-12-25 23:21:24 -- STEP: 814/2207 -- GLOBAL_STEP: 41850[0m
|
| 4430 |
+
| > loss_text_ce: 0.02977011911571026 (0.02955492887719184)
|
| 4431 |
+
| > loss_mel_ce: 3.614575147628784 (3.603410256289733)
|
| 4432 |
+
| > loss: 0.0569428950548172 (0.0567650809627786)
|
| 4433 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4434 |
+
| > step_time: 0.2662 (0.2779962682606839)
|
| 4435 |
+
| > loader_time: 0.0087 (0.008490941741249768)
|
| 4436 |
+
|
| 4437 |
+
|
| 4438 |
+
[1m --> TIME: 2025-12-25 23:21:33 -- STEP: 839/2207 -- GLOBAL_STEP: 41875[0m
|
| 4439 |
+
| > loss_text_ce: 0.031444892287254333 (0.02956062940496583)
|
| 4440 |
+
| > loss_mel_ce: 3.551025152206421 (3.6002840220004644)
|
| 4441 |
+
| > loss: 0.05597609281539917 (0.056716322623575115)
|
| 4442 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4443 |
+
| > step_time: 0.2868 (0.27806115292536604)
|
| 4444 |
+
| > loader_time: 0.0078 (0.0084913640823637)
|
| 4445 |
+
|
| 4446 |
+
|
| 4447 |
+
[1m --> TIME: 2025-12-25 23:21:43 -- STEP: 864/2207 -- GLOBAL_STEP: 41900[0m
|
| 4448 |
+
| > loss_text_ce: 0.030650192871689796 (0.02956611708375729)
|
| 4449 |
+
| > loss_mel_ce: 3.5646896362304688 (3.5992751491290558)
|
| 4450 |
+
| > loss: 0.056177183985710144 (0.056700644730072886)
|
| 4451 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4452 |
+
| > step_time: 0.295 (0.27810406933228154)
|
| 4453 |
+
| > loader_time: 0.0082 (0.008486436197051295)
|
| 4454 |
+
|
| 4455 |
+
|
| 4456 |
+
[1m --> TIME: 2025-12-25 23:21:53 -- STEP: 889/2207 -- GLOBAL_STEP: 41925[0m
|
| 4457 |
+
| > loss_text_ce: 0.02855757251381874 (0.029559336956778984)
|
| 4458 |
+
| > loss_mel_ce: 3.715897560119629 (3.5998992375501504)
|
| 4459 |
+
| > loss: 0.05850711092352867 (0.0567102901704601)
|
| 4460 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4461 |
+
| > step_time: 0.2974 (0.27816501403954474)
|
| 4462 |
+
| > loader_time: 0.0085 (0.00849082579897011)
|
| 4463 |
+
|
| 4464 |
+
|
| 4465 |
+
[1m --> TIME: 2025-12-25 23:22:02 -- STEP: 914/2207 -- GLOBAL_STEP: 41950[0m
|
| 4466 |
+
| > loss_text_ce: 0.02868977189064026 (0.029573501073459827)
|
| 4467 |
+
| > loss_mel_ce: 3.403611898422241 (3.600536287483032)
|
| 4468 |
+
| > loss: 0.053629714995622635 (0.0567204654053734)
|
| 4469 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4470 |
+
| > step_time: 0.2694 (0.2781048731417833)
|
| 4471 |
+
| > loader_time: 0.0088 (0.008485719724609247)
|
| 4472 |
+
|
| 4473 |
+
|
| 4474 |
+
[1m --> TIME: 2025-12-25 23:22:12 -- STEP: 939/2207 -- GLOBAL_STEP: 41975[0m
|
| 4475 |
+
| > loss_text_ce: 0.0286455936729908 (0.029580683363115572)
|
| 4476 |
+
| > loss_mel_ce: 3.444798707962036 (3.601398053133704)
|
| 4477 |
+
| > loss: 0.05427256599068642 (0.056734042720228466)
|
| 4478 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4479 |
+
| > step_time: 0.2693 (0.2783675079528515)
|
| 4480 |
+
| > loader_time: 0.0088 (0.008484023574569106)
|
| 4481 |
+
|
| 4482 |
+
|
| 4483 |
+
[1m --> TIME: 2025-12-25 23:22:21 -- STEP: 964/2207 -- GLOBAL_STEP: 42000[0m
|
| 4484 |
+
| > loss_text_ce: 0.029374901205301285 (0.029570941685764622)
|
| 4485 |
+
| > loss_mel_ce: 3.535006284713745 (3.6007267899533035)
|
| 4486 |
+
| > loss: 0.055693455040454865 (0.056723402013144304)
|
| 4487 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4488 |
+
| > step_time: 0.2873 (0.2783847396799142)
|
| 4489 |
+
| > loader_time: 0.0085 (0.008483292650879671)
|
| 4490 |
+
|
| 4491 |
+
|
| 4492 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_42000.pth
|
| 4493 |
+
|
| 4494 |
+
[1m --> TIME: 2025-12-25 23:22:56 -- STEP: 989/2207 -- GLOBAL_STEP: 42025[0m
|
| 4495 |
+
| > loss_text_ce: 0.02877601981163025 (0.02957262864564258)
|
| 4496 |
+
| > loss_mel_ce: 3.5507211685180664 (3.6016918101373414)
|
| 4497 |
+
| > loss: 0.05592964217066765 (0.05673850681425587)
|
| 4498 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4499 |
+
| > step_time: 0.2917 (0.27828343645024456)
|
| 4500 |
+
| > loader_time: 0.0082 (0.00848140007805897)
|
| 4501 |
+
|
| 4502 |
+
|
| 4503 |
+
[1m --> TIME: 2025-12-25 23:23:05 -- STEP: 1014/2207 -- GLOBAL_STEP: 42050[0m
|
| 4504 |
+
| > loss_text_ce: 0.03058113530278206 (0.029584544376494028)
|
| 4505 |
+
| > loss_mel_ce: 3.402261972427368 (3.6026174594194464)
|
| 4506 |
+
| > loss: 0.05363817512989044 (0.05675315626582803)
|
| 4507 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4508 |
+
| > step_time: 0.2666 (0.27822229942156235)
|
| 4509 |
+
| > loader_time: 0.0093 (0.008486229757350347)
|
| 4510 |
+
|
| 4511 |
+
|
| 4512 |
+
[1m --> TIME: 2025-12-25 23:23:15 -- STEP: 1039/2207 -- GLOBAL_STEP: 42075[0m
|
| 4513 |
+
| > loss_text_ce: 0.030606308951973915 (0.029591388799630185)
|
| 4514 |
+
| > loss_mel_ce: 3.599738597869873 (3.6018914731221208)
|
| 4515 |
+
| > loss: 0.05672413855791092 (0.056741919680440275)
|
| 4516 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4517 |
+
| > step_time: 0.2686 (0.2781383266118538)
|
| 4518 |
+
| > loader_time: 0.0081 (0.008483836006039724)
|
| 4519 |
+
|
| 4520 |
+
|
| 4521 |
+
[1m --> TIME: 2025-12-25 23:23:24 -- STEP: 1064/2207 -- GLOBAL_STEP: 42100[0m
|
| 4522 |
+
| > loss_text_ce: 0.030921196565032005 (0.029586316923141703)
|
| 4523 |
+
| > loss_mel_ce: 3.6529388427734375 (3.601202233841545)
|
| 4524 |
+
| > loss: 0.05756031349301338 (0.05673107106756783)
|
| 4525 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4526 |
+
| > step_time: 0.2619 (0.2781537943764736)
|
| 4527 |
+
| > loader_time: 0.0083 (0.008485158583275363)
|
| 4528 |
+
|
| 4529 |
+
|
| 4530 |
+
[1m --> TIME: 2025-12-25 23:23:34 -- STEP: 1089/2207 -- GLOBAL_STEP: 42125[0m
|
| 4531 |
+
| > loss_text_ce: 0.029492978006601334 (0.02957583624524081)
|
| 4532 |
+
| > loss_mel_ce: 3.5206401348114014 (3.601361174885563)
|
| 4533 |
+
| > loss: 0.05547083169221878 (0.056733390769627884)
|
| 4534 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4535 |
+
| > step_time: 0.2687 (0.27818116581253866)
|
| 4536 |
+
| > loader_time: 0.0087 (0.008484842144534396)
|
| 4537 |
+
|
| 4538 |
+
|
| 4539 |
+
[1m --> TIME: 2025-12-25 23:23:44 -- STEP: 1114/2207 -- GLOBAL_STEP: 42150[0m
|
| 4540 |
+
| > loss_text_ce: 0.027256952598690987 (0.029572197190975806)
|
| 4541 |
+
| > loss_mel_ce: 3.6689023971557617 (3.601469965982694)
|
| 4542 |
+
| > loss: 0.05775249004364014 (0.05673503376797669)
|
| 4543 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4544 |
+
| > step_time: 0.2976 (0.27826024835370816)
|
| 4545 |
+
| > loader_time: 0.0082 (0.008485612150154602)
|
| 4546 |
+
|
| 4547 |
+
|
| 4548 |
+
[1m --> TIME: 2025-12-25 23:23:54 -- STEP: 1139/2207 -- GLOBAL_STEP: 42175[0m
|
| 4549 |
+
| > loss_text_ce: 0.02795250155031681 (0.029575957361481192)
|
| 4550 |
+
| > loss_mel_ce: 3.2562546730041504 (3.6013087275993207)
|
| 4551 |
+
| > loss: 0.05131573602557182 (0.05673257317116009)
|
| 4552 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4553 |
+
| > step_time: 0.2868 (0.2783841270433382)
|
| 4554 |
+
| > loader_time: 0.0084 (0.008486073513215222)
|
| 4555 |
+
|
| 4556 |
+
|
| 4557 |
+
[1m --> TIME: 2025-12-25 23:24:03 -- STEP: 1164/2207 -- GLOBAL_STEP: 42200[0m
|
| 4558 |
+
| > loss_text_ce: 0.02844488061964512 (0.029562839188475385)
|
| 4559 |
+
| > loss_mel_ce: 3.6800124645233154 (3.6007591389708504)
|
| 4560 |
+
| > loss: 0.0579446442425251 (0.056723780878835506)
|
| 4561 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4562 |
+
| > step_time: 0.2907 (0.2784255165414714)
|
| 4563 |
+
| > loader_time: 0.0083 (0.008487004185050637)
|
| 4564 |
+
|
| 4565 |
+
|
| 4566 |
+
[1m --> TIME: 2025-12-25 23:24:13 -- STEP: 1189/2207 -- GLOBAL_STEP: 42225[0m
|
| 4567 |
+
| > loss_text_ce: 0.03105485998094082 (0.029563062251729565)
|
| 4568 |
+
| > loss_mel_ce: 3.4255211353302 (3.6015043970513885)
|
| 4569 |
+
| > loss: 0.054009001702070236 (0.05673542902926317)
|
| 4570 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4571 |
+
| > step_time: 0.2875 (0.2783987117074339)
|
| 4572 |
+
| > loader_time: 0.008 (0.008488745324445247)
|
| 4573 |
+
|
| 4574 |
+
|
| 4575 |
+
[1m --> TIME: 2025-12-25 23:24:22 -- STEP: 1214/2207 -- GLOBAL_STEP: 42250[0m
|
| 4576 |
+
| > loss_text_ce: 0.028729740530252457 (0.029564312930775834)
|
| 4577 |
+
| > loss_mel_ce: 3.380117893218994 (3.6012649683426003)
|
| 4578 |
+
| > loss: 0.05326324328780174 (0.056731707491655695)
|
| 4579 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4580 |
+
| > step_time: 0.2865 (0.2785123900880142)
|
| 4581 |
+
| > loader_time: 0.0083 (0.008485794852946515)
|
| 4582 |
+
|
| 4583 |
+
|
| 4584 |
+
[1m --> TIME: 2025-12-25 23:24:32 -- STEP: 1239/2207 -- GLOBAL_STEP: 42275[0m
|
| 4585 |
+
| > loss_text_ce: 0.03004319593310356 (0.029563165243483538)
|
| 4586 |
+
| > loss_mel_ce: 3.65985107421875 (3.599763808277367)
|
| 4587 |
+
| > loss: 0.057654596865177155 (0.056708233931595686)
|
| 4588 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4589 |
+
| > step_time: 0.2918 (0.27849108620552054)
|
| 4590 |
+
| > loader_time: 0.0083 (0.00848392193311642)
|
| 4591 |
+
|
| 4592 |
+
|
| 4593 |
+
[1m --> TIME: 2025-12-25 23:24:42 -- STEP: 1264/2207 -- GLOBAL_STEP: 42300[0m
|
| 4594 |
+
| > loss_text_ce: 0.03055543825030327 (0.029573862602981404)
|
| 4595 |
+
| > loss_mel_ce: 3.399517774581909 (3.5997410069538067)
|
| 4596 |
+
| > loss: 0.05359489470720291 (0.056708044811635244)
|
| 4597 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4598 |
+
| > step_time: 0.2663 (0.2784884217419206)
|
| 4599 |
+
| > loader_time: 0.0086 (0.008483402525322363)
|
| 4600 |
+
|
| 4601 |
+
|
| 4602 |
+
[1m --> TIME: 2025-12-25 23:24:51 -- STEP: 1289/2207 -- GLOBAL_STEP: 42325[0m
|
| 4603 |
+
| > loss_text_ce: 0.02813180536031723 (0.029572238489592423)
|
| 4604 |
+
| > loss_mel_ce: 3.480710744857788 (3.598701111746907)
|
| 4605 |
+
| > loss: 0.054825663566589355 (0.0566917710711412)
|
| 4606 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4607 |
+
| > step_time: 0.2693 (0.2784063414698709)
|
| 4608 |
+
| > loader_time: 0.0089 (0.008479687111427672)
|
| 4609 |
+
|
| 4610 |
+
|
| 4611 |
+
[1m --> TIME: 2025-12-25 23:25:01 -- STEP: 1314/2207 -- GLOBAL_STEP: 42350[0m
|
| 4612 |
+
| > loss_text_ce: 0.027433201670646667 (0.0295668862833779)
|
| 4613 |
+
| > loss_mel_ce: 3.567831516265869 (3.598705720139421)
|
| 4614 |
+
| > loss: 0.05617601051926613 (0.05669175945165674)
|
| 4615 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4616 |
+
| > step_time: 0.2966 (0.27848474039516163)
|
| 4617 |
+
| > loader_time: 0.0084 (0.008478812431091602)
|
| 4618 |
+
|
| 4619 |
+
|
| 4620 |
+
[1m --> TIME: 2025-12-25 23:25:10 -- STEP: 1339/2207 -- GLOBAL_STEP: 42375[0m
|
| 4621 |
+
| > loss_text_ce: 0.03003629669547081 (0.029575146296526893)
|
| 4622 |
+
| > loss_mel_ce: 3.5849788188934326 (3.599264433706225)
|
| 4623 |
+
| > loss: 0.056484609842300415 (0.05670061841595963)
|
| 4624 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4625 |
+
| > step_time: 0.2684 (0.27855046103479647)
|
| 4626 |
+
| > loader_time: 0.0084 (0.008481283344556556)
|
| 4627 |
+
|
| 4628 |
+
|
| 4629 |
+
[1m --> TIME: 2025-12-25 23:25:20 -- STEP: 1364/2207 -- GLOBAL_STEP: 42400[0m
|
| 4630 |
+
| > loss_text_ce: 0.030226610600948334 (0.029577514211322212)
|
| 4631 |
+
| > loss_mel_ce: 3.8958935737609863 (3.599298962749694)
|
| 4632 |
+
| > loss: 0.06134562939405441 (0.05670119492921572)
|
| 4633 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4634 |
+
| > step_time: 0.2663 (0.2785459048936797)
|
| 4635 |
+
| > loader_time: 0.0078 (0.008481966610178584)
|
| 4636 |
+
|
| 4637 |
+
|
| 4638 |
+
[1m --> TIME: 2025-12-25 23:25:29 -- STEP: 1389/2207 -- GLOBAL_STEP: 42425[0m
|
| 4639 |
+
| > loss_text_ce: 0.03182559087872505 (0.029584931318304197)
|
| 4640 |
+
| > loss_mel_ce: 3.6650683879852295 (3.599335428276364)
|
| 4641 |
+
| > loss: 0.0577639676630497 (0.056701880602784235)
|
| 4642 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4643 |
+
| > step_time: 0.2654 (0.2784240020960679)
|
| 4644 |
+
| > loader_time: 0.0085 (0.008483852217573338)
|
| 4645 |
+
|
| 4646 |
+
|
| 4647 |
+
[1m --> TIME: 2025-12-25 23:25:39 -- STEP: 1414/2207 -- GLOBAL_STEP: 42450[0m
|
| 4648 |
+
| > loss_text_ce: 0.028797058388590813 (0.02958683400576911)
|
| 4649 |
+
| > loss_mel_ce: 3.1696295738220215 (3.599183698523331)
|
| 4650 |
+
| > loss: 0.04997541755437851 (0.05669953955714147)
|
| 4651 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4652 |
+
| > step_time: 0.2903 (0.2784354845233147)
|
| 4653 |
+
| > loader_time: 0.0081 (0.008480801777927352)
|
| 4654 |
+
|
| 4655 |
+
|
| 4656 |
+
[1m --> TIME: 2025-12-25 23:25:48 -- STEP: 1439/2207 -- GLOBAL_STEP: 42475[0m
|
| 4657 |
+
| > loss_text_ce: 0.028589699417352676 (0.02958510335873945)
|
| 4658 |
+
| > loss_mel_ce: 3.5768468379974365 (3.599196351510605)
|
| 4659 |
+
| > loss: 0.05633494630455971 (0.05669971021659841)
|
| 4660 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4661 |
+
| > step_time: 0.3036 (0.2784237182330888)
|
| 4662 |
+
| > loader_time: 0.008 (0.008476556548648123)
|
| 4663 |
+
|
| 4664 |
+
|
| 4665 |
+
[1m --> TIME: 2025-12-25 23:25:58 -- STEP: 1464/2207 -- GLOBAL_STEP: 42500[0m
|
| 4666 |
+
| > loss_text_ce: 0.030285343527793884 (0.02958682348539473)
|
| 4667 |
+
| > loss_mel_ce: 3.851951837539673 (3.599921690798848)
|
| 4668 |
+
| > loss: 0.06065995618700981 (0.05671107052708338)
|
| 4669 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4670 |
+
| > step_time: 0.2667 (0.278392049295655)
|
| 4671 |
+
| > loader_time: 0.0084 (0.008473553618446733)
|
| 4672 |
+
|
| 4673 |
+
|
| 4674 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_42500.pth
|
| 4675 |
+
|
| 4676 |
+
[1m --> TIME: 2025-12-25 23:26:32 -- STEP: 1489/2207 -- GLOBAL_STEP: 42525[0m
|
| 4677 |
+
| > loss_text_ce: 0.030694296583533287 (0.02958516967955094)
|
| 4678 |
+
| > loss_mel_ce: 3.585581064224243 (3.6000041769526003)
|
| 4679 |
+
| > loss: 0.056504301726818085 (0.05671233353577177)
|
| 4680 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4681 |
+
| > step_time: 0.2863 (0.27834421088024647)
|
| 4682 |
+
| > loader_time: 0.0081 (0.00847521974705786)
|
| 4683 |
+
|
| 4684 |
+
|
| 4685 |
+
[1m --> TIME: 2025-12-25 23:26:42 -- STEP: 1514/2207 -- GLOBAL_STEP: 42550[0m
|
| 4686 |
+
| > loss_text_ce: 0.03040524572134018 (0.029581509330008512)
|
| 4687 |
+
| > loss_mel_ce: 3.537659168243408 (3.5998623156956984)
|
| 4688 |
+
| > loss: 0.0557510070502758 (0.05671005976070184)
|
| 4689 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4690 |
+
| > step_time: 0.2909 (0.27840917507282525)
|
| 4691 |
+
| > loader_time: 0.0085 (0.008478734893244522)
|
| 4692 |
+
|
| 4693 |
+
|
| 4694 |
+
[1m --> TIME: 2025-12-25 23:26:52 -- STEP: 1539/2207 -- GLOBAL_STEP: 42575[0m
|
| 4695 |
+
| > loss_text_ce: 0.03251832351088524 (0.029576580182366596)
|
| 4696 |
+
| > loss_mel_ce: 4.2960968017578125 (3.599331544389037)
|
| 4697 |
+
| > loss: 0.06763461232185364 (0.05670168944298638)
|
| 4698 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4699 |
+
| > step_time: 0.2664 (0.2784521189349439)
|
| 4700 |
+
| > loader_time: 0.0083 (0.008477347326247713)
|
| 4701 |
+
|
| 4702 |
+
|
| 4703 |
+
[1m --> TIME: 2025-12-25 23:27:01 -- STEP: 1564/2207 -- GLOBAL_STEP: 42600[0m
|
| 4704 |
+
| > loss_text_ce: 0.03227081522345543 (0.029573800061326812)
|
| 4705 |
+
| > loss_mel_ce: 3.716618299484253 (3.599369705790449)
|
| 4706 |
+
| > loss: 0.058576393872499466 (0.0567022422781152)
|
| 4707 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4708 |
+
| > step_time: 0.2681 (0.2784213542633352)
|
| 4709 |
+
| > loader_time: 0.008 (0.008472342472856926)
|
| 4710 |
+
|
| 4711 |
+
|
| 4712 |
+
[1m --> TIME: 2025-12-25 23:27:11 -- STEP: 1589/2207 -- GLOBAL_STEP: 42625[0m
|
| 4713 |
+
| > loss_text_ce: 0.02894115447998047 (0.029575394229732502)
|
| 4714 |
+
| > loss_mel_ce: 3.6639161109924316 (3.598584911671729)
|
| 4715 |
+
| > loss: 0.05770089477300644 (0.05669000477320821)
|
| 4716 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4717 |
+
| > step_time: 0.2895 (0.27842069047288087)
|
| 4718 |
+
| > loader_time: 0.0088 (0.008471206271325823)
|
| 4719 |
+
|
| 4720 |
+
|
| 4721 |
+
[1m --> TIME: 2025-12-25 23:27:20 -- STEP: 1614/2207 -- GLOBAL_STEP: 42650[0m
|
| 4722 |
+
| > loss_text_ce: 0.030559048056602478 (0.029575852775320766)
|
| 4723 |
+
| > loss_mel_ce: 3.5500946044921875 (3.59878020038392)
|
| 4724 |
+
| > loss: 0.05594771355390549 (0.056693063328755816)
|
| 4725 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4726 |
+
| > step_time: 0.287 (0.27839821274248067)
|
| 4727 |
+
| > loader_time: 0.0084 (0.008469286609020974)
|
| 4728 |
+
|
| 4729 |
+
|
| 4730 |
+
[1m --> TIME: 2025-12-25 23:27:30 -- STEP: 1639/2207 -- GLOBAL_STEP: 42675[0m
|
| 4731 |
+
| > loss_text_ce: 0.0281391404569149 (0.029570235982719108)
|
| 4732 |
+
| > loss_mel_ce: 3.8394227027893066 (3.598065768021356)
|
| 4733 |
+
| > loss: 0.060430653393268585 (0.05668181255959823)
|
| 4734 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4735 |
+
| > step_time: 0.269 (0.2784356612705328)
|
| 4736 |
+
| > loader_time: 0.0087 (0.008468279684368182)
|
| 4737 |
+
|
| 4738 |
+
|
| 4739 |
+
[1m --> TIME: 2025-12-25 23:27:39 -- STEP: 1664/2207 -- GLOBAL_STEP: 42700[0m
|
| 4740 |
+
| > loss_text_ce: 0.030656512826681137 (0.02957000245917768)
|
| 4741 |
+
| > loss_mel_ce: 3.4719624519348145 (3.597907980187581)
|
| 4742 |
+
| > loss: 0.054728422313928604 (0.0566793434800974)
|
| 4743 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4744 |
+
| > step_time: 0.261 (0.2784170211794288)
|
| 4745 |
+
| > loader_time: 0.0083 (0.008468554570124706)
|
| 4746 |
+
|
| 4747 |
+
|
| 4748 |
+
[1m --> TIME: 2025-12-25 23:27:49 -- STEP: 1689/2207 -- GLOBAL_STEP: 42725[0m
|
| 4749 |
+
| > loss_text_ce: 0.02871408313512802 (0.02957116631496546)
|
| 4750 |
+
| > loss_mel_ce: 3.3278684616088867 (3.5970638652324958)
|
| 4751 |
+
| > loss: 0.052446603775024414 (0.056666172372564313)
|
| 4752 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4753 |
+
| > step_time: 0.2674 (0.27836334274816005)
|
| 4754 |
+
| > loader_time: 0.0085 (0.008467691618002975)
|
| 4755 |
+
|
| 4756 |
+
|
| 4757 |
+
[1m --> TIME: 2025-12-25 23:27:58 -- STEP: 1714/2207 -- GLOBAL_STEP: 42750[0m
|
| 4758 |
+
| > loss_text_ce: 0.03059106506407261 (0.029571687544032048)
|
| 4759 |
+
| > loss_mel_ce: 3.564976692199707 (3.597073941414387)
|
| 4760 |
+
| > loss: 0.056180745363235474 (0.05666633795559961)
|
| 4761 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4762 |
+
| > step_time: 0.2656 (0.2783931440900897)
|
| 4763 |
+
| > loader_time: 0.0081 (0.008467214249396136)
|
| 4764 |
+
|
| 4765 |
+
|
| 4766 |
+
[1m --> TIME: 2025-12-25 23:28:08 -- STEP: 1739/2207 -- GLOBAL_STEP: 42775[0m
|
| 4767 |
+
| > loss_text_ce: 0.029922181740403175 (0.02957340050006715)
|
| 4768 |
+
| > loss_mel_ce: 3.630336046218872 (3.5969347867422776)
|
| 4769 |
+
| > loss: 0.05719153583049774 (0.05666419043451495)
|
| 4770 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4771 |
+
| > step_time: 0.2898 (0.27837559983537036)
|
| 4772 |
+
| > loader_time: 0.0084 (0.008465426073737915)
|
| 4773 |
+
|
| 4774 |
+
|
| 4775 |
+
[1m --> TIME: 2025-12-25 23:28:18 -- STEP: 1764/2207 -- GLOBAL_STEP: 42800[0m
|
| 4776 |
+
| > loss_text_ce: 0.030503809452056885 (0.029574879581349443)
|
| 4777 |
+
| > loss_mel_ce: 3.599358320236206 (3.597540051082905)
|
| 4778 |
+
| > loss: 0.056716594845056534 (0.05667367080002684)
|
| 4779 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4780 |
+
| > step_time: 0.2875 (0.2784005922525112)
|
| 4781 |
+
| > loader_time: 0.0081 (0.008464619280799992)
|
| 4782 |
+
|
| 4783 |
+
|
| 4784 |
+
[1m --> TIME: 2025-12-25 23:28:27 -- STEP: 1789/2207 -- GLOBAL_STEP: 42825[0m
|
| 4785 |
+
| > loss_text_ce: 0.02997908554971218 (0.02957231300071407)
|
| 4786 |
+
| > loss_mel_ce: 3.9292876720428467 (3.597976316214274)
|
| 4787 |
+
| > loss: 0.06186354160308838 (0.05668044733711298)
|
| 4788 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4789 |
+
| > step_time: 0.2552 (0.2783788134093528)
|
| 4790 |
+
| > loader_time: 0.0082 (0.00846528433900495)
|
| 4791 |
+
|
| 4792 |
+
|
| 4793 |
+
[1m --> TIME: 2025-12-25 23:28:37 -- STEP: 1814/2207 -- GLOBAL_STEP: 42850[0m
|
| 4794 |
+
| > loss_text_ce: 0.030859747901558876 (0.02957415586294321)
|
| 4795 |
+
| > loss_mel_ce: 3.393686532974243 (3.5970252693159446)
|
| 4796 |
+
| > loss: 0.05350853502750397 (0.05666561602371463)
|
| 4797 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4798 |
+
| > step_time: 0.2895 (0.27832105009926517)
|
| 4799 |
+
| > loader_time: 0.0084 (0.008463889524718498)
|
| 4800 |
+
|
| 4801 |
+
|
| 4802 |
+
[1m --> TIME: 2025-12-25 23:28:46 -- STEP: 1839/2207 -- GLOBAL_STEP: 42875[0m
|
| 4803 |
+
| > loss_text_ce: 0.028500942513346672 (0.02956993839297851)
|
| 4804 |
+
| > loss_mel_ce: 3.4204230308532715 (3.59739816856488)
|
| 4805 |
+
| > loss: 0.0538894385099411 (0.05667137667787923)
|
| 4806 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4807 |
+
| > step_time: 0.2671 (0.27831257912955826)
|
| 4808 |
+
| > loader_time: 0.0084 (0.00846286206351732)
|
| 4809 |
+
|
| 4810 |
+
|
| 4811 |
+
[1m --> TIME: 2025-12-25 23:28:56 -- STEP: 1864/2207 -- GLOBAL_STEP: 42900[0m
|
| 4812 |
+
| > loss_text_ce: 0.03010193444788456 (0.029573831843140302)
|
| 4813 |
+
| > loss_mel_ce: 3.2266414165496826 (3.5968967596860404)
|
| 4814 |
+
| > loss: 0.0508866161108017 (0.05666360299426725)
|
| 4815 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4816 |
+
| > step_time: 0.2667 (0.27829414375861883)
|
| 4817 |
+
| > loader_time: 0.0084 (0.008463309941885296)
|
| 4818 |
+
|
| 4819 |
+
|
| 4820 |
+
[1m --> TIME: 2025-12-25 23:29:05 -- STEP: 1889/2207 -- GLOBAL_STEP: 42925[0m
|
| 4821 |
+
| > loss_text_ce: 0.028747040778398514 (0.02957620279042099)
|
| 4822 |
+
| > loss_mel_ce: 3.624959945678711 (3.5972354839944916)
|
| 4823 |
+
| > loss: 0.057089172303676605 (0.05666893260733925)
|
| 4824 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4825 |
+
| > step_time: 0.2897 (0.2782995338601501)
|
| 4826 |
+
| > loader_time: 0.0084 (0.00846654173556682)
|
| 4827 |
+
|
| 4828 |
+
|
| 4829 |
+
[1m --> TIME: 2025-12-25 23:29:15 -- STEP: 1914/2207 -- GLOBAL_STEP: 42950[0m
|
| 4830 |
+
| > loss_text_ce: 0.029753303155303 (0.02957458005947535)
|
| 4831 |
+
| > loss_mel_ce: 3.1703855991363525 (3.597058830216387)
|
| 4832 |
+
| > loss: 0.050002168864011765 (0.05666614703779558)
|
| 4833 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4834 |
+
| > step_time: 0.2671 (0.27827787187530506)
|
| 4835 |
+
| > loader_time: 0.0082 (0.008465207731711333)
|
| 4836 |
+
|
| 4837 |
+
|
| 4838 |
+
[1m --> TIME: 2025-12-25 23:29:24 -- STEP: 1939/2207 -- GLOBAL_STEP: 42975[0m
|
| 4839 |
+
| > loss_text_ce: 0.028644071891903877 (0.029570777708872995)
|
| 4840 |
+
| > loss_mel_ce: 3.465646982192993 (3.596742140134012)
|
| 4841 |
+
| > loss: 0.05459829792380333 (0.05666113934561785)
|
| 4842 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4843 |
+
| > step_time: 0.2912 (0.2783261578743336)
|
| 4844 |
+
| > loader_time: 0.0088 (0.00846441545825819)
|
| 4845 |
+
|
| 4846 |
+
|
| 4847 |
+
[1m --> TIME: 2025-12-25 23:29:34 -- STEP: 1964/2207 -- GLOBAL_STEP: 43000[0m
|
| 4848 |
+
| > loss_text_ce: 0.027508949860930443 (0.029569518872842104)
|
| 4849 |
+
| > loss_mel_ce: 3.246126413345337 (3.5958170134521064)
|
| 4850 |
+
| > loss: 0.05115055292844772 (0.05664666457378632)
|
| 4851 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4852 |
+
| > step_time: 0.2672 (0.2783618617931845)
|
| 4853 |
+
| > loader_time: 0.0094 (0.00846497577950814)
|
| 4854 |
+
|
| 4855 |
+
|
| 4856 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/checkpoint_43000.pth
|
| 4857 |
+
|
| 4858 |
+
[1m --> TIME: 2025-12-25 23:30:09 -- STEP: 1989/2207 -- GLOBAL_STEP: 43025[0m
|
| 4859 |
+
| > loss_text_ce: 0.028456682339310646 (0.029571294483938013)
|
| 4860 |
+
| > loss_mel_ce: 3.6640565395355225 (3.5960998846814523)
|
| 4861 |
+
| > loss: 0.05769551917910576 (0.05665111218664799)
|
| 4862 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4863 |
+
| > step_time: 0.2691 (0.2783945800671935)
|
| 4864 |
+
| > loader_time: 0.0086 (0.008471240345229454)
|
| 4865 |
+
|
| 4866 |
+
|
| 4867 |
+
[1m --> TIME: 2025-12-25 23:30:19 -- STEP: 2014/2207 -- GLOBAL_STEP: 43050[0m
|
| 4868 |
+
| > loss_text_ce: 0.028763320297002792 (0.029570565616310593)
|
| 4869 |
+
| > loss_mel_ce: 3.8959906101226807 (3.5956691391063185)
|
| 4870 |
+
| > loss: 0.06132427975535393 (0.05664437039835745)
|
| 4871 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4872 |
+
| > step_time: 0.2662 (0.2783633520255146)
|
| 4873 |
+
| > loader_time: 0.0083 (0.008468708191751372)
|
| 4874 |
+
|
| 4875 |
+
|
| 4876 |
+
[1m --> TIME: 2025-12-25 23:30:28 -- STEP: 2039/2207 -- GLOBAL_STEP: 43075[0m
|
| 4877 |
+
| > loss_text_ce: 0.028858451172709465 (0.02956718143190807)
|
| 4878 |
+
| > loss_mel_ce: 3.644981622695923 (3.5961525387597937)
|
| 4879 |
+
| > loss: 0.05740375071763992 (0.056651870642896944)
|
| 4880 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4881 |
+
| > step_time: 0.2973 (0.27841832007538686)
|
| 4882 |
+
| > loader_time: 0.0076 (0.008465924527260417)
|
| 4883 |
+
|
| 4884 |
+
|
| 4885 |
+
[1m --> TIME: 2025-12-25 23:30:38 -- STEP: 2064/2207 -- GLOBAL_STEP: 43100[0m
|
| 4886 |
+
| > loss_text_ce: 0.02978721633553505 (0.02957052259630156)
|
| 4887 |
+
| > loss_mel_ce: 3.81852650642395 (3.596414100638656)
|
| 4888 |
+
| > loss: 0.06012990325689316 (0.05665600975405048)
|
| 4889 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4890 |
+
| > step_time: 0.2667 (0.2783928455770483)
|
| 4891 |
+
| > loader_time: 0.0081 (0.008466232885686006)
|
| 4892 |
+
|
| 4893 |
+
|
| 4894 |
+
[1m --> TIME: 2025-12-25 23:30:47 -- STEP: 2089/2207 -- GLOBAL_STEP: 43125[0m
|
| 4895 |
+
| > loss_text_ce: 0.028963157907128334 (0.02957591548733246)
|
| 4896 |
+
| > loss_mel_ce: 3.6387929916381836 (3.595594869725952)
|
| 4897 |
+
| > loss: 0.05730868875980377 (0.0566432935311898)
|
| 4898 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4899 |
+
| > step_time: 0.2669 (0.27839279391652655)
|
| 4900 |
+
| > loader_time: 0.0081 (0.008465035786863835)
|
| 4901 |
+
|
| 4902 |
+
|
| 4903 |
+
[1m --> TIME: 2025-12-25 23:30:57 -- STEP: 2114/2207 -- GLOBAL_STEP: 43150[0m
|
| 4904 |
+
| > loss_text_ce: 0.030970850959420204 (0.02957318797566582)
|
| 4905 |
+
| > loss_mel_ce: 3.37962007522583 (3.594812859424241)
|
| 4906 |
+
| > loss: 0.053290482610464096 (0.0566310320038951)
|
| 4907 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4908 |
+
| > step_time: 0.2876 (0.2784062750292145)
|
| 4909 |
+
| > loader_time: 0.0085 (0.008462523218586183)
|
| 4910 |
+
|
| 4911 |
+
|
| 4912 |
+
[1m --> TIME: 2025-12-25 23:31:07 -- STEP: 2139/2207 -- GLOBAL_STEP: 43175[0m
|
| 4913 |
+
| > loss_text_ce: 0.02873104251921177 (0.029576002317556936)
|
| 4914 |
+
| > loss_mel_ce: 3.5327610969543457 (3.59504180592095)
|
| 4915 |
+
| > loss: 0.0556483156979084 (0.05663465326908817)
|
| 4916 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4917 |
+
| > step_time: 0.256 (0.27841871884895797)
|
| 4918 |
+
| > loader_time: 0.0082 (0.00846067941762845)
|
| 4919 |
+
|
| 4920 |
+
|
| 4921 |
+
[1m --> TIME: 2025-12-25 23:31:16 -- STEP: 2164/2207 -- GLOBAL_STEP: 43200[0m
|
| 4922 |
+
| > loss_text_ce: 0.027894100174307823 (0.029574418035912535)
|
| 4923 |
+
| > loss_mel_ce: 3.232783794403076 (3.5945909760355277)
|
| 4924 |
+
| > loss: 0.050948090851306915 (0.05662758429767134)
|
| 4925 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4926 |
+
| > step_time: 0.2689 (0.2784275403745519)
|
| 4927 |
+
| > loader_time: 0.0094 (0.008461141872758565)
|
| 4928 |
+
|
| 4929 |
+
|
| 4930 |
+
[1m --> TIME: 2025-12-25 23:31:26 -- STEP: 2189/2207 -- GLOBAL_STEP: 43225[0m
|
| 4931 |
+
| > loss_text_ce: 0.03053663671016693 (0.029576369917361418)
|
| 4932 |
+
| > loss_mel_ce: 3.5480470657348633 (3.5942495688285487)
|
| 4933 |
+
| > loss: 0.05591537058353424 (0.05662228030570842)
|
| 4934 |
+
| > current_lr: 1.9999950898758486e-05
|
| 4935 |
+
| > step_time: 0.2864 (0.2784464403939285)
|
| 4936 |
+
| > loader_time: 0.0084 (0.008461966368639072)
|
| 4937 |
+
|
| 4938 |
+
|
| 4939 |
+
[1m > EVALUATION [0m
|
| 4940 |
+
|
| 4941 |
+
|
| 4942 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 4943 |
+
| > avg_loader_time:[91m 0.09240441547857747 [0m(+0.009598021571700643)
|
| 4944 |
+
| > avg_loss_text_ce:[92m 0.02861217343928041 [0m(-0.00012105615255800228)
|
| 4945 |
+
| > avg_loss_mel_ce:[92m 3.7077841372103304 [0m(-0.011657279890936678)
|
| 4946 |
+
| > avg_loss:[92m 3.7363963143245593 [0m(-0.01177833370260295)
|
| 4947 |
+
|
| 4948 |
+
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-25-2025_09+29PM-0000000/best_model_43243.pth
|