Checkpoint at step 7416
Browse files- best_model.pth +1 -1
- best_model_7416.pth +3 -0
- checkpoint_6000.pth +3 -0
- checkpoint_7000.pth +3 -0
- events.out.tfevents.1766186420.0d38972e308b.2948.1 +2 -2
- trainer_0_log.txt +418 -0
best_model.pth
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 5607926677
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c07f5634cad45064f6d39b3d36e2842d36467c17445636802e005b5cd5ed8ddd
|
| 3 |
size 5607926677
|
best_model_7416.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c07f5634cad45064f6d39b3d36e2842d36467c17445636802e005b5cd5ed8ddd
|
| 3 |
+
size 5607926677
|
checkpoint_6000.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c6c8b02bb60ddb68bc48bd6bf9ec4a1eb08f37cc2dcd6bd58897b022ac69954c
|
| 3 |
+
size 5607926677
|
checkpoint_7000.pth
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3466bb027d66d48980f01cca1f820e25451365999b62a19f99ad2cfc5324dbc2
|
| 3 |
+
size 5607926677
|
events.out.tfevents.1766186420.0d38972e308b.2948.1
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f5171fe9b37fd5c8f6a3661fb42dfb567d2547010cb6a7be6ba761a42f29d1ab
|
| 3 |
+
size 15538
|
trainer_0_log.txt
CHANGED
|
@@ -32,3 +32,421 @@
|
|
| 32 |
| > avg_loss: 4.072864298884932 [0m(+0)
|
| 33 |
|
| 34 |
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-19-2025_11+20PM-0000000/best_model_5209.pth
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
| > avg_loss: 4.072864298884932 [0m(+0)
|
| 33 |
|
| 34 |
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-19-2025_11+20PM-0000000/best_model_5209.pth
|
| 35 |
+
|
| 36 |
+
[4m[1m > EPOCH: 1/6[0m
|
| 37 |
+
--> /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-19-2025_11+20PM-0000000
|
| 38 |
+
|
| 39 |
+
[1m > TRAINING (2025-12-19 23:23:38) [0m
|
| 40 |
+
|
| 41 |
+
[1m --> TIME: 2025-12-19 23:23:58 -- STEP: 41/2207 -- GLOBAL_STEP: 5250[0m
|
| 42 |
+
| > loss_text_ce: 0.033330123871564865 (0.03260848189635974)
|
| 43 |
+
| > loss_mel_ce: 4.138209342956543 (4.13933496358918)
|
| 44 |
+
| > loss: 0.06518030166625977 (0.06518661657848009)
|
| 45 |
+
| > current_lr: 1e-05
|
| 46 |
+
| > step_time: 0.2912 (0.2791317788566031)
|
| 47 |
+
| > loader_time: 0.0083 (0.010119118341585485)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
[1m --> TIME: 2025-12-19 23:24:17 -- STEP: 91/2207 -- GLOBAL_STEP: 5300[0m
|
| 51 |
+
| > loss_text_ce: 0.032737262547016144 (0.032768114295962075)
|
| 52 |
+
| > loss_mel_ce: 3.9631412029266357 (4.132967893893904)
|
| 53 |
+
| > loss: 0.06243560090661049 (0.065089625030101)
|
| 54 |
+
| > current_lr: 1e-05
|
| 55 |
+
| > step_time: 0.2703 (0.28123985542045854)
|
| 56 |
+
| > loader_time: 0.008 (0.009950027361020936)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
[1m --> TIME: 2025-12-19 23:24:37 -- STEP: 141/2207 -- GLOBAL_STEP: 5350[0m
|
| 60 |
+
| > loss_text_ce: 0.03263312578201294 (0.03275274914682754)
|
| 61 |
+
| > loss_mel_ce: 4.11068868637085 (4.126875228070198)
|
| 62 |
+
| > loss: 0.0647394061088562 (0.06499418716692758)
|
| 63 |
+
| > current_lr: 1e-05
|
| 64 |
+
| > step_time: 0.2669 (0.2816294220322413)
|
| 65 |
+
| > loader_time: 0.0088 (0.009555863995924065)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
[1m --> TIME: 2025-12-19 23:24:56 -- STEP: 191/2207 -- GLOBAL_STEP: 5400[0m
|
| 69 |
+
| > loss_text_ce: 0.033539168536663055 (0.032767798459030544)
|
| 70 |
+
| > loss_mel_ce: 3.9542629718780518 (4.11662355642668)
|
| 71 |
+
| > loss: 0.06230940669775009 (0.06483424000481042)
|
| 72 |
+
| > grad_norm: 0 (0.0)
|
| 73 |
+
| > current_lr: 1e-05
|
| 74 |
+
| > step_time: 0.31 (0.2820051752459942)
|
| 75 |
+
| > loader_time: 0.008 (0.009320748414044603)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
[1m --> TIME: 2025-12-19 23:25:15 -- STEP: 241/2207 -- GLOBAL_STEP: 5450[0m
|
| 79 |
+
| > loss_text_ce: 0.03252074122428894 (0.03277294945234578)
|
| 80 |
+
| > loss_mel_ce: 4.288866996765137 (4.110062500253256)
|
| 81 |
+
| > loss: 0.06752168387174606 (0.06473180396613738)
|
| 82 |
+
| > current_lr: 1e-05
|
| 83 |
+
| > step_time: 0.2924 (0.2812330376557791)
|
| 84 |
+
| > loader_time: 0.008 (0.009211289932124347)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
[1m --> TIME: 2025-12-19 23:25:34 -- STEP: 291/2207 -- GLOBAL_STEP: 5500[0m
|
| 88 |
+
| > loss_text_ce: 0.0315721370279789 (0.03275071695139728)
|
| 89 |
+
| > loss_mel_ce: 4.302829742431641 (4.108650724502771)
|
| 90 |
+
| > loss: 0.0677250325679779 (0.06470939760253193)
|
| 91 |
+
| > current_lr: 1e-05
|
| 92 |
+
| > step_time: 0.2699 (0.28055939395812796)
|
| 93 |
+
| > loader_time: 0.0078 (0.009046153104592025)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
[1m --> TIME: 2025-12-19 23:25:54 -- STEP: 341/2207 -- GLOBAL_STEP: 5550[0m
|
| 97 |
+
| > loss_text_ce: 0.03177058324217796 (0.03277641668631708)
|
| 98 |
+
| > loss_mel_ce: 3.9036824703216553 (4.110928934690188)
|
| 99 |
+
| > loss: 0.061491452157497406 (0.06474539622271169)
|
| 100 |
+
| > current_lr: 1e-05
|
| 101 |
+
| > step_time: 0.2694 (0.2810395141500883)
|
| 102 |
+
| > loader_time: 0.008 (0.008892246704996507)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
[1m --> TIME: 2025-12-19 23:26:13 -- STEP: 391/2207 -- GLOBAL_STEP: 5600[0m
|
| 106 |
+
| > loss_text_ce: 0.031645581126213074 (0.0327805225901744)
|
| 107 |
+
| > loss_mel_ce: 4.2700514793396 (4.109736498054638)
|
| 108 |
+
| > loss: 0.06721401959657669 (0.06472682855699381)
|
| 109 |
+
| > current_lr: 1e-05
|
| 110 |
+
| > step_time: 0.289 (0.2811980326767166)
|
| 111 |
+
| > loader_time: 0.008 (0.00883669011733111)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
[1m --> TIME: 2025-12-19 23:26:32 -- STEP: 441/2207 -- GLOBAL_STEP: 5650[0m
|
| 115 |
+
| > loss_text_ce: 0.031354326754808426 (0.03274994752371934)
|
| 116 |
+
| > loss_mel_ce: 4.180124282836914 (4.105678926519797)
|
| 117 |
+
| > loss: 0.06580435484647751 (0.06466295119890289)
|
| 118 |
+
| > current_lr: 1e-05
|
| 119 |
+
| > step_time: 0.2701 (0.28069895249105103)
|
| 120 |
+
| > loader_time: 0.0077 (0.008764371309691276)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
[1m --> TIME: 2025-12-19 23:26:51 -- STEP: 491/2207 -- GLOBAL_STEP: 5700[0m
|
| 124 |
+
| > loss_text_ce: 0.031878069043159485 (0.03277220907801282)
|
| 125 |
+
| > loss_mel_ce: 3.976787805557251 (4.106889547010062)
|
| 126 |
+
| > loss: 0.0626354068517685 (0.06468221494385533)
|
| 127 |
+
| > current_lr: 1e-05
|
| 128 |
+
| > step_time: 0.2708 (0.28033951343684005)
|
| 129 |
+
| > loader_time: 0.008 (0.008695911244316644)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
[1m --> TIME: 2025-12-19 23:27:10 -- STEP: 541/2207 -- GLOBAL_STEP: 5750[0m
|
| 133 |
+
| > loss_text_ce: 0.0341142900288105 (0.03278052015316135)
|
| 134 |
+
| > loss_mel_ce: 4.066941738128662 (4.106222606630732)
|
| 135 |
+
| > loss: 0.06407900154590607 (0.06467192390048801)
|
| 136 |
+
| > current_lr: 1e-05
|
| 137 |
+
| > step_time: 0.2888 (0.2802339895816036)
|
| 138 |
+
| > loader_time: 0.008 (0.008648681552485046)
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
[1m --> TIME: 2025-12-19 23:27:29 -- STEP: 591/2207 -- GLOBAL_STEP: 5800[0m
|
| 142 |
+
| > loss_text_ce: 0.03217954561114311 (0.032738844865646266)
|
| 143 |
+
| > loss_mel_ce: 3.9995813369750977 (4.106167038886884)
|
| 144 |
+
| > loss: 0.0629962608218193 (0.06467040447288165)
|
| 145 |
+
| > current_lr: 1e-05
|
| 146 |
+
| > step_time: 0.2909 (0.2802896427018994)
|
| 147 |
+
| > loader_time: 0.0081 (0.008587077991208046)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
[1m --> TIME: 2025-12-19 23:27:48 -- STEP: 641/2207 -- GLOBAL_STEP: 5850[0m
|
| 151 |
+
| > loss_text_ce: 0.03290794417262077 (0.03274818261367576)
|
| 152 |
+
| > loss_mel_ce: 4.050463676452637 (4.102661906464055)
|
| 153 |
+
| > loss: 0.06380268186330795 (0.06461578270481462)
|
| 154 |
+
| > current_lr: 1e-05
|
| 155 |
+
| > step_time: 0.2891 (0.2799590459665905)
|
| 156 |
+
| > loader_time: 0.0076 (0.008560780244014934)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
[1m --> TIME: 2025-12-19 23:28:08 -- STEP: 691/2207 -- GLOBAL_STEP: 5900[0m
|
| 160 |
+
| > loss_text_ce: 0.033096350729465485 (0.03273934032094115)
|
| 161 |
+
| > loss_mel_ce: 4.230457305908203 (4.103105344234086)
|
| 162 |
+
| > loss: 0.06661802530288696 (0.06462257329034916)
|
| 163 |
+
| > current_lr: 1e-05
|
| 164 |
+
| > step_time: 0.2888 (0.28009270345077864)
|
| 165 |
+
| > loader_time: 0.0079 (0.008544543372566901)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
[1m --> TIME: 2025-12-19 23:28:27 -- STEP: 741/2207 -- GLOBAL_STEP: 5950[0m
|
| 169 |
+
| > loss_text_ce: 0.03447591885924339 (0.03272524028362937)
|
| 170 |
+
| > loss_mel_ce: 3.9467613697052 (4.099739815059457)
|
| 171 |
+
| > loss: 0.06220683082938194 (0.06456976656911358)
|
| 172 |
+
| > current_lr: 1e-05
|
| 173 |
+
| > step_time: 0.2708 (0.28011725209502575)
|
| 174 |
+
| > loader_time: 0.0085 (0.008504998828718061)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
[1m --> TIME: 2025-12-19 23:28:46 -- STEP: 791/2207 -- GLOBAL_STEP: 6000[0m
|
| 178 |
+
| > loss_text_ce: 0.031701214611530304 (0.03270154030977099)
|
| 179 |
+
| > loss_mel_ce: 3.9861724376678467 (4.099287290488723)
|
| 180 |
+
| > loss: 0.06277927756309509 (0.06456232558366029)
|
| 181 |
+
| > current_lr: 1e-05
|
| 182 |
+
| > step_time: 0.2628 (0.2801193307835597)
|
| 183 |
+
| > loader_time: 0.0081 (0.008462230414717302)
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-19-2025_11+20PM-0000000/checkpoint_6000.pth
|
| 187 |
+
|
| 188 |
+
[1m --> TIME: 2025-12-19 23:29:29 -- STEP: 841/2207 -- GLOBAL_STEP: 6050[0m
|
| 189 |
+
| > loss_text_ce: 0.03385571017861366 (0.03268137809407413)
|
| 190 |
+
| > loss_mel_ce: 4.031688690185547 (4.097443981204673)
|
| 191 |
+
| > loss: 0.06352413445711136 (0.06453320885898818)
|
| 192 |
+
| > current_lr: 1e-05
|
| 193 |
+
| > step_time: 0.2884 (0.28015128298973807)
|
| 194 |
+
| > loader_time: 0.0077 (0.008446370237080577)
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
[1m --> TIME: 2025-12-19 23:29:48 -- STEP: 891/2207 -- GLOBAL_STEP: 6100[0m
|
| 198 |
+
| > loss_text_ce: 0.032470740377902985 (0.032681752594164844)
|
| 199 |
+
| > loss_mel_ce: 3.865196466445923 (4.095037801230928)
|
| 200 |
+
| > loss: 0.06090104952454567 (0.06449561814467121)
|
| 201 |
+
| > current_lr: 1e-05
|
| 202 |
+
| > step_time: 0.2699 (0.27991642561290686)
|
| 203 |
+
| > loader_time: 0.0079 (0.008440049007685494)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
[1m --> TIME: 2025-12-19 23:30:08 -- STEP: 941/2207 -- GLOBAL_STEP: 6150[0m
|
| 207 |
+
| > loss_text_ce: 0.03259256109595299 (0.032686683451252044)
|
| 208 |
+
| > loss_mel_ce: 4.243472099304199 (4.092387335966801)
|
| 209 |
+
| > loss: 0.06681351363658905 (0.06445428167690513)
|
| 210 |
+
| > current_lr: 1e-05
|
| 211 |
+
| > step_time: 0.2985 (0.2799384107498512)
|
| 212 |
+
| > loader_time: 0.0083 (0.008427470700773756)
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
[1m --> TIME: 2025-12-19 23:30:27 -- STEP: 991/2207 -- GLOBAL_STEP: 6200[0m
|
| 216 |
+
| > loss_text_ce: 0.033036742359399796 (0.03268012933350354)
|
| 217 |
+
| > loss_mel_ce: 4.026215076446533 (4.090084233028976)
|
| 218 |
+
| > loss: 0.06342580914497375 (0.06441819329119239)
|
| 219 |
+
| > current_lr: 1e-05
|
| 220 |
+
| > step_time: 0.2664 (0.27988023449987287)
|
| 221 |
+
| > loader_time: 0.0089 (0.00843105017838156)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
[1m --> TIME: 2025-12-19 23:30:46 -- STEP: 1041/2207 -- GLOBAL_STEP: 6250[0m
|
| 225 |
+
| > loss_text_ce: 0.033400777727365494 (0.0326768395926635)
|
| 226 |
+
| > loss_mel_ce: 4.0089898109436035 (4.089549863716366)
|
| 227 |
+
| > loss: 0.06316235661506653 (0.06440979238030552)
|
| 228 |
+
| > current_lr: 1e-05
|
| 229 |
+
| > step_time: 0.2701 (0.27983240641503265)
|
| 230 |
+
| > loader_time: 0.0085 (0.008462739838188828)
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
[1m --> TIME: 2025-12-19 23:31:05 -- STEP: 1091/2207 -- GLOBAL_STEP: 6300[0m
|
| 234 |
+
| > loss_text_ce: 0.031533099710941315 (0.03267257152027923)
|
| 235 |
+
| > loss_mel_ce: 4.511966705322266 (4.088653826691708)
|
| 236 |
+
| > loss: 0.07099218666553497 (0.06439572509197226)
|
| 237 |
+
| > current_lr: 1e-05
|
| 238 |
+
| > step_time: 0.2671 (0.27962432510802576)
|
| 239 |
+
| > loader_time: 0.0077 (0.008466678185816975)
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
[1m --> TIME: 2025-12-19 23:31:24 -- STEP: 1141/2207 -- GLOBAL_STEP: 6350[0m
|
| 243 |
+
| > loss_text_ce: 0.03148523345589638 (0.032665948479009436)
|
| 244 |
+
| > loss_mel_ce: 3.9132697582244873 (4.085592917869427)
|
| 245 |
+
| > loss: 0.061636798083782196 (0.0643477948981613)
|
| 246 |
+
| > current_lr: 1e-05
|
| 247 |
+
| > step_time: 0.29 (0.2796653886305254)
|
| 248 |
+
| > loader_time: 0.0079 (0.008451628329772682)
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
[1m --> TIME: 2025-12-19 23:31:43 -- STEP: 1191/2207 -- GLOBAL_STEP: 6400[0m
|
| 252 |
+
| > loss_text_ce: 0.03140372782945633 (0.03266367260100808)
|
| 253 |
+
| > loss_mel_ce: 4.076885223388672 (4.08433563062666)
|
| 254 |
+
| > loss: 0.06419201195240021 (0.06432811422922557)
|
| 255 |
+
| > current_lr: 1e-05
|
| 256 |
+
| > step_time: 0.2867 (0.27976663486983605)
|
| 257 |
+
| > loader_time: 0.0086 (0.008452464911639444)
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
[1m --> TIME: 2025-12-19 23:32:02 -- STEP: 1241/2207 -- GLOBAL_STEP: 6450[0m
|
| 261 |
+
| > loss_text_ce: 0.03408047929406166 (0.03266447187100415)
|
| 262 |
+
| > loss_mel_ce: 4.360547065734863 (4.084415867864647)
|
| 263 |
+
| > loss: 0.06866605579853058 (0.0643293804111595)
|
| 264 |
+
| > current_lr: 1e-05
|
| 265 |
+
| > step_time: 0.2875 (0.279699163413836)
|
| 266 |
+
| > loader_time: 0.0079 (0.0084403975177061)
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
[1m --> TIME: 2025-12-19 23:32:22 -- STEP: 1291/2207 -- GLOBAL_STEP: 6500[0m
|
| 270 |
+
| > loss_text_ce: 0.031175393611192703 (0.03264621023224311)
|
| 271 |
+
| > loss_mel_ce: 4.075261116027832 (4.083227412632316)
|
| 272 |
+
| > loss: 0.06416307389736176 (0.06431052546679841)
|
| 273 |
+
| > current_lr: 1e-05
|
| 274 |
+
| > step_time: 0.2701 (0.2796680572695926)
|
| 275 |
+
| > loader_time: 0.008 (0.008421044124363926)
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
[1m --> TIME: 2025-12-19 23:32:41 -- STEP: 1341/2207 -- GLOBAL_STEP: 6550[0m
|
| 279 |
+
| > loss_text_ce: 0.033355582505464554 (0.0326451789712745)
|
| 280 |
+
| > loss_mel_ce: 4.339946746826172 (4.0817988941155585)
|
| 281 |
+
| > loss: 0.06833285093307495 (0.06428818874988965)
|
| 282 |
+
| > current_lr: 1e-05
|
| 283 |
+
| > step_time: 0.3145 (0.2795609117532119)
|
| 284 |
+
| > loader_time: 0.0077 (0.008406509666812676)
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
[1m --> TIME: 2025-12-19 23:33:00 -- STEP: 1391/2207 -- GLOBAL_STEP: 6600[0m
|
| 288 |
+
| > loss_text_ce: 0.03157105669379234 (0.03262844312558443)
|
| 289 |
+
| > loss_mel_ce: 3.9514052867889404 (4.080753967455179)
|
| 290 |
+
| > loss: 0.06223400682210922 (0.06427160027556764)
|
| 291 |
+
| > current_lr: 1e-05
|
| 292 |
+
| > step_time: 0.2671 (0.2795383186326623)
|
| 293 |
+
| > loader_time: 0.0087 (0.008389373891735833)
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
[1m --> TIME: 2025-12-19 23:33:19 -- STEP: 1441/2207 -- GLOBAL_STEP: 6650[0m
|
| 297 |
+
| > loss_text_ce: 0.03227873519062996 (0.032622452478889404)
|
| 298 |
+
| > loss_mel_ce: 3.937110185623169 (4.079359220349892)
|
| 299 |
+
| > loss: 0.06202170252799988 (0.06424971374164105)
|
| 300 |
+
| > current_lr: 1e-05
|
| 301 |
+
| > step_time: 0.2667 (0.2795068340778024)
|
| 302 |
+
| > loader_time: 0.0081 (0.0083797916118508)
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
[1m --> TIME: 2025-12-19 23:33:39 -- STEP: 1491/2207 -- GLOBAL_STEP: 6700[0m
|
| 306 |
+
| > loss_text_ce: 0.03508618474006653 (0.032608193004944774)
|
| 307 |
+
| > loss_mel_ce: 4.201953887939453 (4.078866420537328)
|
| 308 |
+
| > loss: 0.06620375066995621 (0.06424179094038436)
|
| 309 |
+
| > current_lr: 1e-05
|
| 310 |
+
| > step_time: 0.2621 (0.2797854189741622)
|
| 311 |
+
| > loader_time: 0.008 (0.008368091723968483)
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
[1m --> TIME: 2025-12-19 23:33:58 -- STEP: 1541/2207 -- GLOBAL_STEP: 6750[0m
|
| 315 |
+
| > loss_text_ce: 0.03175294026732445 (0.03260520016828921)
|
| 316 |
+
| > loss_mel_ce: 3.958707094192505 (4.078298873053517)
|
| 317 |
+
| > loss: 0.062350936233997345 (0.06423287624511387)
|
| 318 |
+
| > current_lr: 1e-05
|
| 319 |
+
| > step_time: 0.2906 (0.2797788922311735)
|
| 320 |
+
| > loader_time: 0.0079 (0.008361948379675025)
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
[1m --> TIME: 2025-12-19 23:34:17 -- STEP: 1591/2207 -- GLOBAL_STEP: 6800[0m
|
| 324 |
+
| > loss_text_ce: 0.03180113434791565 (0.03259606553811753)
|
| 325 |
+
| > loss_mel_ce: 3.947313070297241 (4.076905974646468)
|
| 326 |
+
| > loss: 0.06217366084456444 (0.06421096947940934)
|
| 327 |
+
| > current_lr: 1e-05
|
| 328 |
+
| > step_time: 0.2888 (0.2797808211079042)
|
| 329 |
+
| > loader_time: 0.01 (0.008361990537229825)
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
[1m --> TIME: 2025-12-19 23:34:36 -- STEP: 1641/2207 -- GLOBAL_STEP: 6850[0m
|
| 333 |
+
| > loss_text_ce: 0.031015565618872643 (0.032597589773613364)
|
| 334 |
+
| > loss_mel_ce: 4.251021862030029 (4.075699189172724)
|
| 335 |
+
| > loss: 0.06690683215856552 (0.06419213727970165)
|
| 336 |
+
| > current_lr: 1e-05
|
| 337 |
+
| > step_time: 0.2867 (0.27970779399767254)
|
| 338 |
+
| > loader_time: 0.0078 (0.00836158728614076)
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
[1m --> TIME: 2025-12-19 23:34:55 -- STEP: 1691/2207 -- GLOBAL_STEP: 6900[0m
|
| 342 |
+
| > loss_text_ce: 0.032971665263175964 (0.032580885582337404)
|
| 343 |
+
| > loss_mel_ce: 4.157569408416748 (4.074683435142281)
|
| 344 |
+
| > loss: 0.06547720730304718 (0.06417600510698117)
|
| 345 |
+
| > current_lr: 1e-05
|
| 346 |
+
| > step_time: 0.2659 (0.2796958701135421)
|
| 347 |
+
| > loader_time: 0.0079 (0.008353771788350643)
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
[1m --> TIME: 2025-12-19 23:35:15 -- STEP: 1741/2207 -- GLOBAL_STEP: 6950[0m
|
| 351 |
+
| > loss_text_ce: 0.03412429243326187 (0.03258349168735418)
|
| 352 |
+
| > loss_mel_ce: 3.9358301162719727 (4.073978549649421)
|
| 353 |
+
| > loss: 0.06203053891658783 (0.06416503198568956)
|
| 354 |
+
| > current_lr: 1e-05
|
| 355 |
+
| > step_time: 0.2867 (0.2796876093630543)
|
| 356 |
+
| > loader_time: 0.0079 (0.008347673843401302)
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
[1m --> TIME: 2025-12-19 23:35:34 -- STEP: 1791/2207 -- GLOBAL_STEP: 7000[0m
|
| 360 |
+
| > loss_text_ce: 0.03215445205569267 (0.03257655235873722)
|
| 361 |
+
| > loss_mel_ce: 4.294580936431885 (4.072014079554524)
|
| 362 |
+
| > loss: 0.06760524213314056 (0.06413422870626008)
|
| 363 |
+
| > grad_norm: 0 (0.0)
|
| 364 |
+
| > current_lr: 1e-05
|
| 365 |
+
| > step_time: 0.3286 (0.27962819615400836)
|
| 366 |
+
| > loader_time: 0.0094 (0.008355755036533493)
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
> CHECKPOINT : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-19-2025_11+20PM-0000000/checkpoint_7000.pth
|
| 370 |
+
|
| 371 |
+
[1m --> TIME: 2025-12-19 23:36:17 -- STEP: 1841/2207 -- GLOBAL_STEP: 7050[0m
|
| 372 |
+
| > loss_text_ce: 0.03530712425708771 (0.03257391380934299)
|
| 373 |
+
| > loss_mel_ce: 4.284851551055908 (4.070986293956415)
|
| 374 |
+
| > loss: 0.0675024762749672 (0.06411812833466335)
|
| 375 |
+
| > current_lr: 1e-05
|
| 376 |
+
| > step_time: 0.2648 (0.2795902267219839)
|
| 377 |
+
| > loader_time: 0.0078 (0.008350239701403134)
|
| 378 |
+
|
| 379 |
+
|
| 380 |
+
[1m --> TIME: 2025-12-19 23:36:36 -- STEP: 1891/2207 -- GLOBAL_STEP: 7100[0m
|
| 381 |
+
| > loss_text_ce: 0.03220251947641373 (0.03257080443431966)
|
| 382 |
+
| > loss_mel_ce: 4.281186103820801 (4.070231137737148)
|
| 383 |
+
| > loss: 0.06739670038223267 (0.0641062804382041)
|
| 384 |
+
| > current_lr: 1e-05
|
| 385 |
+
| > step_time: 0.2962 (0.2795604967301893)
|
| 386 |
+
| > loader_time: 0.0077 (0.008338792811998402)
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
[1m --> TIME: 2025-12-19 23:36:55 -- STEP: 1941/2207 -- GLOBAL_STEP: 7150[0m
|
| 390 |
+
| > loss_text_ce: 0.030263639986515045 (0.03256799163376465)
|
| 391 |
+
| > loss_mel_ce: 3.867950201034546 (4.068827845889346)
|
| 392 |
+
| > loss: 0.06090959161520004 (0.06408431003659984)
|
| 393 |
+
| > current_lr: 1e-05
|
| 394 |
+
| > step_time: 0.2866 (0.27960251705478245)
|
| 395 |
+
| > loader_time: 0.0078 (0.008333902024903394)
|
| 396 |
+
|
| 397 |
+
|
| 398 |
+
[1m --> TIME: 2025-12-19 23:37:14 -- STEP: 1991/2207 -- GLOBAL_STEP: 7200[0m
|
| 399 |
+
| > loss_text_ce: 0.033825602382421494 (0.03256503755478388)
|
| 400 |
+
| > loss_mel_ce: 3.9784209728240967 (4.067005923739147)
|
| 401 |
+
| > loss: 0.0626913532614708 (0.06405579635152145)
|
| 402 |
+
| > current_lr: 1e-05
|
| 403 |
+
| > step_time: 0.2888 (0.27955440909703266)
|
| 404 |
+
| > loader_time: 0.0082 (0.008326804081079992)
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
[1m --> TIME: 2025-12-19 23:37:33 -- STEP: 2041/2207 -- GLOBAL_STEP: 7250[0m
|
| 408 |
+
| > loss_text_ce: 0.0330999493598938 (0.032551859421970364)
|
| 409 |
+
| > loss_mel_ce: 4.061833381652832 (4.065660371084173)
|
| 410 |
+
| > loss: 0.06398333609104156 (0.06403456617918804)
|
| 411 |
+
| > current_lr: 1e-05
|
| 412 |
+
| > step_time: 0.2625 (0.2794767479754033)
|
| 413 |
+
| > loader_time: 0.0078 (0.008326117750594903)
|
| 414 |
+
|
| 415 |
+
|
| 416 |
+
[1m --> TIME: 2025-12-19 23:37:52 -- STEP: 2091/2207 -- GLOBAL_STEP: 7300[0m
|
| 417 |
+
| > loss_text_ce: 0.03444935381412506 (0.03255168464977251)
|
| 418 |
+
| > loss_mel_ce: 3.84794282913208 (4.064449771404943)
|
| 419 |
+
| > loss: 0.06066237762570381 (0.06401564783026077)
|
| 420 |
+
| > current_lr: 1e-05
|
| 421 |
+
| > step_time: 0.2682 (0.2794808959459245)
|
| 422 |
+
| > loader_time: 0.0078 (0.008326130241711461)
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
[1m --> TIME: 2025-12-19 23:38:12 -- STEP: 2141/2207 -- GLOBAL_STEP: 7350[0m
|
| 426 |
+
| > loss_text_ce: 0.03163890913128853 (0.03255556912684707)
|
| 427 |
+
| > loss_mel_ce: 3.6894760131835938 (4.063137030055838)
|
| 428 |
+
| > loss: 0.058142419904470444 (0.06399519694555815)
|
| 429 |
+
| > current_lr: 1e-05
|
| 430 |
+
| > step_time: 0.2652 (0.2795219843435491)
|
| 431 |
+
| > loader_time: 0.0082 (0.008321784133946753)
|
| 432 |
+
|
| 433 |
+
|
| 434 |
+
[1m --> TIME: 2025-12-19 23:38:31 -- STEP: 2191/2207 -- GLOBAL_STEP: 7400[0m
|
| 435 |
+
| > loss_text_ce: 0.03182921186089516 (0.03254674895764727)
|
| 436 |
+
| > loss_mel_ce: 3.9978630542755127 (4.062054034728858)
|
| 437 |
+
| > loss: 0.06296394020318985 (0.06397813732686077)
|
| 438 |
+
| > current_lr: 1e-05
|
| 439 |
+
| > step_time: 0.2911 (0.2795370181147665)
|
| 440 |
+
| > loader_time: 0.0082 (0.008315004207727725)
|
| 441 |
+
|
| 442 |
+
|
| 443 |
+
[1m > EVALUATION [0m
|
| 444 |
+
|
| 445 |
+
|
| 446 |
+
[1m--> EVAL PERFORMANCE[0m
|
| 447 |
+
| > avg_loader_time:[91m 0.16947253330333814 [0m(+0.08974874502903706)
|
| 448 |
+
| > avg_loss_text_ce:[92m 0.03120868057767682 [0m(-0.0006401909993508717)
|
| 449 |
+
| > avg_loss_mel_ce:[92m 3.949908100269936 [0m(-0.09110731530833593)
|
| 450 |
+
| > avg_loss:[92m 3.9811167942511068 [0m(-0.0917475046338252)
|
| 451 |
+
|
| 452 |
+
> BEST MODEL : /workspace/training_output/XTTS_Dahih_OmarSamir_Finetune-December-19-2025_11+20PM-0000000/best_model_7416.pth
|