| Running 1 job |
| { |
| "type": "diffusion_trainer", |
| "training_folder": "/teamspace/studios/this_studio/ai-toolkit/output", |
| "sqlite_db_path": "/teamspace/studios/this_studio/ai-toolkit/aitk_db.db", |
| "device": "cuda", |
| "trigger_word": "nishazitx1", |
| "performance_log_every": 10, |
| "network": { |
| "type": "lora", |
| "linear": 32, |
| "linear_alpha": 32, |
| "conv": 16, |
| "conv_alpha": 16, |
| "lokr_full_rank": true, |
| "lokr_factor": -1, |
| "network_kwargs": { |
| "ignore_if_contains": [] |
| } |
| }, |
| "save": { |
| "dtype": "bf16", |
| "save_every": 250, |
| "max_step_saves_to_keep": 4, |
| "save_format": "diffusers", |
| "push_to_hub": false |
| }, |
| "datasets": [ |
| { |
| "folder_path": "/teamspace/studios/this_studio/ai-toolkit/datasets/nisha", |
| "mask_path": null, |
| "mask_min_value": 0.1, |
| "default_caption": "", |
| "caption_ext": "txt", |
| "caption_dropout_rate": 0.03, |
| "cache_latents_to_disk": false, |
| "is_reg": false, |
| "network_weight": 1, |
| "resolution": [ |
| 1024, |
| 768, |
| 512 |
| ], |
| "controls": [], |
| "shrink_video_to_frames": true, |
| "num_frames": 1, |
| "flip_x": false, |
| "flip_y": false, |
| "num_repeats": 1 |
| } |
| ], |
| "train": { |
| "batch_size": 1, |
| "bypass_guidance_embedding": false, |
| "steps": 3000, |
| "gradient_accumulation": 1, |
| "train_unet": true, |
| "train_text_encoder": false, |
| "gradient_checkpointing": true, |
| "noise_scheduler": "flowmatch", |
| "optimizer": "adamw8bit", |
| "timestep_type": "weighted", |
| "content_or_style": "balanced", |
| "optimizer_params": { |
| "weight_decay": 0.0001 |
| }, |
| "unload_text_encoder": false, |
| "cache_text_embeddings": false, |
| "lr": 0.0001, |
| "ema_config": { |
| "use_ema": false, |
| "ema_decay": 0.99 |
| }, |
| "skip_first_sample": false, |
| "force_first_sample": false, |
| "disable_sampling": false, |
| "dtype": "bf16", |
| "diff_output_preservation": false, |
| "diff_output_preservation_multiplier": 1, |
| "diff_output_preservation_class": "person", |
| "switch_boundary_every": 1, |
| "loss_type": "mse" |
| }, |
| "logging": { |
| "log_every": 1, |
| "use_ui_logger": true |
| }, |
| "model": { |
| "name_or_path": "Tongyi-MAI/Z-Image-Turbo", |
| "quantize": true, |
| "qtype": "qfloat8", |
| "quantize_te": true, |
| "qtype_te": "qfloat8", |
| "arch": "zimage:turbo", |
| "low_vram": true, |
| "model_kwargs": {}, |
| "layer_offloading": false, |
| "layer_offloading_text_encoder_percent": 1, |
| "layer_offloading_transformer_percent": 1, |
| "assistant_lora_path": "ostris/zimage_turbo_training_adapter/zimage_turbo_training_adapter_v2.safetensors" |
| }, |
| "sample": { |
| "sampler": "flowmatch", |
| "sample_every": 250, |
| "width": 1024, |
| "height": 1024, |
| "samples": [ |
| { |
| "prompt": "ultra realistic portrait of a beautiful Indian 20y woman, soft natural skin texture, detailed face, expressive eyes, subtle makeup, cinematic lighting, 85mm lens, shallow depth of field, high detail, photorealistic, natural skin pores" |
| }, |
| { |
| "prompt": "full body shot of a young Indian 20y nude woman standing naturally, perfect body proportions, realistic anatomy, long legs, natural pose, detailed hands and fingers, soft lighting, high resolution, photorealistic, balanced composition" |
| }, |
| { |
| "prompt": "anal cumshot. A photograph of a sexual act. A naked indian 20yo woman with ligt brown skin, long black hair, and is on all fours on a white bed, looking back over her shoulder with her mouth slightly open. A man with light skin and a hairy arm is behind her, holding his erect penis against her anus. Her anus is dripping semen. She has cum on her anus and vagina. The woman's vulva is visible, with some pubic hair. The bed has a white quilted cover, and the background is a plain white wall. The lighting is bright and even, highlighting the subjects and creating minimal shadows. The man's hand is gripping his penis, and the woman's expression is one of surprise or pleasure." |
| }, |
| { |
| "prompt": "\u8d85\u9ad8\u6e05, \u6781\u81f4\u7ec6\u8282, \u7535\u5f71\u611f, \u771f\u5b9e\u611f\u7206\u68da, \u9876\u7ea7\u753b\u8d28, \u5149\u5f71\u5927\u5e08, \u7ec6\u817b\u76ae\u80a4, \u8d28\u611f\u62c9\u6ee1, masterpiece, best quality, ultra detailed, 8k, photorealistic, cinematic lighting, sharp focus, Lifted clothes. A 20yo indian woman with long, dark hair. She has fair skin, brown eyes, and is wearing a light grey off-the-shoulder top pulled up to reveal her breasts. Her nipples are visible, and her breasts are saggy and medium-sized. She's also wearing a pearl necklace. She's sitting on a bed with dark blue bedding, and there's a pillow with a heart pattern visible in the background. The background also includes a string of warm yellow lights, suggesting a cozy, intimate setting. The overall atmosphere is sensual and suggestive, with the focus on the woman's exposed breasts and ahegao expression." |
| }, |
| { |
| "prompt": "\u8d85\u9ad8\u6e05, \u6781\u81f4\u7ec6\u8282, \u7535\u5f71\u611f, \u771f\u5b9e\u611f\u7206\u68da, \u9876\u7ea7\u753b\u8d28, \u5149\u5f71\u5927\u5e08, \u7ec6\u817b\u76ae\u80a4, \u8d28\u611f\u62c9\u6ee1, masterpiece, best quality, ultra detailed, 8k, photorealistic, cinematic lighting, sharp focus, Photograph of a 20yo indian woman performing oral sex on a man, deepthroat. The woman has midium brown skin, black hair, and is looking up with her eyes partially closed. She has black eyeliner and is wearing no visible clothing. Her lips are wrapped around the man's erect penis, which has a dark brown color. The man's pubic area has a dense patch of dark hair. The background is out of focus, with indistinct dark and light colors. The woman's face is on the left side of the image, and the man's penis is on the right side." |
| } |
| ], |
| "neg": "", |
| "seed": 42, |
| "walk_seed": true, |
| "guidance_scale": 1, |
| "sample_steps": 8, |
| "num_frames": 1, |
| "fps": 1 |
| } |
| } |
| Using SQLite database at /teamspace/studios/this_studio/ai-toolkit/aitk_db.db |
| Job ID: "72431abf-5e0f-490a-a7f3-8c19918b3c7f" |
|
|
| ############################################# |
| # Running job: my_first_lora_v2 |
| ############################################# |
|
|
|
|
| Running 1 process |
| Loading ZImage model |
| Loading transformer |
|
Loading checkpoint shards: 0%| | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards: 0%| | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards: 33%|###3 | 1/3 [00:00<00:01, 1.11it/s]
Loading checkpoint shards: 33%|###3 | 1/3 [00:00<00:01, 1.11it/s]
Loading checkpoint shards: 67%|######6 | 2/3 [00:01<00:00, 1.10it/s]
Loading checkpoint shards: 67%|######6 | 2/3 [00:01<00:00, 1.10it/s]
Loading checkpoint shards: 100%|##########| 3/3 [00:02<00:00, 1.33it/s]
Loading checkpoint shards: 100%|##########| 3/3 [00:02<00:00, 1.33it/s]
Loading checkpoint shards: 100%|##########| 3/3 [00:02<00:00, 1.26it/s]
Loading checkpoint shards: 100%|##########| 3/3 [00:02<00:00, 1.26it/s] |
|
|
| Loading assistant LoRA |
| create LoRA network. base dim (rank): 64, alpha: 64 |
| neuron dropout: p=None, rank dropout: p=None, module dropout: p=None |
| create LoRA for Text Encoder: 0 modules. |
| create LoRA for U-Net: 276 modules. |
| enable LoRA for U-Net |
| Merging in assistant LoRA |
| Missing keys: [] |
| Quantizing Transformer |
| - quantizing 30 transformer blocks |
|
0%| | 0/30 [00:00<?, ?it/s]
0%| | 0/30 [00:00<?, ?it/s]
3%|3 | 1/30 [00:00<00:04, 6.65it/s]
3%|3 | 1/30 [00:00<00:04, 6.65it/s]
7%|6 | 2/30 [00:00<00:03, 7.61it/s]
7%|6 | 2/30 [00:00<00:03, 7.61it/s]
10%|# | 3/30 [00:00<00:03, 8.03it/s]
10%|# | 3/30 [00:00<00:03, 8.03it/s]
13%|#3 | 4/30 [00:00<00:03, 8.65it/s]
13%|#3 | 4/30 [00:00<00:03, 8.65it/s]
20%|## | 6/30 [00:00<00:02, 9.36it/s]
20%|## | 6/30 [00:00<00:02, 9.36it/s]
23%|##3 | 7/30 [00:00<00:02, 9.14it/s]
23%|##3 | 7/30 [00:00<00:02, 9.14it/s]
30%|### | 9/30 [00:00<00:02, 9.55it/s]
30%|### | 9/30 [00:00<00:02, 9.55it/s]
33%|###3 | 10/30 [00:01<00:02, 9.32it/s]
33%|###3 | 10/30 [00:01<00:02, 9.32it/s]
40%|#### | 12/30 [00:01<00:01, 9.41it/s]
40%|#### | 12/30 [00:01<00:01, 9.41it/s]
43%|####3 | 13/30 [00:01<00:01, 9.25it/s]
43%|####3 | 13/30 [00:01<00:01, 9.25it/s]
47%|####6 | 14/30 [00:01<00:01, 9.12it/s]
47%|####6 | 14/30 [00:01<00:01, 9.12it/s]
53%|#####3 | 16/30 [00:01<00:01, 9.35it/s]
53%|#####3 | 16/30 [00:01<00:01, 9.35it/s]
60%|###### | 18/30 [00:01<00:01, 9.41it/s]
60%|###### | 18/30 [00:01<00:01, 9.41it/s]
67%|######6 | 20/30 [00:02<00:01, 9.72it/s]
67%|######6 | 20/30 [00:02<00:01, 9.72it/s]
70%|####### | 21/30 [00:02<00:00, 9.53it/s]
70%|####### | 21/30 [00:02<00:00, 9.53it/s]
77%|#######6 | 23/30 [00:02<00:00, 9.82it/s]
77%|#######6 | 23/30 [00:02<00:00, 9.82it/s]
80%|######## | 24/30 [00:02<00:00, 9.40it/s]
80%|######## | 24/30 [00:02<00:00, 9.40it/s]
83%|########3 | 25/30 [00:02<00:00, 9.45it/s]
83%|########3 | 25/30 [00:02<00:00, 9.45it/s]
87%|########6 | 26/30 [00:02<00:00, 9.20it/s]
87%|########6 | 26/30 [00:02<00:00, 9.20it/s]
90%|######### | 27/30 [00:02<00:00, 9.03it/s]
90%|######### | 27/30 [00:02<00:00, 9.03it/s]
93%|#########3| 28/30 [00:03<00:00, 8.97it/s]
93%|#########3| 28/30 [00:03<00:00, 8.97it/s]
97%|#########6| 29/30 [00:03<00:00, 8.74it/s]
97%|#########6| 29/30 [00:03<00:00, 8.74it/s]
100%|##########| 30/30 [00:03<00:00, 8.73it/s]
100%|##########| 30/30 [00:03<00:00, 8.73it/s]
100%|##########| 30/30 [00:03<00:00, 9.14it/s]
100%|##########| 30/30 [00:03<00:00, 9.14it/s] |
|
|
| - quantizing extras |
| Moving transformer to CPU |
| Text Encoder |
| `torch_dtype` is deprecated! Use `dtype` instead! |
| `torch_dtype` is deprecated! Use `dtype` instead! |
|
Loading checkpoint shards: 0%| | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards: 0%| | 0/3 [00:00<?, ?it/s]
Loading checkpoint shards: 100%|##########| 3/3 [00:00<00:00, 52.91it/s]
Loading checkpoint shards: 100%|##########| 3/3 [00:00<00:00, 52.91it/s] |
|
|
| Quantizing Text Encoder |
| Loading VAE |
| Making pipe |
| Preparing Model |
| Model Loaded |
| create LoRA network. base dim (rank): 32, alpha: 32 |
| neuron dropout: p=None, rank dropout: p=None, module dropout: p=None |
| apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16 |
| create LoRA for Text Encoder: 0 modules. |
| create LoRA for U-Net: 240 modules. |
| enable LoRA for U-Net |
| Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/nisha |
| - Preprocessing image dimensions |
|
0%| | 0/6 [00:00<?, ?it/s]
0%| | 0/6 [00:00<?, ?it/s]
100%|##########| 6/6 [00:00<00:00, 116.65it/s]
100%|##########| 6/6 [00:00<00:00, 116.65it/s] |
|
|
| - Found 6 images |
| Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/nisha: |
| 704x1472: 6 files |
| 1 buckets made |
| Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/nisha |
| - Preprocessing image dimensions |
|
0%| | 0/6 [00:00<?, ?it/s]
0%| | 0/6 [00:00<?, ?it/s]
100%|##########| 6/6 [00:00<00:00, 25944.15it/s]
100%|##########| 6/6 [00:00<00:00, 25944.15it/s] |
|
|
| - Found 6 images |
| Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/nisha: |
| 512x1088: 6 files |
| 1 buckets made |
| Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/nisha |
| - Preprocessing image dimensions |
|
0%| | 0/6 [00:00<?, ?it/s]
0%| | 0/6 [00:00<?, ?it/s]
100%|##########| 6/6 [00:00<00:00, 30283.78it/s]
100%|##########| 6/6 [00:00<00:00, 30283.78it/s] |
|
|
| - Found 6 images |
| Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/nisha: |
| 352x736: 6 files |
| 1 buckets made |
| Generating baseline samples before training |
| Unloading assistant lora |
|
Generating Images: 0%| | 0/5 [00:00<?, ?it/s]
Generating Images: 0%| | 0/5 [00:00<?, ?it/s]
Generating Images: 20%|## | 1/5 [00:07<00:29, 7.50s/it]
Generating Images: 20%|## | 1/5 [00:07<00:29, 7.50s/it]
Generating Images: 40%|#### | 2/5 [00:14<00:22, 7.47s/it]
Generating Images: 40%|#### | 2/5 [00:14<00:22, 7.47s/it]
Generating Images: 60%|###### | 3/5 [00:22<00:15, 7.57s/it]
Generating Images: 60%|###### | 3/5 [00:22<00:15, 7.57s/it]
Generating Images: 80%|######## | 4/5 [00:30<00:07, 7.65s/it]
Generating Images: 80%|######## | 4/5 [00:30<00:07, 7.65s/it]
Generating Images: 100%|##########| 5/5 [00:38<00:00, 7.69s/it]
Generating Images: 100%|##########| 5/5 [00:38<00:00, 7.69s/it]
Loading assistant lora |
|
my_first_lora_v2: 0%| | 0/3000 [00:00<?, ?it/s]
my_first_lora_v2: 0%| | 0/3000 [00:00<?, ?it/s]
my_first_lora_v2: 0%| | 0/3000 [00:03<?, ?it/s, lr: 1.0e-04 loss: 4.727e-01]
my_first_lora_v2: 0%| | 0/3000 [00:03<?, ?it/s, lr: 1.0e-04 loss: 4.727e-01]
my_first_lora_v2: 0%| | 0/3000 [00:03<?, ?it/s, lr: 1.0e-04 loss: 4.727e-01]
my_first_lora_v2: 0%| | 0/3000 [00:03<?, ?it/s, lr: 1.0e-04 loss: 4.727e-01]
my_first_lora_v2: 0%| | 0/3000 [00:04<?, ?it/s, lr: 1.0e-04 loss: 2.128e-01]
my_first_lora_v2: 0%| | 0/3000 [00:04<?, ?it/s, lr: 1.0e-04 loss: 2.128e-01]
my_first_lora_v2: 0%| | 1/3000 [00:05<4:36:17, 5.53s/it, lr: 1.0e-04 loss: 1.373e-01]
my_first_lora_v2: 0%| | 1/3000 [00:05<4:36:17, 5.53s/it, lr: 1.0e-04 loss: 1.373e-01]
my_first_lora_v2: 0%| | 2/3000 [00:06<2:37:58, 3.16s/it, lr: 1.0e-04 loss: 4.070e-01]
my_first_lora_v2: 0%| | 2/3000 [00:06<2:37:58, 3.16s/it, lr: 1.0e-04 loss: 4.070e-01]
my_first_lora_v2: 0%| | 3/3000 [00:07<1:58:20, 2.37s/it, lr: 1.0e-04 loss: 5.340e-01]
my_first_lora_v2: 0%| | 3/3000 [00:07<1:58:20, 2.37s/it, lr: 1.0e-04 loss: 5.340e-01]
my_first_lora_v2: 0%| | 4/3000 [00:07<1:38:10, 1.97s/it, lr: 1.0e-04 loss: 1.474e-01]
my_first_lora_v2: 0%| | 4/3000 [00:07<1:38:10, 1.97s/it, lr: 1.0e-04 loss: 1.474e-01]
my_first_lora_v2: 0%| | 5/3000 [00:09<1:32:37, 1.86s/it, lr: 1.0e-04 loss: 1.076e-01]
my_first_lora_v2: 0%| | 5/3000 [00:09<1:32:37, 1.86s/it, lr: 1.0e-04 loss: 1.076e-01]
my_first_lora_v2: 0%| | 6/3000 [00:10<1:23:36, 1.68s/it, lr: 1.0e-04 loss: 2.589e-01]
my_first_lora_v2: 0%| | 6/3000 [00:10<1:23:36, 1.68s/it, lr: 1.0e-04 loss: 2.589e-01]
my_first_lora_v2: 0%| | 7/3000 [00:11<1:21:39, 1.64s/it, lr: 1.0e-04 loss: 2.158e-01]
my_first_lora_v2: 0%| | 7/3000 [00:11<1:21:39, 1.64s/it, lr: 1.0e-04 loss: 2.158e-01]
my_first_lora_v2: 0%| | 8/3000 [00:14<1:29:17, 1.79s/it, lr: 1.0e-04 loss: 6.402e-01]
my_first_lora_v2: 0%| | 8/3000 [00:14<1:29:17, 1.79s/it, lr: 1.0e-04 loss: 6.402e-01]
my_first_lora_v2: 0%| | 9/3000 [00:17<1:35:14, 1.91s/it, lr: 1.0e-04 loss: 5.290e-01]
my_first_lora_v2: 0%| | 9/3000 [00:17<1:35:14, 1.91s/it, lr: 1.0e-04 loss: 5.290e-01]
my_first_lora_v2: 0%| | 10/3000 [00:18<1:32:47, 1.86s/it, lr: 1.0e-04 loss: 1.946e-02]
my_first_lora_v2: 0%| | 10/3000 [00:18<1:32:47, 1.86s/it, lr: 1.0e-04 loss: 1.946e-02]
my_first_lora_v2: 0%| | 11/3000 [00:21<1:37:17, 1.95s/it, lr: 1.0e-04 loss: 4.394e-01]
my_first_lora_v2: 0%| | 11/3000 [00:21<1:37:17, 1.95s/it, lr: 1.0e-04 loss: 4.394e-01]
my_first_lora_v2: 0%| | 12/3000 [00:23<1:36:02, 1.93s/it, lr: 1.0e-04 loss: 5.838e-02]
my_first_lora_v2: 0%| | 12/3000 [00:23<1:36:02, 1.93s/it, lr: 1.0e-04 loss: 5.838e-02]
my_first_lora_v2: 0%| | 13/3000 [00:24<1:34:14, 1.89s/it, lr: 1.0e-04 loss: 3.231e-01]
my_first_lora_v2: 0%| | 13/3000 [00:24<1:34:14, 1.89s/it, lr: 1.0e-04 loss: 3.231e-01]
my_first_lora_v2: 0%| | 14/3000 [00:27<1:37:40, 1.96s/it, lr: 1.0e-04 loss: 1.357e-01]
my_first_lora_v2: 0%| | 14/3000 [00:27<1:37:40, 1.96s/it, lr: 1.0e-04 loss: 1.357e-01]
my_first_lora_v2: 0%| | 15/3000 [00:30<1:40:39, 2.02s/it, lr: 1.0e-04 loss: 4.001e-01]
my_first_lora_v2: 0%| | 15/3000 [00:30<1:40:39, 2.02s/it, lr: 1.0e-04 loss: 4.001e-01]
my_first_lora_v2: 1%| | 16/3000 [00:31<1:36:41, 1.94s/it, lr: 1.0e-04 loss: 7.292e-02]
my_first_lora_v2: 1%| | 16/3000 [00:31<1:36:41, 1.94s/it, lr: 1.0e-04 loss: 7.292e-02]
my_first_lora_v2: 1%| | 17/3000 [00:32<1:35:37, 1.92s/it, lr: 1.0e-04 loss: 6.517e-01]
my_first_lora_v2: 1%| | 17/3000 [00:32<1:35:37, 1.92s/it, lr: 1.0e-04 loss: 6.517e-01]
my_first_lora_v2: 1%| | 18/3000 [00:35<1:38:13, 1.98s/it, lr: 1.0e-04 loss: 6.511e-01]
my_first_lora_v2: 1%| | 18/3000 [00:35<1:38:13, 1.98s/it, lr: 1.0e-04 loss: 6.511e-01]
my_first_lora_v2: 1%| | 19/3000 [00:38<1:40:34, 2.02s/it, lr: 1.0e-04 loss: 2.807e-01]
my_first_lora_v2: 1%| | 19/3000 [00:38<1:40:34, 2.02s/it, lr: 1.0e-04 loss: 2.807e-01]
my_first_lora_v2: 1%| | 20/3000 [00:39<1:39:03, 1.99s/it, lr: 1.0e-04 loss: 5.139e-01]
my_first_lora_v2: 1%| | 20/3000 [00:39<1:39:03, 1.99s/it, lr: 1.0e-04 loss: 5.139e-01]
my_first_lora_v2: 1%| | 21/3000 [00:42<1:41:07, 2.04s/it, lr: 1.0e-04 loss: 1.673e-01]
my_first_lora_v2: 1%| | 21/3000 [00:42<1:41:07, 2.04s/it, lr: 1.0e-04 loss: 1.673e-01]
my_first_lora_v2: 1%| | 22/3000 [00:43<1:38:13, 1.98s/it, lr: 1.0e-04 loss: 4.545e-02]
my_first_lora_v2: 1%| | 22/3000 [00:43<1:38:13, 1.98s/it, lr: 1.0e-04 loss: 4.545e-02]
my_first_lora_v2: 1%| | 23/3000 [00:44<1:35:34, 1.93s/it, lr: 1.0e-04 loss: 2.307e-01]
my_first_lora_v2: 1%| | 23/3000 [00:44<1:35:34, 1.93s/it, lr: 1.0e-04 loss: 2.307e-01]
my_first_lora_v2: 1%| | 24/3000 [00:47<1:37:29, 1.97s/it, lr: 1.0e-04 loss: 4.169e-01]
my_first_lora_v2: 1%| | 24/3000 [00:47<1:37:29, 1.97s/it, lr: 1.0e-04 loss: 4.169e-01]
my_first_lora_v2: 1%| | 25/3000 [00:47<1:35:08, 1.92s/it, lr: 1.0e-04 loss: 5.428e-01]
my_first_lora_v2: 1%| | 25/3000 [00:47<1:35:08, 1.92s/it, lr: 1.0e-04 loss: 5.428e-01]
my_first_lora_v2: 1%| | 26/3000 [00:49<1:34:48, 1.91s/it, lr: 1.0e-04 loss: 5.455e-02]
my_first_lora_v2: 1%| | 26/3000 [00:49<1:34:48, 1.91s/it, lr: 1.0e-04 loss: 5.455e-02]
my_first_lora_v2: 1%| | 27/3000 [00:51<1:33:57, 1.90s/it, lr: 1.0e-04 loss: 1.836e-01]
my_first_lora_v2: 1%| | 27/3000 [00:51<1:33:57, 1.90s/it, lr: 1.0e-04 loss: 1.836e-01]
my_first_lora_v2: 1%| | 28/3000 [00:51<1:31:56, 1.86s/it, lr: 1.0e-04 loss: 2.375e-01]
my_first_lora_v2: 1%| | 28/3000 [00:51<1:31:56, 1.86s/it, lr: 1.0e-04 loss: 2.375e-01]
my_first_lora_v2: 1%| | 29/3000 [00:52<1:30:07, 1.82s/it, lr: 1.0e-04 loss: 7.300e-01]
my_first_lora_v2: 1%| | 29/3000 [00:52<1:30:07, 1.82s/it, lr: 1.0e-04 loss: 7.300e-01]
my_first_lora_v2: 1%|1 | 30/3000 [00:55<1:31:51, 1.86s/it, lr: 1.0e-04 loss: 1.546e-01]
my_first_lora_v2: 1%|1 | 30/3000 [00:55<1:31:51, 1.86s/it, lr: 1.0e-04 loss: 1.546e-01]
my_first_lora_v2: 1%|1 | 31/3000 [00:57<1:31:09, 1.84s/it, lr: 1.0e-04 loss: 3.833e-01]
my_first_lora_v2: 1%|1 | 31/3000 [00:57<1:31:09, 1.84s/it, lr: 1.0e-04 loss: 3.833e-01]
my_first_lora_v2: 1%|1 | 32/3000 [00:57<1:29:29, 1.81s/it, lr: 1.0e-04 loss: 7.611e-01]
my_first_lora_v2: 1%|1 | 32/3000 [00:57<1:29:29, 1.81s/it, lr: 1.0e-04 loss: 7.611e-01]
my_first_lora_v2: 1%|1 | 33/3000 [01:00<1:31:03, 1.84s/it, lr: 1.0e-04 loss: 4.325e-01]
my_first_lora_v2: 1%|1 | 33/3000 [01:00<1:31:03, 1.84s/it, lr: 1.0e-04 loss: 4.325e-01]
my_first_lora_v2: 1%|1 | 34/3000 [01:02<1:30:24, 1.83s/it, lr: 1.0e-04 loss: 1.497e-01]
my_first_lora_v2: 1%|1 | 34/3000 [01:02<1:30:24, 1.83s/it, lr: 1.0e-04 loss: 1.497e-01]
my_first_lora_v2: 1%|1 | 35/3000 [01:05<1:32:03, 1.86s/it, lr: 1.0e-04 loss: 5.635e-01]
my_first_lora_v2: 1%|1 | 35/3000 [01:05<1:32:03, 1.86s/it, lr: 1.0e-04 loss: 5.635e-01]Error running job: Job stopped |
|
|
| ======================================== |
| Result: |
| - 0 completed jobs |
| - 1 failure |
| ======================================== |
| Traceback (most recent call last): |
| Traceback (most recent call last): |
| File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module> |
| File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module> |
| main()main() |
|
|
| File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main |
| File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main |
| raise eraise e |
|
|
| File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main |
| File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main |
| job.run()job.run() |
|
|
| File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run |
| File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run |
| process.run()process.run() |
|
|
| File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run |
| File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run |
| self.end_step_hook()self.end_step_hook() |
|
|
| File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook |
| File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook |
| self.maybe_stop()self.maybe_stop() |
|
|
| File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop |
| File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop |
| raise Exception("Job stopped")raise Exception("Job stopped") |
|
|
| ExceptionException: : Job stoppedJob stopped |
|
|
|
my_first_lora_v2: 1%|1 | 36/3000 [01:06<1:31:26, 1.85s/it, lr: 1.0e-04 loss: 5.635e-01]
my_first_lora_v2: 1%|1 | 36/3000 [01:06<1:31:26, 1.85s/it, lr: 1.0e-04 loss: 5.635e-01] |
|
|
|
|
