Upload logs/4_log.txt with huggingface_hub

logs/4_log.txt

@@ -0,0 +1,257 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/teamspace/studios/this_studio/ai-toolkit/output",
+    "sqlite_db_path": "/teamspace/studios/this_studio/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": "xtishaz1",
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 32,
+        "linear_alpha": 32,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 500,
+        "max_step_saves_to_keep": 8,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/teamspace/studios/this_studio/ai-toolkit/datasets/tisha",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "Indian 18y nude girl",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                1024
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "do_i2v": true,
+            "flip_x": false,
+            "flip_y": false
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 3000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "weighted",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": true,
+        "cache_text_embeddings": false,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse",
+        "do_differential_guidance": true,
+        "differential_guidance_scale": 3
+    },
+    "model": {
+        "name_or_path": "Tongyi-MAI/Z-Image-Turbo",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "zimage:turbo",
+        "low_vram": true,
+        "model_kwargs": {},
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1,
+        "assistant_lora_path": "ostris/zimage_turbo_training_adapter/zimage_turbo_training_adapter_v1.safetensors"
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [
+            {
+                "prompt": "xtishaz1, A young nude woman with a soft oval face shape, smooth light-to-medium warm skin tone, and clear glowing complexion, large expressive almond-shaped dark brown eyes, well-defined natural eyebrows, a small straight nose, and full pink lips with a gentle smile, long straight silky black hair styled with volume at the crown (slight puffed front) flowing past the shoulders, with a youthful, fresh, and naturally elegant appearance and balanced facial symmetry, is captured in a moment of intense intimacy as she leans forward with her eyes closed in pleasure, her tongue delicately exploring the base of a man\u2019s erect penis, her head tilted back slightly, her left hand gently cradling the back of her head while her right arm rests along his torso, the man\u2019s tattooed forearm adorned with a black dagger design rests possessively on her head, his body partially visible with defined musculature and natural body hair, the scene bathed in warm, golden-orange ambient lighting that casts soft shadows and highlights the contours of their skin, the background blurred into a smooth gradient of rich amber tones with subtle bokeh lights, creating a shallow depth of field that draws focus entirely to the couple\u2019s connection, the camera positioned at a close, slightly low angle to emphasize the intimacy and raw emotion of the moment, the woman\u2019s delicate gold hoop earring and thin gold chain necklace catching the light, adding a touch of elegance to the sensual scene, the overall composition exudes a sense of passion, trust, and natural beauty, rendered in ultra-high detail with a realistic, NSFW, adult, Z-Image Turbo optimized aesthetic."
+            },
+            {
+                "prompt": "xtishaz1, A young nude woman with a soft oval face shape, smooth light-to-medium warm skin tone, and clear glowing complexion, large expressive almond-shaped dark brown eyes, well-defined natural eyebrows, a small straight nose, and full pink lips with a gentle smile, long straight silky black hair styled with volume at the crown (slight puffed front) flowing past the shoulders, with a youthful, fresh, and naturally elegant appearance and balanced facial symmetry, is captured in a moment of intense intimacy, kneeling on a plush white quilted bed, her body arched gracefully as she leans forward, one hand resting possessively on her hip while the other is gently placed on her thigh, her skin glistening with natural moisture under the soft, diffused natural light filtering through sheer white curtains and a wooden-framed window in the background, the warm beige walls and a wicker basket subtly visible in the corner adding to the serene, private atmosphere, the camera angle is low and intimate, emphasizing the curves of her body and the sensual tension of the moment, the depth of field is shallow, blurring the background slightly to draw focus to her form and the subtle interplay of light and shadow across her skin, creating a highly detailed, realistic, NSFW, adult, Z-Image Turbo optimized scene that exudes passion, elegance, and natural beauty."
+            },
+            {
+                "prompt": "xtishaz1, \"Photo of a nude Indian woman 21y old standing in a bedroom, facing           the camera, medium wide shot, full body, black hair, dark eyes, medium breasts,   slender physique, medium skin tone, brown eyes, dark eyebrows, small areola,  standing with her legs together , perfect feet, hands at her sides, perfect fingers, looking at the camera, natural lighting, bright and airy room, white walls, large window with sheer           white curtains, white bed with a wooden frame, white bed linens, wooden floor, potted plant in the background, casual and relaxed pose, confident  expression, soft shadows, high quality, jpeg artifacts, no watermark, perfectly optimized for Z-Image Turbo.\""
+            },
+            {
+                "prompt": "xtishaz1, A young nude woman with a soft oval face shape, smooth light-to-medium warm skin tone, and clear glowing complexion, large expressive almond-shaped dark brown eyes, well-defined natural eyebrows, a small straight nose, and full pink lips with a gentle smile, long straight silky black hair styled with volume at the crown (slight puffed front) flowing past the shoulders, with a youthful, fresh, and naturally elegant appearance and balanced facial symmetry, reclines provocatively atop a plush, deep purple satin couch, her legs spread wide in a bold, sensual pose that accentuates her curves and natural beauty, her tongue playfully extended in a teasing expression, the rich, velvety texture of the couch contrasting with the smooth, glistening skin of her body, the ambient lighting casting soft, warm highlights across her form, the background revealing a cozy, intimate room with a purple wall, a bookshelf filled with books and decorative items, and a vintage-style lamp casting a gentle glow, the depth of field keeps the focus sharply on her figure while softly blurring the background elements, creating a luxurious, NSFW, adult, Z-Image Turbo optimized scene that exudes confidence, allure, and raw sensuality."
+            },
+            {
+                "prompt": "xtishaz1, A young nude woman with a soft oval face shape, smooth light-to-medium warm skin tone, and clear glowing complexion, large expressive almond-shaped dark brown eyes, well-defined natural eyebrows, a small straight nose, and full pink lips with a gentle smile, long straight silky black hair styled with volume at the crown (slight puffed front) flowing past the shoulders, with a youthful, fresh, and naturally elegant appearance and balanced facial symmetry, kneels gracefully on gray tiled bathroom floor in a poised, slightly arched position, her body glistening with subtle sheen under soft, diffused ambient lighting that highlights the natural curves and contours of her torso, hips, and thighs, her left hand resting gently on her inner thigh while her right hand lightly touches her knee, her gaze directed confidently toward the viewer with an air of serene sensuality, behind her, a pristine white ceramic toilet stands against a backdrop of muted gray wall tiles with clean, minimalist grout lines, the scene framed by a slightly elevated camera angle that captures the full length of her form from mid-thighs upward, creating a sense of depth and intimacy, the shallow depth of field softly blurs the background elements while keeping her figure sharply in focus, evoking a quiet, private moment of vulnerability and elegance within a modern, sterile bathroom environment, the overall composition exudes a natural, unposed beauty with a touch of NSFW allure, rendered in ultra-high detail with realistic textures and lighting, optimized for Z-Image Turbo."
+            }
+        ],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 1,
+        "sample_steps": 8,
+        "num_frames": 1,
+        "fps": 1
+    },
+    "logging": {
+        "log_every": 1,
+        "use_ui_logger": true
+    }
+}
+Using SQLite database at /teamspace/studios/this_studio/ai-toolkit/aitk_db.db
+Job ID: "f7eb6cbb-a54a-4f3a-92b0-66455621336c"
+
+#############################################
+# Running job: my_first_lora_v6
+#############################################
+
+
+Running  1 process
+Loading ZImage model
+Loading transformer
+
+
+Loading assistant LoRA
+create LoRA network. base dim (rank): 32, alpha: 32
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 276 modules.
+enable LoRA for U-Net
+Merging in assistant LoRA
+Missing keys: []
+Quantizing Transformer
+ - quantizing 30 transformer blocks
+
  0%|          | 0/30 [00:00<?, ?it/s]
  0%|          | 0/30 [00:00<?, ?it/s]
  3%|3         | 1/30 [00:00<00:04,  6.60it/s]
  3%|3         | 1/30 [00:00<00:04,  6.60it/s]
  7%|6         | 2/30 [00:00<00:03,  7.72it/s]
  7%|6         | 2/30 [00:00<00:03,  7.72it/s]
 10%|#         | 3/30 [00:00<00:03,  8.10it/s]
 10%|#         | 3/30 [00:00<00:03,  8.10it/s]
 13%|#3        | 4/30 [00:00<00:03,  8.30it/s]
 13%|#3        | 4/30 [00:00<00:03,  8.30it/s]
 17%|#6        | 5/30 [00:00<00:02,  8.47it/s]
 17%|#6        | 5/30 [00:00<00:02,  8.47it/s]
 20%|##        | 6/30 [00:00<00:02,  8.55it/s]
 20%|##        | 6/30 [00:00<00:02,  8.55it/s]
 23%|##3       | 7/30 [00:00<00:02,  8.60it/s]
 23%|##3       | 7/30 [00:00<00:02,  8.60it/s]
 27%|##6       | 8/30 [00:00<00:02,  8.66it/s]
 27%|##6       | 8/30 [00:00<00:02,  8.66it/s]
 30%|###       | 9/30 [00:01<00:02,  8.68it/s]
 30%|###       | 9/30 [00:01<00:02,  8.68it/s]
 33%|###3      | 10/30 [00:01<00:02,  8.72it/s]
 33%|###3      | 10/30 [00:01<00:02,  8.72it/s]
 37%|###6      | 11/30 [00:01<00:02,  8.74it/s]
 37%|###6      | 11/30 [00:01<00:02,  8.74it/s]
 40%|####      | 12/30 [00:01<00:02,  8.78it/s]
 40%|####      | 12/30 [00:01<00:02,  8.78it/s]
 43%|####3     | 13/30 [00:01<00:01,  8.81it/s]
 43%|####3     | 13/30 [00:01<00:01,  8.81it/s]
 47%|####6     | 14/30 [00:01<00:01,  8.83it/s]
 47%|####6     | 14/30 [00:01<00:01,  8.83it/s]
 50%|#####     | 15/30 [00:01<00:01,  8.83it/s]
 50%|#####     | 15/30 [00:01<00:01,  8.83it/s]
 53%|#####3    | 16/30 [00:01<00:01,  8.86it/s]
 53%|#####3    | 16/30 [00:01<00:01,  8.86it/s]
 57%|#####6    | 17/30 [00:01<00:01,  8.87it/s]
 57%|#####6    | 17/30 [00:01<00:01,  8.87it/s]
 60%|######    | 18/30 [00:02<00:01,  8.86it/s]
 60%|######    | 18/30 [00:02<00:01,  8.86it/s]
 63%|######3   | 19/30 [00:02<00:01,  8.87it/s]
 63%|######3   | 19/30 [00:02<00:01,  8.87it/s]
 67%|######6   | 20/30 [00:02<00:01,  8.90it/s]
 67%|######6   | 20/30 [00:02<00:01,  8.90it/s]
 70%|#######   | 21/30 [00:02<00:01,  8.89it/s]
 70%|#######   | 21/30 [00:02<00:01,  8.89it/s]
 73%|#######3  | 22/30 [00:02<00:00,  8.89it/s]
 73%|#######3  | 22/30 [00:02<00:00,  8.89it/s]
 77%|#######6  | 23/30 [00:02<00:00,  8.89it/s]
 77%|#######6  | 23/30 [00:02<00:00,  8.89it/s]
 83%|########3 | 25/30 [00:02<00:00,  9.19it/s]
 83%|########3 | 25/30 [00:02<00:00,  9.19it/s]
 87%|########6 | 26/30 [00:02<00:00,  9.12it/s]
 87%|########6 | 26/30 [00:02<00:00,  9.12it/s]
 90%|######### | 27/30 [00:03<00:00,  9.07it/s]
 90%|######### | 27/30 [00:03<00:00,  9.07it/s]
 93%|#########3| 28/30 [00:03<00:00,  9.03it/s]
 93%|#########3| 28/30 [00:03<00:00,  9.03it/s]
 97%|#########6| 29/30 [00:03<00:00,  9.02it/s]
 97%|#########6| 29/30 [00:03<00:00,  9.02it/s]
+
+ - quantizing extras
+Moving transformer to CPU
+Text Encoder
+`torch_dtype` is deprecated! Use `dtype` instead!
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+
+Quantizing Text Encoder
+Loading VAE
+Making pipe
+Preparing Model
+Model Loaded
+create LoRA network. base dim (rank): 32, alpha: 32
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 240 modules.
+enable LoRA for U-Net
+#### IMPORTANT RESUMING FROM /teamspace/studios/this_studio/ai-toolkit/output/my_first_lora_v6/my_first_lora_v6_000002000.safetensors ####
+Loading from /teamspace/studios/this_studio/ai-toolkit/output/my_first_lora_v6/my_first_lora_v6_000002000.safetensors
+Missing keys: []
+Found step 2000 in metadata, starting from there
+Loading optimizer state from /teamspace/studios/this_studio/ai-toolkit/output/my_first_lora_v6/optimizer.pt
+Updating optimizer LR from params
+Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/tisha
+  -  Preprocessing image dimensions
+
  0%|          | 0/9 [00:00<?, ?it/s]
  0%|          | 0/9 [00:00<?, ?it/s]
+
+  -  Found 9 images
+Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/tisha:
+832x1152: 4 files
+896x1152: 1 files
+512x832: 2 files
+864x864: 1 files
+768x1056: 1 files
+5 buckets made
+Caching embeddings and unloading text encoder
+
+***** UNLOADING TEXT ENCODER *****
+This will train only with a blank prompt or trigger word, if set
+If this is not what you want, remove the unload_text_encoder flag
+***********************************
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                
                                                                Loading assistant lora
+
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module>
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run
+        self.end_step_hook()self.end_step_hook()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+        self.maybe_stop()self.maybe_stop()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+        raise Exception("Job stopped")raise Exception("Job stopped")
+
+ExceptionException: : Job stoppedJob stopped
+
+
+