Upload logs/0_log.txt with huggingface_hub

logs/0_log.txt

@@ -0,0 +1,243 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/teamspace/studios/this_studio/ai-toolkit/output",
+    "sqlite_db_path": "/teamspace/studios/this_studio/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": "nishanix1",
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 32,
+        "linear_alpha": 32,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 250,
+        "max_step_saves_to_keep": 4,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/teamspace/studios/this_studio/ai-toolkit/datasets/nishanix1",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                512,
+                768,
+                1024
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 1
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 2500,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "weighted",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": false,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse"
+    },
+    "logging": {
+        "log_every": 1,
+        "use_ui_logger": true
+    },
+    "model": {
+        "name_or_path": "Tongyi-MAI/Z-Image-Turbo",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "zimage:turbo",
+        "low_vram": true,
+        "model_kwargs": {},
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1,
+        "assistant_lora_path": "ostris/zimage_turbo_training_adapter/zimage_turbo_training_adapter_v2.safetensors"
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [
+            {
+                "prompt": "nishanix1, High-resolution, realistic, 85mm camera lens capture of an 18-year-old Indian girl standing fully naked in a poised, seductive stance, her long, dark wavy hair cascading over her shoulders as she gazes directly into the camera with an alluring, confident expression; her body is sculpted with natural curves and detailed anatomy \u2014 full, perky breasts with subtle areolas, a toned midriff with a soft navel, and smooth, glistening skin that highlights the subtle contours of her hips, thighs, and legs; her arms hang loosely at her sides, fingers slightly curled, accentuating the elegant lines of her shoulders and upper arms; a striking red rose tattoo with the phrase \u201cFucked Me\u201d adorns her left hip, adding a bold, sensual touch to her natural beauty; the lighting is soft and diffused, casting gentle shadows that enhance the three-dimensional form of her body, while the neutral beige background ensures all attention remains on her; the composition is framed precisely to include her toes at the bottom and her head at the top, leaving no gaps, creating an immersive, intimate portrait that exudes raw, erotic allure and unapologetic sensuality."
+            },
+            {
+                "prompt": "nishanix1, High-resolution, realistic, 85mm camera lens capture of an 18-year-old Indian girl, fully naked and exuding sensual confidence as she sits provocatively on a vibrant yellow couch, her body sculpted with natural curves and flawless skin, her dark hair cascading over her shoulders, her gaze locked with the viewer with a seductive, knowing smile, her left hand resting on her hip while her right hand delicately touches her chin, her toned abdomen and defined waistline accentuated by the soft, warm lighting, her legs elegantly spread to reveal a meticulously trimmed, glistening pussy with a detailed, moist vagina visible, her pubic mound perfectly shaped and inviting, her entire form radiating youthful allure and erotic magnetism, every contour and curve meticulously rendered in hyper-realistic detail, capturing the essence of raw, unfiltered beauty and sensual power."
+            },
+            {
+                "prompt": "nishanix1, High-resolution, realistic, 85mm camera lens capture of an 18-year-old Indian girl, fully naked and exuding seductive allure, kneeling provocatively on a sleek gray tiled bathroom floor with her back arched and hips thrust forward, showcasing her perfectly sculpted, curvaceous anatomy \u2014 the smooth, glistening contours of her toned thighs, the firm, rounded curves of her buttocks, the subtle definition of her waist, and the natural, unadorned beauty of her skin \u2014 her long, dark hair cascading over her shoulders as she gazes over her shoulder with a sultry, confident expression, her body radiating erotic magnetism against the minimalist backdrop of a white toilet and neutral gray tiles, every detail rendered with cinematic precision to highlight the sensual interplay of light, shadow, and form."
+            },
+            {
+                "prompt": "nishanix1, High-resolution, realistic, 85mm camera lens capture of an 18-year-old Indian girl, fully naked, her body sculpted with natural curves and smooth, sun-kissed skin, her dark, wavy hair cascading over her shoulders, her eyes locked with intense, seductive gaze as she passionately performs oral sex, her lips wrapped tightly around the erect penis, her tongue expertly flicking and swirling, her hand gripping the base of the shaft with delicate yet firm pressure, her fingernails painted a soft pink, her neck adorned with a delicate silver chain, her chest bare and inviting, her skin glowing with natural warmth and subtle freckles, the scene charged with raw, erotic energy, the green couch behind her adding a pop of color to the intimate moment, every detail rendered with hyper-realistic precision, capturing the essence of sensual desire and uninhibited passion."
+            },
+            {
+                "prompt": "nishanix1, High-resolution, realistic, 85mm camera lens capture of an 18-year-old Indian girl, fully naked, kneeling provocatively on a sleek gray tiled bathroom floor with her body sculpted in perfect, sensual curves \u2014 her long, dark, wavy hair cascading over her shoulders, her full, lush breasts prominently displayed with natural areolas, her toned abdomen and defined hip bones accentuating her hourglass figure, her thighs parted slightly as she gazes directly into the camera with a sultry, confident expression, her left hand resting gently on her knee while her right hand trails along her inner thigh, the background simplified to plain gray tiled walls to emphasize her flawless, sun-kissed skin and the erotic allure of her pose, evoking a sense of intimate vulnerability and seductive power, every detail meticulously rendered to highlight the natural beauty and eroticism of her form."
+            }
+        ],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 1,
+        "sample_steps": 8,
+        "num_frames": 1,
+        "fps": 1
+    }
+}
+Using SQLite database at /teamspace/studios/this_studio/ai-toolkit/aitk_db.db
+Job ID: "3068a505-5fd1-4aad-9ca4-dee86c50e02e"
+
+#############################################
+# Running job: Nishani_ZIT_loras_x
+#############################################
+
+
+Running  1 process
+Loading ZImage model
+Loading transformer
+
+
+Loading assistant LoRA
+create LoRA network. base dim (rank): 64, alpha: 64
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 276 modules.
+enable LoRA for U-Net
+Merging in assistant LoRA
+Missing keys: []
+Quantizing Transformer
+ - quantizing 30 transformer blocks
+
  0%|          | 0/30 [00:00<?, ?it/s]
  0%|          | 0/30 [00:00<?, ?it/s]
  3%|3         | 1/30 [00:00<00:04,  6.59it/s]
  3%|3         | 1/30 [00:00<00:04,  6.59it/s]
  7%|6         | 2/30 [00:00<00:03,  7.68it/s]
  7%|6         | 2/30 [00:00<00:03,  7.68it/s]
 10%|#         | 3/30 [00:00<00:03,  8.15it/s]
 10%|#         | 3/30 [00:00<00:03,  8.15it/s]
 17%|#6        | 5/30 [00:00<00:02,  9.20it/s]
 17%|#6        | 5/30 [00:00<00:02,  9.20it/s]
 20%|##        | 6/30 [00:00<00:02,  9.07it/s]
 20%|##        | 6/30 [00:00<00:02,  9.07it/s]
 23%|##3       | 7/30 [00:00<00:02,  8.96it/s]
 23%|##3       | 7/30 [00:00<00:02,  8.96it/s]
 30%|###       | 9/30 [00:01<00:02,  9.48it/s]
 30%|###       | 9/30 [00:01<00:02,  9.48it/s]
 37%|###6      | 11/30 [00:01<00:02,  9.47it/s]
 37%|###6      | 11/30 [00:01<00:02,  9.47it/s]
 40%|####      | 12/30 [00:01<00:01,  9.32it/s]
 40%|####      | 12/30 [00:01<00:01,  9.32it/s]
 47%|####6     | 14/30 [00:01<00:01,  9.41it/s]
 47%|####6     | 14/30 [00:01<00:01,  9.41it/s]
 53%|#####3    | 16/30 [00:01<00:01,  9.46it/s]
 53%|#####3    | 16/30 [00:01<00:01,  9.46it/s]
 60%|######    | 18/30 [00:01<00:01,  9.72it/s]
 60%|######    | 18/30 [00:01<00:01,  9.72it/s]
 67%|######6   | 20/30 [00:02<00:01,  9.93it/s]
 67%|######6   | 20/30 [00:02<00:01,  9.93it/s]
 73%|#######3  | 22/30 [00:02<00:00, 10.36it/s]
 73%|#######3  | 22/30 [00:02<00:00, 10.36it/s]
 80%|########  | 24/30 [00:02<00:00,  9.87it/s]
 80%|########  | 24/30 [00:02<00:00,  9.87it/s]
 83%|########3 | 25/30 [00:02<00:00,  9.61it/s]
 83%|########3 | 25/30 [00:02<00:00,  9.61it/s]
 87%|########6 | 26/30 [00:02<00:00,  9.39it/s]
 87%|########6 | 26/30 [00:02<00:00,  9.39it/s]
 93%|#########3| 28/30 [00:02<00:00,  9.37it/s]
 93%|#########3| 28/30 [00:02<00:00,  9.37it/s]
 97%|#########6| 29/30 [00:03<00:00,  9.24it/s]
 97%|#########6| 29/30 [00:03<00:00,  9.24it/s]
+
+ - quantizing extras
+Moving transformer to CPU
+Text Encoder
+`torch_dtype` is deprecated! Use `dtype` instead!
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+
+Quantizing Text Encoder
+Loading VAE
+Making pipe
+Preparing Model
+Model Loaded
+create LoRA network. base dim (rank): 32, alpha: 32
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 240 modules.
+enable LoRA for U-Net
+Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/nishanix1
+  -  Preprocessing image dimensions
+
  0%|          | 0/7 [00:00<?, ?it/s]
  0%|          | 0/7 [00:00<?, ?it/s]
 14%|#4        | 1/7 [00:00<00:00,  6.88it/s]
 14%|#4        | 1/7 [00:00<00:00,  6.88it/s]
 29%|##8       | 2/7 [00:00<00:00,  6.74it/s]
 29%|##8       | 2/7 [00:00<00:00,  6.74it/s]
 43%|####2     | 3/7 [00:00<00:00,  6.78it/s]
 43%|####2     | 3/7 [00:00<00:00,  6.78it/s]
 57%|#####7    | 4/7 [00:00<00:00,  6.86it/s]
 57%|#####7    | 4/7 [00:00<00:00,  6.86it/s]
 71%|#######1  | 5/7 [00:00<00:00,  6.89it/s]
 71%|#######1  | 5/7 [00:00<00:00,  6.89it/s]
 86%|########5 | 6/7 [00:00<00:00,  6.92it/s]
 86%|########5 | 6/7 [00:00<00:00,  6.92it/s]
+
+  -  Found 7 images
+Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/nishanix1:
+416x576: 7 files
+1 buckets made
+Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/nishanix1
+  -  Preprocessing image dimensions
+
  0%|          | 0/7 [00:00<?, ?it/s]
  0%|          | 0/7 [00:00<?, ?it/s]
+
+  -  Found 7 images
+Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/nishanix1:
+672x864: 7 files
+1 buckets made
+Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/nishanix1
+  -  Preprocessing image dimensions
+
  0%|          | 0/7 [00:00<?, ?it/s]
  0%|          | 0/7 [00:00<?, ?it/s]
+
+  -  Found 7 images
+Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/nishanix1:
+832x1152: 7 files
+1 buckets made
+Generating baseline samples before training
+Unloading assistant lora
+
                                                                
                                                                
+
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module>
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run
+        self.end_step_hook()self.end_step_hook()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+        self.maybe_stop()self.maybe_stop()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+        raise Exception("Job stopped")raise Exception("Job stopped")
+
+ExceptionException: : Job stoppedJob stopped
+
+
+