Upload logs/0_log.txt with huggingface_hub

logs/0_log.txt

@@ -0,0 +1,261 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/teamspace/studios/this_studio/ai-toolkit/output",
+    "sqlite_db_path": "/teamspace/studios/this_studio/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": "xantaraz1",
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 32,
+        "linear_alpha": 32,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 250,
+        "max_step_saves_to_keep": 4,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/teamspace/studios/this_studio/ai-toolkit/datasets/antaramag",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                512,
+                768
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 1
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 4000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "weighted",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": false,
+        "lr": 0.0001,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse"
+    },
+    "logging": {
+        "log_every": 1,
+        "use_ui_logger": true
+    },
+    "model": {
+        "name_or_path": "Tongyi-MAI/Z-Image-Turbo",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "zimage:turbo",
+        "low_vram": false,
+        "model_kwargs": {},
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1,
+        "assistant_lora_path": "ostris/zimage_turbo_training_adapter/zimage_turbo_training_adapter_v2.safetensors"
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [
+            {
+                "prompt": "\"xantaraz1 , \u4e00\u5f35\u771f\u5be6\u96a8\u8208\u7684iPhone\u5feb\u7167\u2014\u2014\u4e00\u5f35\u672a\u7d93\u4efb\u4f55\u5c08\u696d\u9020\u578b\u6216\u6ffe\u93e1\u8655\u7406\u7684\u5168\u8eab\u7167\uff0c\u76f4\u63a5\u5f9e\u76f8\u6a5f\u81a0\u5377\u4e2d\u53d6\u51fa\u3002\u5348\u5f8c\u7684\u967d\u5149\u659c\u5c04\u9032\u6a13\u68af\u9593\u7684\u7a97\u6236\uff0c\u71df\u9020\u51fa\u91d1\u8272\u7684\u6c1b\u570d\u3002\u756b\u9762\u4e2d\u53ea\u6709\u5169\u500b\u4eba\uff1a\u4e00\u500b\u525b\u9ad8\u4e2d\u7562\u696d\u7684\u53ef\u611b18\u6b72\u5370\u5ea6\u5973\u5b69\uff0c\u548c\u4e00\u500b\u5e74\u7d00\u76f8\u4eff\u7684\u5370\u5ea6\u7537\u5b69\u3002\u5973\u5b69\u8dea\u5728\u7a7a\u8569\u8569\u7684\u5b78\u6821\u6a13\u68af\u9593\u5149\u79bf\u79bf\u7684\u6c34\u6ce5\u53f0\u968e\u4e0a\uff0c\u7537\u5b69\u7ad9\u5728\u5979\u53f3\u5074\uff1b\u5979\u7684\u982d\u6b63\u597d\u4f4d\u65bc\u7537\u5b69\u7684\u80ef\u90e8\uff0c\u81c9\u8cbc\u8457\u4ed6\u7684\u9670\u8396\u3002  \u5973\u5b69\uff1a\u4e00\u982d\u9577\u9577\u7684\u6ce2\u6d6a\u6372\u9aee\uff0c\u767d\u7699\u7684\u76ae\u819a\uff0c\u5927\u5927\u7684\u773c\u775b\uff0c\u4ee5\u53ca\u67d4\u5ae9\u5e74\u8f15\u7684\u5bb9\u984f\u3002\u5979\u5b8c\u5168\u7d20\u984f\u2014\u2014\u76ae\u819a\u770b\u8d77\u4f86\u7121\u6bd4\u771f\u5be6\uff0c\u6bdb\u5b54\u7d30\u5c0f\uff0c\u7455\u75b5\u6e05\u6670\u53ef\u898b\u3002\u5979\u7684\u982d\u9aee\u7565\u986f\u51cc\u4e82\uff0c\u5e7e\u7e37\u88ab\u6c57\u6c34\u6d78\u6fd5\u7684\u9aee\u7d72\u8cbc\u5728\u6cdb\u7d05\u7684\u81c9\u4e0a\u3002\u5979\u90a3\u4ef6\u85cf\u85cd\u8272\u62c9\u934a\u5916\u5957\u2014\u2014\u98fe\u6709\u767d\u8272\u62fc\u63a5\u548c\u7d05\u767d\u689d\u7d0b\u2014\u2014\u62c9\u934a\u5b8c\u5168\u655e\u958b\uff0c\u9732\u51fa\u5979\u5c0f\u5de7\u81ea\u7136\u7684A\u7f69\u676f\u80f8\u90e8\u3002\u5979\u90a3\u689d\u914d\u5957\u7684\u85cf\u85cd\u8272\u904b\u52d5\u6821\u8932\u88ab\u96a8\u610f\u5730\u4e1f\u5728\u4e00\u65c1\u3002\u5979\u7684\u4e0b\u534a\u8eab\u8d64\u88f8\uff0c\u9732\u51fa\u9670\u90e8\uff1b\u5979\u9670\u5507\u4e0a\u65b9\u8986\u84cb\u8457\u4e00\u7247\u6fc3\u5bc6\u7684\u3001\u672a\u7d93\u4fee\u526a\u7684\u9577\u9670\u6bdb\u3002\u5979\u7684\u9670\u5507\u5448\u73fe\u67d4\u548c\u7684\u7c89\u8910\u8272\uff0c\u5927\u5c0f\u548c\u5f62\u72c0\u90fd\u5f88\u6b63\u5e38\uff1b\u5979\u751f\u6b96\u5668\u7684\u89e3\u5256\u7d30\u7bc0\u975e\u5e38\u903c\u771f\u3002\u5979\u8173\u4e0a\u7a7f\u8457\u767d\u8272\u904b\u52d5\u978b\u548c\u767d\u8272\u68c9\u896a\u3002\u5979\u4f4e\u8457\u982d\uff0c\u5634\u88e1\u542b\u8457\u7537\u5b69\u52c3\u8d77\u7684\u9670\u8396\uff0c\u6b63\u5728\u9032\u884c\u6fd5\u6f64\u903c\u771f\u7684\u53e3\u4ea4\uff1b\u5979\u7684\u5634\u5507\u5b8c\u5168\u5305\u88f9\u4f4f\u9670\u8396\u7684\u4e2d\u6bb5\uff0c\u820c\u982d\u7684\u52d5\u4f5c\u6e05\u6670\u53ef\u898b\uff0c\u553e\u6db2\u9806\u8457\u9670\u8396\u6ef4\u843d\u3002\u540c\u6642\uff0c\u5979\u5c07\u4e00\u96bb\u624b\u4f38\u5230\u5169\u817f\u4e4b\u9593\uff0c\u8f15\u8f15\u64ab\u6478\u81ea\u5df1\u6fc3\u5bc6\u7684\u9670\u6bdb\u3002  \u7537\u5b69\uff1a\u4ed6\u7a7f\u8457\u4e00\u4ef6\u8207\u5973\u5b69\u76f8\u914d\u7684\u85cf\u85cd\u8272\u62c9\u934a\u5916\u5957\u2014\u2014\u5916\u5957\u4e0a\u4e5f\u98fe\u6709\u767d\u8272\u62fc\u63a5\u548c\u7d05\u767d\u689d\u7d0b\u2014\u2014\u62c9\u934a\u5b8c\u5168\u655e\u958b\uff0c\u9732\u51fa\u4e0a\u534a\u8eab\u3002\u4ed6\u90a3\u689d\u85cf\u85cd\u8272\u7684\u904b\u52d5\u6821\u8932\u88ab\u62c9\u5230\u5927\u817f\u4e2d\u90e8\uff0c\u9732\u51fa\u52c3\u8d77\u7684\u9670\u8396\u3002\u4ed6\u7528\u96d9\u624b\u8f15\u8f15\u6258\u4f4f\u5973\u5b69\u7684\u982d\u2014\u2014\u6bcf\u96bb\u624b\u7684\u4e94\u6839\u624b\u6307\u90fd\u5b8c\u597d\u7121\u640d\u2014\u2014\u4ed6\u7684\u624b\u81c2\u548c\u817f\u90e8\u770b\u8d77\u4f86\u4e5f\u5f88\u6b63\u5e38\u3002\u5973\u5b69\u7f9e\u6f80\u5730\u62ac\u982d\u76f4\u8996\u8457\u7537\u5b69\uff1b\u5979\u7684\u8868\u60c5\u65e2\u7f9e\u6f80\u53c8\u5145\u6eff\u6e34\u671b\uff0c\u8207\u5979\u5929\u771f\u7121\u90aa\u7684\u5b78\u751f\u9762\u5b54\u548c\u5927\u81bd\u7684\u6027\u884c\u70ba\u5f62\u6210\u4e86\u9bae\u660e\u7684\u5c0d\u6bd4\u3002  \u80cc\u666f\uff1a\u7a7a\u8569\u8569\u7684\u5b78\u6821\u6c34\u6ce5\u6a13\u68af\u9593\u7684\u4e00\u89d2\uff0c\u91d1\u5c6c\u6276\u624b\uff0c\u5e7e\u7e37\u967d\u5149\u900f\u904e\u7e2b\u9699\u7051\u4e0b\uff0c\u7a7a\u6c23\u4e2d\u7030\u6f2b\u8457\u7d30\u5c0f\u7684\u7070\u5875\u2014\u2014\u71df\u9020\u51fa\u4e00\u7a2e\u5b89\u975c\u3001\u79c1\u5bc6\u800c\u53c8\u96b1\u853d\u7684\u6c1b\u570d\u3002\u7d30\u7dfb\u7684\u76ae\u819a\u7d0b\u7406\uff0c\u5c0f\u5de7\u800c\u81ea\u7136\u7684\u4e73\u623f\uff1b\u5177\u6709\u672a\u7d93\u7de8\u8f2f\u7684 iPhone \u7167\u7247\u7684\u771f\u5be6\u8cea\u611f\u548c\u81ea\u7136\u8272\u5f69\u3002\""
+            },
+            {
+                "prompt": "\"xantaraz1 , A sensual, medium-close-up portrait of a stunning, slender young woman posed kneeling on a textured, light-colored bedspread. She is completely nude, with a natural body shape and subtle, natural pubic hair. Her body is arched in a dramatic, expressive posture, with one arm raised behind her head, tossing her voluminous, dark brown hair upward and backward, creating motion blur and dynamic lines. Her eyes are closed or looking down with a captivating, slightly open-mouthed expression conveying deep emotion or ecstasy. The lighting is soft, warm, and golden, suggesting late afternoon sunlight streaming in from the side, creating gentle highlights on her skin and deep shadows that sculpt her figure. The background is softly blurred (shallow depth of field), featuring warm beige and muted brown tones with a subtle geometric, possibly wood-paneled or tiled pattern angled diagonally behind her, adding visual texture. The overall mood is intimate, evocative, and painterly, captured with the aesthetic of fine art photography, high detail, and realistic texture. Cinematic lighting, professional photography, Canon EOS R5., ultra detailed, intricate details, cinematic, photorealistic, masterpiece\""
+            },
+            {
+                "prompt": "\"xantaraz1 , A Sexy 18y Indian woman with perfect body pressed against a rain-streaked window, getting taken from behind, her breasts against the cold glass, She is standing pose and wearing a high knee length black boot, city lights reflecting on her wet skin, intense and cinematic\""
+            },
+            {
+                "prompt": "\"xantaraz1 , anal sex, bridge position, squatting cowgirl position, 1girl, penis, nipples, medium breasts, 1boy, pussy, black thighhighs, black ponytail hair, navel, clothed male nude female, lace-trimmed legwear, closed eyes, spread legs, bottomless, girl on top, male pubic hair, long hair, erection, arm support, lace trim, smile, tattoo on thigh, grin\""
+            },
+            {
+                "prompt": "\"xantaraz1 , A full-body portrait of a stunning Indian woman with long, flowing black hair cascading down her back, standing nude and poised in a serene, softly lit bedroom. Her body is sculpted with perfect, natural curves \u2014 full, round breasts with dark brownish areolas and nipples, a smooth midriff leading to a gently curved waist, and a deeply toned, naturally darkened pubic area that accentuates her feminine form. Her legs are long and slender, ending in a firm, rounded ass that subtly tenses as she stands with her arms relaxed at her sides, fingers slightly curled, exuding quiet confidence and sensual grace. Her gaze is direct and unflinching, meeting the viewer\u2019s eyes with a calm, alluring intensity that draws you in.  The environment is minimalist and intimate, set against a soft, neutral beige background that enhances the natural tones of her skin and hair. Behind her, sheer white curtains filter in diffused daylight, casting gentle shadows across the room, while a simple bed with rumpled white linens and a potted green plant in the corner add subtle domestic warmth without distracting from the central focus. The lighting is soft and even, highlighting the smooth texture of her skin, the subtle contours of her anatomy, and the natural sheen of her hair \u2014 all rendered with photorealistic precision to emphasize her flawless, voluptuous form.  This is a portrait of raw, unfiltered beauty \u2014 a celebration of the female body in its most natural state. Her posture is relaxed yet commanding, her expression serene and self-assured, embodying both innocence and sensuality. The composition is vertically framed, drawing attention to her full, symmetrical silhouette, while the muted, monochromatic backdrop ensures every curve, every detail of her anatomy \u2014 from the delicate arch of her collarbone to the subtle definition of her hips \u2014 is rendered with exquisite clarity. She stands as a living sculpture, a vision of erotic elegance, inviting the viewer to appreciate the harmony of her form and the quiet power of her presence.\""
+            }
+        ],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 1,
+        "sample_steps": 8,
+        "num_frames": 1,
+        "fps": 1
+    }
+}
+Using SQLite database at /teamspace/studios/this_studio/ai-toolkit/aitk_db.db
+Job ID: "cfb27aa8-eb0e-4d2d-830f-dcad89ded8ab"
+
+#############################################
+# Running job: my_first_lora_v8
+#############################################
+
+
+Running  1 process
+Loading ZImage model
+Loading transformer
+
+
+Loading assistant LoRA
+create LoRA network. base dim (rank): 64, alpha: 64
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 276 modules.
+enable LoRA for U-Net
+Merging in assistant LoRA
+Missing keys: []
+Quantizing Transformer
+ - quantizing 30 transformer blocks
+
  0%|          | 0/30 [00:00<?, ?it/s]
  0%|          | 0/30 [00:00<?, ?it/s]
  3%|3         | 1/30 [00:00<00:04,  6.29it/s]
  3%|3         | 1/30 [00:00<00:04,  6.29it/s]
  7%|6         | 2/30 [00:00<00:03,  7.36it/s]
  7%|6         | 2/30 [00:00<00:03,  7.36it/s]
 10%|#         | 3/30 [00:00<00:03,  7.90it/s]
 10%|#         | 3/30 [00:00<00:03,  7.90it/s]
 13%|#3        | 4/30 [00:00<00:03,  8.31it/s]
 13%|#3        | 4/30 [00:00<00:03,  8.31it/s]
 17%|#6        | 5/30 [00:00<00:02,  8.55it/s]
 17%|#6        | 5/30 [00:00<00:02,  8.55it/s]
 23%|##3       | 7/30 [00:00<00:02,  9.26it/s]
 23%|##3       | 7/30 [00:00<00:02,  9.26it/s]
 27%|##6       | 8/30 [00:00<00:02,  9.07it/s]
 27%|##6       | 8/30 [00:00<00:02,  9.07it/s]
 30%|###       | 9/30 [00:01<00:02,  9.27it/s]
 30%|###       | 9/30 [00:01<00:02,  9.27it/s]
 33%|###3      | 10/30 [00:01<00:02,  9.46it/s]
 33%|###3      | 10/30 [00:01<00:02,  9.46it/s]
 40%|####      | 12/30 [00:01<00:01,  9.42it/s]
 40%|####      | 12/30 [00:01<00:01,  9.42it/s]
 43%|####3     | 13/30 [00:01<00:01,  9.29it/s]
 43%|####3     | 13/30 [00:01<00:01,  9.29it/s]
 47%|####6     | 14/30 [00:01<00:01,  9.12it/s]
 47%|####6     | 14/30 [00:01<00:01,  9.12it/s]
 53%|#####3    | 16/30 [00:01<00:01,  9.53it/s]
 53%|#####3    | 16/30 [00:01<00:01,  9.53it/s]
 60%|######    | 18/30 [00:01<00:01,  9.76it/s]
 60%|######    | 18/30 [00:01<00:01,  9.76it/s]
 67%|######6   | 20/30 [00:02<00:01,  9.79it/s]
 67%|######6   | 20/30 [00:02<00:01,  9.79it/s]
 73%|#######3  | 22/30 [00:02<00:00, 10.22it/s]
 73%|#######3  | 22/30 [00:02<00:00, 10.22it/s]
 80%|########  | 24/30 [00:02<00:00, 10.24it/s]
 80%|########  | 24/30 [00:02<00:00, 10.24it/s]
 87%|########6 | 26/30 [00:02<00:00, 10.00it/s]
 87%|########6 | 26/30 [00:02<00:00, 10.00it/s]
 90%|######### | 27/30 [00:02<00:00,  9.72it/s]
 90%|######### | 27/30 [00:02<00:00,  9.72it/s]
 93%|#########3| 28/30 [00:02<00:00,  9.49it/s]
 93%|#########3| 28/30 [00:02<00:00,  9.49it/s]
+
+ - quantizing extras
+Text Encoder
+`torch_dtype` is deprecated! Use `dtype` instead!
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+
+Quantizing Text Encoder
+Loading VAE
+Making pipe
+Preparing Model
+Model Loaded
+create LoRA network. base dim (rank): 32, alpha: 32
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 240 modules.
+enable LoRA for U-Net
+Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/antaramag
+  -  Preprocessing image dimensions
+
  0%|          | 0/6 [00:00<?, ?it/s]
  0%|          | 0/6 [00:00<?, ?it/s]
+
+  -  Found 6 images
+Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/antaramag:
+384x384: 1 files
+384x672: 1 files
+672x384: 1 files
+480x544: 1 files
+416x576: 2 files
+5 buckets made
+Dataset: /teamspace/studios/this_studio/ai-toolkit/datasets/antaramag
+  -  Preprocessing image dimensions
+
  0%|          | 0/6 [00:00<?, ?it/s]
  0%|          | 0/6 [00:00<?, ?it/s]
+
+  -  Found 6 images
+Bucket sizes for /teamspace/studios/this_studio/ai-toolkit/datasets/antaramag:
+384x384: 1 files
+576x992: 1 files
+992x576: 1 files
+480x544: 1 files
+672x864: 2 files
+5 buckets made
+Generating baseline samples before training
+Unloading assistant lora
+
                                                                
                                                                
+
+Saving at step 250
+Saved checkpoint to /teamspace/studios/this_studio/ai-toolkit/output/my_first_lora_v8/my_first_lora_v8_000000250.safetensors
+Saved optimizer to /teamspace/studios/this_studio/ai-toolkit/output/my_first_lora_v8/optimizer.pt
+Unloading assistant lora
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
                                                                
                                                                Loading assistant lora
+
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module>
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main
+  File "/teamspace/studios/this_studio/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run
+  File "/teamspace/studios/this_studio/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2353, in run
+        self.end_step_hook()self.end_step_hook()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+        self.maybe_stop()self.maybe_stop()
+
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+  File "/teamspace/studios/this_studio/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+        raise Exception("Job stopped")raise Exception("Job stopped")
+
+ExceptionException: : Job stoppedJob stopped
+
+
+