Upload 8 files

app.py

@@ -38,12 +38,17 @@ from civitai_to_hf import (
     update_civitai_gallery_mode,
     begin_probe_civitai,
     clear_retry_state,
+    safe_retry_probe,
+    safe_upload_verify_probe,
+    safe_summary_probe,
+    create_report_zip,
     CIVITAI_TYPE,
     CIVITAI_BASEMODEL,
     CIVITAI_SORT,
     CIVITAI_PERIOD,
     CIVITAI_FILETYPE,
     uniq_urls,
+    HF_UPLOAD_RETRY_POLICY_CHOICES,
 )
 
 
@@ -141,6 +146,7 @@ with gr.Blocks(fill_width=True, delete_cache=(60, 3600)) as demo:
                 is_private = gr.Checkbox(label="Create private repo", value=True)
                 is_info = gr.Checkbox(label="Upload Civitai information files", value=False)
                 is_rename = gr.Checkbox(label="Auto rename", value=True)
+            hf_upload_retry_policy = gr.Dropdown(label="HF upload retry policy", choices=HF_UPLOAD_RETRY_POLICY_CHOICES, value="Auto", info="Auto is a cautious default. Use Patient for repeated HF 429/503/LFS commit errors.")
             bucket_mode_note = gr.Markdown(value="", visible=False, elem_classes="info")
         with gr.Row():
             run_button = gr.Button(value="Download and Upload", variant="primary")
@@ -160,6 +166,12 @@ with gr.Blocks(fill_width=True, delete_cache=(60, 3600)) as demo:
             with gr.Row():
                 smoke_test_button = gr.Button(value="Smoke Test", variant="secondary")
                 probe_button = gr.Button("Probe Civitai", variant="secondary")
+                safe_retry_probe_button = gr.Button("Safe Retry Probe", variant="secondary")
+                safe_upload_verify_probe_button = gr.Button("Safe Upload Verify Probe", variant="secondary")
+                safe_summary_probe_button = gr.Button("Safe Summary Probe", variant="secondary")
+            with gr.Row():
+                report_zip_button = gr.Button("Create Report ZIP", variant="secondary")
+                report_zip_file = gr.File(label="Report ZIP", visible=True)
         gr.DuplicateButton(value="Duplicate Space")
 
     heavy_outputs = [uploaded_urls, urls_md, urls_remain, urls_failed, civitai_key_status, session_state]
@@ -170,7 +182,7 @@ with gr.Blocks(fill_width=True, delete_cache=(60, 3600)) as demo:
     download_evt = gr.on(
         triggers=[run_button.click],
         fn=download_civitai,
-        inputs=[dl_url, civitai_key, hf_token, uploaded_urls, newrepo_id, newrepo_type, is_private, is_info, is_rename, session_state],
+        inputs=[dl_url, civitai_key, hf_token, uploaded_urls, newrepo_id, newrepo_type, is_private, is_info, is_rename, session_state, hf_upload_retry_policy],
         outputs=heavy_outputs,
         queue=True,
         concurrency_limit=1,
@@ -179,7 +191,7 @@ with gr.Blocks(fill_width=True, delete_cache=(60, 3600)) as demo:
     smoke_evt = gr.on(
         triggers=[smoke_test_button.click],
         fn=smoke_test_civitai,
-        inputs=[civitai_key, hf_token, uploaded_urls, newrepo_type, is_private, is_info, is_rename, session_state],
+        inputs=[civitai_key, hf_token, uploaded_urls, newrepo_type, is_private, is_info, is_rename, session_state, hf_upload_retry_policy],
         outputs=heavy_outputs,
         queue=True,
         api_visibility="undocumented",
@@ -215,6 +227,10 @@ with gr.Blocks(fill_width=True, delete_cache=(60, 3600)) as demo:
     search_civitai_user.change(refresh_civitai_creators, [search_civitai_user, civitai_key], [search_civitai_user], queue=False, api_visibility="undocumented")
     civitai_key.change(refresh_civitai_key_status, [civitai_key], [civitai_key_status], queue=False, api_visibility="undocumented")
     probe_button.click(begin_probe_civitai, [], [probe_md], queue=False, api_visibility="undocumented").then(probe_civitai_api, [search_civitai_query, civitai_key], [probe_md, civitai_key_status], queue=False, api_visibility="undocumented")
+    safe_retry_probe_button.click(safe_retry_probe, [hf_upload_retry_policy], [probe_md], queue=False, api_visibility="undocumented")
+    safe_upload_verify_probe_button.click(safe_upload_verify_probe, [hf_upload_retry_policy], [probe_md], queue=False, api_visibility="undocumented")
+    safe_summary_probe_button.click(safe_summary_probe, [], [probe_md], queue=False, api_visibility="undocumented")
+    report_zip_button.click(create_report_zip, [session_state, state], [report_zip_file], queue=False, api_visibility="undocumented")
 
 
 demo.queue()

civitai_to_hf.py

@@ -14,7 +14,10 @@ from utils import (get_token, set_token, is_repo_exists, get_user_agent, get_dow
                    create_retry_session, retry_call, ensure_repo, parse_civitai_api_keys,
                    should_switch_civitai_key, resolve_civitai_download_url, suppress_hf_hub_progress_bars,
                    reset_civitai_key_status, update_civitai_key_status, get_civitai_key_status,
-                   sanitize_url_for_log)
+                   sanitize_url_for_log, HF_UPLOAD_RETRY_POLICY_CHOICES,
+                   get_hf_upload_retry_policy_config, hf_upload_retry_call,
+                   is_retryable_hf_upload_exception, parse_hf_retry_delay_from_headers,
+                   format_hf_rate_limit_hint, format_error_short)
 from bucket_ops import (is_bucket_api_available, ensure_bucket, upload_file_to_bucket,
                         get_safe_bucket_filename, get_bucket_url)
 import re
@@ -29,6 +32,10 @@ import shutil
 import random
 import subprocess
 import threading
+import sys
+import platform
+import zipfile
+from datetime import datetime, timezone
 from io import BytesIO
 
 
@@ -67,6 +74,9 @@ CREATOR_SUGGEST_LOCK = threading.Lock()
 CIVITAI_ACTIVE_API_ORIGIN = ""
 CIVITAI_ACTIVE_API_BASE = ""
 _CIVITAI_API_LOCK = threading.Lock()
+REPORT_EVENT_LIMIT = 800
+REPORT_TEXT_LIMIT = 4000
+REPORT_ZIP_PREFIX = "civitai_to_hf_report"
 
 
 def canonicalize_civitai_netloc(netloc: str):
@@ -394,6 +404,187 @@ def log_line(prefix: str, message: str):
     tag = str(prefix or "info").strip() or "info"
     print(f"[{tag}] {message}")
 
+def utc_timestamp():
+    try:
+        return datetime.now(timezone.utc).isoformat(timespec="seconds")
+    except Exception:
+        return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+
+def redact_report_value(value):
+    if isinstance(value, dict):
+        return {str(k): redact_report_value(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple, set)):
+        return [redact_report_value(v) for v in value]
+    text = str(value if value is not None else "")
+    if not text:
+        return ""
+    text = re.sub(r"hf_[A-Za-z0-9]{20,}", "[redacted-hf-token]", text)
+    text = re.sub(r"(?i)(token=)[^\s&]+", r"\1[redacted-token]", text)
+    text = re.sub(r"(?i)(Authorization:\s*Bearer\s+)[^\s]+", r"\1[redacted-token]", text)
+    text = re.sub(r"(?i)(xet-read-token/)[^\s/?#]+", r"\1[redacted-xet-token]", text)
+    text = re.sub(r"(?i)(X-Amz-Signature=)[^\s&]+", r"\1[redacted-signature]", text)
+    text = re.sub(r"(?i)(X-Amz-Credential=)[^\s&]+", r"\1[redacted-credential]", text)
+    text = re.sub(r"(?i)(Key-Pair-Id=)[^\s&]+", r"\1[redacted-key-pair]", text)
+    if len(text) > REPORT_TEXT_LIMIT:
+        return text[:REPORT_TEXT_LIMIT] + "...[truncated]"
+    return text
+
+
+def append_report_event(session_state, event: str, **fields):
+    state = ensure_session_state(session_state)
+    events = list(state.get("report_events") or [])
+    session_events = list(state.get("session_report_events") or [])
+    explicit_run_id = fields.pop("run_id", "") if "run_id" in fields else ""
+    run_id = str(explicit_run_id or state.get("active_run_id") or state.get("last_run_id") or "")
+    clean = {"ts": utc_timestamp(), "event": str(event or "event")}
+    if run_id:
+        clean["run_id"] = run_id
+    for key, value in fields.items():
+        if key in {"hf_token", "civitai_key", "authorization", "cookie"}:
+            clean[str(key)] = "[redacted]" if value else ""
+        else:
+            clean[str(key)] = redact_report_value(value)
+    events.append(clean)
+    session_events.append(clean)
+    if len(events) > REPORT_EVENT_LIMIT:
+        events = events[-REPORT_EVENT_LIMIT:]
+    session_event_limit = max(REPORT_EVENT_LIMIT * 4, REPORT_EVENT_LIMIT)
+    if len(session_events) > session_event_limit:
+        session_events = session_events[-session_event_limit:]
+    session_state_update(state, report_events=events, session_report_events=session_events)
+    return state
+
+
+def report_write_text(zipf, name: str, text: str):
+    zipf.writestr(name, redact_report_value(text))
+
+
+def safe_json_dumps(value):
+    return json.dumps(redact_report_value(value), ensure_ascii=False, indent=2, sort_keys=True)
+
+
+def list_report_events_for_run(events, run_id: str):
+    target = str(run_id or "")
+    if not target:
+        return list(events or [])
+    return [ev for ev in list(events or []) if str(ev.get("run_id") or "") == target]
+
+
+def summarize_report_runs(run_records):
+    records = list(run_records or [])
+    summary = {
+        "runs": len(records),
+        "done": 0,
+        "incomplete": 0,
+        "failed": 0,
+        "cancelled": 0,
+        "input_urls": 0,
+        "downloaded": 0,
+        "uploaded": 0,
+        "skipped_duplicate": 0,
+        "failed_download": 0,
+        "failed_upload": 0,
+        "verified_after_error": 0,
+        "remaining": 0,
+        "failed_urls": 0,
+    }
+    for record in records:
+        run_summary = record.get("summary") if isinstance(record, dict) else {}
+        if not isinstance(run_summary, dict):
+            run_summary = {}
+        stage = str(run_summary.get("stage") or record.get("stage") or "").lower()
+        if stage == "done":
+            summary["done"] += 1
+        elif stage == "incomplete":
+            summary["incomplete"] += 1
+        elif stage == "failed":
+            summary["failed"] += 1
+        elif stage == "cancelled":
+            summary["cancelled"] += 1
+        for key in ("input_urls", "downloaded", "uploaded", "skipped_duplicate", "failed_download", "failed_upload", "verified_after_error", "remaining"):
+            try:
+                summary[key] += int(run_summary.get(key, 0) or 0)
+            except Exception:
+                pass
+        try:
+            summary["failed_urls"] += int(run_summary.get("failed", 0) or 0)
+        except Exception:
+            pass
+    return summary
+
+
+def build_report_run_record(run_id: str, summary: dict, events, remaining, failed, uploaded, smoke_lines=None, failure_reasons=None):
+    clean_summary = redact_report_value(dict(summary or {}))
+    clean_events = redact_report_value(list(events or []))
+    clean_remaining = redact_report_value(list(remaining or []))
+    clean_failed = redact_report_value(list(failed or []))
+    clean_uploaded = redact_report_value(list(uploaded or []))
+    record = {
+        "run_id": str(run_id or clean_summary.get("run_id") or ""),
+        "mode": clean_summary.get("mode") or "",
+        "stage": clean_summary.get("stage") or "",
+        "repo_id": clean_summary.get("repo_id") or "",
+        "repo_type": clean_summary.get("repo_type") or "",
+        "summary": clean_summary,
+        "events": clean_events,
+        "remaining_urls": clean_remaining,
+        "failed_urls": clean_failed,
+        "uploaded_urls": clean_uploaded,
+        "smoke_lines": redact_report_value(list(smoke_lines or [])),
+        "failure_reasons": redact_report_value(dict(failure_reasons or {})),
+        "advice": build_report_advice(clean_summary),
+    }
+    return record
+
+
+def append_session_run_record(session_state, record, limit: int = 20):
+    state = ensure_session_state(session_state)
+    records = list(state.get("session_run_records") or [])
+    run_id = str(record.get("run_id") or "")
+    if run_id:
+        records = [r for r in records if str(r.get("run_id") or "") != run_id]
+    records.append(redact_report_value(record))
+    if len(records) > limit:
+        records = records[-limit:]
+    session_state_update(state, session_run_records=records, last_run_record=record)
+    return records
+
+
+def get_package_version(package_name: str):
+    try:
+        from importlib import metadata
+        return metadata.version(package_name)
+    except Exception:
+        return "unknown"
+
+
+def build_report_advice(summary: dict):
+    summary = summary if isinstance(summary, dict) else {}
+    failed_upload = int(summary.get("failed_upload", 0) or 0)
+    failed_download = int(summary.get("failed_download", 0) or 0)
+    remaining = int(summary.get("remaining", 0) or 0)
+    skipped = int(summary.get("skipped_duplicate", 0) or 0)
+    verified = int(summary.get("verified_after_error", 0) or 0)
+    lines = ["# What to try next", ""]
+    if remaining:
+        lines.append(f"- {remaining} URL(s) remained unprocessed. Use the remaining URL list and rerun. This usually means interruption/cancel/timeout before all URLs were processed.")
+    if failed_upload:
+        lines.append(f"- {failed_upload} item(s) failed after Civitai download. This points to HF upload/LFS/commit side. Retry failed only; if repeated, wait a few minutes or use the Patient HF upload retry policy.")
+    if failed_download:
+        lines.append(f"- {failed_download} item(s) failed before upload. Retry later; if repeated, check Civitai visibility/login/API-key status for those specific items.")
+    if skipped:
+        lines.append(f"- {skipped} item(s) were skipped as duplicates by SHA256. This is expected and usually does not need retry.")
+    if verified:
+        lines.append(f"- {verified} upload error(s) were recovered because the remote file existed after the API error. This suggests HF commit/LFS returned an error after partial success.")
+    if not any([remaining, failed_upload, failed_download]):
+        lines.append("- No retry is needed based on the recorded summary.")
+    lines.append("")
+    lines.append("# Notes")
+    lines.append("- Tokens, signed URLs, Authorization headers, and transient redirect tokens are redacted.")
+    lines.append("- This report contains structured Space-side state, not raw container stdout.")
+    return "\n".join(lines) + "\n"
+
 def summarize_failure_text(message: str, url: str=""):
     text = str(message or "").strip().replace("\n", " ")
     text = re.sub(r"\s+", " ", text)
@@ -663,6 +854,74 @@ def stage_detail(label: str, enabled: bool):
     return "enabled" if enabled else f"skipped ({label} off)"
 
 
+def new_run_stats(total_urls: int):
+    return {
+        "input_urls": int(total_urls or 0),
+        "downloaded": 0,
+        "uploaded": 0,
+        "skipped_duplicate": 0,
+        "failed_download": 0,
+        "failed_upload": 0,
+        "failed_info": 0,
+        "verified_after_error": 0,
+    }
+
+
+def increment_run_stat(stats: dict, key: str, amount: int=1):
+    if isinstance(stats, dict):
+        stats[key] = int(stats.get(key, 0) or 0) + int(amount)
+    return stats
+
+
+def build_run_summary_lines(stats: dict, remain_urls, failed_urls, final_stage: str):
+    stats = stats if isinstance(stats, dict) else {}
+    remain_count = len(remain_urls or [])
+    failed_count = len(failed_urls or [])
+    lines = [
+        "",
+        "### Run summary",
+        f"- Status: **{final_stage}**",
+        f"- Input URLs: {int(stats.get('input_urls', 0) or 0)}",
+        f"- Downloaded: {int(stats.get('downloaded', 0) or 0)}",
+        f"- Uploaded: {int(stats.get('uploaded', 0) or 0)}",
+        f"- Skipped duplicate: {int(stats.get('skipped_duplicate', 0) or 0)}",
+        f"- Failed download: {int(stats.get('failed_download', 0) or 0)}",
+        f"- Failed upload: {int(stats.get('failed_upload', 0) or 0)}",
+        f"- Remaining: {remain_count}",
+        f"- Failed URL list: {failed_count}",
+    ]
+    verified_after_error = int(stats.get('verified_after_error', 0) or 0)
+    if verified_after_error:
+        lines.append(f"- Upload verified after API error: {verified_after_error}")
+    lines.append("")
+    lines.append("### What to try next")
+    if remain_count:
+        lines.append("- Some URLs are still remaining. Use **Use Remaining URLs** and run again; this usually means the run was interrupted or stopped before all URLs were processed.")
+    if int(stats.get('failed_upload', 0) or 0):
+        lines.append("- Some Civitai downloads completed but HF upload failed. Use **Retry Failed Only**; if it repeats, wait a few minutes or switch HF upload retry policy to **Patient**.")
+    if int(stats.get('failed_download', 0) or 0):
+        lines.append("- Some Civitai downloads failed or produced no file. Retry them later; if repeated, check Civitai visibility/login/API-key settings for those files.")
+    if int(stats.get('skipped_duplicate', 0) or 0):
+        lines.append("- Some files were skipped because matching SHA256 already exists in the target repo. This is expected and does not require retry.")
+    if not remain_count and not failed_count and not int(stats.get('failed_upload', 0) or 0) and not int(stats.get('failed_download', 0) or 0):
+        lines.append("- No retry is needed.")
+    return lines
+
+
+def log_run_summary(run_mode: str, final_stage: str, stats: dict, remain_urls, failed_urls):
+    stats = stats if isinstance(stats, dict) else {}
+    log_line(
+        "cleanup",
+        "summary "
+        f"mode={run_mode} stage={final_stage.lower()} input={int(stats.get('input_urls', 0) or 0)} "
+        f"downloaded={int(stats.get('downloaded', 0) or 0)} uploaded={int(stats.get('uploaded', 0) or 0)} "
+        f"skipped_duplicate={int(stats.get('skipped_duplicate', 0) or 0)} "
+        f"failed_download={int(stats.get('failed_download', 0) or 0)} "
+        f"failed_upload={int(stats.get('failed_upload', 0) or 0)} "
+        f"remaining={len(remain_urls or [])} failed={len(failed_urls or [])}"
+    )
+
+
 def verify_repo_upload(repo_id: str, repo_type: str, filename: str, api: HfApi | None = None, hf_token=None):
     if hf_token is None: hf_token = get_token()
     if api is None: api = HfApi(token=hf_token)
@@ -710,18 +969,31 @@ def civitai_get(session, url: str, *, api_key: str="", params=None, timeout=(7.0
         r.close()
     return last_response
 
-def upload_safetensors_to_repo(filename, repo_id, repo_type, is_private, repo_ready=False, api: HfApi | None = None, hf_token=None, progress=gr.Progress(track_tqdm=False)):
+def upload_safetensors_to_repo(filename, repo_id, repo_type, is_private, repo_ready=False, api: HfApi | None = None, hf_token=None, progress=gr.Progress(track_tqdm=False), hf_retry_policy="Auto"):
     output_filename = Path(filename).name
     if hf_token is None: hf_token = get_token()
     if api is None: api = HfApi(token=hf_token)
+    policy_config = get_hf_upload_retry_policy_config(hf_retry_policy)
     try:
         if not repo_ready and not is_repo_exists(repo_id, repo_type): ensure_repo(api, repo_id=repo_id, repo_type=repo_type, is_private=is_private, hf_token=hf_token)
         progress(0, desc=f"Start uploading... {filename} to {repo_id}")
         with suppress_hf_hub_progress_bars():
-            retry_call(lambda: api.upload_file(path_or_fileobj=filename, path_in_repo=output_filename, repo_type=repo_type, revision="main", token=hf_token, repo_id=repo_id), action=f'upload_file {repo_id}:{output_filename}')
+            hf_upload_retry_call(lambda: api.upload_file(path_or_fileobj=filename, path_in_repo=output_filename, repo_type=repo_type, revision="main", token=hf_token, repo_id=repo_id), policy=hf_retry_policy, action=f'upload_file {repo_id}:{output_filename}')
+        post_sleep = float(policy_config.get("post_upload_sleep", 0.0) or 0.0)
+        if post_sleep > 0:
+            time.sleep(post_sleep)
         progress(1, desc="Uploaded.")
         url = hf_hub_url(repo_id=repo_id, repo_type=repo_type, filename=output_filename)
     except Exception as e:
+        verified_after_error = False
+        try:
+            verified_after_error = verify_repo_upload(repo_id, repo_type, output_filename, api=api, hf_token=hf_token)
+        except Exception:
+            verified_after_error = False
+        if verified_after_error:
+            log_line("retry", f"upload error but remote file exists: {repo_id}:{output_filename}")
+            progress(1, desc="Uploaded.")
+            return hf_hub_url(repo_id=repo_id, repo_type=repo_type, filename=output_filename)
         print(f"Error: Failed to upload to {repo_id}. {e}")
         gr.Warning(f"Error: Failed to upload to {repo_id}. {e}")
         return None
@@ -1047,7 +1319,7 @@ def pick_smoke_test_civitai_item(api_key: str = "", progress=gr.Progress(track_t
     raise gr.Error(f"Smoke test candidate not found within {round(SMOKE_TEST_MAX_SIZE_KB / 1000.0, 2)}MB.")
 
 
-def smoke_test_civitai(civitai_key, hf_token, urls, repo_type="model", is_private=True, is_info=False, is_rename=True, session_state=None, progress=gr.Progress(track_tqdm=False)):
+def smoke_test_civitai(civitai_key, hf_token, urls, repo_type="model", is_private=True, is_info=False, is_rename=True, session_state=None, hf_retry_policy="Auto", progress=gr.Progress(track_tqdm=False)):
     session_state = prepare_new_run_state(session_state)
     reset_civitai_key_status(civitai_key, source="smoke")
     repo_id = str(os.environ.get("HF_REPO", "") or "").strip()
@@ -1077,7 +1349,7 @@ def smoke_test_civitai(civitai_key, hf_token, urls, repo_type="model", is_privat
         gr.Info(f"Smoke Test target: {selected.get('name', 'LoRA')} / {round(float(selected.get('size_kb', 0.0)) / 1000.0, 2)}MB")
         print(f"SMOKE TEST: repo={repo_id} type={repo_type} url={selected_url}")
         run_context = {"mode": "smoke", "smoke_lines": smoke_lines, "selected": selected, "resolved_host": resolved_host}
-        yield from download_civitai(selected_url, api_key, resolved_hf_token, urls, repo_id, repo_type, is_private, is_info, is_rename, session_state=session_state, run_context=run_context, progress=progress)
+        yield from download_civitai(selected_url, api_key, resolved_hf_token, urls, repo_id, repo_type, is_private, is_info, is_rename, session_state=session_state, hf_retry_policy=hf_retry_policy, run_context=run_context, progress=progress)
         return
     except Exception as e:
         detail = f"{type(e).__name__}: {e}"
@@ -1091,7 +1363,7 @@ def smoke_test_civitai(civitai_key, hf_token, urls, repo_type="model", is_privat
 
 
 def download_civitai(dl_url, civitai_key, hf_token, urls,
-                     newrepo_id, repo_type="model", is_private=True, is_info=False, is_rename=True, session_state=None, run_context=None, progress=gr.Progress(track_tqdm=False)):
+                     newrepo_id, repo_type="model", is_private=True, is_info=False, is_rename=True, session_state=None, hf_retry_policy="Auto", run_context=None, progress=gr.Progress(track_tqdm=False)):
     session_state = prepare_new_run_state(session_state)
     run_context = run_context if isinstance(run_context, dict) else {}
     run_mode = run_context.get("mode", "manual")
@@ -1113,6 +1385,9 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
     dl_urls = normalize_url_entries(dl_url)
     remain_urls = dl_urls.copy()
     failed_urls = []
+    failure_reasons = {}
+    run_stats = new_run_stats(len(dl_urls))
+    hf_retry_config = get_hf_upload_retry_policy_config(hf_retry_policy)
     result_lines = []
     error_message = ""
     cancelled = False
@@ -1124,8 +1399,9 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
     repo_header = ""
     hashes = set()
     api = None
-    session_state_update(session_state, current_run_mode=run_mode, current_repo_id=newrepo_id, current_repo_type=repo_type, current_run_temp_dir=run_temp_dir, current_remaining_urls=remain_urls.copy(), current_failed_urls=failed_urls.copy(), current_uploaded_urls=urls.copy(), current_smoke_lines=smoke_lines.copy(), active_run_id=run_id, cancel_requested=False, last_error="", last_failure_summary="", run_started_at=time.time(), run_elapsed_sec=0.0)
-    log_line("info", f"starting {run_mode} run target={newrepo_id} type={repo_type} urls={len(dl_urls)} repos={len(normalize_repo_entries(dl_url))} info={'on' if is_info else 'off'} rename={'on' if is_rename else 'off'}")
+    session_state_update(session_state, current_run_mode=run_mode, current_run_id=run_id, current_repo_id=newrepo_id, current_repo_type=repo_type, current_run_temp_dir=run_temp_dir, current_remaining_urls=remain_urls.copy(), current_failed_urls=failed_urls.copy(), current_uploaded_urls=urls.copy(), current_smoke_lines=smoke_lines.copy(), current_hf_retry_policy=hf_retry_config.get('key', 'auto'), report_events=[], active_run_id=run_id, cancel_requested=False, last_error="", last_failure_summary="", run_started_at=time.time(), run_elapsed_sec=0.0)
+    append_report_event(session_state, "run_started", run_id=run_id, mode=run_mode, target=newrepo_id, repo_type=repo_type, input_urls=len(dl_urls), repo_inputs=len(normalize_repo_entries(dl_url)), info=is_info, rename=is_rename, hf_retry=hf_retry_config.get('key', 'auto'))
+    log_line("info", f"starting {run_mode} run target={newrepo_id} type={repo_type} urls={len(dl_urls)} repos={len(normalize_repo_entries(dl_url))} info={'on' if is_info else 'off'} rename={'on' if is_rename else 'off'} hf_retry={hf_retry_config.get('key', 'auto')}")
     update_run_stage(session_state, "Preparing", f"target {newrepo_id}")
     try:
         set_stage_progress(progress, 0, max(len(dl_urls), 1), f"Preparing target {newrepo_id}...")
@@ -1165,6 +1441,8 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
                 set_stage_progress(progress, index - 1, total_urls, f"Processing {index}/{total_urls}")
                 civitai_sha256 = get_civitai_sha256(u, civitai_key) if repo_type != "bucket" else None
                 if repo_type != "bucket" and civitai_sha256 and civitai_sha256 in hashes:
+                    increment_run_stat(run_stats, "skipped_duplicate")
+                    append_report_event(session_state, "skipped_duplicate", url=u, index=index, total=total_urls, sha256=bool(civitai_sha256))
                     log_line("retry", f"skip duplicate in target repo: {sanitize_url_for_log(u)}")
                     if run_mode == "smoke":
                         smoke_lines.append(smoke_stage_line("Duplicate/skip", "ok", "same SHA256 already exists in target repo"))
@@ -1178,11 +1456,16 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
                 check_run_cancel(run_id, session_state=session_state)
                 update_run_stage(session_state, "Downloading", "fetching from Civitai", index=index, total=total_urls, current_url=u)
                 set_stage_progress(progress, index - 1, total_urls, f"Downloading {index}/{total_urls}")
+                append_report_event(session_state, "download_started", url=u, index=index, total=total_urls)
                 current_file = download_file(u, civitai_key, temp_dir=run_temp_dir, progress=progress)
                 file_ok, file_detail = summarize_downloaded_file(current_file)
+                append_report_event(session_state, "download_completed" if file_ok else "download_failed", url=u, index=index, total=total_urls, detail=file_detail, file=Path(str(current_file)).name if current_file else "")
                 if run_mode == "smoke":
                     smoke_lines.append(smoke_stage_line("Download verify", "ok" if file_ok else "fail", file_detail))
                 if not file_ok:
+                    increment_run_stat(run_stats, "failed_download")
+                    append_report_event(session_state, "download_failed", url=u, index=index, total=total_urls, detail=file_detail)
+                    failure_reasons[u] = "download"
                     if u not in failed_urls:
                         failed_urls.append(u)
                     set_last_failure_summary(session_state, "download failed or file missing", url=u)
@@ -1192,6 +1475,7 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
                     session_state_update(session_state, current_remaining_urls=remain_urls.copy(), current_failed_urls=failed_urls.copy(), current_uploaded_urls=urls.copy(), current_smoke_lines=smoke_lines.copy())
                     yield build_run_outputs(urls, md, remain_urls, failed_urls, civitai_key, session_state)
                     continue
+                increment_run_stat(run_stats, "downloaded")
                 check_run_cancel(run_id, session_state=session_state)
                 if is_rename:
                     update_run_stage(session_state, "Renaming", "checking target name", index=index, total=total_urls, current_url=u)
@@ -1202,7 +1486,8 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
                 uploaded_name = Path(current_file).name
                 update_run_stage(session_state, "Uploading", uploaded_name, index=index, total=total_urls, current_url=u)
                 set_stage_progress(progress, index - 1, total_urls, f"Uploading {index}/{total_urls}")
-                url = upload_safetensors_to_bucket(current_file, newrepo_id, bucket_ready=bucket_ready, progress=progress) if repo_type == "bucket" else upload_safetensors_to_repo(current_file, newrepo_id, repo_type, is_private, repo_ready=repo_ready, api=api, hf_token=hf_token_value, progress=progress)
+                append_report_event(session_state, "upload_started", url=u, filename=uploaded_name, index=index, total=total_urls, repo_id=newrepo_id, repo_type=repo_type, hf_retry=hf_retry_config.get('key', 'auto'))
+                url = upload_safetensors_to_bucket(current_file, newrepo_id, bucket_ready=bucket_ready, progress=progress) if repo_type == "bucket" else upload_safetensors_to_repo(current_file, newrepo_id, repo_type, is_private, repo_ready=repo_ready, api=api, hf_token=hf_token_value, progress=progress, hf_retry_policy=hf_retry_policy)
                 if url:
                     upload_verified = True
                     upload_detail = f"{uploaded_name} -> {newrepo_id}"
@@ -1211,6 +1496,19 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
                         upload_detail = uploaded_name
                     if run_mode == "smoke":
                         smoke_lines.append(smoke_stage_line("Upload verify", "ok" if upload_verified else "fail", upload_detail))
+                    if repo_type != "bucket" and not upload_verified:
+                        increment_run_stat(run_stats, "failed_upload")
+                        append_report_event(session_state, "upload_verify_failed", url=u, filename=uploaded_name, index=index, total=total_urls, repo_id=newrepo_id, repo_type=repo_type)
+                        failure_reasons[u] = "upload verify"
+                        if u not in failed_urls:
+                            failed_urls.append(u)
+                        result_lines.append(f"- Failed [{str(u)}]({str(u)}) (upload verify)")
+                        md = build_run_markdown(repo_header, result_lines, smoke_lines)
+                        session_state_update(session_state, current_remaining_urls=remain_urls.copy(), current_failed_urls=failed_urls.copy(), current_uploaded_urls=urls.copy(), current_smoke_lines=smoke_lines.copy())
+                        yield build_run_outputs(urls, md, remain_urls, failed_urls, civitai_key, session_state)
+                        continue
+                    increment_run_stat(run_stats, "uploaded")
+                    append_report_event(session_state, "upload_completed", url=u, filename=uploaded_name, index=index, total=total_urls, repo_id=newrepo_id, repo_type=repo_type, upload_url=url)
                     if civitai_sha256:
                         hashes.add(civitai_sha256)
                         if repo_type != "bucket":
@@ -1246,6 +1544,9 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
                     if u in failed_urls:
                         failed_urls.remove(u)
                 else:
+                    increment_run_stat(run_stats, "failed_upload")
+                    append_report_event(session_state, "upload_failed", url=u, filename=uploaded_name, index=index, total=total_urls, repo_id=newrepo_id, repo_type=repo_type)
+                    failure_reasons[u] = "upload"
                     if u not in failed_urls:
                         failed_urls.append(u)
                     if run_mode == "smoke":
@@ -1260,6 +1561,13 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
                 break
             except Exception as e:
                 log_line("fail", f"error while processing {sanitize_url_for_log(u)}: {type(e).__name__}: {e}")
+                failure_reasons[u] = f"exception:{type(e).__name__}"
+                if current_file and Path(str(current_file)).exists():
+                    increment_run_stat(run_stats, "failed_upload")
+                    append_report_event(session_state, "item_failed", url=u, phase="upload_or_post_download", error_type=type(e).__name__, error=str(e))
+                else:
+                    increment_run_stat(run_stats, "failed_download")
+                    append_report_event(session_state, "item_failed", url=u, phase="download_or_pre_download", error_type=type(e).__name__, error=str(e))
                 set_last_failure_summary(session_state, f"{type(e).__name__}: {e}", url=u)
                 if u not in failed_urls:
                     failed_urls.append(u)
@@ -1302,7 +1610,8 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
             store_session_repo_hash_cache(session_state, newrepo_id, repo_type, hashes)
         cleanup_run_temp_dir(run_temp_dir)
         unregister_run(run_id)
-        final_stage = "Cancelled" if cancelled else ("Failed" if error_message else "Done")
+        incomplete = bool(remain_urls) or bool(failed_urls)
+        final_stage = "Cancelled" if cancelled else ("Failed" if error_message else ("Incomplete" if incomplete else "Done"))
         final_detail = f"remaining={len(remain_urls)} failed={len(failed_urls)}"
         update_run_stage(session_state, final_stage, final_detail, index=len(dl_urls), total=max(len(dl_urls), 1))
         if run_mode == "smoke":
@@ -1312,15 +1621,26 @@ def download_civitai(dl_url, civitai_key, hf_token, urls,
             set_last_failure_summary(session_state, "cancelled by user")
         elif error_message and not str(session_state.get("last_failure_summary") or "").strip():
             set_last_failure_summary(session_state, error_message)
+        if final_stage == "Incomplete":
+            log_line("cleanup", f"run incomplete remaining={len(remain_urls)} failed={len(failed_urls)} reasons={json.dumps(failure_reasons, ensure_ascii=False)[:1000]}")
         log_line("cleanup", f"finished {run_mode} run stage={final_stage.lower()} remaining={len(remain_urls)} failed={len(failed_urls)}")
-        session_state_update(session_state, current_run_mode="idle", current_repo_id=newrepo_id, current_repo_type=repo_type, current_run_temp_dir="", current_remaining_urls=remain_urls.copy(), current_failed_urls=failed_urls.copy(), current_uploaded_urls=urls.copy(), current_smoke_lines=smoke_lines.copy(), last_run_mode=run_mode, last_repo_id=newrepo_id, last_repo_type=repo_type, last_remaining_urls=remain_urls.copy(), last_failed_urls=failed_urls.copy(), last_uploaded_urls=urls.copy(), last_smoke_lines=smoke_lines.copy(), last_error=error_message, active_run_id="", cancel_requested=False)
+        log_run_summary(run_mode, final_stage, run_stats, remain_urls, failed_urls)
+        last_run_summary = dict(run_stats)
+        last_run_summary.update({"run_id": run_id, "stage": final_stage, "mode": run_mode, "repo_id": newrepo_id, "repo_type": repo_type, "remaining": len(remain_urls), "failed": len(failed_urls), "cancelled": bool(cancelled), "error": bool(error_message), "hf_retry_policy": hf_retry_config.get('key', 'auto')})
+        append_report_event(session_state, "run_finished", **last_run_summary)
+        current_events = list(session_state.get("report_events") or [])
+        last_run_record = build_report_run_record(run_id, last_run_summary, current_events, remain_urls, failed_urls, urls, smoke_lines=smoke_lines, failure_reasons=failure_reasons)
+        session_records = append_session_run_record(session_state, last_run_record)
+        session_summary = summarize_report_runs(session_records)
+        session_state_update(session_state, current_run_mode="idle", current_run_id=run_id, current_repo_id=newrepo_id, current_repo_type=repo_type, current_run_temp_dir="", current_remaining_urls=remain_urls.copy(), current_failed_urls=failed_urls.copy(), current_uploaded_urls=urls.copy(), current_smoke_lines=smoke_lines.copy(), last_run_id=run_id, last_run_mode=run_mode, last_repo_id=newrepo_id, last_repo_type=repo_type, last_remaining_urls=remain_urls.copy(), last_failed_urls=failed_urls.copy(), last_uploaded_urls=urls.copy(), last_smoke_lines=smoke_lines.copy(), last_run_summary=last_run_summary, last_run_record=last_run_record, session_summary=session_summary, last_failure_reasons=dict(failure_reasons), last_hf_retry_policy=hf_retry_config.get('key', 'auto'), last_error=error_message, active_run_id="", cancel_requested=False)
         gc.collect()
-    md = build_run_markdown(repo_header if repo_header else "", result_lines, smoke_lines)
+    summary_lines = build_run_summary_lines(run_stats, remain_urls, failed_urls, final_stage)
+    md = build_run_markdown(repo_header if repo_header else "", result_lines + summary_lines, smoke_lines)
     if cancelled:
-        md = build_run_markdown(repo_header if repo_header else "", result_lines + ["- Cancelled by user."], smoke_lines)
+        md = build_run_markdown(repo_header if repo_header else "", result_lines + ["- Cancelled by user."] + summary_lines, smoke_lines)
     elif error_message and not result_lines:
-        md = build_run_markdown(repo_header if repo_header else "", [f"- Failed ({error_message})"], smoke_lines)
-    set_stage_progress(progress, 1, 1, "Cancelled" if cancelled else "Done")
+        md = build_run_markdown(repo_header if repo_header else "", [f"- Failed ({error_message})"] + summary_lines, smoke_lines)
+    set_stage_progress(progress, 1, 1, final_stage)
     yield build_run_outputs(urls, md, remain_urls, failed_urls, civitai_key, session_state, remain_visible=bool(remain_urls) or bool(error_message) or bool(cancelled), failed_visible=bool(failed_urls))
 
 
@@ -1812,15 +2132,26 @@ def search_on_civitai(query: str, types: list[str], allow_model: list[str] = [],
         print(e)
     items = []
     origin = get_civitai_display_origin().rstrip('/')
+    api_item_count = 0
+    version_count = 0
+    file_count = 0
+    filtered_version_count = 0
+    filtered_file_count = 0
+    has_next_page = False
     for r in rs:
         if not r.ok:
             continue
         json = get_civitai_response_json(r, default={}) or {}
+        if isinstance(json.get('metadata', {}), dict) and json.get('metadata', {}).get('nextPage'):
+            has_next_page = True
         if 'items' not in json:
             continue
+        api_item_count += len(json.get('items') or [])
         for j in json['items']:
             for model in j.get('modelVersions', []):
+                version_count += 1
                 if len(allow_model) != 0 and model.get('baseModel', '') not in set(allow_model):
+                    filtered_version_count += 1
                     continue
                 base_item = {
                     'name': j.get('name', ''),
@@ -1851,12 +2182,14 @@ def search_on_civitai(query: str, types: list[str], allow_model: list[str] = [],
                 )
                 files = model.get('files', []) if isinstance(model.get('files', []), list) else []
                 if files:
+                    file_count += len(files)
                     for f in files:
                         item = base_item.copy()
                         item['dl_url'] = normalize_civitai_download_api_url(f.get('downloadUrl', ''))
                         item['size_kb'] = f.get('sizeKB', 0.0)
                         item['file_type'] = f.get('type', '')
                         if len(filetype) != 0 and f.get('type', '') not in set(filetype):
+                            filtered_file_count += 1
                             continue
                         items.append(item)
                 else:
@@ -1868,6 +2201,7 @@ def search_on_civitai(query: str, types: list[str], allow_model: list[str] = [],
             items = sorted(items, key=lambda x: x.get('size_kb', 0.0), reverse=True)
         elif sort == "Size (from smallest)":
             items = sorted(items, key=lambda x: x.get('size_kb', 0.0))
+    log_line("search", f"summary responses={len(rs)} api_items={api_item_count} versions={version_count} files={file_count} filtered_versions={filtered_version_count} filtered_files={filtered_file_count} selected_files={len(items)} page_mode={'all' if page == 0 else 'single'} has_next_page={str(has_next_page).lower()}")
     return items if len(items) > 0 else None
 
 
@@ -2183,6 +2517,231 @@ def select_civitai_all_item_fast(button_name: str, api_key: str, state: dict):
     return render_civitai_state(api_key, state, build_missing=False)
 
 
+class _FakeResponse:
+    def __init__(self, headers=None):
+        self.headers = headers or {}
+
+
+class _FakeHFError(Exception):
+    def __init__(self, message, headers=None):
+        super().__init__(message)
+        self.response = _FakeResponse(headers=headers)
+
+
+class _FakeUploadApi:
+    def __init__(self, exists_after_error=False):
+        self.exists_after_error = bool(exists_after_error)
+        self.upload_calls = 0
+        self.exists_calls = 0
+
+    def upload_file(self, **kwargs):
+        self.upload_calls += 1
+        raise _FakeHFError("Bad request for commit endpoint: Unexpected internal error hook: lfs-verify")
+
+    def file_exists(self, **kwargs):
+        self.exists_calls += 1
+        return self.exists_after_error
+
+
+def safe_retry_probe(hf_retry_policy="Auto"):
+    config = get_hf_upload_retry_policy_config(hf_retry_policy)
+    cases = [
+        ("429 with Retry-After", _FakeHFError("HTTP Error 429 Too Many Requests", {"Retry-After": "3", "RateLimit": "api|r=0;t=183"})),
+        ("503 LFS batch", _FakeHFError("HTTP Error 503 while requesting POST /info/lfs/objects/batch")),
+        ("lfs-verify hook", _FakeHFError("Bad request for commit endpoint: Unexpected internal error hook: lfs-verify")),
+        ("403 permission", _FakeHFError("403 Forbidden: permission denied")),
+        ("repo not found", _FakeHFError("Repo not found")),
+    ]
+    lines = ["### Safe Retry Probe", "- network: none", f"- policy: {config.get('key')} attempts={config.get('attempts')} base_wait={config.get('base_wait')} max_wait={config.get('max_wait')}"]
+    for label, exc in cases:
+        retryable = is_retryable_hf_upload_exception(exc)
+        delay, delay_source = parse_hf_retry_delay_from_headers(exc)
+        hint = format_hf_rate_limit_hint(exc)
+        parts = [f"- {label}: {'retryable' if retryable else 'not retryable'}"]
+        if delay is not None:
+            parts.append(f"delay={delay:g}s source={delay_source}")
+        if hint:
+            parts.append(hint)
+        lines.append(" | ".join(parts))
+    return gr.update(value="\n".join(lines), visible=True)
+
+
+def safe_upload_verify_probe(hf_retry_policy="Auto"):
+    config = get_hf_upload_retry_policy_config(hf_retry_policy)
+    lines = ["### Safe Upload Verify Probe", "- network: none", "- upload_file failure is simulated", f"- policy: {config.get('key')} (no sleeps, no real retries)"]
+    for exists_after_error in (True, False):
+        fake_api = _FakeUploadApi(exists_after_error=exists_after_error)
+        try:
+            try:
+                fake_api.upload_file()
+            except Exception as e:
+                retryable = is_retryable_hf_upload_exception(e)
+                recovered = fake_api.file_exists(repo_id="user/repo", filename="file.safetensors", repo_type="model", token="[redacted]")
+                error_short = format_error_short(e)
+            else:
+                retryable = False
+                recovered = True
+                error_short = ""
+            state = "recovered" if recovered else "failed"
+            lines.append(f"- remote_exists_after_error={exists_after_error}: {state} retryable={retryable} upload_calls={fake_api.upload_calls} file_exists_calls={fake_api.exists_calls} error={error_short}")
+        except Exception as e:
+            lines.append(f"- remote_exists_after_error={exists_after_error}: probe error {type(e).__name__}: {format_error_short(e)}")
+    return gr.update(value="\n".join(lines), visible=True)
+
+
+def safe_summary_probe():
+    lines = ["### Safe Summary Probe", "- network: none"]
+    scenarios = [
+        ("clean", {"input_urls": 3, "downloaded": 3, "uploaded": 3, "skipped_duplicate": 0, "failed_download": 0, "failed_upload": 0, "verified_after_error": 0}, [], [], "Done"),
+        ("interrupted", {"input_urls": 5, "downloaded": 2, "uploaded": 2, "skipped_duplicate": 0, "failed_download": 0, "failed_upload": 0, "verified_after_error": 0}, ["https://civitai.com/api/download/models/1"], [], "Incomplete"),
+        ("hf upload failures", {"input_urls": 4, "downloaded": 4, "uploaded": 2, "skipped_duplicate": 0, "failed_download": 0, "failed_upload": 2, "verified_after_error": 1}, [], ["https://civitai.com/api/download/models/2"], "Incomplete"),
+        ("duplicates", {"input_urls": 4, "downloaded": 1, "uploaded": 1, "skipped_duplicate": 3, "failed_download": 0, "failed_upload": 0, "verified_after_error": 0}, [], [], "Done"),
+    ]
+    for name, stats, remaining, failed, stage in scenarios:
+        lines.append("")
+        lines.append(f"#### {name}")
+        lines.extend(build_run_summary_lines(stats, remaining, failed, stage))
+    return gr.update(value="\n".join(lines), visible=True)
+
+
+def create_report_zip(session_state=None, search_state=None):
+    state = ensure_session_state(session_state)
+    search_state = search_state if isinstance(search_state, dict) else {}
+    report_dir = Path(tempfile.mkdtemp(prefix="civitai_report_", dir=TEMP_DIR))
+    timestamp = time.strftime("%Y%m%d_%H%M%S", time.gmtime())
+    zip_path = report_dir / f"{REPORT_ZIP_PREFIX}_{timestamp}.zip"
+    session_events = list(state.get("session_report_events") or [])
+    current_events = list(state.get("report_events") or [])
+    run_records = list(state.get("session_run_records") or [])
+    last_record = state.get("last_run_record") if isinstance(state.get("last_run_record"), dict) else {}
+    if last_record and not any(str(r.get("run_id") or "") == str(last_record.get("run_id") or "") for r in run_records):
+        run_records.append(last_record)
+    remaining = list(state.get("last_remaining_urls") or state.get("current_remaining_urls") or [])
+    failed = list(state.get("last_failed_urls") or state.get("current_failed_urls") or [])
+    uploaded = list(state.get("last_uploaded_urls") or state.get("current_uploaded_urls") or [])
+    smoke_lines = list(state.get("last_smoke_lines") or state.get("current_smoke_lines") or [])
+    summary = dict(state.get("last_run_summary") or {})
+    if not summary:
+        summary = {
+            "stage": str(state.get("current_stage") or ""),
+            "repo_id": str(state.get("last_repo_id") or state.get("current_repo_id") or ""),
+            "repo_type": str(state.get("last_repo_type") or state.get("current_repo_type") or ""),
+            "remaining": len(remaining),
+            "failed": len(failed),
+            "uploaded": len(uploaded),
+        }
+    current_run_id = str(summary.get("run_id") or state.get("last_run_id") or state.get("current_run_id") or "")
+    if not last_record:
+        run_events = list_report_events_for_run(session_events or current_events, current_run_id)
+        last_record = build_report_run_record(current_run_id, summary, run_events or current_events, remaining, failed, uploaded, smoke_lines=smoke_lines, failure_reasons=state.get("last_failure_reasons") or {})
+    summary = redact_report_value(summary)
+    session_summary = summarize_report_runs(run_records)
+    if not run_records and last_record:
+        run_records = [last_record]
+        session_summary = summarize_report_runs(run_records)
+    search_summary = {
+        "last_choices": len(search_state.get("civitai_last_choices") or []),
+        "last_results": len(search_state.get("civitai_last_results") or {}),
+        "last_selects": len(search_state.get("civitai_last_selects") or []),
+        "last_items": len(search_state.get("civitai_last_items") or []),
+        "visible_count": int(search_state.get("civitai_visible_count") or 0),
+        "gallery_enabled": bool(search_state.get("civitai_gallery_enabled", True)),
+        "detail_url": search_state.get("civitai_detail_url") or "",
+        "preview_fail_urls": list(search_state.get("civitai_preview_fail_urls") or []),
+    }
+    selected_urls = []
+    for value in list(search_state.get("civitai_last_selects") or []):
+        if value:
+            selected_urls.append(str(value))
+    env = {
+        "created_at_utc": utc_timestamp(),
+        "python": sys.version.split()[0],
+        "platform": platform.platform(),
+        "gradio": get_package_version("gradio"),
+        "huggingface_hub": get_package_version("huggingface_hub"),
+        "requests": get_package_version("requests"),
+        "civitai_hf_debug": bool(os.environ.get("CIVITAI_HF_DEBUG")),
+        "hf_upload_retry_policy": str(state.get("last_hf_retry_policy") or state.get("current_hf_retry_policy") or ""),
+    }
+    run_table_lines = ["| run_id | mode | stage | input | uploaded | skipped | failed download | failed upload | remaining |", "|---|---:|---:|---:|---:|---:|---:|---:|---:|"]
+    for record in run_records:
+        run_summary = record.get("summary") if isinstance(record, dict) else {}
+        if not isinstance(run_summary, dict):
+            run_summary = {}
+        rid = str(record.get("run_id") or run_summary.get("run_id") or "")
+        run_table_lines.append(
+            f"| {redact_report_value(rid)} | {redact_report_value(record.get('mode') or run_summary.get('mode') or '')} | {redact_report_value(record.get('stage') or run_summary.get('stage') or '')} | {int(run_summary.get('input_urls', 0) or 0)} | {int(run_summary.get('uploaded', 0) or 0)} | {int(run_summary.get('skipped_duplicate', 0) or 0)} | {int(run_summary.get('failed_download', 0) or 0)} | {int(run_summary.get('failed_upload', 0) or 0)} | {int(run_summary.get('remaining', 0) or 0)} |"
+        )
+    report_md = [
+        "# Civitai to HF Diagnostic Report",
+        "",
+        "## Current Run Summary",
+        "```json",
+        safe_json_dumps(summary),
+        "```",
+        "",
+        build_report_advice(summary),
+        "",
+        "## Session Summary",
+        "```json",
+        safe_json_dumps(session_summary),
+        "```",
+        "",
+        "## Runs",
+        "",
+        *run_table_lines,
+    ]
+    if smoke_lines:
+        report_md.extend(["", "## Smoke / Probe Lines", ""])
+        report_md.extend([f"- {redact_report_value(line)}" for line in smoke_lines])
+    with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+        report_write_text(zf, "report.md", "\n".join(report_md).strip() + "\n")
+        report_write_text(zf, "current_run/summary.json", safe_json_dumps(summary) + "\n")
+        report_write_text(zf, "current_run/events.jsonl", "\n".join(json.dumps(redact_report_value(ev), ensure_ascii=False, sort_keys=True) for ev in (last_record.get("events") or current_events)) + ("\n" if (last_record.get("events") or current_events) else ""))
+        report_write_text(zf, "current_run/remaining_urls.txt", "\n".join(redact_report_value(u) for u in (last_record.get("remaining_urls") or remaining)) + ("\n" if (last_record.get("remaining_urls") or remaining) else ""))
+        report_write_text(zf, "current_run/failed_urls.txt", "\n".join(redact_report_value(u) for u in (last_record.get("failed_urls") or failed)) + ("\n" if (last_record.get("failed_urls") or failed) else ""))
+        report_write_text(zf, "current_run/uploaded_urls.txt", "\n".join(redact_report_value(u) for u in (last_record.get("uploaded_urls") or uploaded)) + ("\n" if (last_record.get("uploaded_urls") or uploaded) else ""))
+        report_write_text(zf, "current_run/advice.md", build_report_advice(summary))
+        report_write_text(zf, "session_summary.json", safe_json_dumps(session_summary) + "\n")
+        report_write_text(zf, "session_events.jsonl", "\n".join(json.dumps(redact_report_value(ev), ensure_ascii=False, sort_keys=True) for ev in session_events) + ("\n" if session_events else ""))
+        report_write_text(zf, "environment.json", safe_json_dumps(env) + "\n")
+        report_write_text(zf, "search_summary.json", safe_json_dumps(search_summary) + "\n")
+        report_write_text(zf, "selected_search_urls.txt", "\n".join(redact_report_value(u) for u in selected_urls) + ("\n" if selected_urls else ""))
+        # Legacy top-level files kept for quick manual inspection.
+        report_write_text(zf, "summary.json", safe_json_dumps(summary) + "\n")
+        report_write_text(zf, "events.jsonl", "\n".join(json.dumps(redact_report_value(ev), ensure_ascii=False, sort_keys=True) for ev in current_events) + ("\n" if current_events else ""))
+        report_write_text(zf, "remaining_urls.txt", "\n".join(redact_report_value(u) for u in remaining) + ("\n" if remaining else ""))
+        report_write_text(zf, "failed_urls.txt", "\n".join(redact_report_value(u) for u in failed) + ("\n" if failed else ""))
+        report_write_text(zf, "uploaded_urls.txt", "\n".join(redact_report_value(u) for u in uploaded) + ("\n" if uploaded else ""))
+        for record in run_records:
+            run_id = str(record.get("run_id") or "run") or "run"
+            safe_run_id = re.sub(r"[^A-Za-z0-9_.-]+", "_", run_id)[:80] or "run"
+            prefix = f"runs/{safe_run_id}"
+            run_summary = record.get("summary") if isinstance(record.get("summary"), dict) else {}
+            report_write_text(zf, f"{prefix}/summary.json", safe_json_dumps(run_summary) + "\n")
+            report_write_text(zf, f"{prefix}/events.jsonl", "\n".join(json.dumps(redact_report_value(ev), ensure_ascii=False, sort_keys=True) for ev in list(record.get("events") or [])) + ("\n" if record.get("events") else ""))
+            report_write_text(zf, f"{prefix}/remaining_urls.txt", "\n".join(redact_report_value(u) for u in list(record.get("remaining_urls") or [])) + ("\n" if record.get("remaining_urls") else ""))
+            report_write_text(zf, f"{prefix}/failed_urls.txt", "\n".join(redact_report_value(u) for u in list(record.get("failed_urls") or [])) + ("\n" if record.get("failed_urls") else ""))
+            report_write_text(zf, f"{prefix}/uploaded_urls.txt", "\n".join(redact_report_value(u) for u in list(record.get("uploaded_urls") or [])) + ("\n" if record.get("uploaded_urls") else ""))
+            report_write_text(zf, f"{prefix}/advice.md", str(record.get("advice") or build_report_advice(run_summary)))
+        curated_state = {
+            "last_run_id": state.get("last_run_id"),
+            "last_run_mode": state.get("last_run_mode"),
+            "last_repo_id": state.get("last_repo_id"),
+            "last_repo_type": state.get("last_repo_type"),
+            "last_error": state.get("last_error"),
+            "last_failure_summary": state.get("last_failure_summary"),
+            "last_run_summary": state.get("last_run_summary"),
+            "session_summary": session_summary,
+            "session_run_count": len(run_records),
+            "current_stage": state.get("current_stage"),
+            "current_stage_detail": state.get("current_stage_detail"),
+            "run_elapsed_sec": state.get("run_elapsed_sec"),
+        }
+        report_write_text(zf, "session_state_curated.json", safe_json_dumps(curated_state) + "\n")
+    log_line("probe", f"created report zip: {zip_path}")
+    return str(zip_path)
+
 def refresh_civitai_key_status(api_key: str = ""):
     reset_civitai_key_status(api_key, source="input")
     return format_civitai_key_status_md(api_key)

utils.py

@@ -254,6 +254,116 @@ def retry_call(func, attempts: int = 4, base_wait: float = 1.0, action: str = 'o
             time.sleep(delay)
     if last_error is not None:
         raise last_error
+
+HF_UPLOAD_RETRY_POLICY_CHOICES = ["Auto", "Gentle", "Standard", "Patient"]
+HF_UPLOAD_RETRY_POLICIES = {
+    "auto": {"attempts": 4, "base_wait": 2.0, "max_wait": 30.0, "post_upload_sleep": 1.0},
+    "gentle": {"attempts": 2, "base_wait": 2.0, "max_wait": 12.0, "post_upload_sleep": 1.5},
+    "standard": {"attempts": 4, "base_wait": 2.0, "max_wait": 25.0, "post_upload_sleep": 1.0},
+    "patient": {"attempts": 5, "base_wait": 3.0, "max_wait": 60.0, "post_upload_sleep": 2.5},
+}
+
+
+def normalize_hf_upload_retry_policy(policy: Any):
+    value = str(policy or "Auto").strip().lower()
+    if value not in HF_UPLOAD_RETRY_POLICIES:
+        return "auto"
+    return value
+
+
+def get_hf_upload_retry_policy_config(policy: Any):
+    key = normalize_hf_upload_retry_policy(policy)
+    config = dict(HF_UPLOAD_RETRY_POLICIES.get(key) or HF_UPLOAD_RETRY_POLICIES["auto"])
+    config["key"] = key
+    return config
+
+
+def _get_exception_headers(exc: Exception):
+    response = getattr(exc, "response", None)
+    headers = getattr(response, "headers", None) if response is not None else None
+    return headers or {}
+
+
+def parse_hf_retry_delay_from_headers(exc: Exception):
+    headers = _get_exception_headers(exc)
+    retry_after = str(headers.get("Retry-After", "") or "").strip()
+    if retry_after:
+        try:
+            return max(0.0, float(retry_after)), "retry-after"
+        except ValueError:
+            pass
+    rate_limit = str(headers.get("RateLimit", "") or "")
+    match = re.search(r"(?:^|[;,\s])t=(\d+)", rate_limit)
+    if match:
+        try:
+            return max(0.0, float(match.group(1))), "ratelimit-reset"
+        except ValueError:
+            pass
+    return None, ""
+
+
+def format_hf_rate_limit_hint(exc: Exception):
+    headers = _get_exception_headers(exc)
+    parts = []
+    for key in ("RateLimit", "RateLimit-Policy", "Retry-After"):
+        value = str(headers.get(key, "") or "").strip()
+        if value:
+            parts.append(f"{key}={value}")
+    return " ".join(parts)
+
+
+def is_retryable_hf_upload_exception(exc: Exception):
+    msg = f"{type(exc).__name__}: {exc}".lower()
+    non_retryable = [
+        "401 unauthorized", "403 forbidden", "invalid token", "permission",
+        "repo not found", "not found", "invalid repo",
+    ]
+    if any(token in msg for token in non_retryable):
+        return False
+    retryable = [
+        "429", "408", "425", "500", "502", "503", "504",
+        "too many requests", "slow down", "rate limit",
+        "timed out", "timeout", "connection", "temporarily unavailable",
+        "remote end closed", "reset by peer", "server error", "service unavailable",
+        "gateway timeout", "network",
+        "lfs-verify", "commit endpoint", "internal error hook",
+        "/info/lfs/objects/batch", "lfs/objects/batch",
+    ]
+    return any(token in msg for token in retryable)
+
+
+def hf_upload_retry_call(func, policy: Any="Auto", action: str="hf_upload"):
+    config = get_hf_upload_retry_policy_config(policy)
+    attempts = max(1, int(config.get("attempts", 1)))
+    base_wait = float(config.get("base_wait", 2.0))
+    max_wait = float(config.get("max_wait", 30.0))
+    last_error = None
+    for attempt in range(1, attempts + 1):
+        try:
+            return func()
+        except Exception as e:
+            last_error = e
+            if attempt >= attempts or not is_retryable_hf_upload_exception(e):
+                hint = format_hf_rate_limit_hint(e)
+                suffix = f" {hint}" if hint else ""
+                print(f"[fail] action={action} policy={config['key']} error={format_error_short(e)}{suffix}")
+                raise
+            header_delay, delay_source = parse_hf_retry_delay_from_headers(e)
+            if header_delay is not None:
+                if header_delay > max_wait:
+                    print(f"[fail] action={action} policy={config['key']} wait_hint={round(header_delay, 2)}s exceeds max_wait={round(max_wait, 2)}s error={format_error_short(e)}")
+                    raise
+                delay = header_delay
+            else:
+                delay = min(max_wait, base_wait * (2 ** (attempt - 1))) + random.uniform(0.0, 0.5)
+                delay_source = "backoff"
+            hint = format_hf_rate_limit_hint(e)
+            suffix = f" {hint}" if hint else ""
+            print(f"[retry] action={action} policy={config['key']} attempt={attempt}/{attempts} wait={round(delay, 2)}s source={delay_source} error={format_error_short(e)}{suffix}")
+            time.sleep(delay)
+    if last_error is not None:
+        raise last_error
+
 
 
 def resolve_civitai_download_url(url: str, civitai_api_key: str, max_tries: int = 3):