Upload 8 files

packages.txt

@@ -1,3 +1,2 @@
 git-lfs
-aria2
 ffmpeg

requirements.txt

@@ -1,5 +1,3 @@
 huggingface-hub
 gdown
-hf_xet
-#hf_transfer
-#pydantic==2.10.6
+#pydantic==2.10.6

utils.py

@@ -129,29 +129,16 @@ def get_user_agent():
 
 CIVITAI_REFERER = 'https://civitai.com/'
 RETRYABLE_STATUS_CODES = frozenset([408, 409, 425, 429, 500, 502, 503, 504])
-ARIA2_CIVITAI_OPTIONS = [
-    "--console-log-level=notice",
-    "--log-level=notice",
-    "--summary-interval=0",
-    "--stderr=true",
-    "--log=-",
-    "--check-certificate=true",
-    "--connect-timeout=15",
-    "--timeout=60",
-    "--max-tries=1",
+WGET_CIVITAI_OPTIONS = [
     "-c",
-    "-x", "1",
-    "-s", "1",
-    "-k", "20M",
+    "-nv",
+    "--content-disposition",
 ]
-ARIA2_GENERIC_OPTIONS = [
-    "--console-log-level=error",
-    "--summary-interval=10",
+WGET_GENERIC_OPTIONS = [
     "-c",
-    "-x", "16",
-    "-k", "1M",
-    "-s", "16",
+    "-nv",
 ]
+
 
 
 def build_retry(total: int = 5, backoff_factor: float = 1.0, status_forcelist=RETRYABLE_STATUS_CODES):
@@ -220,7 +207,7 @@ def sanitize_url_for_log(url: str):
         query = urllib.parse.parse_qsl(parts.query, keep_blank_values=True)
         safe_query = []
         for key, value in query:
-            if key.lower() in {"token", "authorization", "auth", "key", "api_key"}:
+            if key.lower() in {"token", "authorization", "auth", "key", "api_key", "x-amz-signature", "x-amz-credential", "x-amz-security-token", "policy", "signature", "key-pair-id"}:
                 safe_query.append((key, "***"))
             else:
                 safe_query.append((key, value))
@@ -229,15 +216,28 @@ def sanitize_url_for_log(url: str):
         return url
 
 
+def sanitize_sensitive_log_text(value: Any):
+    text = str(value or "")
+    def repl(match):
+        return f"{match.group(1)}=***"
+    return re.sub(r"(?i)(token|authorization|auth|key|api_key|x-amz-signature|x-amz-credential|x-amz-security-token|policy|signature|key-pair-id)=([^\s&]+)", repl, text)
+
+
 def log_subprocess_tail(label: str, result: subprocess.CompletedProcess, success_tail: int = 1200, failure_tail: int = 4000):
     tail = success_tail if result.returncode == 0 else failure_tail
     if result.stdout:
-        print(f"{label} stdout:\n", result.stdout[-tail:])
+        print(f"{label} stdout:\n", sanitize_sensitive_log_text(result.stdout[-tail:]))
     if result.stderr:
-        print(f"{label} stderr:\n", result.stderr[-tail:])
+        print(f"{label} stderr:\n", sanitize_sensitive_log_text(result.stderr[-tail:]))
     print(f"{label} returncode:", result.returncode)
 
 
+def ensure_wget_available():
+    if shutil.which("wget") is None:
+        raise FileNotFoundError("wget is required for direct URL downloads but was not found")
+
+
+
 def retry_call(func, attempts: int = 4, base_wait: float = 1.0, action: str = 'operation'):
     last_error = None
     for attempt in range(1, attempts + 1):
@@ -506,9 +506,12 @@ def download_hf_file(directory, url, progress=gr.Progress(track_tqdm=True)):
         return None
 
 
-def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(track_tqdm=True)): # requires aria2, gdown
+def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(track_tqdm=True)): # requires wget, gdown
     try:
         url = url.strip()
+        if not url:
+            print("Skipping empty download URL.")
+            return None
         if "drive.google.com" in url:
             original_dir = os.getcwd()
             try:
@@ -521,6 +524,7 @@ def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(trac
             if "/blob/" in url: url = url.replace("/blob/", "/resolve/")
             download_hf_file(directory, url)
         elif "civitai.com" in url:
+            ensure_wget_available()
             keys = parse_civitai_api_keys(civitai_api_key)
             if keys:
                 user_agent = get_user_agent()
@@ -538,23 +542,23 @@ def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(trac
                         signed_host = urllib.parse.urlparse(signed_url).netloc
                         print(f"Downloading {sanitize_url_for_log(url)} -> {signed_host} (key {key_index}/{len(keys)})")
                         cmd = [
-                            "aria2c",
-                            *ARIA2_CIVITAI_OPTIONS,
+                            "wget",
+                            *WGET_CIVITAI_OPTIONS,
                             "--user-agent", user_agent,
                             "--referer", CIVITAI_REFERER,
-                            "-d", directory,
+                            "-P", directory,
                             signed_url,
                         ]
                         print("Running:", " ".join(cmd[:-1]), "[signed URL omitted]")
                         result = subprocess.run(cmd, capture_output=True, text=True)
-                        log_subprocess_tail("aria2", result)
+                        log_subprocess_tail("wget", result)
                         if result.returncode == 0:
                             last_error = None
                             success = True
                             break
-                        last_error = RuntimeError(f"aria2c failed with rc={result.returncode} host={signed_host}")
+                        last_error = RuntimeError(f"wget failed with rc={result.returncode} host={signed_host}")
                         if len(keys) > 1 and key_index < len(keys):
-                            print(f"Switching Civitai key {key_index}/{len(keys)} after aria2 failure.")
+                            print(f"Switching Civitai key {key_index}/{len(keys)} after wget failure.")
                     if success:
                         break
                     if i + 1 < 3:
@@ -564,12 +568,15 @@ def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(trac
             else:
                 print("You need an API key to download Civitai models.")
         else:
-            cmd = ["aria2c", *ARIA2_GENERIC_OPTIONS, "-d", directory, url]
-            subprocess.run(cmd, check=False)
+            ensure_wget_available()
+            cmd = ["wget", *WGET_GENERIC_OPTIONS, "-P", directory, url]
+            print("Running:", " ".join(cmd[:-1]), sanitize_url_for_log(cmd[-1]))
+            result = subprocess.run(cmd, capture_output=True, text=True)
+            log_subprocess_tail("wget", result)
     except Exception as e:
         print(f"Failed to download: {format_error_short(e)}")
-
-
+
+
 def get_local_file_list(dir_path, recursive=False):
     file_list = []
     pattern = "**/*.*" if recursive else "*/*.*"
@@ -686,7 +693,7 @@ BASE_DIR = str(Path(__file__).resolve().parent.resolve())
 CIVITAI_API_KEY = os.environ.get("CIVITAI_API_KEY")
 
 
-def get_file(url: str, path: str): # requires aria2, gdown
+def get_file(url: str, path: str): # requires wget, gdown
     print(f"Downloading {url} to {path}...")
     get_download_file(path, url, CIVITAI_API_KEY)