Spaces:
Running
Running
Use canonical Storage dataset root only
Browse files
api.py
CHANGED
|
@@ -37,7 +37,7 @@ ASSETS_RUNTIME_ROOT = os.environ.get("SPORALIZE_ASSETS_DIR", os.path.join(RUNTIM
|
|
| 37 |
WEIGHTS_RUNTIME_ROOT = os.environ.get("SPORALIZE_WEIGHTS_DIR", os.path.join(RUNTIME_ROOT, "weights"))
|
| 38 |
DEFAULT_LOCAL_STORAGE_ROOT = os.path.join(CURRENT_DIR, "Storage")
|
| 39 |
if not os.path.isdir("/data") and not os.access(CURRENT_DIR, os.W_OK):
|
| 40 |
-
DEFAULT_LOCAL_STORAGE_ROOT = os.path.join(RUNTIME_ROOT, "
|
| 41 |
STORAGE_ROOT = os.environ.get(
|
| 42 |
"SPORALIZE_STORAGE_DIR",
|
| 43 |
os.path.join("/data", "sporalize_storage") if os.path.isdir("/data") else DEFAULT_LOCAL_STORAGE_ROOT,
|
|
@@ -131,13 +131,9 @@ def hf_storage_path(*parts: str) -> str:
|
|
| 131 |
return "/".join(segment for segment in normalized if segment)
|
| 132 |
|
| 133 |
|
| 134 |
-
def storage_dataset_paths(
|
| 135 |
paths = []
|
| 136 |
candidates = [STORAGE_DATASET_PATH]
|
| 137 |
-
if include_legacy:
|
| 138 |
-
# The private dataset originally used "Storage". HF repo paths are
|
| 139 |
-
# case-sensitive, so deletes must cover both roots during migration.
|
| 140 |
-
candidates.extend(["storage", "Storage"])
|
| 141 |
for candidate in candidates:
|
| 142 |
normalized = str(candidate or "").strip("/").strip()
|
| 143 |
if normalized and normalized not in paths:
|
|
@@ -152,7 +148,7 @@ def hf_storage_paths_for_delete(*parts: str):
|
|
| 152 |
for part in parts
|
| 153 |
if part is not None and str(part).strip("/")
|
| 154 |
]
|
| 155 |
-
for root in storage_dataset_paths(
|
| 156 |
path = "/".join([root, *suffix])
|
| 157 |
if path not in paths:
|
| 158 |
paths.append(path)
|
|
@@ -186,7 +182,7 @@ def sync_storage_from_hf(force: bool = False):
|
|
| 186 |
if os.path.isdir(local_repo_dir):
|
| 187 |
shutil.rmtree(local_repo_dir, ignore_errors=True)
|
| 188 |
|
| 189 |
-
storage_roots = storage_dataset_paths(
|
| 190 |
snapshot_download(
|
| 191 |
repo_id=STORAGE_DATASET_REPO_ID,
|
| 192 |
repo_type=STORAGE_DATASET_REPO_TYPE,
|
|
|
|
| 37 |
WEIGHTS_RUNTIME_ROOT = os.environ.get("SPORALIZE_WEIGHTS_DIR", os.path.join(RUNTIME_ROOT, "weights"))
|
| 38 |
DEFAULT_LOCAL_STORAGE_ROOT = os.path.join(CURRENT_DIR, "Storage")
|
| 39 |
if not os.path.isdir("/data") and not os.access(CURRENT_DIR, os.W_OK):
|
| 40 |
+
DEFAULT_LOCAL_STORAGE_ROOT = os.path.join(RUNTIME_ROOT, "Storage")
|
| 41 |
STORAGE_ROOT = os.environ.get(
|
| 42 |
"SPORALIZE_STORAGE_DIR",
|
| 43 |
os.path.join("/data", "sporalize_storage") if os.path.isdir("/data") else DEFAULT_LOCAL_STORAGE_ROOT,
|
|
|
|
| 131 |
return "/".join(segment for segment in normalized if segment)
|
| 132 |
|
| 133 |
|
| 134 |
+
def storage_dataset_paths():
|
| 135 |
paths = []
|
| 136 |
candidates = [STORAGE_DATASET_PATH]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 137 |
for candidate in candidates:
|
| 138 |
normalized = str(candidate or "").strip("/").strip()
|
| 139 |
if normalized and normalized not in paths:
|
|
|
|
| 148 |
for part in parts
|
| 149 |
if part is not None and str(part).strip("/")
|
| 150 |
]
|
| 151 |
+
for root in storage_dataset_paths():
|
| 152 |
path = "/".join([root, *suffix])
|
| 153 |
if path not in paths:
|
| 154 |
paths.append(path)
|
|
|
|
| 182 |
if os.path.isdir(local_repo_dir):
|
| 183 |
shutil.rmtree(local_repo_dir, ignore_errors=True)
|
| 184 |
|
| 185 |
+
storage_roots = storage_dataset_paths()
|
| 186 |
snapshot_download(
|
| 187 |
repo_id=STORAGE_DATASET_REPO_ID,
|
| 188 |
repo_type=STORAGE_DATASET_REPO_TYPE,
|