Shoraky commited on
Commit
8dc3ee4
·
1 Parent(s): 6f8fe42

Use canonical Storage dataset root only

Browse files
Files changed (1) hide show
  1. api.py +4 -8
api.py CHANGED
@@ -37,7 +37,7 @@ ASSETS_RUNTIME_ROOT = os.environ.get("SPORALIZE_ASSETS_DIR", os.path.join(RUNTIM
37
  WEIGHTS_RUNTIME_ROOT = os.environ.get("SPORALIZE_WEIGHTS_DIR", os.path.join(RUNTIME_ROOT, "weights"))
38
  DEFAULT_LOCAL_STORAGE_ROOT = os.path.join(CURRENT_DIR, "Storage")
39
  if not os.path.isdir("/data") and not os.access(CURRENT_DIR, os.W_OK):
40
- DEFAULT_LOCAL_STORAGE_ROOT = os.path.join(RUNTIME_ROOT, "storage")
41
  STORAGE_ROOT = os.environ.get(
42
  "SPORALIZE_STORAGE_DIR",
43
  os.path.join("/data", "sporalize_storage") if os.path.isdir("/data") else DEFAULT_LOCAL_STORAGE_ROOT,
@@ -131,13 +131,9 @@ def hf_storage_path(*parts: str) -> str:
131
  return "/".join(segment for segment in normalized if segment)
132
 
133
 
134
- def storage_dataset_paths(include_legacy: bool = True):
135
  paths = []
136
  candidates = [STORAGE_DATASET_PATH]
137
- if include_legacy:
138
- # The private dataset originally used "Storage". HF repo paths are
139
- # case-sensitive, so deletes must cover both roots during migration.
140
- candidates.extend(["storage", "Storage"])
141
  for candidate in candidates:
142
  normalized = str(candidate or "").strip("/").strip()
143
  if normalized and normalized not in paths:
@@ -152,7 +148,7 @@ def hf_storage_paths_for_delete(*parts: str):
152
  for part in parts
153
  if part is not None and str(part).strip("/")
154
  ]
155
- for root in storage_dataset_paths(include_legacy=True):
156
  path = "/".join([root, *suffix])
157
  if path not in paths:
158
  paths.append(path)
@@ -186,7 +182,7 @@ def sync_storage_from_hf(force: bool = False):
186
  if os.path.isdir(local_repo_dir):
187
  shutil.rmtree(local_repo_dir, ignore_errors=True)
188
 
189
- storage_roots = storage_dataset_paths(include_legacy=True)
190
  snapshot_download(
191
  repo_id=STORAGE_DATASET_REPO_ID,
192
  repo_type=STORAGE_DATASET_REPO_TYPE,
 
37
  WEIGHTS_RUNTIME_ROOT = os.environ.get("SPORALIZE_WEIGHTS_DIR", os.path.join(RUNTIME_ROOT, "weights"))
38
  DEFAULT_LOCAL_STORAGE_ROOT = os.path.join(CURRENT_DIR, "Storage")
39
  if not os.path.isdir("/data") and not os.access(CURRENT_DIR, os.W_OK):
40
+ DEFAULT_LOCAL_STORAGE_ROOT = os.path.join(RUNTIME_ROOT, "Storage")
41
  STORAGE_ROOT = os.environ.get(
42
  "SPORALIZE_STORAGE_DIR",
43
  os.path.join("/data", "sporalize_storage") if os.path.isdir("/data") else DEFAULT_LOCAL_STORAGE_ROOT,
 
131
  return "/".join(segment for segment in normalized if segment)
132
 
133
 
134
+ def storage_dataset_paths():
135
  paths = []
136
  candidates = [STORAGE_DATASET_PATH]
 
 
 
 
137
  for candidate in candidates:
138
  normalized = str(candidate or "").strip("/").strip()
139
  if normalized and normalized not in paths:
 
148
  for part in parts
149
  if part is not None and str(part).strip("/")
150
  ]
151
+ for root in storage_dataset_paths():
152
  path = "/".join([root, *suffix])
153
  if path not in paths:
154
  paths.append(path)
 
182
  if os.path.isdir(local_repo_dir):
183
  shutil.rmtree(local_repo_dir, ignore_errors=True)
184
 
185
+ storage_roots = storage_dataset_paths()
186
  snapshot_download(
187
  repo_id=STORAGE_DATASET_REPO_ID,
188
  repo_type=STORAGE_DATASET_REPO_TYPE,