Spaces:
Sleeping
Sleeping
Use write token for HF storage commits
Browse files
api.py
CHANGED
|
@@ -88,8 +88,37 @@ runtime_state = {
|
|
| 88 |
}
|
| 89 |
|
| 90 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
def get_hf_token():
|
| 92 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
|
| 94 |
|
| 95 |
def hf_storage_enabled():
|
|
@@ -184,13 +213,15 @@ def sync_storage_from_hf(force: bool = False):
|
|
| 184 |
def push_session_to_hf(player_id: str, session_id: str, session_dir: str):
|
| 185 |
if not hf_storage_enabled():
|
| 186 |
return
|
| 187 |
-
|
|
|
|
| 188 |
api.upload_folder(
|
| 189 |
repo_id=STORAGE_DATASET_REPO_ID,
|
| 190 |
repo_type=STORAGE_DATASET_REPO_TYPE,
|
| 191 |
folder_path=session_dir,
|
| 192 |
path_in_repo=hf_storage_path(safe_name(player_id), safe_name(session_id)),
|
| 193 |
commit_message=f"Add session {safe_name(session_id)} for player {safe_name(player_id)}",
|
|
|
|
| 194 |
)
|
| 195 |
|
| 196 |
|
|
@@ -201,7 +232,6 @@ def _hf_files_under(api, path_in_repo: str):
|
|
| 201 |
repo_id=STORAGE_DATASET_REPO_ID,
|
| 202 |
repo_type=STORAGE_DATASET_REPO_TYPE,
|
| 203 |
revision="main",
|
| 204 |
-
token=get_hf_token(),
|
| 205 |
)
|
| 206 |
return [f for f in files if f == path_in_repo or f.startswith(prefix)]
|
| 207 |
|
|
@@ -217,7 +247,8 @@ def _delete_folder_from_hf(path_in_repo: str, commit_message: str):
|
|
| 217 |
if not hf_storage_enabled():
|
| 218 |
return True, None
|
| 219 |
|
| 220 |
-
|
|
|
|
| 221 |
|
| 222 |
try:
|
| 223 |
targets = _hf_files_under(api, path_in_repo)
|
|
@@ -237,10 +268,10 @@ def _delete_folder_from_hf(path_in_repo: str, commit_message: str):
|
|
| 237 |
repo_type=STORAGE_DATASET_REPO_TYPE,
|
| 238 |
operations=operations,
|
| 239 |
commit_message=commit_message,
|
| 240 |
-
token=
|
| 241 |
)
|
| 242 |
except Exception as error:
|
| 243 |
-
delete_error =
|
| 244 |
print(f"[SPORALIZE] HF delete commit failed for '{path_in_repo}' ({len(targets)} files): {delete_error}", flush=True)
|
| 245 |
# Confirm by end state anyway — a benign race may still have removed it.
|
| 246 |
try:
|
|
|
|
| 88 |
}
|
| 89 |
|
| 90 |
|
| 91 |
+
def get_env_token(*names):
|
| 92 |
+
for name in names:
|
| 93 |
+
value = os.environ.get(name)
|
| 94 |
+
if value:
|
| 95 |
+
return value
|
| 96 |
+
return None
|
| 97 |
+
|
| 98 |
+
|
| 99 |
def get_hf_token():
|
| 100 |
+
return get_env_token("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HUGGINGFACE_HUB_TOKEN", "HUGGINGFACEHUB_API_TOKEN")
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def get_hf_storage_write_token():
|
| 104 |
+
return get_env_token(
|
| 105 |
+
"SPORALIZE_STORAGE_WRITE_TOKEN",
|
| 106 |
+
"HF_WRITE_TOKEN",
|
| 107 |
+
"HUGGING_FACE_HUB_TOKEN",
|
| 108 |
+
"HF_TOKEN",
|
| 109 |
+
"HUGGINGFACE_HUB_TOKEN",
|
| 110 |
+
"HUGGINGFACEHUB_API_TOKEN",
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def add_hf_write_token_hint(error):
|
| 115 |
+
message = str(error)
|
| 116 |
+
if "403" in message or "Forbidden" in message or "correct permissions" in message:
|
| 117 |
+
return (
|
| 118 |
+
f"{message} Set SPORALIZE_STORAGE_WRITE_TOKEN, HF_WRITE_TOKEN, or HF_TOKEN "
|
| 119 |
+
"to a Hugging Face token with write access to the storage dataset."
|
| 120 |
+
)
|
| 121 |
+
return message
|
| 122 |
|
| 123 |
|
| 124 |
def hf_storage_enabled():
|
|
|
|
| 213 |
def push_session_to_hf(player_id: str, session_id: str, session_dir: str):
|
| 214 |
if not hf_storage_enabled():
|
| 215 |
return
|
| 216 |
+
token = get_hf_storage_write_token()
|
| 217 |
+
api = HfApi(token=token)
|
| 218 |
api.upload_folder(
|
| 219 |
repo_id=STORAGE_DATASET_REPO_ID,
|
| 220 |
repo_type=STORAGE_DATASET_REPO_TYPE,
|
| 221 |
folder_path=session_dir,
|
| 222 |
path_in_repo=hf_storage_path(safe_name(player_id), safe_name(session_id)),
|
| 223 |
commit_message=f"Add session {safe_name(session_id)} for player {safe_name(player_id)}",
|
| 224 |
+
token=token,
|
| 225 |
)
|
| 226 |
|
| 227 |
|
|
|
|
| 232 |
repo_id=STORAGE_DATASET_REPO_ID,
|
| 233 |
repo_type=STORAGE_DATASET_REPO_TYPE,
|
| 234 |
revision="main",
|
|
|
|
| 235 |
)
|
| 236 |
return [f for f in files if f == path_in_repo or f.startswith(prefix)]
|
| 237 |
|
|
|
|
| 247 |
if not hf_storage_enabled():
|
| 248 |
return True, None
|
| 249 |
|
| 250 |
+
token = get_hf_storage_write_token()
|
| 251 |
+
api = HfApi(token=token)
|
| 252 |
|
| 253 |
try:
|
| 254 |
targets = _hf_files_under(api, path_in_repo)
|
|
|
|
| 268 |
repo_type=STORAGE_DATASET_REPO_TYPE,
|
| 269 |
operations=operations,
|
| 270 |
commit_message=commit_message,
|
| 271 |
+
token=token,
|
| 272 |
)
|
| 273 |
except Exception as error:
|
| 274 |
+
delete_error = add_hf_write_token_hint(error)
|
| 275 |
print(f"[SPORALIZE] HF delete commit failed for '{path_in_repo}' ({len(targets)} files): {delete_error}", flush=True)
|
| 276 |
# Confirm by end state anyway — a benign race may still have removed it.
|
| 277 |
try:
|