Shoraky commited on
Commit
3c4d5d2
·
1 Parent(s): 302d607

Use write token for HF storage commits

Browse files
Files changed (1) hide show
  1. api.py +37 -6
api.py CHANGED
@@ -88,8 +88,37 @@ runtime_state = {
88
  }
89
 
90
 
 
 
 
 
 
 
 
 
91
  def get_hf_token():
92
- return os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93
 
94
 
95
  def hf_storage_enabled():
@@ -184,13 +213,15 @@ def sync_storage_from_hf(force: bool = False):
184
  def push_session_to_hf(player_id: str, session_id: str, session_dir: str):
185
  if not hf_storage_enabled():
186
  return
187
- api = HfApi(token=get_hf_token())
 
188
  api.upload_folder(
189
  repo_id=STORAGE_DATASET_REPO_ID,
190
  repo_type=STORAGE_DATASET_REPO_TYPE,
191
  folder_path=session_dir,
192
  path_in_repo=hf_storage_path(safe_name(player_id), safe_name(session_id)),
193
  commit_message=f"Add session {safe_name(session_id)} for player {safe_name(player_id)}",
 
194
  )
195
 
196
 
@@ -201,7 +232,6 @@ def _hf_files_under(api, path_in_repo: str):
201
  repo_id=STORAGE_DATASET_REPO_ID,
202
  repo_type=STORAGE_DATASET_REPO_TYPE,
203
  revision="main",
204
- token=get_hf_token(),
205
  )
206
  return [f for f in files if f == path_in_repo or f.startswith(prefix)]
207
 
@@ -217,7 +247,8 @@ def _delete_folder_from_hf(path_in_repo: str, commit_message: str):
217
  if not hf_storage_enabled():
218
  return True, None
219
 
220
- api = HfApi(token=get_hf_token())
 
221
 
222
  try:
223
  targets = _hf_files_under(api, path_in_repo)
@@ -237,10 +268,10 @@ def _delete_folder_from_hf(path_in_repo: str, commit_message: str):
237
  repo_type=STORAGE_DATASET_REPO_TYPE,
238
  operations=operations,
239
  commit_message=commit_message,
240
- token=get_hf_token(),
241
  )
242
  except Exception as error:
243
- delete_error = str(error)
244
  print(f"[SPORALIZE] HF delete commit failed for '{path_in_repo}' ({len(targets)} files): {delete_error}", flush=True)
245
  # Confirm by end state anyway — a benign race may still have removed it.
246
  try:
 
88
  }
89
 
90
 
91
+ def get_env_token(*names):
92
+ for name in names:
93
+ value = os.environ.get(name)
94
+ if value:
95
+ return value
96
+ return None
97
+
98
+
99
  def get_hf_token():
100
+ return get_env_token("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HUGGINGFACE_HUB_TOKEN", "HUGGINGFACEHUB_API_TOKEN")
101
+
102
+
103
+ def get_hf_storage_write_token():
104
+ return get_env_token(
105
+ "SPORALIZE_STORAGE_WRITE_TOKEN",
106
+ "HF_WRITE_TOKEN",
107
+ "HUGGING_FACE_HUB_TOKEN",
108
+ "HF_TOKEN",
109
+ "HUGGINGFACE_HUB_TOKEN",
110
+ "HUGGINGFACEHUB_API_TOKEN",
111
+ )
112
+
113
+
114
+ def add_hf_write_token_hint(error):
115
+ message = str(error)
116
+ if "403" in message or "Forbidden" in message or "correct permissions" in message:
117
+ return (
118
+ f"{message} Set SPORALIZE_STORAGE_WRITE_TOKEN, HF_WRITE_TOKEN, or HF_TOKEN "
119
+ "to a Hugging Face token with write access to the storage dataset."
120
+ )
121
+ return message
122
 
123
 
124
  def hf_storage_enabled():
 
213
  def push_session_to_hf(player_id: str, session_id: str, session_dir: str):
214
  if not hf_storage_enabled():
215
  return
216
+ token = get_hf_storage_write_token()
217
+ api = HfApi(token=token)
218
  api.upload_folder(
219
  repo_id=STORAGE_DATASET_REPO_ID,
220
  repo_type=STORAGE_DATASET_REPO_TYPE,
221
  folder_path=session_dir,
222
  path_in_repo=hf_storage_path(safe_name(player_id), safe_name(session_id)),
223
  commit_message=f"Add session {safe_name(session_id)} for player {safe_name(player_id)}",
224
+ token=token,
225
  )
226
 
227
 
 
232
  repo_id=STORAGE_DATASET_REPO_ID,
233
  repo_type=STORAGE_DATASET_REPO_TYPE,
234
  revision="main",
 
235
  )
236
  return [f for f in files if f == path_in_repo or f.startswith(prefix)]
237
 
 
247
  if not hf_storage_enabled():
248
  return True, None
249
 
250
+ token = get_hf_storage_write_token()
251
+ api = HfApi(token=token)
252
 
253
  try:
254
  targets = _hf_files_under(api, path_in_repo)
 
268
  repo_type=STORAGE_DATASET_REPO_TYPE,
269
  operations=operations,
270
  commit_message=commit_message,
271
+ token=token,
272
  )
273
  except Exception as error:
274
+ delete_error = add_hf_write_token_hint(error)
275
  print(f"[SPORALIZE] HF delete commit failed for '{path_in_repo}' ({len(targets)} files): {delete_error}", flush=True)
276
  # Confirm by end state anyway — a benign race may still have removed it.
277
  try: