Spaces:
Running on CPU Upgrade
Running on CPU Upgrade
Commit ·
60fec81
1
Parent(s): 7679d4b
Add missing dependencies to test tokenizer in model validation
Browse filesFixes https://huggingface.co/spaces/fr-gouv-coordination-ia/llm_leaderboard_fr/discussions/5
backend/app/utils/model_validation.py
CHANGED
|
@@ -196,7 +196,8 @@ class ModelValidator:
|
|
| 196 |
)
|
| 197 |
except ValueError as e:
|
| 198 |
return False, f"The tokenizer is not available in an official Transformers release: {e}", None
|
| 199 |
-
except Exception:
|
|
|
|
| 200 |
return False, "The tokenizer cannot be loaded. Ensure the tokenizer class is part of a stable Transformers release and correctly configured.", None
|
| 201 |
|
| 202 |
return True, None, config
|
|
|
|
| 196 |
)
|
| 197 |
except ValueError as e:
|
| 198 |
return False, f"The tokenizer is not available in an official Transformers release: {e}", None
|
| 199 |
+
except Exception as e:
|
| 200 |
+
logger.error("The tokenizer cannot be loaded", e)
|
| 201 |
return False, "The tokenizer cannot be loaded. Ensure the tokenizer class is part of a stable Transformers release and correctly configured.", None
|
| 202 |
|
| 203 |
return True, None, config
|
backend/pyproject.toml
CHANGED
|
@@ -20,6 +20,8 @@ aiofiles = "^24.1.0"
|
|
| 20 |
fastapi-cache2 = "^0.2.1"
|
| 21 |
python-dotenv = "^1.0.1"
|
| 22 |
sentry-sdk = {extras = ["fastapi"], version = "^2.20.0"}
|
|
|
|
|
|
|
| 23 |
|
| 24 |
[tool.poetry.group.dev.dependencies]
|
| 25 |
pytest = "^8.3.4"
|
|
|
|
| 20 |
fastapi-cache2 = "^0.2.1"
|
| 21 |
python-dotenv = "^1.0.1"
|
| 22 |
sentry-sdk = {extras = ["fastapi"], version = "^2.20.0"}
|
| 23 |
+
protobuf = "^5.29.3"
|
| 24 |
+
sentencepiece = "^0.2.0"
|
| 25 |
|
| 26 |
[tool.poetry.group.dev.dependencies]
|
| 27 |
pytest = "^8.3.4"
|