updated indexer to handle colors

CLAUDE.md

@@ -0,0 +1,85 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What This Is
+
+A FastAPI application deployed as a Hugging Face Space (Gradio SDK). It acts as an indexer and API server for a climbing holds dataset hosted on Hugging Face Hub. It handles:
+- Webhook-triggered re-indexation when the HF dataset changes
+- Vote processing for community-driven hold identification
+- Anonymous hold contributions
+
+## Running Locally
+
+```bash
+python -m venv .venv && source .venv/bin/activate
+pip install -r requirements.txt
+uvicorn app:app --reload
+```
+
+Required environment variables (see `.gitignore` — use a `.env` file):
+- `HF_TOKEN` — write access to the main dataset
+- `WEBHOOK_SECRET` — validates incoming HF webhooks
+- `HF_REVISION` — branch to read/write (defaults to `staging`)
+- `HF_ANONYMOUS_TOKEN` — write access to the anonymous contributions repo
+- `HF_ANONYMOUS_REPO_ID` — anonymous contributions repo ID
+- `CORS_ALLOWED_ORIGINS` — comma-separated additional CORS origins
+
+## Manual Re-indexation
+
+```bash
+# Dry run (default) — shows what would change without committing
+python reindex.py
+
+# Commit the reindex to the dataset
+python reindex.py --commit
+
+# Target a specific repo or revision
+python reindex.py --repo-id owner/dataset --revision main --commit
+```
+
+## Architecture
+
+### Data Flow: Webhook-triggered Re-indexation
+1. HF dataset webhook fires → `webhooks.py` verifies secret and receives payload
+2. `global_index.py` loads the current `global_index.json` from the HF dataset
+3. `hf_repo.py` lists all `metadata.json` files in the dataset
+4. `holds.py` normalizes/validates each hold's metadata, infers mesh presence, tracks attention items
+5. A new `train.jsonl` is built from all holds
+6. `hf_repo.py` commits if content hash changed (idempotent via SHA-256)
+
+### Data Flow: Vote Processing
+1. POST `/webhooks/vote` → `votes.py` validates payload (rating 1–5, ISO8601 timestamp)
+2. Voter fingerprint computed from HF username or hashed IP
+3. Votes stored in per-hold `votes.json`; duplicate votes rejected
+4. Dominant manufacturer/model inferred from vote aggregate
+5. Hold `metadata.json` updated if dominant values changed
+
+### Data Flow: Anonymous Contributions
+1. POST multipart form → `contributions.py` sanitizes filenames and deduplicates
+2. Files committed to anonymous HF repo under `pending/<uuid>/`
+
+### Key Files
+- `app.py` — FastAPI app, mounts Gradio UI, wires endpoints
+- `config.py` — all constants: repo IDs, file path names, mesh extensions, managed keys
+- `global_index.py` — index bootstrap, reference validation (manufacturers, hold types, status), attention tracking
+- `hf_repo.py` — all Hugging Face Hub I/O (load/save JSON, list files, commit)
+- `holds.py` — metadata normalization, reference validation with fuzzy-match suggestions, `train.jsonl` builder
+- `votes.py` — vote validation, fingerprinting, aggregation
+- `webhooks.py` — webhook route handlers, CORS setup
+- `contributions.py` — anonymous submission handler
+- `reindex.py` — standalone CLI for manual full reindex
+
+### Dataset Structure on HF Hub
+- `global_index.json` — top-level index with allowed references and attention sets
+- `<hold_id>/metadata.json` — per-hold metadata
+- `<hold_id>/votes.json` — per-hold community votes
+- `train.jsonl` — flat JSONL export of all normalized holds
+
+## No Tests
+
+There are no automated tests in this project.
+
+## Deployment
+
+This runs as a Hugging Face Space (see the YAML frontmatter in `README.md`). The `app_file` is `app.py`. Pushing to the Space repo triggers a redeploy.

config.py

@@ -13,6 +13,7 @@ METADATA_FILENAME = "metadata.json"
 VOTES_FILENAME = "votes.json"
 MESH_EXTENSIONS = {".glb", ".gltf", ".obj", ".stl"}
 MANAGED_ATTENTION_KEYS = {
+    "invalid_color_reference",
     "invalid_hold_type_reference",
     "invalid_manufacturer_reference",
     "invalid_metadata",

global_index.py

@@ -31,6 +31,7 @@ def bootstrap_global_index(repo_id: str) -> dict[str, Any]:
             "manufacturers": [],
             "hold_types": [],
             "status": ["to_render", "to_clean", "to_identify"],
+            "colors": [],
         },
         "stats": {"total_holds": 0, "to_identify": 0},
         "needs_attention": {},
@@ -45,6 +46,7 @@ def ensure_allowed_references(global_index: dict[str, Any]) -> dict[str, set[str
     manufacturers = allowed_references.setdefault("manufacturers", [])
     hold_types = allowed_references.setdefault("hold_types", [])
     statuses = allowed_references.setdefault("status", [])
+    colors = allowed_references.setdefault("colors", [])
 
     if not isinstance(manufacturers, list) or not isinstance(hold_types, list):
         raise RuntimeError(
@@ -55,11 +57,16 @@ def ensure_allowed_references(global_index: dict[str, Any]) -> dict[str, set[str
         raise RuntimeError(
             "global_index.json must define a list value for 'allowed_references.status'."
         )
+    if not isinstance(colors, list):
+        raise RuntimeError(
+            "global_index.json must define a list value for 'allowed_references.colors'."
+        )
 
     return {
         "manufacturers": {value for value in map(normalize_reference_value, manufacturers) if value},
         "hold_types": {value for value in map(normalize_reference_value, hold_types) if value},
         "status": {value for value in map(normalize_reference_value, statuses) if value},
+        "colors": {value for value in map(normalize_reference_value, colors) if value},
     }
 
 

holds.py

@@ -147,6 +147,21 @@ def validate_metadata(
             attention_bucket=needs_attention["invalid_status_reference"],
         )
 
+    allowed_colors = allowed_references.get("colors", set())
+    if allowed_colors:
+        available_colors = metadata.get("available_colors") or []
+        if isinstance(available_colors, list):
+            for color_value in available_colors:
+                color_ref = global_index.normalize_reference_value(color_value)
+                if color_ref is not None and color_ref not in allowed_colors:
+                    warn_about_reference(
+                        hold_id=hold_id,
+                        field_name="available_colors",
+                        value=color_value,
+                        allowed_values=allowed_colors,
+                        attention_bucket=needs_attention["invalid_color_reference"],
+                    )
+
 
 def canonical_metadata_defaults() -> list[tuple[str, Any]]:
     return [