# SPDX-License-Identifier: Apache-2.0
# tt-model-manager container manifest (schema 5.1) for NVIDIA LocateAnything-3B on one Blackhole p150a.
#
# Run every tt-model command FROM THIS DIRECTORY (`extra_code.root: code` is CWD-relative):
#   source $ROOT/bin/docker-env.sh
#   tt-model package --container tt-model.yaml --out $ROOT/build      # amd64 + Docker>=25, hours cold
#   tt-model serve   $ROOT/build/locate-anything-3b-p150/tt_kernel_manifest.json
#   tt-model push    $ROOT/build/locate-anything-3b-p150 --publish
#
# The port validated against tt-metal main 8b98410e730 (v0.78.0-dev20260820-25): torch 2.11.0+cpu,
# transformers 5.12.1. source.tt_metal is that local checkout (absolute; supplies the torch pin).
schema: "5.1"

repo: changh95/locate-anything-3b-p150
name: locate-anything-3b-p150

# A POINTER, pinned. `tt-model serve` downloads it into the consumer's HF cache before boot;
# the app resolves the same repo/revision offline (TT_WEIGHTS_REVISION below) and extracts the
# vanilla Qwen2.5-3B LLM checkpoint into ~/.cache/tt-model/locate-anything-3b-p150/weights.
weights:
  repo: nvidia/LocateAnything-3B
  revision: c32291ca5e996f5a7a485845b4f57a233936bba0
  ignore_patterns: ["assets/*", "training_args.bin"]

kind: tt-dit-server
arch: blackhole

source:
  tt_metal: /home/deepgadget/experiments/gbp-tt/tt-metal
  # tt-metal packages the port really imports (module-level closure incl. models/common).
  code:
    - models/common
    - models/tt_transformers
    - models/demos/qwen25_vl
  # This repo's code (does not live in the tt-metal tree). Everything listed survives `push`.
  # `models` holds ONE overlay file, code/models/tt_transformers/tt/mlp.py: extra_code is staged
  # after source.code and wins on collision, so it replaces the stock file in the image. Stock
  # mlp.py fails on a single p150a at the first decode step ("Statically allocated circular
  # buffers ... clash with L1 buffers", w3 linear); the overlay spills the decode w1/w3 outputs
  # to DRAM before the next matmul, the change the port validated with (PCC 0.9928).
  extra_code:
    - root: code
      paths:
        - locate_anything
        - models
        - scripts
        - conftest.py
  ubuntu: "22.04"
  python: "3.12"

runtime:
  app: locate_anything.server.app:app
  mesh_shape_env: TT_MESH_SHAPE
  # On top of the kind's defaults (fastapi, uvicorn, pydantic>=2, pillow) and the auto torch pin.
  # transformers must equal tt-metal's own pin (ModelArgs loads the Qwen2 checkpoint through it);
  # loguru/tqdm/pytest/pyyaml are module-level imports of models.tt_transformers / models.common.
  packages:
    - "numpy>=1.24.4,<2"
    - "transformers==5.12.1"
    - safetensors
    - huggingface_hub
    - loguru
    - tqdm
    - pytest
    - pyyaml

serve:
  port: 20000
  hardware: p150
  mesh_device: P150
  env:
    TT_WEIGHTS_REVISION: "c32291ca5e996f5a7a485845b4f57a233936bba0"
    TT_METAL_VISIBLE_DEVICES: "0"
    LA_IN_TOKEN_LIMIT: "1024"
    LA_PREC: "accuracy"
    LA_TRACE: "1"
    LA_WARMUP_RUNS: "2"
    # Fused device paths (code default since the 2026-09-13 device validation; explicit here so the
    # served configuration is visible in `tt-model info`). "0" = the 2026-09-12 legacy graph.
    TT_FUSED: "1"

# Build-time assertions, run INSIDE the finished image as uid 1000 with no device and no weights.
verify:
  - "import locate_anything.server.app as a; assert a.app"
  - "import locate_anything.tt.pipeline, locate_anything.tt.vision, locate_anything.tt.model_la, locate_anything.reference.la_inputs, locate_anything.reference.extract_llm_checkpoint"
  - "import models.tt_transformers.tt.model_config, models.tt_transformers.tt.generator, models.demos.qwen25_vl.tt.common, models.common.utility_functions"
  - "import transformers; assert transformers.__version__ == '5.12.1', transformers.__version__"
  - "import safetensors, huggingface_hub, loguru, tqdm, pytest, yaml, numpy; assert numpy.__version__.startswith('1.'), numpy.__version__"
  - "from pathlib import Path; assert Path('/opt/tt-metal/models/tt_transformers/tt/prefetcher/prefetcher_config.yaml').is_file()"
  - "from pathlib import Path; assert Path('/opt/tt-metal/locate_anything/server/smoke_test.py').is_file()"
  - "from pathlib import Path; assert 'spill_ff1_3' in Path('/opt/tt-metal/models/tt_transformers/tt/mlp.py').read_text(), 'mlp.py overlay (decode w1/w3 DRAM spill) missing'"

card:
  description: >
    NVIDIA LocateAnything-3B (Eagle-family visual grounding / open-vocabulary detection VLM: MoonViT-SO-400M vision tower + Qwen2.5-3B-Instruct with a detection vocabulary) running entirely on one Tenstorrent Blackhole p150a via tt-nn: image + free-text query in, labelled boxes out.

    Weights: [nvidia/LocateAnything-3B](https://huggingface.co/nvidia/LocateAnything-3B) ·
    Paper: [arXiv:2605.27365](https://arxiv.org/abs/2605.27365) ·
    Upstream code: [NVlabs/Eagle (Embodied)](https://github.com/NVlabs/Eagle/tree/main/Embodied) ·
    Port: [changh95/tt-locate-anything](https://github.com/changh95/tt-locate-anything)
  quickstart: |
    ### Run with tt-cli

    ```bash
    tt serve changh95/locate-anything-3b-p150
    { printf '{"query":"car","image":"'; base64 -w0 media/demo_input.png; printf '"}'; } > req.json
    curl -s localhost:20000/predict -H 'Content-Type: application/json' -d @req.json
    tt model stop changh95/locate-anything-3b-p150
    ```

    - `POST /predict`: `image` (base64 PNG/JPEG, one image), `query` (free text, 1-1000 chars; join categories with `</c>`, e.g. `person</c>car`); optional `max_new_tokens` (128, cap 1024), `return_overlay` (false).
    - `GET /health`, `GET /info`.

    ### Response

    ```json
    {"query": "car", "width": 1920, "height": 1080, "canonical_size": [616, 336], "grid_hw": [24, 44],
     "raw_text": "<ref>car</ref><box><282><414><606><794></box><|im_end|>",
     "detections": [{"label": "car", "box": [541.44, 447.12, 1163.52, 857.52], "box_norm": [282, 414, 606, 794]}],
     "points": [], "num_generated_tokens": 10, "stopped_on_eos": true, "decode_mode": "ar_greedy_trace_device_sampling",
     "timing_ms": {"vision": 1.4, "prefill": 86.3, "decode": 202.1, "total": 303.9, "decode_tok_s": 44.53}}
    ```

    - `box` is `[x1, y1, x2, y2]` in original image pixels; `box_norm` is the model's own 0..1000 output over the squashed `canonical_size` view. `points` holds 2-coordinate outputs the same way.
    - `return_overlay: true` adds `overlay_png_b64`, the boxes drawn on your image as a base64 PNG.

    ### Demo

    | Input (`media/demo_input.png`), query `car` | Greedy decode on p150a (`media/demo_ar.png`) |
    |:---:|:---:|
    | ![](media/demo_input.png) | ![](media/demo_ar.png) |

    ### Accuracy and speed

    | Metric | Value |
    |---|---:|
    | Full on-device logits PCC vs torch-CPU reference (vision → LLM) | 0.9919 (gate ≥ 0.99; vision projector 0.9924, LLM prefill 0.9915) — measured 2026-09-13 on the p150a against goldens regenerated with the port's torch 2.7.1 / transformers 4.53.0; identical with `TT_FUSED=0` (the fused default keeps the vision numerics bit for bit) |
    | Decoded box string on the demo image | `<ref>car</ref><box><282><414><606><794></box>`, unchanged from the 2026-09-12 package (the HF torch-CPU reference generated with the port's torch 2.7.1 / transformers 4.53.0 gives `<282><414><607><797>`, IoU 0.989) |
    | `/predict` served over HTTP (warm, batch 1, `car`, 10 tokens; median of 100 requests, 2026-09-13) | **~305 ms** server-side (304–306; vision-trace submit 2 · prefill incl. the vision wait 86 · decode 202 ms, ~44.5 tok/s) · ~0.34 s wall. `TT_FUSED=0` (the 2026-09-12 graph): ~357 ms (351–365; vision 48 · prefill 62 · decode 232 ms, ~39 tok/s) · ~0.40 s wall |
    | Same forward on an RTX 5090 (same host, port's torch reference, eager PyTorch, batch 1, incl. H2D/D2H; bf16 / fp16 autocast) | bf16 native weights 158 ms (83 tok/s decode) → GPU 1.8× faster than the p150a's 290 ms (vision + prefill + decode, 44.5 tok/s); bf16/fp16 autocast over fp32 weights 218 / 215 ms (1.3×); fp32-strict 216 ms (1.3×; vision+prefill 92 ms vs p150a 88 ms — p150a faster); best `torch.compile` 137 ms (2.1×) |

    ### Caveats

    - One image + one query per request, batch 1, requests are serialized. Every image is squash-resized onto a fixed 24×44-patch grid (616×336, 16:9; `LA_IN_TOKEN_LIMIT=1024`, upstream default 25600): non-16:9 images are distorted before the model sees them, boxes still map back to original pixels.
    - bf16 vision, BF16 attention + BFP8 MLP LLM; the package ships an `mlp.py` overlay (`code/models/tt_transformers/tt/mlp.py`, decode `w1`/`w3` spilled to DRAM) so the LLM fits L1 on one p150a. Fused device paths (MoonViT as one metal trace with the patch merger on device, device vision→LLM merge, traced prefill, one-row first token, on-device greedy argmax) are on by default since 2026-09-13; `TT_FUSED=0` restores the 2026-09-12 graph, and the vision numerics are unchanged bit for bit either way. Only greedy AR decode is served; the experimental Parallel Box Decoding (MTP) is not.
    - Weights are public and ungated but under NVIDIA's own license (not OSI); ~7.7 GB download, and the first boot extracts a 6.8 GB Qwen2.5-3B checkpoint, converts it to BFP8 and JIT-compiles (~100 s on the very first boot, ~50 s with cached weights but an empty kernel cache, ~17 s warm; measured 2026-09-13).
    - Not an OpenAI-compatible API; `GET /v1/models` is a stub so the tt-model ready card does not 404.
    - Validated on tt-metal `v0.78.0-dev20260820` (main `8b98410e730`), single p150a only.
    - GPU comparison: GPU (Qwen2.5-VL-3B in HF transformers) bf16 is 1.8× faster overall; on vision + prefill the p150a is slightly faster than fp32-strict GPU (88 vs 92 ms) and the gap is in decode (44.5 vs 83 tok/s). RTX 5090 rows (2026-09-14): same host, the port's own torch reference (same weights) run eagerly in PyTorch 2.11 cu128 with fp32 weights + autocast unless stated, no TensorRT; medians of 50 iterations after warm-up, H2D/D2H included; the p150a rows are the served bf16 fused path incl. upload/readback. p150a power was not measured, so no efficiency comparison is made. Full table: [`GPU_COMPARISON.md`](GPU_COMPARISON.md).

    ### Licensing

    - Weights: [nvidia/LocateAnything-3B](https://huggingface.co/nvidia/LocateAnything-3B), `other` ([NVIDIA license](https://huggingface.co/nvidia/LocateAnything-3B/blob/main/LICENSE), not OSI); fetched from upstream, not redistributed here.
    - Port and serving code (`code/locate_anything`, `code/scripts`): Apache-2.0 SPDX headers, from [changh95/tt-locate-anything](https://github.com/changh95/tt-locate-anything), published under the same upstream terms; the vendored [tt-metal](https://github.com/tenstorrent/tt-metal) overlay `code/models/tt_transformers/tt/mlp.py` is Apache-2.0.