tt-model.yaml

# SPDX-License-Identifier: Apache-2.0
# tt-model-manager container manifest (schema 5.1): FRNet (Autoware lidar_frnet) on one Blackhole p150.
# Conventions: research/BUNDLE_CONVENTIONS.md (workspace) -- section 3 explains every field.
#
# Run EVERY tt-model command FROM THIS DIRECTORY: source.tt_metal and extra_code[].root resolve against the
# process CWD, not this file; --out must point OUTSIDE this directory (package deletes <out>/<name> first).
#   python -c "from tt_kernel.container_manifest import load_container_manifest as L; L('tt-model.yaml', check_sources=True)"
#   sg docker -c "tt-model package --container tt-model.yaml --out /home/ubuntu/experiments/tt-models/build"
#   (serve + smoke + stop under bin/devrun, once per serve profile: code/scripts/container_smoke.sh, SERVING.md 2)
#   HF_TOKEN=<from env> tt-model push /home/ubuntu/experiments/tt-models/build/frnet-p150 --publish
schema: "5.1"

repo: "changh95/frnet-p150"
name: "frnet-p150"

# A POINTER to the network Autoware deploys (ansible artifacts role: lidar_frnet @ v2.0), never baked into
# the image. Pinned to the COMMIT the tag points to -- HfApi().model_info(repo, revision="v2.0").sha, not
# list_repo_refs().tags[].target_commit (that is the annotated-tag object) -- because tags can move and `package`
# does not resolve weights refs. `serve` / `pull --with-weights` pre-download exactly these files at this sha; the
# launcher exports the sha as TT_MODEL_WEIGHTS_REVISION (and serve.env repeats it as TT_WEIGHTS_REVISION).
weights:
  repo: "AutowareFoundation/lidar_frnet"
  revision: "a53a1c11b0b28d31fd50a03abef5db0e830a5b3c"
  allow_patterns: ["frnet_*.onnx", "ml_package_frnet_*.param.yaml", "deploy_metadata.yaml"]

kind: tt-dit-server        # the model's own FastAPI app under uvicorn (no vLLM); see the conventions doc
arch: blackhole

source:
  # tt-metal main 44d6650 (v0.80.0-dev20261006) WITH patches/tt-metal-eth-dispatch.patch applied: a dirty tree,
  # packaged as-is (the image is built from it, so it carries ETH dispatch). torch 2.11.0 pin read from it.
  tt_metal: "/home/ubuntu/experiments/tt-models/tt-metal"
  # Schema minimum (>= 1 path relative to the tt-metal tree). The port imports nothing from tt-metal's models/.
  code:
    - models/common/lightweightmodule.py
  # The port itself. code/ on the Hub becomes EXACTLY this set (+ the file above) at every push, and it is
  # COPY'd over /opt/tt-metal/<path> in the image (PYTHONPATH=/opt/tt-metal) BEFORE tt-metal's C++ build and its
  # `uv pip install -e /opt/tt-metal`. So never list a file that tt-metal's root also has and its build reads
  # (pyproject.toml, setup.py, setup.cfg, MANIFEST.in, README.md, uv.toml, CMakeLists.txt, build_metal.sh, ...) or
  # anything under its build trees (tt_metal/, ttnn/, cmake/, tools/, ...): check_bundle.py refuses them. The pip
  # project is the repo-ROOT pyproject.toml (`pip install -e .`), published as an overlay file, never from here.
  # conftest.py is deliberate: it replaces tt-metal's root conftest.py in the builder stage, which no build step
  # reads, and the runtime stage never copies tt-metal's, so in the image it is the only root conftest.
  extra_code:
    - root: code
      paths:
        - "tt_frnet"
        - scripts
        - conftest.py
        - PYTHON.md
  ubuntu: "22.04"
  python: "3.12"

runtime:
  app: "tt_frnet.server.app:app"
  mesh_shape_env: TT_MESH_SHAPE
  # On top of the kind defaults (fastapi, uvicorn, pydantic>=2, pillow) and the auto-pinned torch==2.11.0+cpu.
  # numpy stays < 2 (ttnn). Add per model: scipy, opencv-python-headless>=4.9,<4.12, ...
  packages:
    - "numpy>=1.24.4,<2"
    - huggingface_hub
    - safetensors
    - pyyaml
    - "onnx>=1.17,<2"
  # After the first green build: copy <out>/frnet-p150/requirements.lock next to this file, check it, and
  # uncomment (resolved relative to THIS file):
  # lock: requirements.lock

serve:
  port: 20000
  hardware: p150
  mesh_device: P150
  env:
    TT_WEIGHTS_REVISION: "a53a1c11b0b28d31fd50a03abef5db0e830a5b3c"      # older tt-model exports no revision; the app also reads TT_MODEL_WEIGHTS_REVISION
    TT_METAL_VISIBLE_DEVICES: "0"
    # The weights repos are public and ungated: never send a token. serve mounts the host HF cache (which may hold a
    # stored token file) at /hf, so tell huggingface_hub not to pick one up implicitly (PACKAGING_PILOT.md problem 10).
    HF_HUB_DISABLE_IMPLICIT_TOKEN: "1"
    # Pinned dispatch: ETH, the 12x10 grid every published number uses (D14). Profiles must not override it:
    # check_bundle.py refuses any *_DISPATCH other than eth, and the container smoke fails unless /info reports
    # dispatch=eth, grid=12x10 and these pins.
    "FRNET_DISPATCH": "eth"
    "FRNET_NUM_CQS": "1"
    "FRNET_VARIANT": "ot128"
    # Knob defaults of tt_frnet.tt.model.KNOBS (asserted by tests/test_bundle_host.py): the two-term bf16 point island
    # (FRNET_POINT_TERMS=1, fp32 x fp32, misses the OT128 e2e gates on nuScenes frames: PORT_LOG.md "M1 device results")
    "FRNET_POINT_TERMS": "2"
    # fp32 image-branch maps (conv operands bf16): with bf16 maps every conv / HardSwish / add output is rounded and
    # OT128 misses the e2e argmax gate on nuScenes 0916 kf40 (98.28 %; PORT_LOG.md "M3 precision follow-up")
    "FRNET_MAP_DTYPE": "float32"
    "FRNET_CONV_TERMS": "1"

# OPTIONAL: load-time variants of ONE image as serve profiles (`tt-model serve <repo> --profile <name>`), e.g. the
# CenterPoint base / tiny pair. Each profile deep-merges over serve: (hardware / mesh_device are inherited) and is
# smoke-tested on its own (`container_smoke.sh <staged> <port> --profile <name>`).
serve_profiles:
  - name: ot128
    description: FRNet for the Hesai OT128 (Autoware's default sensor_model; 1024x128 frustum grid)
    env:
      "FRNET_VARIANT": ot128
  - name: qt128
    description: FRNet for the Hesai QT128 (256x128 frustum grid, +-52.6 deg FOV)
    env:
      "FRNET_VARIANT": qt128
default_profile: ot128

# Build-time assertions run INSIDE the finished image as uid 1000, no device, no weights.
verify:
  # the image must carry the ETH-dispatch patch (12x10 grid): fail the build otherwise (PLAN.md 6.2 step 1)
  - "assert 'single_chip_arch_1cq_no_dispatch_s' in open('/opt/tt-metal/tt_metal/impl/dispatch/topology.cpp').read()"
  - "from tt_frnet.ttaw.device import eth_dispatch_patch_present as p; assert p(), 'ttaw does not see the ETH patch'"
  - "import tt_frnet.server.app as a; assert a.app"
  - "from tt_frnet.server.app import parse_mesh_shape as p; assert p('1x1') == p('(1, 1)') == p('1,1') == (1, 1)"
  - "import numpy; assert int(numpy.__version__.split('.')[0]) < 2, numpy.__version__"
  - "import onnx, yaml, huggingface_hub, safetensors, PIL"
  - "from tt_frnet.api import FRNet; assert FRNet.DEFAULT_REVISION == 'a53a1c11b0b28d31fd50a03abef5db0e830a5b3c'"
  - "import ttnn; assert hasattr(ttnn, 'DispatchCoreConfig') and hasattr(ttnn, 'begin_trace_capture') and hasattr(ttnn, 'execute_trace') and hasattr(ttnn, 'release_trace')"
  # the shipped samples (synthetic scans, code/scripts/make_sample.py), their calibration presets and stored CPU
  # references: the container smoke posts the sample of the served profile and compares with its reference
  - "from pathlib import Path; d = Path('/opt/tt-metal/tt_frnet'); assert all((d / 'samples' / f'synthetic_{v}.npz').is_file() and (d / 'samples' / f'synthetic_{v}.{v}.reference.json').is_file() and (d / 'calib' / f'synthetic_{v}.json').is_file() for v in ('ot128', 'qt128'))"
  # the K1 segment_reduce kernel source (ttaw.ops.segment; JIT-compiled at run time)
  - "from pathlib import Path; assert Path('/opt/tt-metal/tt_frnet/ttaw/ops/kernels/segment_reduce_dm.cpp').is_file()"
  - "from tt_frnet.ttaw.ops import kernel_path; kernel_path('segment_reduce_dm.cpp')"

card:
  # Lede under the title of the GENERATED card (tt-model package). The published README.md is the hand-written
  # card of this directory (overlaid before push); keep the facts here and there identical.
  description: >
    FRNet (Autoware lidar_frnet): the network Autoware deploys (autoware_lidar_frnet, weights AutowareFoundation/lidar_frnet
    v2.0) running on one Tenstorrent Blackhole p150 via tt-nn: one LiDAR sweep in the sensor frame (x, y, z, raw 0-255 intensity) in, a class id and probability for every point (27 T4 classes; Autoware's interpolated points included) out.
  architecture: "FRNet: frustum feature encoder (point MLP 8->64->128->256->256, per-cell max, 256->16) + frustum-range 2D CNN (stem, 4 residual stages with point-pixel fusion and attention, align-corners decoder) + per-point head (27-class softmax); one ONNX graph per sensor profile"
  status: Community port (first release, baseline; optimization pending), validated against the fp32 CPU reference on public driving data
  intended_use: >
    LiDAR 3D semantic segmentation for Autoware-style sensor setups (one Hesai OT128 or QT128 sweep in the sensor frame) on one Blackhole p150, as a drop-in for the TensorRT node's network.
  out_of_scope_use: >
    Safety-critical driving decisions (this is not a certified Autoware component); multi-chip meshes; batch > 1;
    inputs outside the Autoware sensor configuration the weights were trained for. 
  quickstart: |
    The first start compiles the kernels and captures the traces (minutes on a cold cache, seconds later); the
    server is ready when its log shows `Application startup complete`. Build a request from a file with
    `python3 code/tt_frnet/server/client.py --points code/tt_frnet/samples/synthetic_ot128.npz --calib-preset synthetic_ot128 --out req.json` (your own cloud: `--calib my_calib.json`); the QT128 network is the `qt128` profile (`tt-model serve --profile qt128 <repo>`), then
    `curl -s localhost:20000/predict -H 'Content-Type: application/json' -d @req.json`.
  usage: |
    `POST /predict` (JSON): `points` (base64 `.bin` / `.npy` / `.npz` / `.pcd`, sensor frame, fields x, y, z, intensity 0-255); `calibration` (`T_base_link_from_sensor` for the ego crop box, or `{"preset": name}`; optional `ego_crop_box`, `null` disables the box); optional `params` (`class_probability_threshold` 0.05, `filter_classes` "drivable_surface"), `output_format`. `GET /health`, `GET /info`, `GET /v1/models` (stub).
    Full request / response contract: `SERVING.md` section 3.
  performance: |
    | Metric | Performance |
    |---|---:|
    | Class agreement with the fp32 CPU reference, shipped sample | OT128 99.55 % · QT128 99.62 % of the points |
    | Class agreement, 113 public frames per profile (PandaSet 32, nuScenes mini_val 81) | OT128 99.66 % / 99.19 % pooled (worst frame 98.37 %) · QT128 99.29 % / 99.44 % (worst 98.10 %) |
    | Frozen device gates (17 OT128 / 5 QT128 frames; PCC >= 0.998, agreement >= 98.5 %) | all pass; OT128 min agreement 98.907 % (nuScenes 0916 kf40, 0.41 pt margin) |
    | Python `model()` call, shipped sample | OT128 292.6 ms · QT128 221.1 ms p50 |
    | Served `/predict` `timing_ms.total`, shipped sample | OT128 310.4 ms · QT128 228.9 ms median |
    | Device trace, one forward / back-to-back | OT128 210.02 / 209.95 ms · QT128 133.53 / 133.43 ms |
    | `from_pretrained` load, empty / warm JIT cache | OT128 156.6 / 7.5 s · QT128 151.9 / 6.6 s |

    ETH dispatch, 1 CQ, 12x10 grid, batch 1, 2026-10-09; details in README.md "Demo & Performances".
  limitations: |
    - First release: baseline port, optimization pending (OPT_REPORT.md); the point-wise layers kept at two-term bf16 precision are 63 % (OT128) / 75 % (QT128) of the device time.
    - Narrow accuracy margin: 98.907 % agreement on the hardest gated frame (gate 98.5 %); 3 of 226 frame-profile pairs of the wider public set fall to 98.10-98.499 % (nuScenes scene-0916, out of domain).
    - FRNet is an opt-in, standalone Autoware node; this bundle is not a ROS 2 node and not a certified Autoware component.
    - Trained on T4 data from Hesai OT128 / QT128 sensors (not public); other sensors (PandaSet Pandar64, nuScenes HDL-32E) show a domain gap in the reference itself.
    - Batch 1, one sweep per request, one variant per process; fixed device shapes per profile (160,000 / 120,000 points, 60,000 cells).
    - Single p150 only (ETH dispatch patch; no multi-chip mesh); dispatch=worker is an A/B opt-in the numbers do not cover.
    - Not an OpenAI-compatible API; p150 power not measured.
  risks: >
    Device numerics: per-point probabilities differ slightly from the fp32 reference (see the agreement figures);
    points near a class boundary can change class. Validated against the fp32 reference on public driving data
    (no public OT128 / QT128 data exists); the mapped-label mIoU is a sanity check, not an accuracy claim.
  licensing: |
    - Weights: [AutowareFoundation/lidar_frnet](https://huggingface.co/AutowareFoundation/lidar_frnet) at tag `v2.0` (commit `a53a1c11b0b28d31fd50a03abef5db0e830a5b3c`), Apache-2.0 per its model card; not redistributed here. Trained by TIER IV with AWML on about 16,000 frames of the T4Dataset (Hesai OT128 / QT128 sensors; not public).
    - Pre-/post-processing ported from autoware_universe `perception/autoware_lidar_frnet` @ `9ceaccf026c31ffc5319bc9eeb4bd7bede0af3fd` (Apache-2.0).
    - Port and serving code (`code/`): Apache-2.0. Shipped samples: synthetic scans generated by `code/scripts/make_sample.py` (Apache-2.0).
    - Demo media: PandaSet renders (CC BY 4.0 + PandaSet Dataset Terms, attribution in media/ATTRIBUTION.md) and nuScenes renders (`*_NC`, CC BY-NC-SA 4.0, non-commercial).
  related: |
    Other Autoware models on Blackhole: the "Autoware" collection of https://huggingface.co/changh95
  license:
    id: apache-2.0
  pipeline_tag: "image-segmentation"
  base_model:
    - "AutowareFoundation/lidar_frnet"