{
  "source_filename": "milo_benchmark_20260819T173809Z.json",
  "leaderboard_columns": [
    "planner",
    "goal_success",
    "execution_success",
    "tier1_locate",
    "tier2_pickup",
    "tier3_store",
    "avg_wall_clock_ms",
    "llm_calls_per_episode",
    "approx_tokens_per_episode",
    "llm_retries_per_episode"
  ],
  "leaderboard_headers": {
    "planner": "Planner",
    "goal_success": "Goal success (overall)",
    "execution_success": "Execution success (overall)",
    "tier1_locate": "tier1_locate",
    "tier2_pickup": "tier2_pickup",
    "tier3_store": "tier3_store",
    "avg_wall_clock_ms": "Avg wall-clock ms/episode",
    "llm_calls_per_episode": "LLM calls/episode",
    "approx_tokens_per_episode": "Approx tokens/episode",
    "llm_retries_per_episode": "LLM retries/episode"
  },
  "leaderboard_rows": [
    {
      "planner": "rule_based (deterministic, no LLM)",
      "goal_success": "24/25 (96%)",
      "execution_success": "24/25 (96%)",
      "tier1_locate": "10/10 (100%)",
      "tier2_pickup": "10/10 (100%)",
      "tier3_store": "4/5 (80%)",
      "avg_wall_clock_ms": "620",
      "llm_retries_per_episode": "n/a",
      "llm_calls_per_episode": "n/a",
      "approx_tokens_per_episode": "n/a"
    },
    {
      "planner": "behavior_tree (deterministic, no LLM)",
      "goal_success": "24/25 (96%)",
      "execution_success": "24/25 (96%)",
      "tier1_locate": "10/10 (100%)",
      "tier2_pickup": "10/10 (100%)",
      "tier3_store": "4/5 (80%)",
      "avg_wall_clock_ms": "633",
      "llm_retries_per_episode": "n/a",
      "llm_calls_per_episode": "n/a",
      "approx_tokens_per_episode": "n/a"
    },
    {
      "planner": "htn (deterministic, no LLM -- HTN task-network engine)",
      "goal_success": "24/25 (96%)",
      "execution_success": "24/25 (96%)",
      "tier1_locate": "10/10 (100%)",
      "tier2_pickup": "10/10 (100%)",
      "tier3_store": "4/5 (80%)",
      "avg_wall_clock_ms": "647",
      "llm_retries_per_episode": "n/a",
      "llm_calls_per_episode": "n/a",
      "approx_tokens_per_episode": "n/a"
    },
    {
      "planner": "react (LLM-driven)",
      "goal_success": "20/25 (80%)",
      "execution_success": "20/25 (80%)",
      "tier1_locate": "10/10 (100%)",
      "tier2_pickup": "10/10 (100%)",
      "tier3_store": "0/5 (0%)",
      "avg_wall_clock_ms": "5744",
      "llm_retries_per_episode": "0.00",
      "llm_calls_per_episode": "2.8",
      "approx_tokens_per_episode": "\u2014"
    }
  ],
  "source_caption": "Source: `milo_benchmark_20260819T173809Z.json` \u2014 Simulator (real AI2-THOR/Unity, restarted between episodes), planners: rule_based, behavior_tree, htn, react. `react` model: `qwen2.5:7b` via `qwen` (local Ollama, no external quota). `n/a` = not applicable to this planner (no LLM involved). `\u2014` = field not present in this source JSON (older runs don't have every column; the loader renders what's available rather than guessing).",
  "episodes": [
    {
      "title": "[SUCCESS] rule_based \u2014 milo-v1-fp1-t3a \u2014 Put the bread away in the fridge.",
      "planner": "rule_based",
      "task_id": "milo-v1-fp1-t3a",
      "reason_picked": "rule_based success on a tier3_store task (bread -> fridge).",
      "scene": "FloorPlan1",
      "room_type": "kitchen",
      "difficulty_tier": "tier3_store",
      "instruction": "Put the bread away in the fridge.",
      "goal": "store",
      "object": "bread",
      "target": "fridge",
      "plan_success": true,
      "execution_success": true,
      "goal_success": true,
      "wall_clock_ms": 745.4245300004914,
      "failure_cause": null,
      "llm_retry_attempts": 0,
      "model_label": "rule_based \u2014 deterministic symbolic planner, no LLM",
      "plan_trace": [
        "locate(bread)",
        "navigate(to=bread)",
        "pick_up(bread)",
        "navigate(to=fridge)",
        "open(fridge)  [only if target is openable]",
        "place(bread, fridge)",
        "close(fridge)  [only if opened above]"
      ],
      "screenshots": [
        "live-01-instruction-typed.png",
        "live-02-task-in-progress.png",
        "live-03-task-complete.png"
      ]
    },
    {
      "title": "[SUCCESS] behavior_tree \u2014 milo-v1-fp5-t3a \u2014 Put the mug away in the cabinet.",
      "planner": "behavior_tree",
      "task_id": "milo-v1-fp5-t3a",
      "reason_picked": "behavior_tree success on a different scene's tier3_store task (mug -> cabinet), for scene variety.",
      "scene": "FloorPlan5",
      "room_type": "kitchen",
      "difficulty_tier": "tier3_store",
      "instruction": "Put the mug away in the cabinet.",
      "goal": "store",
      "object": "mug",
      "target": "cabinet",
      "plan_success": true,
      "execution_success": true,
      "goal_success": true,
      "wall_clock_ms": 669.150754999464,
      "failure_cause": null,
      "llm_retry_attempts": 0,
      "model_label": "behavior_tree \u2014 deterministic BT composition of the same goal templates, no LLM",
      "plan_trace": [
        "locate(mug)",
        "navigate(to=mug)",
        "pick_up(mug)",
        "navigate(to=cabinet)",
        "open(cabinet)  [only if target is openable]",
        "place(mug, cabinet)",
        "close(cabinet)  [only if opened above]"
      ],
      "screenshots": []
    },
    {
      "title": "[SUCCESS] htn \u2014 milo-v1-fp401-t3a \u2014 Put the spray bottle away on the shelf.",
      "planner": "htn",
      "task_id": "milo-v1-fp401-t3a",
      "reason_picked": "htn success on a tier3_store task requiring real open/place decomposition (spray bottle -> shelf, non-openable target) -- a genuine HTN method-library expansion (DepositObject), not a second implementation of rule_based's control flow, reaching the identical plan shape.",
      "scene": "FloorPlan401",
      "room_type": "bathroom",
      "difficulty_tier": "tier3_store",
      "instruction": "Put the spray bottle away on the shelf.",
      "goal": "store",
      "object": "spraybottle",
      "target": "shelf",
      "plan_success": true,
      "execution_success": true,
      "goal_success": true,
      "wall_clock_ms": 693.6673519994656,
      "failure_cause": null,
      "llm_retry_attempts": 0,
      "model_label": "htn \u2014 deterministic Hierarchical Task Network engine (compound tasks, method library, recursive decomposition), no LLM",
      "plan_trace": [
        "locate(spraybottle)",
        "navigate(to=spraybottle)",
        "pick_up(spraybottle)",
        "navigate(to=shelf)",
        "open(shelf)  [only if target is openable]",
        "place(spraybottle, shelf)",
        "close(shelf)  [only if opened above]"
      ],
      "screenshots": []
    },
    {
      "title": "[SUCCESS] react \u2014 milo-v1-fp1-t2a \u2014 Pick up the apple.",
      "planner": "react",
      "task_id": "milo-v1-fp1-t2a",
      "reason_picked": "react's strongest tier: a genuine tier2_pickup success (apple), plan produced and executed for real.",
      "scene": "FloorPlan1",
      "room_type": "kitchen",
      "difficulty_tier": "tier2_pickup",
      "instruction": "Pick up the apple.",
      "goal": "pick_up",
      "object": "apple",
      "target": null,
      "plan_success": true,
      "execution_success": true,
      "goal_success": true,
      "wall_clock_ms": 6474.353801000007,
      "failure_cause": null,
      "llm_retry_attempts": 0,
      "model_label": "react \u2014 qwen2.5:7b via Ollama, local (no cloud API, no external quota)",
      "plan_trace": [
        "locate(apple)",
        "navigate(to=apple)",
        "pick_up(apple)"
      ],
      "screenshots": []
    },
    {
      "title": "[FAILED] react \u2014 milo-v1-fp1-t3a \u2014 Put the bread away in the fridge.",
      "planner": "react",
      "task_id": "milo-v1-fp1-t3a",
      "reason_picked": "react's typical tier3_store failure mode: mis-sequenced action rejected by the precondition validator, not an infrastructure error.",
      "scene": "FloorPlan1",
      "room_type": "kitchen",
      "difficulty_tier": "tier3_store",
      "instruction": "Put the bread away in the fridge.",
      "goal": "store",
      "object": "bread",
      "target": "fridge",
      "plan_success": false,
      "execution_success": false,
      "goal_success": false,
      "wall_clock_ms": 9746.627000999979,
      "failure_cause": "LLM produced malformed output after 3 attempt(s): Proposed action 'place' violates precondition(s): holding_target, container_ready.",
      "llm_retry_attempts": 0,
      "model_label": "react \u2014 qwen2.5:7b via Ollama, local (no cloud API, no external quota)",
      "plan_trace": [
        "locate(bread)",
        "navigate(to=bread)",
        "pick_up(bread)",
        "navigate(to=fridge)",
        "open(fridge)  [only if target is openable]",
        "place(bread, fridge)  <-- rejected: LLM produced malformed output after 3 attempt(s): Proposed action 'place' violates precondition(s): holding_target, container_ready."
      ],
      "screenshots": []
    },
    {
      "title": "[FAILED] rule_based \u2014 milo-v1-fp301-t3a \u2014 Put the book away in the drawer.",
      "planner": "rule_based",
      "task_id": "milo-v1-fp301-t3a",
      "reason_picked": "The known, still-failing FloorPlan301 book->drawer case: a real AI2-THOR geometry limit (the drawer has no room for this book), not a planner defect. Documented in the dataset README's 'Known limitations' and reproduced identically by all three symbolic planners (rule_based, behavior_tree, htn).",
      "scene": "FloorPlan301",
      "room_type": "bedroom",
      "difficulty_tier": "tier3_store",
      "instruction": "Put the book away in the drawer.",
      "goal": "store",
      "object": "book",
      "target": "drawer",
      "plan_success": true,
      "execution_success": false,
      "goal_success": false,
      "wall_clock_ms": 3476.1707779998687,
      "failure_cause": "No valid positions to place object found",
      "llm_retry_attempts": 0,
      "model_label": "rule_based \u2014 deterministic symbolic planner, no LLM",
      "plan_trace": [
        "locate(book)",
        "navigate(to=book)",
        "pick_up(book)",
        "navigate(to=drawer)",
        "open(drawer)  [only if target is openable]",
        "place(book, drawer)  <-- execution failed: No valid positions to place object found"
      ],
      "screenshots": []
    }
  ]
}