{ "reproducibility": { "generated_at_utc": "20260817T154347Z", "dataset": "milo_benchmark v1.0", "simulator": "Simulator (real AI2-THOR/Unity, restarted between episodes)", "planners": [ "rule_based", "behavior_tree", "react" ], "react_llm_provider": "gemini", "react_llm_model": "gemini-flash-latest", "react_transient_retry_policy": { "max_extra_attempts": 2, "retry_trigger_markers": [ "503", "unavailable", "429", "resource_exhausted", "rate limit", "quota" ], "episodes_that_needed_a_retry": 23, "total_extra_attempts_used": 45, "episodes_still_failing_after_all_retries": 22 }, "memory_enabled": false }, "summary_by_planner": { "rule_based": { "overall": { "n": 25, "goal_success_rate": 0.96, "execution_success_rate": 0.96 }, "by_tier": { "tier1_locate": { "n": 10, "goal_success_rate": 1.0, "execution_success_rate": 1.0 }, "tier2_pickup": { "n": 10, "goal_success_rate": 1.0, "execution_success_rate": 1.0 }, "tier3_store": { "n": 5, "goal_success_rate": 0.8, "execution_success_rate": 0.8 } } }, "behavior_tree": { "overall": { "n": 25, "goal_success_rate": 0.96, "execution_success_rate": 0.96 }, "by_tier": { "tier1_locate": { "n": 10, "goal_success_rate": 1.0, "execution_success_rate": 1.0 }, "tier2_pickup": { "n": 10, "goal_success_rate": 1.0, "execution_success_rate": 1.0 }, "tier3_store": { "n": 5, "goal_success_rate": 0.8, "execution_success_rate": 0.8 } } }, "react": { "overall": { "n": 25, "goal_success_rate": 0.44, "execution_success_rate": 0.08 }, "by_tier": { "tier1_locate": { "n": 10, "goal_success_rate": 1.0, "execution_success_rate": 0.1 }, "tier2_pickup": { "n": 10, "goal_success_rate": 0.1, "execution_success_rate": 0.1 }, "tier3_store": { "n": 5, "goal_success_rate": 0.0, "execution_success_rate": 0.0 } } } }, "episodes": [ { "planner": "rule_based", "task_id": "milo-v1-fp1-t1a", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the mug.", "goal": "find", "object": "mug", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 134.86986800035083, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp1-t1b", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the tomato.", "goal": "find", "object": "tomato", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 159.76903499995387, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp1-t2a", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the apple.", "goal": "pick_up", "object": "apple", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 209.8083669998232, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp1-t2b", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the spoon.", "goal": "pick_up", "object": "spoon", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 221.480129000156, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp1-t3a", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier3_store", "instruction": "Put the bread away in the fridge.", "goal": "store", "object": "bread", "target": "fridge", "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 8, "plan_step_count": 8, "failure_cause": null, "wall_clock_ms": 922.80972799972, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp5-t1a", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the bowl.", "goal": "find", "object": "bowl", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 150.50396200012983, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp5-t1b", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the kettle.", "goal": "find", "object": "kettle", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 127.46781900023052, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp5-t2a", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the potato.", "goal": "pick_up", "object": "potato", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 183.12880899975426, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp5-t2b", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the pan.", "goal": "pick_up", "object": "pan", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 158.54242800014617, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp5-t3a", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier3_store", "instruction": "Put the mug away in the cabinet.", "goal": "store", "object": "mug", "target": "cabinet", "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 8, "plan_step_count": 8, "failure_cause": null, "wall_clock_ms": 866.2697129998378, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp201-t1a", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier1_locate", "instruction": "Find the laptop.", "goal": "find", "object": "laptop", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 239.07422999991468, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp201-t1b", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier1_locate", "instruction": "Find the vase.", "goal": "find", "object": "vase", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 179.67469599989272, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp201-t2a", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the newspaper.", "goal": "pick_up", "object": "newspaper", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 308.9128229999005, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp201-t2b", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the pillow.", "goal": "pick_up", "object": "pillow", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 396.58132000022306, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp201-t3a", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier3_store", "instruction": "Put the remote control away in the drawer.", "goal": "store", "object": "remotecontrol", "target": "drawer", "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 8, "plan_step_count": 8, "failure_cause": null, "wall_clock_ms": 1263.3034229997975, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp301-t1a", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier1_locate", "instruction": "Find the alarm clock.", "goal": "find", "object": "alarmclock", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 3306.682566999825, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp301-t1b", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier1_locate", "instruction": "Find the cell phone.", "goal": "find", "object": "cellphone", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 2980.515125000238, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp301-t2a", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the boots.", "goal": "pick_up", "object": "boots", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 147.34704500006046, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp301-t2b", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the CD.", "goal": "pick_up", "object": "cd", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 197.85034000005908, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp301-t3a", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier3_store", "instruction": "Put the book away in the drawer.", "goal": "store", "object": "book", "target": "drawer", "plan_success": true, "execution_success": false, "goal_success": false, "action_count": 8, "plan_step_count": 8, "failure_cause": "No valid positions to place object found", "wall_clock_ms": 4508.782035999957, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp401-t1a", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier1_locate", "instruction": "Find the towel.", "goal": "find", "object": "towel", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 394.1123040003731, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp401-t1b", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier1_locate", "instruction": "Find the candle.", "goal": "find", "object": "candle", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 109.60298100008004, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp401-t2a", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the soap bar.", "goal": "pick_up", "object": "soapbar", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 216.23726000007082, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp401-t2b", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the toilet paper.", "goal": "pick_up", "object": "toiletpaper", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 174.54068400002143, "llm_retry_attempts": 0 }, { "planner": "rule_based", "task_id": "milo-v1-fp401-t3a", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier3_store", "instruction": "Put the spray bottle away on the shelf.", "goal": "store", "object": "spraybottle", "target": "shelf", "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 6, "plan_step_count": 6, "failure_cause": null, "wall_clock_ms": 674.779596999997, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp1-t1a", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the mug.", "goal": "find", "object": "mug", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 139.701352999964, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp1-t1b", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the tomato.", "goal": "find", "object": "tomato", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 200.81045200004155, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp1-t2a", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the apple.", "goal": "pick_up", "object": "apple", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 337.0999069998106, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp1-t2b", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the spoon.", "goal": "pick_up", "object": "spoon", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 155.31059300019479, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp1-t3a", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier3_store", "instruction": "Put the bread away in the fridge.", "goal": "store", "object": "bread", "target": "fridge", "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 8, "plan_step_count": 8, "failure_cause": null, "wall_clock_ms": 817.3430079996251, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp5-t1a", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the bowl.", "goal": "find", "object": "bowl", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 150.820554999882, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp5-t1b", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the kettle.", "goal": "find", "object": "kettle", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 154.57777599976907, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp5-t2a", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the potato.", "goal": "pick_up", "object": "potato", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 202.00574399996185, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp5-t2b", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the pan.", "goal": "pick_up", "object": "pan", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 163.99158600006558, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp5-t3a", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier3_store", "instruction": "Put the mug away in the cabinet.", "goal": "store", "object": "mug", "target": "cabinet", "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 8, "plan_step_count": 8, "failure_cause": null, "wall_clock_ms": 823.9831179998873, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp201-t1a", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier1_locate", "instruction": "Find the laptop.", "goal": "find", "object": "laptop", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 179.55906199995297, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp201-t1b", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier1_locate", "instruction": "Find the vase.", "goal": "find", "object": "vase", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 158.53934500000832, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp201-t2a", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the newspaper.", "goal": "pick_up", "object": "newspaper", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 335.07921499995064, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp201-t2b", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the pillow.", "goal": "pick_up", "object": "pillow", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 344.94742200013206, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp201-t3a", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier3_store", "instruction": "Put the remote control away in the drawer.", "goal": "store", "object": "remotecontrol", "target": "drawer", "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 8, "plan_step_count": 8, "failure_cause": null, "wall_clock_ms": 877.9208089999884, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp301-t1a", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier1_locate", "instruction": "Find the alarm clock.", "goal": "find", "object": "alarmclock", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 3247.3547829999916, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp301-t1b", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier1_locate", "instruction": "Find the cell phone.", "goal": "find", "object": "cellphone", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 3204.4693449997794, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp301-t2a", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the boots.", "goal": "pick_up", "object": "boots", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 170.40390099964497, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp301-t2b", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the CD.", "goal": "pick_up", "object": "cd", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 278.0093179999312, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp301-t3a", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier3_store", "instruction": "Put the book away in the drawer.", "goal": "store", "object": "book", "target": "drawer", "plan_success": true, "execution_success": false, "goal_success": false, "action_count": 8, "plan_step_count": 8, "failure_cause": "No valid positions to place object found", "wall_clock_ms": 4378.925735999928, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp401-t1a", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier1_locate", "instruction": "Find the towel.", "goal": "find", "object": "towel", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 505.69612700019206, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp401-t1b", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier1_locate", "instruction": "Find the candle.", "goal": "find", "object": "candle", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 2, "plan_step_count": 2, "failure_cause": null, "wall_clock_ms": 106.44875500020134, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp401-t2a", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the soap bar.", "goal": "pick_up", "object": "soapbar", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 208.67725500011147, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp401-t2b", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the toilet paper.", "goal": "pick_up", "object": "toiletpaper", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 187.66953900012595, "llm_retry_attempts": 0 }, { "planner": "behavior_tree", "task_id": "milo-v1-fp401-t3a", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier3_store", "instruction": "Put the spray bottle away on the shelf.", "goal": "store", "object": "spraybottle", "target": "shelf", "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 6, "plan_step_count": 6, "failure_cause": null, "wall_clock_ms": 1016.9352389998494, "llm_retry_attempts": 0 }, { "planner": "react", "task_id": "milo-v1-fp1-t1a", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the mug.", "goal": "find", "object": "mug", "target": null, "plan_success": false, "execution_success": false, "goal_success": true, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: LLM request to provider 'gemini' failed after 3 attempt(s): HTTPSConnectionPool(host='generativelanguage.googleapis.com', port=443): Max retries exceeded with url: /v1beta/models/gemini-flash-latest:generateContent (Caused by NameResolutionError(\"HTTPSConnection(host='generativelanguage.googleapis.com', port=443): Failed to resolve 'generativelanguage.googleapis.com' ([Errno -3] Temporary failure in name resolution)\"))", "wall_clock_ms": 30032.81878900043, "llm_retry_attempts": 0 }, { "planner": "react", "task_id": "milo-v1-fp1-t1b", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the tomato.", "goal": "find", "object": "tomato", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 1, "plan_step_count": 1, "failure_cause": null, "wall_clock_ms": 13180.638001000261, "llm_retry_attempts": 0 }, { "planner": "react", "task_id": "milo-v1-fp1-t2a", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the apple.", "goal": "pick_up", "object": "apple", "target": null, "plan_success": true, "execution_success": true, "goal_success": true, "action_count": 3, "plan_step_count": 3, "failure_cause": null, "wall_clock_ms": 12092.732995000006, "llm_retry_attempts": 1 }, { "planner": "react", "task_id": "milo-v1-fp1-t2b", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the spoon.", "goal": "pick_up", "object": "spoon", "target": null, "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 503: {\n \"error\": {\n \"code\": 503,\n \"message\": \"This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\",\n \"status\": \"UNAVAILABLE\"\n }\n}\n", "wall_clock_ms": 7147.5733000006585, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp1-t3a", "scene": "FloorPlan1", "room_type": "kitchen", "difficulty_tier": "tier3_store", "instruction": "Put the bread away in the fridge.", "goal": "store", "object": "bread", "target": "fridge", "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 503: {\n \"error\": {\n \"code\": 503,\n \"message\": \"This model is currently experiencing high demand. Spikes in demand are usually temporary. Please try again later.\",\n \"status\": \"UNAVAILABLE\"\n }\n}\n", "wall_clock_ms": 9741.322859999855, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp5-t1a", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the bowl.", "goal": "find", "object": "bowl", "target": null, "plan_success": false, "execution_success": false, "goal_success": true, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 24.69896895s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"d... [truncated]", "wall_clock_ms": 508.71539099989604, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp5-t1b", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier1_locate", "instruction": "Find the kettle.", "goal": "find", "object": "kettle", "target": null, "plan_success": false, "execution_success": false, "goal_success": true, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 1.523806917s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"d... [truncated]", "wall_clock_ms": 508.2156799999211, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp5-t2a", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the potato.", "goal": "pick_up", "object": "potato", "target": null, "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 39.461904907s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 636.5799329996662, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp5-t2b", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the pan.", "goal": "pick_up", "object": "pan", "target": null, "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 17.175515623s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 433.5004699996716, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp5-t3a", "scene": "FloorPlan5", "room_type": "kitchen", "difficulty_tier": "tier3_store", "instruction": "Put the mug away in the cabinet.", "goal": "store", "object": "mug", "target": "cabinet", "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 54.39469611s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"d... [truncated]", "wall_clock_ms": 464.3448050001098, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp201-t1a", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier1_locate", "instruction": "Find the laptop.", "goal": "find", "object": "laptop", "target": null, "plan_success": false, "execution_success": false, "goal_success": true, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 29.705598629s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 670.9617759997855, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp201-t1b", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier1_locate", "instruction": "Find the vase.", "goal": "find", "object": "vase", "target": null, "plan_success": false, "execution_success": false, "goal_success": true, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 5.896598304s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"d... [truncated]", "wall_clock_ms": 361.89151000053243, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp201-t2a", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the newspaper.", "goal": "pick_up", "object": "newspaper", "target": null, "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 43.454455894s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 606.6959919999135, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp201-t2b", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the pillow.", "goal": "pick_up", "object": "pillow", "target": null, "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 21.178983433s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 453.3069869994506, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp201-t3a", "scene": "FloorPlan201", "room_type": "living_room", "difficulty_tier": "tier3_store", "instruction": "Put the remote control away in the drawer.", "goal": "store", "object": "remotecontrol", "target": "drawer", "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 58.981349052s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 386.145848000524, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp301-t1a", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier1_locate", "instruction": "Find the alarm clock.", "goal": "find", "object": "alarmclock", "target": null, "plan_success": false, "execution_success": false, "goal_success": true, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 24.870134139s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 540.2673059998051, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp301-t1b", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier1_locate", "instruction": "Find the cell phone.", "goal": "find", "object": "cellphone", "target": null, "plan_success": false, "execution_success": false, "goal_success": true, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 50.528793625s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 643.6687759996857, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp301-t2a", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the boots.", "goal": "pick_up", "object": "boots", "target": null, "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 15.889383712s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 637.0015139991665, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp301-t2b", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the CD.", "goal": "pick_up", "object": "cd", "target": null, "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 41.299406338s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 579.5582740001919, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp301-t3a", "scene": "FloorPlan301", "room_type": "bedroom", "difficulty_tier": "tier3_store", "instruction": "Put the book away in the drawer.", "goal": "store", "object": "book", "target": "drawer", "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 6.777791973s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"d... [truncated]", "wall_clock_ms": 581.8874960004905, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp401-t1a", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier1_locate", "instruction": "Find the towel.", "goal": "find", "object": "towel", "target": null, "plan_success": false, "execution_success": false, "goal_success": true, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 43.599717839s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 461.1683000002813, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp401-t1b", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier1_locate", "instruction": "Find the candle.", "goal": "find", "object": "candle", "target": null, "plan_success": false, "execution_success": false, "goal_success": true, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 20.48186455s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"d... [truncated]", "wall_clock_ms": 569.7539299999335, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp401-t2a", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the soap bar.", "goal": "pick_up", "object": "soapbar", "target": null, "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 57.236904411s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 571.2872880003488, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp401-t2b", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier2_pickup", "instruction": "Pick up the toilet paper.", "goal": "pick_up", "object": "toiletpaper", "target": null, "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 34.099233662s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 607.7411329997631, "llm_retry_attempts": 2 }, { "planner": "react", "task_id": "milo-v1-fp401-t3a", "scene": "FloorPlan401", "room_type": "bathroom", "difficulty_tier": "tier3_store", "instruction": "Put the spray bottle away on the shelf.", "goal": "store", "object": "spraybottle", "target": "shelf", "plan_success": false, "execution_success": false, "goal_success": false, "action_count": 0, "plan_step_count": 0, "failure_cause": "LLM call failed: Provider 'gemini' returned HTTP 429: {\n \"error\": {\n \"code\": 429,\n \"message\": \"You exceeded your current quota, please check your plan and billing details. For more information on this error, head to: https://ai.google.dev/gemini-api/docs/rate-limits. To monitor your current usage, head to: https://ai.dev/rate-limit. \\n* Quota exceeded for metric: generativelanguage.googleapis.com/generate_content_free_tier_requests, limit: 20, model: gemini-3.7-flash\\nPlease retry in 10.639217753s.\",\n \"status\": \"RESOURCE_EXHAUSTED\",\n \"... [truncated]", "wall_clock_ms": 426.64688199965894, "llm_retry_attempts": 2 } ], "totals": { "total_episodes": 75, "goal_succeeded": 59, "execution_succeeded": 50 } }