papers.jsonl

{"key": "10778628", "title": "JARVIS-1: Open-World Multi-Task Agents With Memory-Augmented Multimodal Language Models", "venue": "IEEE Transactions on Pattern Analysis & Machine Intelligence", "date": "2023-11", "url": "https://doi.ieeecomputersociety.org/10.1109/TPAMI.2024.3511593", "arxiv_id": "2311.05997", "code": "https://github.com/CraftJarvis/JARVIS-1", "citations": 206, "github_stars": 410, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "11334583", "title": "LongCodeZip: Compress Long Context for Code Language Models", "venue": "2025 40th IEEE/ACM International Conference on Automated Software Engineering (ASE)", "date": "2025-10", "url": "https://doi.org/10.1109/ase63991.2025.00020", "arxiv_id": "2510.00446", "code": "https://github.com/YerbaPage/LongCodeZip", "citations": 42, "github_stars": 164, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "anokhin2025herobench", "title": "HeroBench: A Benchmark for Long-Horizon Planning and Structured Reasoning in Virtual Worlds", "venue": "arXiv preprint arXiv:2508.12782", "date": "2025-08", "url": "https://arxiv.org/abs/2508.12782", "arxiv_id": "2508.12782", "code": "https://github.com/stefanrer/HeroBench", "citations": 6, "github_stars": 14, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "bfcl2025", "title": "The Berkeley Function Calling Leaderboard (BFCL): From tool use to agentic evaluation of large language models", "venue": "Proceedings of the 42nd International Conference on Machine Learning", "date": "2025", "url": "https://proceedings.mlr.press/v267/patil25a.html", "arxiv_id": "", "code": "https://github.com/ShishirPatil/gorilla", "citations": 435, "github_stars": 13007, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "bonatti2025windows", "title": "Windows Agent Arena: Evaluating Multi-Modal OS Agents at Scale", "venue": "International Conference on Machine Learning", "date": "2024-09", "url": "https://arxiv.org/abs/2409.08264", "arxiv_id": "2409.08264", "code": "https://github.com/microsoft/WindowsAgentArena", "citations": 194, "github_stars": 889, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "buildbench2025", "title": "BuildBench: Benchmarking LLM Agents on Compiling Real-World Open-Source Software", "venue": "arXiv", "date": "2025-09", "url": "https://arxiv.org/abs/2509.25248", "arxiv_id": "2509.25248", "code": "", "citations": 1, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "cai2026mossselfevolutionsourcelevelrewriting", "title": "MOSS: Self-Evolution through Source-Level Rewriting in Autonomous Agent Systems", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.22794", "arxiv_id": "2605.22794", "code": "https://github.com/hkgai-official/Moss", "citations": 5, "github_stars": 21, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "cemri2026multi", "title": "Why Do Multi-Agent LLM Systems Fail?", "venue": "Advances in Neural Information Processing Systems", "date": "2025-03", "url": "https://proceedings.neurips.cc/paper_files/paper/2025/hash/b1041e52d3be19f0a9bc491657488e4a-Abstract-Datasets_and_Benchmarks_Track.html", "arxiv_id": "2503.13657", "code": "https://github.com/multi-agent-systems-failure-taxonomy/MAST", "citations": 535, "github_stars": 410, "collections": ["benchmarks", "harness-design"], "sections": ["evidence/benchmarks", "targets/harness"]}
{"key": "chan2025mlebench", "title": "MLE-bench: Evaluating machine learning agents on machine learning engineering", "venue": "ICLR 2025", "date": "2024-10", "url": "https://arxiv.org/abs/2410.07095", "arxiv_id": "2410.07095", "code": "https://github.com/openai/mle-bench", "citations": 360, "github_stars": 1716, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "chen2024spin", "title": "SPIN: Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models", "venue": "Proceedings of the 41st International Conference on Machine Learning", "date": "2024", "url": "https://proceedings.mlr.press/v235/chen24j.html", "arxiv_id": "", "code": "https://github.com/uclaml/SPIN", "citations": 621, "github_stars": 1254, "collections": [], "sections": ["targets/weights"]}
{"key": "chen2025iterresearch", "title": "IterResearch: Rethinking Long-Horizon Agents with Interaction Scaling", "venue": "arXiv preprint arXiv:2511.07327", "date": "2025-11", "url": "https://arxiv.org/abs/2511.07327", "arxiv_id": "2511.07327", "code": "https://github.com/Alibaba-NLP/DeepResearch", "citations": 17, "github_stars": 19873, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "chen2025loop", "title": "Reinforcement Learning for Long-Horizon Interactive LLM Agents", "venue": "arXiv preprint arXiv:2502.01600", "date": "2025-02", "url": "https://arxiv.org/abs/2502.01600", "arxiv_id": "2502.01600", "code": "", "citations": 99, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "chen2025mlrbench", "title": "MLR-Bench: Evaluating AI Agents on Open-Ended Machine Learning Research", "venue": "arXiv preprint arXiv:2505.19955", "date": "2025-05", "url": "https://arxiv.org/abs/2505.19955", "arxiv_id": "2505.19955", "code": "https://github.com/chchenhui/mlrbench", "citations": 49, "github_stars": 34, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "chen2026agent", "title": "Agent^2 RL-Bench: Can LLM Agents Engineer Agentic RL Post-Training?", "venue": "arXiv preprint arXiv:2604.10547", "date": "2026-04", "url": "https://arxiv.org/abs/2604.10547", "arxiv_id": "2604.10547", "code": "https://github.com/microsoft/RD-Agent", "citations": 3, "github_stars": 14332, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "chen2026aiscientist", "title": "Toward autonomous long-horizon engineering for ML research", "venue": "arXiv", "date": "2026-04", "url": "https://arxiv.org/abs/2604.13018", "arxiv_id": "2604.13018", "code": "https://github.com/AweAI-Team/AiScientist", "citations": 8, "github_stars": 145, "collections": ["benchmarks", "harness-design"], "sections": ["targets/substrate"]}
{"key": "chen2026harnessxcomposableadaptiveevolvable", "title": "HarnessX: A Composable, Adaptive, and Evolvable Agent Harness Foundry", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.14249", "arxiv_id": "2606.14249", "code": "", "citations": 15, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "chen2026knowu", "title": "KnowU-Bench: Towards Interactive, Proactive, and Personalized Mobile Agent Evaluation", "venue": "arXiv preprint arXiv:2604.08455", "date": "2026-04", "url": "https://arxiv.org/abs/2604.08455", "arxiv_id": "2604.08455", "code": "https://github.com/ZJU-REAL/KnowU-Bench", "citations": 16, "github_stars": 75, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "chen2026pastprologueplugincontroller", "title": "The Past Is Prologue: A Plug-in Controller for Selective Updates in Sequentially Evolving LLM Memory", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.31121", "arxiv_id": "2606.31121", "code": "", "citations": 1, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "chen2026recursiveselfimprovementaibounded", "title": "Recursive Self-Improvement in AI: From Bounded Self-Refinement to Autonomous Research Loops", "venue": "—", "date": "2026-07", "url": "https://arxiv.org/abs/2607.07663", "arxiv_id": "2607.07663", "code": "", "citations": 6, "github_stars": 0, "collections": ["harness-design"], "sections": ["analyses"]}
{"key": "chhikara2025mem0", "title": "Mem0: Building Production-Ready AI Agents with Scalable Long-Term Memory", "venue": "European Conference on Artificial Intelligence (ECAI)", "date": "2025-04", "url": "https://doi.org/10.3233/faia251160", "arxiv_id": "2504.19413", "code": "https://github.com/mem0ai/mem0", "citations": 567, "github_stars": 64001, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "choudhury2025processrewardmodelsllm", "title": "Process Reward Models for LLM Agents: Practical Framework and Directions", "venue": "arXiv", "date": "2025-02", "url": "https://arxiv.org/abs/2502.10325", "arxiv_id": "2502.10325", "code": "https://github.com/sanjibanc/agent_prm", "citations": 79, "github_stars": 59, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "cirepairbench2026", "title": "CI-Repair-Bench: A Repository-Aware Benchmark for Automated Patch Validation via CI Workflows", "venue": "arXiv", "date": "2026-04", "url": "https://arxiv.org/abs/2604.27148", "arxiv_id": "2604.27148", "code": "https://github.com/RabeyaMuna/CI-REPAIR-BENCH", "citations": 0, "github_stars": 1, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "clitoolbench2026", "title": "Evaluating LLM-Based 0-to-1 Software Generation in End-to-End CLI Tool Scenarios", "venue": "arXiv", "date": "2026-04", "url": "https://arxiv.org/abs/2604.06742", "arxiv_id": "2604.06742", "code": "https://github.com/kinesiatricssxilm14/CLI-Tool-Bench", "citations": 1, "github_stars": 2, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "dataenvgym2025", "title": "DataEnvGym: Data Generation Agents in Teacher Environments with Student Feedback", "venue": "International Conference on Learning Representations", "date": "2024-10", "url": "https://arxiv.org/abs/2410.06215", "arxiv_id": "2410.06215", "code": "https://github.com/codezakh/DataEnvGym", "citations": 20, "github_stars": 34, "collections": [], "sections": ["targets/data"]}
{"key": "deepswe2026", "title": "DeepSWE: Measuring Frontier Coding Agents on Original, Long-Horizon Engineering Tasks", "venue": "arXiv preprint arXiv:2607.07946", "date": "2026-07", "url": "https://arxiv.org/abs/2607.07946", "arxiv_id": "2607.07946", "code": "https://github.com/datacurve-ai/deep-swe", "citations": 13, "github_stars": 1488, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "deng2023mind2web", "title": "Mind2Web: Towards a Generalist Agent for the Web", "venue": "Advances in Neural Information Processing Systems", "date": "2023-06", "url": "https://arxiv.org/abs/2306.06070", "arxiv_id": "2306.06070", "code": "https://github.com/OSU-NLP-Group/Mind2Web", "citations": 1370, "github_stars": 1021, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "deng2026swemilestone", "title": "SWE-Milestone: Evaluating AI Agents on Continuous Software Evolution", "venue": "International Conference on Machine Learning", "date": "2026-03", "url": "https://arxiv.org/abs/2603.13428", "arxiv_id": "2603.13428", "code": "https://github.com/DeepCommit-ai/SWE-Milestone", "citations": 6, "github_stars": 70, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "dong2026longhorizon", "title": "Towards Long-Horizon Agents: A Survey", "venue": "Preprints", "date": "2026", "url": "https://doi.org/10.20944/preprints202607.1328.v1", "arxiv_id": "", "code": "", "citations": 0, "github_stars": 0, "collections": [], "sections": ["analyses"]}
{"key": "du2025deepresearch", "title": "DeepResearch Bench: A Comprehensive Benchmark for Deep Research Agents", "venue": "arXiv preprint arXiv:2506.11763", "date": "2025-06", "url": "https://arxiv.org/abs/2506.11763", "arxiv_id": "2506.11763", "code": "https://github.com/Ayanami0730/deep_research_bench", "citations": 212, "github_stars": 816, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "erdogan2025planandact", "title": "Plan-and-Act: Improving Planning of Agents for Long-Horizon Tasks", "venue": "International Conference on Machine Learning", "date": "2025-03", "url": "https://arxiv.org/abs/2503.09572", "arxiv_id": "2503.09572", "code": "https://github.com/SqueezeAILab/plan-and-act", "citations": 197, "github_stars": 45, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "fang-etal-2026-memp", "title": "Memp: Exploring Agent Procedural Memory", "venue": "Findings of the Association for Computational Linguistics: ACL 2026", "date": "2025-08", "url": "https://aclanthology.org/2026.findings-acl.866/", "arxiv_id": "2508.06433", "code": "https://github.com/zjunlp/MemP", "citations": 62, "github_stars": 35, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "featurebench2026", "title": "FeatureBench: Benchmarking Agentic Coding for Complex Feature Development", "venue": "arXiv", "date": "2026-02", "url": "https://arxiv.org/abs/2602.10975", "arxiv_id": "2602.10975", "code": "https://github.com/LiberCoders/FeatureBench", "citations": 26, "github_stars": 87, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "feng2025gigpo", "title": "Group-in-Group Policy Optimization for LLM Agent Training", "venue": "Advances in Neural Information Processing Systems", "date": "2025-05", "url": "https://arxiv.org/abs/2505.10978", "arxiv_id": "2505.10978", "code": "https://github.com/langfengQ/verl-agent", "citations": 375, "github_stars": 2250, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "frontierswe2026", "title": "FrontierSWE", "venue": "Proximal Blog", "date": "2026", "url": "https://frontierswe.com/blog", "arxiv_id": "", "code": "https://github.com/Proximal-Labs/frontier-swe", "citations": 0, "github_stars": 219, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "gaosurvey", "title": "A Survey of Self-Evolving Agents: What, When, How, and Where to Evolve on the Path to Artificial Super Intelligence", "venue": "TMLR 2026", "date": "2025-07", "url": "https://arxiv.org/abs/2507.21046", "arxiv_id": "2507.21046", "code": "https://github.com/CharlesQ9/Self-Evolving-Agents", "citations": 99, "github_stars": 1300, "collections": [], "sections": ["analyses"]}
{"key": "gonzalezpumariega2025robotouille", "title": "Robotouille: An Asynchronous Planning Benchmark for LLM Agents", "venue": "International Conference on Learning Representations", "date": "2025-02", "url": "https://arxiv.org/abs/2502.05227", "arxiv_id": "2502.05227", "code": "https://github.com/portal-cornell/robotouille", "citations": 37, "github_stars": 46, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "gou2024critic", "title": "CRITIC: Large Language Models Can Self-Correct with Tool-Interactive Critiquing", "venue": "International Conference on Learning Representations", "date": "2023-05", "url": "https://arxiv.org/abs/2305.11738", "arxiv_id": "2305.11738", "code": "https://github.com/microsoft/ProphetNet", "citations": 863, "github_stars": 746, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "gou2025mind2web", "title": "Mind2Web 2: Evaluating Agentic Search with Agent-as-a-Judge", "venue": "arXiv preprint arXiv:2506.21506", "date": "2025-06", "url": "https://arxiv.org/abs/2506.21506", "arxiv_id": "2506.21506", "code": "https://github.com/OSU-NLP-Group/Mind2Web-2", "citations": 68, "github_stars": 114, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "gulati2026askearlyasklate", "title": "Ask Early, Ask Late, Ask Right: When Does Clarification Timing Matter for Long-Horizon Agents?", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.07937", "arxiv_id": "2605.07937", "code": "", "citations": 2, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "guo2026questionansweringtaskcompletion", "title": "From Question Answering to Task Completion: A Survey on Agent System and Harness Design", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.20683", "arxiv_id": "2606.20683", "code": "", "citations": 3, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "gutierrez2024hipporag", "title": "HippoRAG: Neurobiologically Inspired Long-Term Memory for Large Language Models", "venue": "Advances in Neural Information Processing Systems", "date": "2024-05", "url": "https://arxiv.org/abs/2405.14831", "arxiv_id": "2405.14831", "code": "https://github.com/OSU-NLP-Group/HippoRAG", "citations": 328, "github_stars": 3963, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "han2026robocerebra", "title": "RoboCerebra: A Large-scale Benchmark for Long-horizon Robotic Manipulation Evaluation", "venue": "Advances in Neural Information Processing Systems", "date": "2025-06", "url": "https://arxiv.org/abs/2506.06677", "arxiv_id": "2506.06677", "code": "https://github.com/buaa-colalab/RoboCerebra", "citations": 30, "github_stars": 75, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "he2024webvoyager", "title": "WebVoyager: Building an End-to-End Web Agent with Large Multimodal Models", "venue": "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", "date": "2024-01", "url": "https://arxiv.org/abs/2401.13919", "arxiv_id": "2401.13919", "code": "https://github.com/MinorJerry/WebVoyager", "citations": 434, "github_stars": 1122, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "hill2023mineplanner", "title": "MinePlanner: A Benchmark for Long-Horizon Planning in Large Minecraft Worlds", "venue": "Proceedings of the 6th ICAPS Workshop on the International Planning Competition (WIPC)", "date": "2023-12", "url": "https://arxiv.org/abs/2312.12891", "arxiv_id": "2312.12891", "code": "https://github.com/IretonLiu/mine-pddl", "citations": 8, "github_stars": 23, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "hou2026single", "title": "Single-Rollout Asynchronous Optimization for Agentic Reinforcement Learning", "venue": "arXiv preprint arXiv:2607.07508", "date": "2026-07", "url": "https://arxiv.org/abs/2607.07508", "arxiv_id": "2607.07508", "code": "", "citations": 7, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "hu2025adas", "title": "Automated design of agentic systems", "venue": "ICLR 2025", "date": "2024-08", "url": "https://arxiv.org/abs/2408.08435", "arxiv_id": "2408.08435", "code": "https://github.com/ShengranHu/ADAS", "citations": 282, "github_stars": 1631, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "hu2025memory", "title": "Memory in the Age of AI Agents", "venue": "arXiv preprint arXiv:2512.13564", "date": "2025-12", "url": "https://arxiv.org/abs/2512.13564", "arxiv_id": "2512.13564", "code": "", "citations": 245, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "hu2025step", "title": "Step-DeepResearch Technical Report", "venue": "arXiv preprint arXiv:2512.20491", "date": "2025-12", "url": "https://arxiv.org/abs/2512.20491", "arxiv_id": "2512.20491", "code": "https://github.com/stepfun-ai/StepDeepResearch", "citations": 12, "github_stars": 571, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "huang2024selfcorrect", "title": "Large Language Models Cannot Self-Correct Reasoning Yet", "venue": "International Conference on Learning Representations", "date": "2023-10", "url": "https://arxiv.org/abs/2310.01798", "arxiv_id": "2310.01798", "code": "", "citations": 1135, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "huang2026rawexperienceskillconsumption", "title": "From Raw Experience to Skill Consumption: A Systematic Study of Model-Generated Agent Skills", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.23899", "arxiv_id": "2605.23899", "code": "", "citations": 13, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "imajuku2025alebench", "title": "ALE-Bench: A Benchmark for Long-Horizon Objective-Driven Algorithm Engineering", "venue": "Advances in Neural Information Processing Systems", "date": "2025-06", "url": "https://arxiv.org/abs/2506.09050", "arxiv_id": "2506.09050", "code": "https://github.com/SakanaAI/ALE-Bench", "citations": 28, "github_stars": 213, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "jansen2025codescientist", "title": "CodeScientist: End-to-End Semi-Automated Scientific Discovery with Code-based Experimentation", "venue": "Findings of the Association for Computational Linguistics: ACL 2025", "date": "2025", "url": "https://aclanthology.org/2025.findings-acl.692/", "arxiv_id": "", "code": "https://github.com/allenai/codescientist", "citations": 55, "github_stars": 348, "collections": [], "sections": ["targets/substrate"]}
{"key": "jiang2025aide", "title": "AIDE: AI-driven exploration in the space of code", "venue": "arXiv", "date": "2025-02", "url": "https://arxiv.org/abs/2502.13138", "arxiv_id": "2502.13138", "code": "https://github.com/WecoAI/aideml", "citations": 174, "github_stars": 1491, "collections": [], "sections": ["targets/data"]}
{"key": "jiang2026darwindynamicagenticallyrewriting", "title": "DARWIN: Dynamic Agentically Rewriting Self-Improving Network", "venue": "arXiv", "date": "2026-02", "url": "https://arxiv.org/abs/2602.05848", "arxiv_id": "2602.05848", "code": "https://github.com/henryyjiang/DARWIN", "citations": 1, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "jimenez2024swe", "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "venue": "ICLR 2024", "date": "2023-10", "url": "https://arxiv.org/abs/2310.06770", "arxiv_id": "2310.06770", "code": "https://github.com/SWE-bench/SWE-bench", "citations": 3453, "github_stars": 5705, "collections": ["benchmarks"], "sections": ["foundations/long-horizon"]}
{"key": "jin2026chainswe", "title": "ChainSWE: Benchmarking Coding Agents on Multi-Bug Software Maintenance", "venue": "arXiv preprint arXiv:2607.02606", "date": "2026-07", "url": "https://arxiv.org/abs/2607.02606", "arxiv_id": "2607.02606", "code": "", "citations": 1, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "jin2026reveal", "title": "ReVeal: Self-Evolving Code Agents via Reliable Self-Verification", "venue": "The Fourteenth International Conference on Learning Representations", "date": "2025-06", "url": "https://arxiv.org/abs/2506.11442", "arxiv_id": "2506.11442", "code": "", "citations": 12, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "kamoi2024can", "title": "When Can LLMs Actually Correct Their Own Mistakes? A Critical Survey of Self-Correction of LLMs", "venue": "Transactions of the Association for Computational Linguistics", "date": "2024-06", "url": "https://arxiv.org/abs/2406.01297", "arxiv_id": "2406.01297", "code": "", "citations": 327, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "kang2025acon", "title": "ACON: Optimizing Context Compression for Long-horizon LLM Agents", "venue": "arXiv preprint arXiv:2510.00615", "date": "2025-10", "url": "https://arxiv.org/abs/2510.00615", "arxiv_id": "2510.00615", "code": "https://github.com/microsoft/acon", "citations": 84, "github_stars": 106, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "kapoor2024omniact", "title": "OmniACT: A Dataset and Benchmark for Enabling Multimodal Generalist Autonomous Agents for Desktop and Web", "venue": "Computer Vision -- ECCV 2024", "date": "2024-02", "url": "https://arxiv.org/abs/2402.17553", "arxiv_id": "2402.17553", "code": "", "citations": 167, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "kapoor2025holistic", "title": "Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation", "venue": "arXiv preprint arXiv:2510.11977", "date": "2025-10", "url": "https://arxiv.org/abs/2510.11977", "arxiv_id": "2510.11977", "code": "https://github.com/princeton-pli/hal-harness", "citations": 56, "github_stars": 311, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "khalifa2026process", "title": "Process Reward Models That Think", "venue": "Transactions on Machine Learning Research", "date": "2025-04", "url": "https://arxiv.org/abs/2504.16828", "arxiv_id": "2504.16828", "code": "https://github.com/mukhal/ThinkPRM", "citations": 103, "github_stars": 91, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "khanal2026beyond", "title": "Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents", "venue": "arXiv preprint arXiv:2603.29231", "date": "2026-03", "url": "https://arxiv.org/abs/2603.29231", "arxiv_id": "2603.29231", "code": "", "citations": 5, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "kim2026sciencescalingagentsystems", "title": "Towards a Science of Scaling Agent Systems", "venue": "arXiv", "date": "2025-12", "url": "https://arxiv.org/abs/2512.08296", "arxiv_id": "2512.08296", "code": "https://github.com/ybkim95/agent-scaling", "citations": 115, "github_stars": 42, "collections": ["benchmarks", "harness-design"], "sections": ["evidence/benchmarks", "targets/harness"]}
{"key": "kimi2026k3", "title": "Kimi K3: Open Frontier Intelligence", "venue": "arXiv", "date": "2026-07", "url": "https://arxiv.org/abs/2607.24653", "arxiv_id": "2607.24653", "code": "https://github.com/MoonshotAI/Kimi-K3", "citations": 8, "github_stars": 8617, "collections": ["benchmarks", "model-design", "harness-design"], "sections": ["evidence/benchmarks", "targets/weights", "targets/harness"]}
{"key": "kirgis2026shadow", "title": "Can AI Agents Conduct Open-Ended AI Research? Early Evidence from Two Case Studies", "venue": "arXiv", "date": "2026-07", "url": "https://arxiv.org/abs/2607.27191", "arxiv_id": "2607.27191", "code": "", "citations": 3, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "koh2024visualwebarena", "title": "VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks", "venue": "Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", "date": "2024-01", "url": "https://arxiv.org/abs/2401.13649", "arxiv_id": "2401.13649", "code": "https://github.com/web-arena-x/visualwebarena", "citations": 0, "github_stars": 485, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "kovacs2026squeez", "title": "Squeez: Task-Conditioned Tool-Output Pruning for Coding Agents", "venue": "arXiv preprint arXiv:2604.04979", "date": "2026-04", "url": "https://arxiv.org/abs/2604.04979", "arxiv_id": "2604.04979", "code": "https://github.com/KRLabsOrg/squeez", "citations": 2, "github_stars": 23, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "kulikov2026autodata", "title": "Autodata: An Agentic Data Scientist to Create High Quality Synthetic Data", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.25996", "arxiv_id": "2606.25996", "code": "", "citations": 6, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/data"]}
{"key": "kwa2026measuring", "title": "Measuring AI Ability to Complete Long Software Tasks", "venue": "NeurIPS 2025", "date": "2025-03", "url": "https://arxiv.org/abs/2503.14499", "arxiv_id": "2503.14499", "code": "https://github.com/METR/eval-analysis-public", "citations": 134, "github_stars": 313, "collections": [], "sections": ["evidence/measurement"]}
{"key": "laban2026llms", "title": "LLMs Get Lost In Multi-Turn Conversation", "venue": "International Conference on Learning Representations", "date": "2025-05", "url": "https://arxiv.org/abs/2505.06120", "arxiv_id": "2505.06120", "code": "https://github.com/microsoft/lost_in_conversation", "citations": 398, "github_stars": 296, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "lange2025shinkaevolve", "title": "ShinkaEvolve: Towards open-ended and sample-efficient program evolution", "venue": "arXiv", "date": "2025-09", "url": "https://openreview.net/forum?id=lKEdGCoDNC", "arxiv_id": "2509.19349", "code": "https://github.com/SakanaAI/ShinkaEvolve", "citations": 139, "github_stars": 1354, "collections": [], "sections": ["targets/substrate"]}
{"key": "lee2024benchmarking", "title": "Benchmarking Mobile Device Control Agents across Diverse Configurations", "venue": "arXiv preprint arXiv:2404.16660", "date": "2024-04", "url": "https://arxiv.org/abs/2404.16660", "arxiv_id": "2404.16660", "code": "https://github.com/jylee425/b-moca", "citations": 47, "github_stars": 33, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "lee2025learning", "title": "Learning to Contextualize Web Pages for Enhanced Decision Making by LLM Agents", "venue": "The Thirteenth International Conference on Learning Representations", "date": "2025-03", "url": "https://arxiv.org/abs/2503.10689", "arxiv_id": "2503.10689", "code": "https://github.com/dgjun32/lcow_iclr2025", "citations": 21, "github_stars": 6, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "lee2026metaharness", "title": "Meta-Harness: End-to-End Optimization of Model Harnesses", "venue": "arXiv", "date": "2026-03", "url": "https://arxiv.org/abs/2603.28052", "arxiv_id": "2603.28052", "code": "https://github.com/stanford-iris-lab/meta-harness-tbench2-artifact", "citations": 141, "github_stars": 1183, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "lee2026rhi", "title": "Recursive harness self-improvement", "venue": "arXiv", "date": "2026-07", "url": "https://arxiv.org/abs/2607.15524", "arxiv_id": "2607.15524", "code": "", "citations": 6, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "li2023behavior", "title": "BEHAVIOR-1K: A benchmark for embodied AI with 1,000 everyday activities and realistic simulation", "venue": "Proceedings of The 6th Conference on Robot Learning", "date": "2023", "url": "https://proceedings.mlr.press/v205/li23a.html", "arxiv_id": "", "code": "https://github.com/StanfordVL/BEHAVIOR-1K", "citations": 382, "github_stars": 1657, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "li2025salt", "title": "SALT: Step-level Advantage Assignment for Long-horizon Agents via Trajectory Graph", "venue": "Findings of the Association for Computational Linguistics: EACL 2026", "date": "2025-10", "url": "https://arxiv.org/abs/2510.20022", "arxiv_id": "2510.20022", "code": "", "citations": 16, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "li2025sorl", "title": "Stabilizing Off-Policy Training for Long-Horizon LLM Agent via Turn-Level Importance Sampling and Clipping-Triggered Normalization", "venue": "arXiv preprint arXiv:2511.20718", "date": "2025-11", "url": "https://arxiv.org/abs/2511.20718", "arxiv_id": "2511.20718", "code": "https://github.com/Cloud0723/SORL", "citations": 4, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "li2025webweaver", "title": "WebWeaver: Structuring Web-Scale Evidence with Dynamic Outlines for Open-Ended Deep Research", "venue": "arXiv preprint arXiv:2509.13312", "date": "2025-09", "url": "https://arxiv.org/abs/2509.13312", "arxiv_id": "2509.13312", "code": "https://github.com/Alibaba-NLP/DeepResearch", "citations": 41, "github_stars": 19873, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "li2026acmagenticcontextmanagement", "title": "ACM: Agentic Context Management for Long Horizon Tasks", "venue": "arXiv", "date": "2026-07", "url": "https://arxiv.org/abs/2607.23809", "arxiv_id": "2607.23809", "code": "https://github.com/lixiaochuan2020/agentic-context-management", "citations": 0, "github_stars": 31, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "li2026autosota", "title": "AutoSOTA: An End-to-End Automated Research System for State-of-the-Art AI Model Discovery", "venue": "arXiv", "date": "2026-04", "url": "https://arxiv.org/abs/2604.05550", "arxiv_id": "2604.05550", "code": "https://github.com/tsinghua-fib-lab/AutoSOTA", "citations": 15, "github_stars": 662, "collections": ["benchmarks"], "sections": ["targets/substrate"]}
{"key": "li2026compactionrlreinforcementlearningcontext", "title": "CompactionRL: Reinforcement Learning with Context Compaction for Long-Horizon Agents", "venue": "arXiv", "date": "2026-07", "url": "https://arxiv.org/abs/2607.05378", "arxiv_id": "2607.05378", "code": "", "citations": 1, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "li2026harness", "title": "Agent Harness Engineering: A Survey", "venue": "—", "date": "2026", "url": "https://picrew.github.io/LLM-Harness/", "arxiv_id": "", "code": "https://github.com/Picrew/LLM-Harness", "citations": 0, "github_stars": 2, "collections": [], "sections": ["analyses"]}
{"key": "li2026weavebench", "title": "WeaveBench: A Long-Horizon, Real-World Benchmark for Computer-Use Agents with Hybrid Interfaces", "venue": "arXiv preprint arXiv:2606.09426", "date": "2026-06", "url": "https://arxiv.org/abs/2606.09426", "arxiv_id": "2606.09426", "code": "", "citations": 4, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "liao2026kernelevolve", "title": "KernelEvolve: Scaling Agentic Kernel Coding for Heterogeneous AI Accelerators at Meta", "venue": "arXiv", "date": "2025-12", "url": "https://doi.org/10.1109/ISCA66397.2026.00063", "arxiv_id": "2512.23236", "code": "", "citations": 2, "github_stars": 0, "collections": [], "sections": ["targets/substrate"]}
{"key": "lindenbauer2025complexity", "title": "The Complexity Trap: Simple Observation Masking Is as Efficient as LLM Summarization for Agent Context Management", "venue": "arXiv preprint arXiv:2508.21433", "date": "2025-08", "url": "https://arxiv.org/abs/2508.21433", "arxiv_id": "2508.21433", "code": "", "citations": 23, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "liu2024agentbench", "title": "AgentBench: Evaluating LLMs as Agents", "venue": "International Conference on Learning Representations", "date": "2023-08", "url": "https://arxiv.org/abs/2308.03688", "arxiv_id": "2308.03688", "code": "https://github.com/THUDM/AgentBench", "citations": 1185, "github_stars": 3691, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "liu2025mlmaster", "title": "ML-Master: Towards AI-for-AI via integration of exploration and reasoning", "venue": "arXiv", "date": "2025-06", "url": "https://arxiv.org/abs/2506.16499", "arxiv_id": "2506.16499", "code": "https://github.com/sjtu-sai-agents/ML-Master", "citations": 54, "github_stars": 447, "collections": ["benchmarks"], "sections": ["targets/data"]}
{"key": "liu2026diveclaudecodedesign", "title": "Dive into Claude Code: The Design Space of Today's and Future AI Agent Systems", "venue": "arXiv", "date": "2026-04", "url": "https://arxiv.org/abs/2604.14228", "arxiv_id": "2604.14228", "code": "https://github.com/VILA-Lab/Dive-into-Claude-Code", "citations": 28, "github_stars": 2078, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "liu2026escherloopmutualevolutionclosedloop", "title": "Escher-Loop: Mutual Evolution by Closed-Loop Self-Referential Optimization", "venue": "arXiv", "date": "2026-04", "url": "https://arxiv.org/abs/2604.23472", "arxiv_id": "2604.23472", "code": "https://github.com/scaling-group/escher-loop", "citations": 5, "github_stars": 7, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "liu2026llms", "title": "Do LLMs Catch Their Own Mistakes? A Comprehensive Benchmark for Reflective Tool Use LLMs", "venue": "Findings of the Association for Computational Linguistics: ACL 2026", "date": "2026", "url": "https://aclanthology.org/2026.findings-acl.86/", "arxiv_id": "", "code": "", "citations": 0, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "longclibench2026", "title": "LongCLI-Bench: A Preliminary Benchmark and Study for Long-horizon Agentic Programming in Command-Line Interfaces", "venue": "arXiv", "date": "2026-02", "url": "https://arxiv.org/abs/2602.14337", "arxiv_id": "2602.14337", "code": "https://github.com/finyorko/longcli-bench", "citations": 21, "github_stars": 46, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "lu-etal-2025-runaway", "title": "Runaway is Ashamed, But Helpful: On the Early-Exit Behavior of Large Language Model-based Agents in Embodied Environments", "venue": "Findings of the Association for Computational Linguistics: EMNLP 2025", "date": "2025-05", "url": "https://aclanthology.org/2025.findings-emnlp.1304/", "arxiv_id": "2505.17616", "code": "https://github.com/Coldmist-Lu/AgentExit", "citations": 7, "github_stars": 2, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "lu2024aiscientist", "title": "The AI Scientist: Towards fully automated open-ended scientific discovery", "venue": "arXiv", "date": "2024-08", "url": "https://arxiv.org/abs/2408.06292", "arxiv_id": "2408.06292", "code": "https://github.com/SakanaAI/AI-Scientist", "citations": 1059, "github_stars": 14442, "collections": [], "sections": ["targets/research"]}
{"key": "lu2026endtoendautomation", "title": "Towards End-to-End Automation of AI Research", "venue": "Nature 2026", "date": "2026-03", "url": "https://doi.org/10.1038/s41586-026-10265-5", "arxiv_id": "", "code": "https://github.com/SakanaAI/AI-Scientist-v2", "citations": 211, "github_stars": 7048, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "lu2026meta", "title": "The Meta-Agent Challenge: Are Current Agents Capable of Autonomous Agent Development?", "venue": "arXiv preprint arXiv:2606.04455", "date": "2026-06", "url": "https://arxiv.org/abs/2606.04455", "arxiv_id": "2606.04455", "code": "https://github.com/ant-research/meta-agent-challenge", "citations": 2, "github_stars": 20, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "lu2402weblinx", "title": "WebLINX: Real-World Website Navigation with Multi-Turn Dialogue", "venue": "International Conference on Machine Learning", "date": "2024-02", "url": "https://arxiv.org/abs/2402.05930", "arxiv_id": "2402.05930", "code": "https://github.com/McGill-NLP/weblinx", "citations": 182, "github_stars": 163, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "lu2408toolsandbox", "title": "ToolSandbox: A Stateful, Conversational, Interactive Evaluation Benchmark for LLM Tool Use Capabilities", "venue": "Findings of the Association for Computational Linguistics: NAACL 2025", "date": "2024-08", "url": "https://aclanthology.org/2025.findings-naacl.65/", "arxiv_id": "2408.04682", "code": "https://github.com/apple/ToolSandbox", "citations": 221, "github_stars": 279, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "lupidi2026airsbench", "title": "AIRS-Bench: A Suite of Tasks for Frontier AI Research Science Agents", "venue": "arXiv", "date": "2026-02", "url": "https://arxiv.org/abs/2602.06855", "arxiv_id": "2602.06855", "code": "https://github.com/facebookresearch/airs-bench", "citations": 19, "github_stars": 111, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "lyu2026mls", "title": "MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI", "venue": "arXiv preprint arXiv:2605.08678", "date": "2026-05", "url": "https://arxiv.org/abs/2605.08678", "arxiv_id": "2605.08678", "code": "https://github.com/Imbernoulli/MLS-Bench", "citations": 6, "github_stars": 105, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "ma2026trex", "title": "TREX: Automating LLM Fine-tuning via Agent-Driven Tree-based Exploration", "venue": "arXiv preprint arXiv:2604.14116", "date": "2026-04", "url": "https://arxiv.org/abs/2604.14116", "arxiv_id": "2604.14116", "code": "", "citations": 1, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "ma2512dover", "title": "DoVer: Intervention-Driven Auto Debugging for LLM Multi-Agent Systems", "venue": "International Conference on Learning Representations", "date": "2025-12", "url": "https://arxiv.org/abs/2512.06749", "arxiv_id": "2512.06749", "code": "https://github.com/microsoft/ACV", "citations": 10, "github_stars": 39, "collections": ["benchmarks", "harness-design"], "sections": ["evidence/benchmarks", "targets/harness"]}
{"key": "madaan2023selfrefine", "title": "Self-Refine: Iterative refinement with self-feedback", "venue": "NeurIPS 2023", "date": "2023-03", "url": "https://arxiv.org/abs/2303.17651", "arxiv_id": "2303.17651", "code": "https://github.com/madaan/self-refine", "citations": 4416, "github_stars": 818, "collections": ["harness-design"], "sections": ["foundations/self-improvement"]}
{"key": "mcaleese2024llm", "title": "LLM Critics Help Catch LLM Bugs", "venue": "arXiv preprint arXiv:2407.00215", "date": "2024-07", "url": "https://arxiv.org/abs/2407.00215", "arxiv_id": "2407.00215", "code": "", "citations": 161, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "meng2026agent", "title": "Agent Harness for Large Language Model Agents: A Survey", "venue": "Preprints", "date": "2026", "url": "https://doi.org/10.20944/preprints202604.0428.v3", "arxiv_id": "", "code": "https://github.com/Gloriaameng/Awesome-Agent-Harness", "citations": 1, "github_stars": 332, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "meng2026rsibenchdata", "title": "RSIBench-Data: Benchmarking Data-Centric Research for Recursive Self-Improvement", "venue": "arXiv", "date": "2026-07", "url": "https://arxiv.org/abs/2607.25886", "arxiv_id": "2607.25886", "code": "https://github.com/evolvent-ai/RSIBench-Data", "citations": 4, "github_stars": 135, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "meyerson2025solving", "title": "Solving a Million-Step LLM Task with Zero Errors", "venue": "arXiv preprint arXiv:2511.09030", "date": "2025-11", "url": "https://arxiv.org/abs/2511.09030", "arxiv_id": "2511.09030", "code": "https://github.com/cognizant-ai-lab/neuro-san-benchmarking", "citations": 22, "github_stars": 46, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "mohammadi2025evaluation", "title": "Evaluation and Benchmarking of LLM Agents: A Survey", "venue": "Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2", "date": "2025-07", "url": "https://arxiv.org/abs/2507.21504", "arxiv_id": "2507.21504", "code": "https://github.com/SAP-samples/llm-agents-eval-tutorial", "citations": 189, "github_stars": 21, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "mozannar2025magenticuihumanintheloopagenticsystems", "title": "Magentic-UI: Towards Human-in-the-loop Agentic Systems", "venue": "arXiv", "date": "2025-07", "url": "https://arxiv.org/abs/2507.22358", "arxiv_id": "2507.22358", "code": "https://github.com/microsoft/magentic-ui", "citations": 48, "github_stars": 10075, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "nakano2022webgpt", "title": "WebGPT: Browser-assisted question-answering with human feedback", "venue": "arXiv preprint arXiv:2112.09332", "date": "2021-12", "url": "https://arxiv.org/abs/2112.09332", "arxiv_id": "2112.09332", "code": "", "citations": 2024, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "ning2026codeagentharness", "title": "Code as Agent Harness", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.18747", "arxiv_id": "2605.18747", "code": "https://github.com/YennNing/Awesome-Code-as-Agent-Harness-Papers", "citations": 18, "github_stars": 655, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "nl2repo2025", "title": "NL2Repo-Bench: Towards Long-Horizon Repository Generation Evaluation of Coding Agents", "venue": "arXiv", "date": "2025-12", "url": "https://arxiv.org/abs/2512.12730", "arxiv_id": "2512.12730", "code": "https://github.com/multimodal-art-projection/NL2RepoBench", "citations": 39, "github_stars": 162, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "novikov2025alphaevolve", "title": "AlphaEvolve: A coding agent for scientific and algorithmic discovery", "venue": "arXiv", "date": "2025-06", "url": "https://arxiv.org/abs/2506.13131", "arxiv_id": "2506.13131", "code": "", "citations": 777, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/substrate"]}
{"key": "officebench2024", "title": "OfficeBench: Benchmarking Language Agents across Multiple Applications for Office Automation", "venue": "arXiv preprint arXiv:2407.19056", "date": "2024-07", "url": "https://arxiv.org/abs/2407.19056", "arxiv_id": "2407.19056", "code": "https://github.com/zlwang-cs/OfficeBench", "citations": 47, "github_stars": 42, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "openai2024introducing", "title": "Introducing SWE-bench Verified", "venue": "—", "date": "2024", "url": "https://openai.com/index/introducing-swe-bench-verified/", "arxiv_id": "", "code": "", "citations": 0, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "ord2025there", "title": "Is there a half-life for the success rates of AI agents?", "venue": "arXiv preprint arXiv:2505.05115", "date": "2025-05", "url": "https://arxiv.org/abs/2505.05115", "arxiv_id": "2505.05115", "code": "", "citations": 5, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "orimo2025parcautonomousselfreflectivecoding", "title": "PARC: An Autonomous Self-Reflective Coding Agent for Robust Execution of Long-Horizon Tasks", "venue": "arXiv", "date": "2025-12", "url": "https://arxiv.org/abs/2512.03549", "arxiv_id": "2512.03549", "code": "", "citations": 3, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "ouyang2026skilloslearningskillcuration", "title": "SkillOS: Learning Skill Curation for Self-Evolving Agents", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.06614", "arxiv_id": "2605.06614", "code": "", "citations": 27, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "packer2023memgpt", "title": "MemGPT: Towards LLMs as Operating Systems", "venue": "arXiv preprint arXiv:2310.08560", "date": "2023-10", "url": "https://arxiv.org/abs/2310.08560", "arxiv_id": "2310.08560", "code": "https://github.com/letta-ai/letta", "citations": 1162, "github_stars": 24432, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "pan2026naturallanguageagentharnesses", "title": "Natural-Language Agent Harnesses", "venue": "arXiv", "date": "2026-03", "url": "https://arxiv.org/abs/2603.25723", "arxiv_id": "2603.25723", "code": "", "citations": 34, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "pan2406webcanvas", "title": "WebCanvas: Benchmarking Web Agents in Online Environments", "venue": "ICML 2024 Workshop on Agentic Markets", "date": "2024-06", "url": "https://arxiv.org/abs/2406.12373", "arxiv_id": "2406.12373", "code": "https://github.com/iMeanAI/WebCanvas", "citations": 119, "github_stars": 280, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "park2023generative", "title": "Generative Agents: Interactive Simulacra of Human Behavior", "venue": "Proceedings of the 36th annual acm symposium on user interface software and technology", "date": "2023-04", "url": "https://arxiv.org/abs/2304.03442", "arxiv_id": "2304.03442", "code": "https://github.com/joonspk-research/generative_agents", "citations": 5257, "github_stars": 21984, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "patil2023gorilla", "title": "Gorilla: Large Language Model Connected with Massive APIs", "venue": "Advances in Neural Information Processing Systems", "date": "2023-05", "url": "https://arxiv.org/abs/2305.15334", "arxiv_id": "2305.15334", "code": "https://github.com/ShishirPatil/gorilla", "citations": 1537, "github_stars": 13007, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "posttrainbench2026", "title": "PostTrainBench: Can LLM Agents Automate LLM Post-Training?", "venue": "arXiv", "date": "2026-03", "url": "https://arxiv.org/abs/2603.08640", "arxiv_id": "2603.08640", "code": "https://github.com/aisa-group/PostTrainBench", "citations": 29, "github_stars": 531, "collections": ["benchmarks", "model-design"], "sections": ["evidence/benchmarks"]}
{"key": "programbench2026", "title": "ProgramBench: Can Language Models Rebuild Programs From Scratch?", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.03546", "arxiv_id": "2605.03546", "code": "https://github.com/facebookresearch/ProgramBench", "citations": 23, "github_stars": 904, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "qu2026bilevel", "title": "Bilevel Autoresearch: Meta-Autoresearching Itself", "venue": "arXiv", "date": "2026-03", "url": "https://arxiv.org/abs/2603.23420", "arxiv_id": "2603.23420", "code": "https://github.com/EdwardOptimization/Bilevel-Autoresearch", "citations": 4, "github_stars": 180, "collections": ["harness-design"], "sections": ["targets/research"]}
{"key": "rakhsha2026lumina", "title": "LUMINA: Long-horizon Understanding for Multi-turn Interactive Agents", "venue": "Findings of the Association for Computational Linguistics: ACL 2026", "date": "2026-01", "url": "https://aclanthology.org/2026.findings-acl.190/", "arxiv_id": "2601.16649", "code": "", "citations": 0, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "rawles2025androidworld", "title": "AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents", "venue": "International Conference on Learning Representations", "date": "2024-05", "url": "https://arxiv.org/abs/2405.14573", "arxiv_id": "2405.14573", "code": "https://github.com/google-research/android_world", "citations": 405, "github_stars": 857, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "ray2026structuredfeedbackimprovesrepair", "title": "Structured Feedback Improves Repair in an LLM Agent Loop", "venue": "arXiv", "date": "2026-07", "url": "https://arxiv.org/abs/2607.14167", "arxiv_id": "2607.14167", "code": "", "citations": 0, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "recube2026", "title": "ReCUBE: Evaluating Repository-Level Context Utilization in Code Generation", "venue": "arXiv", "date": "2026-03", "url": "https://arxiv.org/abs/2603.25770", "arxiv_id": "2603.25770", "code": "https://github.com/JiseungHong/ReCUBE", "citations": 1, "github_stars": 1, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "ren2026selfimprovementsmodernagenticsystems", "title": "Self-Improvements in Modern Agentic Systems: A Survey", "venue": "—", "date": "2026-07", "url": "https://arxiv.org/abs/2607.13104", "arxiv_id": "2607.13104", "code": "https://github.com/selfimproving-agent/Awesome-Self-Improving-Agents", "citations": 5, "github_stars": 406, "collections": ["harness-design"], "sections": ["analyses"]}
{"key": "repogenesis2026", "title": "RepoGenesis: Benchmarking End-to-End Microservice Generation from Readme to Repository", "venue": "arXiv", "date": "2026-01", "url": "https://arxiv.org/abs/2601.13943", "arxiv_id": "2601.13943", "code": "https://github.com/pzy2000/RepoGenesis", "citations": 6, "github_stars": 101, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "roadmapbench2026", "title": "RoadmapBench: Evaluating Long-Horizon Agentic Software Development Across Version Upgrades", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.15846", "arxiv_id": "2605.15846", "code": "https://github.com/UniPat-AI/RoadmapBench", "citations": 2, "github_stars": 14, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "schick2023toolformer", "title": "Toolformer: Language Models Can Teach Themselves to Use Tools", "venue": "Advances in Neural Information Processing Systems", "date": "2023-02", "url": "https://arxiv.org/abs/2302.04761", "arxiv_id": "2302.04761", "code": "", "citations": 5241, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "schmidgall2025agentrxiv", "title": "AgentRxiv: Towards Collaborative Autonomous Research", "venue": "arXiv", "date": "2025-03", "url": "https://arxiv.org/abs/2503.18102", "arxiv_id": "2503.18102", "code": "", "citations": 61, "github_stars": 0, "collections": [], "sections": ["targets/harness"]}
{"key": "seal2025", "title": "Self-Adapting Language Models", "venue": "Advances in Neural Information Processing Systems", "date": "2025-06", "url": "https://arxiv.org/abs/2506.10943", "arxiv_id": "2506.10943", "code": "https://github.com/Continual-Intelligence/SEAL", "citations": 57, "github_stars": 1849, "collections": [], "sections": ["targets/weights"]}
{"key": "shao2026agent", "title": "Do Agent Benchmarks Measure Capability? Protocol Validity in the Age of Agentic AI", "venue": "arXiv preprint arXiv:2607.22368", "date": "2026-07", "url": "https://arxiv.org/abs/2607.22368", "arxiv_id": "2607.22368", "code": "", "citations": 3, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "shi2026swebenchpromaxbenchmarkingagents", "title": "SWE-Bench ProMax: Benchmarking Agents on Large-Scale Multilingual Code Refactoring", "venue": "arXiv", "date": "2026-08", "url": "https://arxiv.org/abs/2608.09802", "arxiv_id": "2608.09802", "code": "", "citations": 3, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "shinn2023reflexion", "title": "Reflexion: language agents with verbal reinforcement learning", "venue": "NeurIPS 2023", "date": "2023-03", "url": "https://arxiv.org/abs/2303.11366", "arxiv_id": "2303.11366", "code": "https://github.com/noahshinn/reflexion", "citations": 4917, "github_stars": 3241, "collections": ["harness-design"], "sections": ["foundations/self-improvement"]}
{"key": "shridhar2020alfred", "title": "ALFRED: A Benchmark for Interpreting Grounded Instructions for Everyday Tasks", "venue": "Proceedings of the IEEE/CVF conference on computer vision and pattern recognition", "date": "2019-12", "url": "https://openaccess.thecvf.com/content_CVPR_2020/html/Shridhar_ALFRED_A_Benchmark_for_Interpreting_Grounded_Instructions_for_Everyday_Tasks_CVPR_2020_paper.html", "arxiv_id": "1912.01734", "code": "https://github.com/askforalfred/alfred", "citations": 1177, "github_stars": 529, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "si2026executiongrounded", "title": "Towards Execution-Grounded Automated AI Research", "venue": "arXiv", "date": "2026-01", "url": "https://arxiv.org/abs/2601.14525", "arxiv_id": "2601.14525", "code": "https://github.com/NoviScl/Automated-AI-Researcher", "citations": 12, "github_stars": 81, "collections": ["model-design"], "sections": ["evidence/measurement"]}
{"key": "sigdel2026schematoolapisllm", "title": "Schema First Tool APIs for LLM Agents: A Controlled Study of Tool Misuse, Recovery, and Budgeted Performance", "venue": "arXiv", "date": "2026-03", "url": "https://arxiv.org/abs/2603.13404", "arxiv_id": "2603.13404", "code": "https://github.com/akgitrepos/schema-first-tool-apis-experiments", "citations": 3, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "starace2025paperbench", "title": "PaperBench: Evaluating AI's ability to replicate AI research", "venue": "arXiv", "date": "2025-04", "url": "https://arxiv.org/abs/2504.01848", "arxiv_id": "2504.01848", "code": "https://github.com/openai/frontier-evals", "citations": 249, "github_stars": 1288, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "starace2026scaffoldeffectsgaiacontrolled", "title": "Scaffold Effects on GAIA: A Controlled Comparison", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.08529", "arxiv_id": "2606.08529", "code": "", "citations": 0, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "sumers2024coala", "title": "Cognitive Architectures for Language Agents", "venue": "Transactions on Machine Learning Research", "date": "2023-09", "url": "https://arxiv.org/abs/2309.02427", "arxiv_id": "2309.02427", "code": "", "citations": 485, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "sun2023adaplanner", "title": "AdaPlanner: Adaptive Planning from Feedback with Language Models", "venue": "Advances in Neural Information Processing Systems", "date": "2023-05", "url": "https://arxiv.org/abs/2305.16653", "arxiv_id": "2305.16653", "code": "https://github.com/haotiansun14/AdaPlanner", "citations": 247, "github_stars": 127, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "sun2025scaling", "title": "Scaling Long-Horizon LLM Agent via Context-Folding", "venue": "arXiv preprint arXiv:2510.11967", "date": "2025-10", "url": "https://arxiv.org/abs/2510.11967", "arxiv_id": "2510.11967", "code": "https://github.com/sunnweiwei/FoldAgent", "citations": 105, "github_stars": 185, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "sweatlas2026", "title": "SWE Atlas: Benchmarking Coding Agents Beyond Issue Resolution", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.08366", "arxiv_id": "2605.08366", "code": "https://github.com/scaleapi/SWE-Atlas", "citations": 4, "github_stars": 70, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "swebenchlive2025", "title": "SWE-bench Goes Live!", "venue": "Advances in Neural Information Processing Systems", "date": "2025-05", "url": "https://arxiv.org/abs/2505.23419", "arxiv_id": "2505.23419", "code": "https://github.com/microsoft/SWE-bench-Live", "citations": 55, "github_stars": 226, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "swebenchpro2025", "title": "SWE-Bench Pro: Can AI Agents Solve Long-Horizon Software Engineering Tasks?", "venue": "arXiv", "date": "2025-09", "url": "https://arxiv.org/abs/2509.16941", "arxiv_id": "2509.16941", "code": "https://github.com/scaleapi/SWE-bench_Pro-os", "citations": 209, "github_stars": 509, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "swechain2026", "title": "SWE-Chain: Benchmarking Coding Agents on Chained Release-Level Package Upgrades", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.14415", "arxiv_id": "2605.14415", "code": "https://github.com/CUHK-ARISE/SWE-Chain", "citations": 4, "github_stars": 14, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "sweci2026", "title": "SWE-CI: Evaluating Agent Capabilities in Maintaining Codebases via Continuous Integration", "venue": "arXiv", "date": "2026-03", "url": "https://arxiv.org/abs/2603.03823", "arxiv_id": "2603.03823", "code": "https://github.com/SKYLENAGE-AI/SWE-CI", "citations": 15, "github_stars": 175, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "swecycle2026", "title": "SWE-Cycle: Benchmarking Code Agents across the Complete Issue Resolution Cycle", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.13139", "arxiv_id": "2605.13139", "code": "https://github.com/tubehao/SWE-Cycle", "citations": 2, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "sweevo2025", "title": "SWE-EVO: Benchmarking Coding Agents in Long-Horizon Software Evolution Scenarios", "venue": "arXiv", "date": "2025-12", "url": "https://arxiv.org/abs/2512.18470", "arxiv_id": "2512.18470", "code": "https://github.com/SWE-EVO/SWE-EVO", "citations": 38, "github_stars": 55, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "sweexplore2026", "title": "SWE-Explore: Benchmarking How Coding Agents Explore Repositories", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.07297", "arxiv_id": "2606.07297", "code": "https://github.com/Qiushao-E/SWE-Explore-Bench", "citations": 11, "github_stars": 42, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "sweinfrabench2026", "title": "SWE-InfraBench: Evaluating Language Models on Cloud Infrastructure Code", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.05249", "arxiv_id": "2606.05249", "code": "", "citations": 0, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "swemarathon2026", "title": "SWE-Marathon: Can Agents Autonomously Complete Ultra-Long-Horizon Software Work?", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.07682", "arxiv_id": "2606.07682", "code": "", "citations": 6, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "swemera2025", "title": "SWE-MERA: A Dynamic Benchmark for Agenticly Evaluating Large Language Models on Software Engineering Tasks", "venue": "Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing: System Demonstrations", "date": "2025-07", "url": "https://arxiv.org/abs/2507.11059", "arxiv_id": "2507.11059", "code": "", "citations": 5, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "swerebenchv22026", "title": "SWE-rebench V2: Language-Agnostic SWE Task Collection at Scale", "venue": "arXiv preprint arXiv:2602.23866", "date": "2026-02", "url": "https://arxiv.org/abs/2602.23866", "arxiv_id": "2602.23866", "code": "", "citations": 13, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "sypherd2024practicalconsiderationsagenticllm", "title": "Practical Considerations for Agentic LLM Systems", "venue": "arXiv", "date": "2024-12", "url": "https://arxiv.org/abs/2412.04093", "arxiv_id": "2412.04093", "code": "", "citations": 16, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "ta-etal-2026-reinforced", "title": "Reinforced Agent: Inference-Time Feedback for Tool-Calling Agents", "venue": "Proceedings of the Fifth Workshop on Generation, Evaluation and Metrics (GEM)", "date": "2026-04", "url": "https://aclanthology.org/2026.gem-main.13/", "arxiv_id": "2604.27233", "code": "", "citations": 3, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "tan2026hcapo", "title": "Hindsight Credit Assignment for Long-Horizon LLM Agents", "venue": "arXiv preprint arXiv:2603.08754", "date": "2026-03", "url": "https://arxiv.org/abs/2603.08754", "arxiv_id": "2603.08754", "code": "", "citations": 35, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "tang2025airesearcher", "title": "AI-Researcher: Autonomous Scientific Innovation", "venue": "Advances in Neural Information Processing Systems", "date": "2025-05", "url": "https://proceedings.neurips.cc/paper_files/paper/2025/hash/0d904d300a105809a2114d727851e759-Abstract-Conference.html", "arxiv_id": "2505.18705", "code": "https://github.com/HKUDS/AI-Researcher", "citations": 83, "github_stars": 5701, "collections": [], "sections": ["targets/research"]}
{"key": "tang2026fars", "title": "FARS: A Fully Automated Research System Deployed at Scale", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.31651", "arxiv_id": "2606.31651", "code": "", "citations": 3, "github_stars": 0, "collections": ["benchmarks"], "sections": ["targets/research"]}
{"key": "tebench2026", "title": "Breaking, Stale, or Missing? Benchmarking Coding Agents on Project-Level Test Evolution", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.06125", "arxiv_id": "2605.06125", "code": "https://github.com/iSEngLab/TEBench", "citations": 1, "github_stars": 5, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "terminalbench2026", "title": "Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces", "venue": "arXiv", "date": "2026-01", "url": "https://arxiv.org/abs/2601.11868", "arxiv_id": "2601.11868", "code": "https://github.com/harbor-framework/terminal-bench-1", "citations": 354, "github_stars": 2552, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "toolllm2023", "title": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs", "venue": "International Conference on Learning Representations", "date": "2023-07", "url": "https://arxiv.org/abs/2307.16789", "arxiv_id": "2307.16789", "code": "https://github.com/OpenBMB/ToolBench", "citations": 2090, "github_stars": 5734, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "trivedi2024appworld", "title": "AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents", "venue": "ACL 2024", "date": "2024-07", "url": "https://arxiv.org/abs/2407.18901", "arxiv_id": "2407.18901", "code": "https://github.com/StonyBrookNLP/appworld", "citations": 286, "github_stars": 490, "collections": ["benchmarks"], "sections": ["foundations/long-horizon"]}
{"key": "ursekar2026harnessopt", "title": "HarnessOpt-Bench: Evaluating LLMs at Harness Optimization", "venue": "arXiv preprint arXiv:2608.06301", "date": "2026-08", "url": "https://arxiv.org/abs/2608.06301", "arxiv_id": "2608.06301", "code": "", "citations": 0, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "wang-etal-2025-swe", "title": "SWE-Dev: Building Software Engineering Agents with Training and Inference Scaling", "venue": "Findings of the Association for Computational Linguistics: ACL 2025", "date": "2025-06", "url": "https://aclanthology.org/2025.findings-acl.193/", "arxiv_id": "2506.07636", "code": "https://github.com/THUDM/SWE-Dev", "citations": 24, "github_stars": 65, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "wang2024voyager", "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models", "venue": "Transactions on Machine Learning Research", "date": "2023-05", "url": "https://arxiv.org/abs/2305.16291", "arxiv_id": "2305.16291", "code": "https://github.com/MineDojo/Voyager", "citations": 2175, "github_stars": 7157, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "wang2025agent", "title": "Agent Workflow Memory", "venue": "Forty-second International Conference on Machine Learning", "date": "2024-09", "url": "https://arxiv.org/abs/2409.07429", "arxiv_id": "2409.07429", "code": "https://github.com/zorazrw/agent-workflow-memory", "citations": 240, "github_stars": 461, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "wang2025hell", "title": "Hell or High Water: Evaluating Agentic Recovery from External Failures", "venue": "Second Conference on Language Modeling", "date": "2025-08", "url": "https://arxiv.org/abs/2508.11027", "arxiv_id": "2508.11027", "code": "https://github.com/JHU-CLSP/hell-or-high-water", "citations": 4, "github_stars": 5, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "wang2025odysseybench", "title": "OdysseyBench: Evaluating LLM Agents on Long-Horizon Complex Office Application Workflows", "venue": "arXiv preprint arXiv:2508.09124", "date": "2025-08", "url": "https://arxiv.org/abs/2508.09124", "arxiv_id": "2508.09124", "code": "https://github.com/microsoft/OdysseyBench", "citations": 45, "github_stars": 14, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "wang2026agent", "title": "Do Agent Optimizers Compound? A Continual-Learning Evaluation on Terminal-Bench 2.0", "venue": "arXiv preprint arXiv:2607.14004", "date": "2026-07", "url": "https://arxiv.org/abs/2607.14004", "arxiv_id": "2607.14004", "code": "https://github.com/relai-ai/Continual-Learning-Terminal-Bench", "citations": 0, "github_stars": 7, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "wang2026context", "title": "Context Compression for LLM Agents: A Survey of Methods, Failure Modes, and Evaluation", "venue": "Preprints", "date": "2026", "url": "https://doi.org/10.20944/preprints202605.2065.v1", "arxiv_id": "", "code": "", "citations": 0, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "wang2026deskcraft", "title": "DeskCraft: Benchmarking Desktop Agents on Professional Workflows and Human-in-the-Loop Collaboration", "venue": "arXiv preprint arXiv:2606.03103", "date": "2026-06", "url": "https://arxiv.org/abs/2606.03103", "arxiv_id": "2606.03103", "code": "https://github.com/mrwwk/DeskCraft", "citations": 1, "github_stars": 91, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "wang2026long", "title": "The Long-Horizon Task Mirage? Diagnosing Where and Why Agentic Systems Break", "venue": "arXiv preprint arXiv:2604.11978", "date": "2026-04", "url": "https://arxiv.org/abs/2604.11978", "arxiv_id": "2604.11978", "code": "", "citations": 18, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "wang2026metaskillevolverecursiveselfimprovementllm", "title": "MetaSkill-Evolve: Recursive Self-Improvement of LLM Agents via Two-Timescale Meta-Skill Evolution", "venue": "arXiv", "date": "2026-07", "url": "https://arxiv.org/abs/2607.05297", "arxiv_id": "2607.05297", "code": "", "citations": 2, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "wang2026naturebench", "title": "NatureBench: Can Coding Agents Match the Published SOTA of Nature-Family Papers?", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.24530", "arxiv_id": "2606.24530", "code": "https://github.com/FrontisAI/NatureBench", "citations": 1, "github_stars": 102, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "wang2026rethinking", "title": "Rethinking the Evaluation of Harness Evolution for Agents", "venue": "arXiv", "date": "2026-07", "url": "https://arxiv.org/abs/2607.12227", "arxiv_id": "2607.12227", "code": "https://github.com/rethinking-harness-evolution/code", "citations": 9, "github_stars": 27, "collections": ["harness-design"], "sections": ["evidence/measurement"]}
{"key": "wang2026steppo", "title": "CAPO: Critic-Guided Action-Aligned Policy Optimization for Advancing LLM Agent Capabilities", "venue": "arXiv preprint arXiv:2604.18401", "date": "2026-04", "url": "https://arxiv.org/abs/2604.18401", "arxiv_id": "2604.18401", "code": "https://github.com/AgentR1/Agent-R1", "citations": 8, "github_stars": 1629, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "wang2026swe", "title": "SWE-Pruner: Self-Adaptive Context Pruning for Coding Agents", "venue": "arXiv preprint arXiv:2601.16746", "date": "2026-01", "url": "https://arxiv.org/abs/2601.16746", "arxiv_id": "2601.16746", "code": "https://github.com/Ayanami1314/swe-pruner", "citations": 32, "github_stars": 314, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "wang2026verificationhorizonsilverbullet", "title": "The Verification Horizon: No Silver Bullet for Coding Agent Rewards", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.26300", "arxiv_id": "2606.26300", "code": "", "citations": 3, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "weco2026aide2", "title": "First Evidence of Recursive Self-Improvement", "venue": "—", "date": "2026", "url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement", "arxiv_id": "", "code": "", "citations": 0, "github_stars": 0, "collections": [], "sections": ["targets/research"]}
{"key": "wei2022chain", "title": "Chain-Of-Thought Prompting Elicits Reasoning in Large Language Models", "venue": "Advances in Neural Information Processing Systems", "date": "2022-01", "url": "https://arxiv.org/abs/2201.11903", "arxiv_id": "2201.11903", "code": "https://github.com/jasonwei20/chain-of-thought-prompting", "citations": 21268, "github_stars": 41, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "wei2026agentloopsstructuredgraphsa", "title": "From Agent Loops to Structured Graphs:A Scheduler-Theoretic Framework for LLM Agent Execution", "venue": "arXiv", "date": "2026-04", "url": "https://arxiv.org/abs/2604.11378", "arxiv_id": "2604.11378", "code": "", "citations": 2, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "weng2026deepscientist", "title": "DeepScientist: Advancing Frontier-Pushing Scientific Findings Progressively", "venue": "International Conference on Learning Representations", "date": "2025-09", "url": "https://openreview.net/forum?id=cZFgsLq8Gs", "arxiv_id": "2509.26603", "code": "https://github.com/ResearAI/DeepScientist", "citations": 48, "github_stars": 3293, "collections": [], "sections": ["targets/substrate"]}
{"key": "wijk2024re", "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts", "venue": "arXiv", "date": "2024-11", "url": "https://arxiv.org/abs/2411.15114", "arxiv_id": "2411.15114", "code": "https://github.com/METR/RE-Bench", "citations": 136, "github_stars": 156, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "workarena2024", "title": "WorkArena: How Capable are Web Agents at Solving Common Knowledge Work Tasks?", "venue": "Proceedings of the 41st International Conference on Machine Learning", "date": "2024-03", "url": "https://proceedings.mlr.press/v235/drouin24a.html", "arxiv_id": "2403.07718", "code": "https://github.com/ServiceNow/WorkArena", "citations": 320, "github_stars": 268, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "worldofbits", "title": "World of Bits: An Open-Domain Platform for Web-Based Agents", "venue": "Proceedings of the 34th International Conference on Machine Learning", "date": "2017", "url": "https://proceedings.mlr.press/v70/shi17a.html", "arxiv_id": "", "code": "", "citations": 352, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "wu2025resum", "title": "ReSum: Unlocking Long-Horizon Search Intelligence via Context Summarization", "venue": "arXiv preprint arXiv:2509.13313", "date": "2025-09", "url": "https://arxiv.org/abs/2509.13313", "arxiv_id": "2509.13313", "code": "https://github.com/Alibaba-NLP/DeepResearch", "citations": 101, "github_stars": 19873, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "wu2026contextbudget", "title": "ContextBudget: Budget-Aware Context Management for Long-Horizon Search Agents", "venue": "arXiv preprint arXiv:2604.01664", "date": "2026-04", "url": "https://arxiv.org/abs/2604.01664", "arxiv_id": "2604.01664", "code": "https://github.com/yw-0311/ContextBudget", "citations": 9, "github_stars": 7, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "wu2026contextweaver", "title": "ContextWeaver: Selective and Dependency-Structured Memory Construction for LLM Agents", "venue": "arXiv preprint arXiv:2604.23069", "date": "2026-04", "url": "https://arxiv.org/abs/2604.23069", "arxiv_id": "2604.23069", "code": "", "citations": 2, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "wu2026history", "title": "When History Lies: Evaluating and Improving Tool Use under Misleading Multi-Turn Histories", "venue": "arXiv preprint arXiv:2608.06057", "date": "2026-08", "url": "https://arxiv.org/abs/2608.06057", "arxiv_id": "2608.06057", "code": "", "citations": 0, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "xi2025agentgymrl", "title": "AgentGym-RL: Training LLM Agents for Long-Horizon Decision Making through Multi-Turn Reinforcement Learning", "venue": "arXiv preprint arXiv:2509.08755", "date": "2025-09", "url": "https://arxiv.org/abs/2509.08755", "arxiv_id": "2509.08755", "code": "https://github.com/WooooDyy/AgentGym-RL", "citations": 64, "github_stars": 852, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "xia2024agentless", "title": "Demystifying LLM-Based Software Engineering Agents", "venue": "Proceedings of the ACM on Software Engineering", "date": "2024-07", "url": "https://doi.org/10.1145/3715754", "arxiv_id": "2407.01489", "code": "https://github.com/OpenAutoCoder/Agentless", "citations": 465, "github_stars": 2103, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "xiao2026reducing", "title": "Reducing Cost of LLM Agents with Trajectory Reduction", "venue": "Proceedings of the ACM on Software Engineering", "date": "2025-09", "url": "https://arxiv.org/abs/2509.23586", "arxiv_id": "2509.23586", "code": "", "citations": 33, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "xie2024osworld", "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "venue": "NeurIPS 2024", "date": "2024-04", "url": "https://arxiv.org/abs/2404.07972", "arxiv_id": "2404.07972", "code": "https://github.com/xlang-ai/OSWorld", "citations": 1056, "github_stars": 3106, "collections": ["benchmarks"], "sections": ["foundations/long-horizon"]}
{"key": "xie2024travelplanner", "title": "TravelPlanner: A Benchmark for Real-World Planning with Language Agents", "venue": "International Conference on Machine Learning", "date": "2024-02", "url": "https://arxiv.org/abs/2402.01622", "arxiv_id": "2402.01622", "code": "https://github.com/OSU-NLP-Group/TravelPlanner", "citations": 454, "github_stars": 541, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "xu2026adaptinginterfacemodelruntime", "title": "Adapting the Interface, Not the Model: Runtime Harness Adaptation for Deterministic LLM Agents", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.22166", "arxiv_id": "2605.22166", "code": "https://github.com/Tianshi-Xu/Life-Harness", "citations": 15, "github_stars": 215, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "xu2026asievolve", "title": "ASI-Evolve: AI Accelerates AI", "venue": "arXiv", "date": "2026-03", "url": "https://arxiv.org/abs/2603.29640", "arxiv_id": "2603.29640", "code": "https://github.com/GAIR-NLP/ASI-Evolve", "citations": 5, "github_stars": 848, "collections": ["model-design"], "sections": ["targets/research"]}
{"key": "xu2026looptrapterminationpoisoningattacks", "title": "LoopTrap: Termination Poisoning Attacks on LLM Agents", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.05846", "arxiv_id": "2605.05846", "code": "", "citations": 1, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "xu2026mem", "title": "A-MEM: Agentic Memory for LLM Agents", "venue": "Advances in Neural Information Processing Systems", "date": "2025-02", "url": "https://arxiv.org/abs/2502.12110", "arxiv_id": "2502.12110", "code": "https://github.com/WujiangXu/A-mem", "citations": 906, "github_stars": 946, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "xu2026theagentcompany", "title": "TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks", "venue": "Advances in Neural Information Processing Systems (NeurIPS) Datasets and Benchmarks Track", "date": "2024-12", "url": "https://arxiv.org/abs/2412.14161", "arxiv_id": "2412.14161", "code": "https://github.com/TheAgentCompany/TheAgentCompany", "citations": 283, "github_stars": 770, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "yamada2025aiscientistv2", "title": "The AI Scientist-v2: Workshop-Level Automated Scientific Discovery via Agentic Tree Search", "venue": "arXiv", "date": "2025-04", "url": "https://arxiv.org/abs/2504.08066", "arxiv_id": "2504.08066", "code": "https://github.com/SakanaAI/AI-Scientist-v2", "citations": 360, "github_stars": 7048, "collections": [], "sections": ["targets/research"]}
{"key": "yang2026beyond", "title": "Beyond Static Summarization: Proactive Memory Extraction for LLM Agents", "venue": "arXiv preprint arXiv:2601.04463", "date": "2026-01", "url": "https://arxiv.org/abs/2601.04463", "arxiv_id": "2601.04463", "code": "", "citations": 14, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "yang2026frontisma1", "title": "Frontis-MA1: Training an AI4AI Model towards Recursive Self-Improvement in Machine Learning Engineering", "venue": "arXiv", "date": "2026-07", "url": "https://arxiv.org/abs/2607.28568", "arxiv_id": "2607.28568", "code": "https://github.com/FrontisAI/OpenRSI", "citations": 2, "github_stars": 568, "collections": ["benchmarks", "model-design", "harness-design"], "sections": ["targets/substrate"]}
{"key": "yao2022webshop", "title": "WebShop: Towards Scalable Real-World Web Interaction with Grounded Language Agents", "venue": "Advances in Neural Information Processing Systems", "date": "2022-07", "url": "https://arxiv.org/abs/2207.01206", "arxiv_id": "2207.01206", "code": "https://github.com/princeton-nlp/WebShop", "citations": 1309, "github_stars": 586, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "yao2023react", "title": "ReAct: Synergizing Reasoning and Acting in Language Models", "venue": "International Conference on Learning Representations (ICLR)", "date": "2022-10", "url": "https://arxiv.org/abs/2210.03629", "arxiv_id": "2210.03629", "code": "https://github.com/ysymyth/ReAct", "citations": 10470, "github_stars": 4120, "collections": ["model-design", "harness-design"], "sections": ["targets/weights", "targets/harness"]}
{"key": "yao2023tot", "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models", "venue": "Advances in Neural Information Processing Systems", "date": "2023-05", "url": "https://arxiv.org/abs/2305.10601", "arxiv_id": "2305.10601", "code": "https://github.com/princeton-nlp/tree-of-thought-llm", "citations": 4728, "github_stars": 6053, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "yao2024tau", "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "venue": "arXiv", "date": "2024-06", "url": "https://arxiv.org/abs/2406.12045", "arxiv_id": "2406.12045", "code": "https://github.com/sierra-research/tau-bench", "citations": 1005, "github_stars": 1403, "collections": ["benchmarks"], "sections": ["foundations/long-horizon"]}
{"key": "yao2026arc", "title": "ARC: Active and Reflection-driven Context Management for Long-Horizon Information Seeking Agents", "venue": "Findings of the Association for Computational Linguistics: ACL 2026", "date": "2026-01", "url": "https://aclanthology.org/2026.findings-acl.930/", "arxiv_id": "2601.12030", "code": "", "citations": 5, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "ye2025agentfold", "title": "AgentFold: Long-Horizon Web Agents with Proactive Context Management", "venue": "arXiv preprint arXiv:2510.24699", "date": "2025-10", "url": "https://arxiv.org/abs/2510.24699", "arxiv_id": "2510.24699", "code": "https://github.com/Alibaba-NLP/DeepResearch", "citations": 69, "github_stars": 19873, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "yehudai2026survey", "title": "A Survey on Evaluation of LLM-based Agents", "venue": "Findings of the Association for Computational Linguistics: ACL 2026", "date": "2025-03", "url": "https://arxiv.org/abs/2503.16416", "arxiv_id": "2503.16416", "code": "", "citations": 212, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "yi2026learningagentcompatiblecontextmanagement", "title": "Learning Agent-Compatible Context Management for Long-Horizon Tasks", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.30785", "arxiv_id": "2605.30785", "code": "", "citations": 3, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "yin2024godelagent", "title": "Godel Agent: A self-referential agent framework for recursive self-improvement", "venue": "arXiv", "date": "2024-10", "url": "https://aclanthology.org/2025.acl-long.1354/", "arxiv_id": "2410.04444", "code": "https://github.com/Arvid-pku/Godel_Agent", "citations": 21, "github_stars": 212, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "yuan2024selfreward", "title": "Self-rewarding language models", "venue": "arXiv", "date": "2024-01", "url": "https://proceedings.mlr.press/v235/yuan24d.html", "arxiv_id": "2401.10020", "code": "", "citations": 691, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "yuan2026osworld2", "title": "OSWorld 2.0: Benchmarking Computer Use Agents on Long-Horizon Real-World Tasks", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.29537", "arxiv_id": "2606.29537", "code": "https://github.com/xlang-ai/OSWorld-V2", "citations": 10, "github_stars": 258, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "zelikman2022star", "title": "STaR: Bootstrapping reasoning with reasoning", "venue": "NeurIPS 2022", "date": "2022-03", "url": "https://proceedings.neurips.cc/paper_files/paper/2022/hash/639a9a172c044fbb64175b5fad42e9a5-Abstract-Conference.html", "arxiv_id": "2203.14465", "code": "", "citations": 1035, "github_stars": 0, "collections": ["model-design"], "sections": ["targets/weights"]}
{"key": "zelikman2024stop", "title": "Self-Taught Optimizer (STOP): Recursively self-improving code generation", "venue": "Conference on Language Modeling", "date": "2023-10", "url": "https://arxiv.org/abs/2310.02304", "arxiv_id": "2310.02304", "code": "https://github.com/microsoft/stop", "citations": 126, "github_stars": 52, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "zhang-etal-2025-di", "title": "DI-BENCH: Benchmarking Large Language Models on Dependency Inference with Testable Repositories at Scale", "venue": "Findings of the Association for Computational Linguistics: ACL 2025", "date": "2025-01", "url": "https://arxiv.org/abs/2501.13699", "arxiv_id": "2501.13699", "code": "https://github.com/microsoft/DI-Bench", "citations": 5, "github_stars": 6, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "zhang2025dgm", "title": "Darwin Godel Machine: Open-ended evolution of self-improving agents", "venue": "arXiv", "date": "2025-05", "url": "https://openreview.net/forum?id=pUpzQZTvGY", "arxiv_id": "2505.22954", "code": "https://github.com/jennyzzt/dgm", "citations": 193, "github_stars": 2250, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "zhang2026agenticcontextengineeringevolving", "title": "Agentic Context Engineering: Evolving Contexts for Self-Improving Language Models", "venue": "arXiv", "date": "2025-10", "url": "https://arxiv.org/abs/2510.04618", "arxiv_id": "2510.04618", "code": "https://github.com/ace-agent/ace", "citations": 266, "github_stars": 1273, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "zhang2026deepplanning", "title": "DeepPlanning: Benchmarking Long-Horizon Agentic Planning with Verifiable Constraints", "venue": "arXiv preprint arXiv:2601.18137", "date": "2026-01", "url": "https://arxiv.org/abs/2601.18137", "arxiv_id": "2601.18137", "code": "", "citations": 26, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "zhang2026researcharena", "title": "How Far Are We From True Auto-Research?", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.19156", "arxiv_id": "2605.19156", "code": "", "citations": 6, "github_stars": 0, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "zhang2026selfharness", "title": "Self-Harness: Harnesses That Improve Themselves", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.09498", "arxiv_id": "2606.09498", "code": "https://github.com/qzzqzzb/Self-Harness", "citations": 24, "github_stars": 82, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "zhang2026stop", "title": "Stop Comparing LLM Agents Without Disclosing the Harness", "venue": "Second Workshop on Agents in the Wild: Safety, Security, and Beyond", "date": "2026", "url": "https://openreview.net/forum?id=ffKHSraOIK", "arxiv_id": "", "code": "", "citations": 8, "github_stars": 0, "collections": ["harness-design"], "sections": ["evidence/measurement"]}
{"key": "zhao2024expel", "title": "ExpeL: LLM Agents Are Experiential Learners", "venue": "AAAI 2024", "date": "2023-08", "url": "https://arxiv.org/abs/2308.10144", "arxiv_id": "2308.10144", "code": "https://github.com/LeapLabTHU/ExpeL", "citations": 825, "github_stars": 237, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "zhao2026specbenchmeasuringrewardhacking", "title": "SpecBench: Measuring Reward Hacking in Long-Horizon Coding Agents", "venue": "arXiv", "date": "2026-05", "url": "https://arxiv.org/abs/2605.21384", "arxiv_id": "2605.21384", "code": "https://github.com/WecoAI/SpecBench", "citations": 11, "github_stars": 12, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "zheng2024naturalplan", "title": "NATURAL PLAN: Benchmarking LLMs on Natural Language Planning", "venue": "arXiv preprint arXiv:2406.04520", "date": "2024-06", "url": "https://arxiv.org/abs/2406.04520", "arxiv_id": "2406.04520", "code": "https://github.com/google-deepmind/natural-plan", "citations": 137, "github_stars": 58, "collections": ["benchmarks"], "sections": ["evidence/benchmarks"]}
{"key": "zhou2024lats", "title": "Language Agent Tree Search Unifies Reasoning Acting and Planning in Language Models", "venue": "International Conference on Machine Learning", "date": "2023-10", "url": "https://arxiv.org/abs/2310.04406", "arxiv_id": "2310.04406", "code": "https://github.com/lapisrocks/LanguageAgentTreeSearch", "citations": 608, "github_stars": 854, "collections": ["model-design", "harness-design"], "sections": ["targets/weights", "targets/harness"]}
{"key": "zhou2024webarena", "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents", "venue": "ICLR 2024", "date": "2023-07", "url": "https://arxiv.org/abs/2307.13854", "arxiv_id": "2307.13854", "code": "https://github.com/web-arena-x/webarena", "citations": 1858, "github_stars": 1587, "collections": ["benchmarks"], "sections": ["foundations/long-horizon"]}
{"key": "zhou2025mem1", "title": "MEM1: Learning to Synergize Memory and Reasoning for Efficient Long-Horizon Agents", "venue": "arXiv preprint arXiv:2506.15841", "date": "2025-06", "url": "https://arxiv.org/abs/2506.15841", "arxiv_id": "2506.15841", "code": "https://github.com/MIT-MI/MEM1", "citations": 190, "github_stars": 331, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "zhou2026externalization", "title": "Externalization in LLM Agents: A Unified Review of Memory, Skills, Protocols and Harness Engineering", "venue": "arXiv preprint arXiv:2604.08224", "date": "2026-04", "url": "https://arxiv.org/abs/2604.08224", "arxiv_id": "2604.08224", "code": "", "citations": 48, "github_stars": 0, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "zhou2026readyagentnativememorysystem", "title": "Are We Ready For An Agent-Native Memory System?", "venue": "arXiv", "date": "2026-06", "url": "https://arxiv.org/abs/2606.24775", "arxiv_id": "2606.24775", "code": "https://github.com/OpenDataBox/MemoryData", "citations": 7, "github_stars": 139, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "zhu2025llmagentsfaillearn", "title": "Where LLM Agents Fail and How They can Learn From Failures", "venue": "arXiv", "date": "2025-09", "url": "https://arxiv.org/abs/2509.25370", "arxiv_id": "2509.25370", "code": "https://github.com/ulab-uiuc/AgentDebug", "citations": 102, "github_stars": 101, "collections": ["harness-design"], "sections": ["targets/harness"]}
{"key": "zhu2026toward", "title": "Toward ultra-long-horizon agentic science: Cognitive accumulation for machine learning engineering", "venue": "arXiv", "date": "2026-01", "url": "https://arxiv.org/abs/2601.10402", "arxiv_id": "2601.10402", "code": "", "citations": 22, "github_stars": 0, "collections": ["benchmarks"], "sections": ["targets/data"]}
{"key": "zhuge2025agentasajudge", "title": "Agent-as-a-Judge: Evaluate Agents with Agents", "venue": "Forty-second International Conference on Machine Learning", "date": "2024-10", "url": "https://arxiv.org/abs/2410.10934", "arxiv_id": "2410.10934", "code": "https://github.com/metauto-ai/agent-as-a-judge", "citations": 205, "github_stars": 821, "collections": ["benchmarks", "harness-design"], "sections": ["evidence/benchmarks", "targets/harness"]}