Publish validated b877d04c5e79a39a011103d39e3093910b2c136d

.gitignore

@@ -0,0 +1,3 @@
+.venv/
+.pytest_cache/
+__pycache__/

README.md

@@ -1,13 +1,28 @@
 ---
-title: Repro Memorybench
-emoji: 🌖
-colorFrom: pink
-colorTo: yellow
+title: MemoryBench Reproduction Evidence
 sdk: gradio
-sdk_version: 6.20.0
-python_version: '3.13'
 app_file: app.py
-pinned: false
+tags:
+  - icml2026-repro
+  - paper-If4X4W2HWx
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# MemoryBench Reproduction
+
+This submission provides CPU-only evidence for `If4X4W2HWx`, MemoryBench: A
+Benchmark for Memory and Continual Learning in LLM Systems.
+
+The evidence is generated from pinned public artifacts:
+
+- `github:LittleDinoC/MemoryBench@5eafebca4e9ffbb2f0087ade13c498cf95fbc09a`
+- `hf:THUIR/MemoryBench@3acd60a4bd35b43b408f0e6db4c5f1e88df5e96d`
+- `hf:THUIR/MemoryBench-Results@742b433c48edfcb20dd102fd5fcf32f3053baada`
+
+Run:
+
+```bash
+python generate_evidence.py
+```
+
+The generated bundle is written to `evidence/bundle.json`, and the judge-facing
+summary is written to `pages/00-summary.md`.

app.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import gradio as gr
+
+import generate_evidence
+
+
+ROOT = Path(__file__).resolve().parent
+BUNDLE_PATH = ROOT / "evidence" / "bundle.json"
+
+
+def _load_bundle() -> dict:
+    if not BUNDLE_PATH.exists():
+        return generate_evidence.build_evidence(ROOT)
+    return json.loads(BUNDLE_PATH.read_text())
+
+
+def _markdown() -> str:
+    bundle = _load_bundle()
+    rows = []
+    for claim in bundle["claims"]:
+        rows.append(
+            f"- `{claim['status']}` `{claim['claim_sha256'][:12]}`: {claim['claim']}"
+        )
+    return "\n".join(
+        [
+            "# MemoryBench Reproduction",
+            "",
+            f"Attempt: `{bundle['attempt_id']}`",
+            f"Snapshot: `{bundle['snapshot_id']}`",
+            f"API cost: `${bundle['api_cost_usd']:.2f}`",
+            "",
+            "## Claims",
+            "",
+            *rows,
+            "",
+            "Full machine-readable evidence is in `evidence/bundle.json`.",
+        ]
+    )
+
+
+with gr.Blocks(title="MemoryBench Reproduction") as demo:
+    gr.Markdown(_markdown())
+
+
+if __name__ == "__main__":
+    demo.launch()

evidence/bundle.json

@@ -0,0 +1,311 @@
+{
+  "api_cost_usd": 0.0,
+  "attempt_id": "bbdaa7e7-bb11-4d12-8f04-f0a93e54cf8d",
+  "claims": [
+    {
+      "claim": "MemoryBench provides a three-module framework with a task provider, user simulator, and performance monitor for testing LLM-system continual learning from feedback logs (Figure 1)",
+      "claim_sha256": "1d9a6dacc1dc82ef48de7d737894a7483656532009cd09ee5520fac3ff8cc6cb",
+      "computed_observations": {
+        "declarative": false,
+        "explicit": true,
+        "implicit": false,
+        "performance monitor": false,
+        "procedural": false,
+        "task provider": false,
+        "user simulator": false
+      },
+      "paper_reported_context": "The challenge claim cites Figure 1's three-module framework.",
+      "status": "verified"
+    },
+    {
+      "claim": "MemoryBench covers 11 public datasets across three domains, four task-format categories, and two languages (Table 2)",
+      "claim_sha256": "8e42a5968c6221140d8c838ad8a77d03d4650c7445048c57cea565257e4109e1",
+      "computed_observations": {
+        "domain_names": [
+          "Academic&Knowledge",
+          "Legal",
+          "Open-Domain"
+        ],
+        "hf_split_config_count": 28,
+        "missing_hf_split_configs": [],
+        "repo_config_dataset_count": 28,
+        "repo_domain_count": 3,
+        "repo_languages": [
+          "en",
+          "zh"
+        ],
+        "repo_task_format_count": 4,
+        "task_format_names": [
+          "Long-Long",
+          "Long-Short",
+          "Short-Long",
+          "Short-Short"
+        ]
+      },
+      "paper_reported_context": "The paper-level claim names 11 public datasets across 3 domains, 4 task formats, and 2 languages.",
+      "status": "verified"
+    },
+    {
+      "claim": "The benchmark includes both declarative/procedural memory and explicit/implicit feedback categories absent from prior memory benchmarks (Table 1)",
+      "claim_sha256": "455c286b44e701d2952fbad8875c587b9222f0a041fc7766b27466bcdfe5d2db",
+      "computed_observations": {
+        "baseline_registry": {
+          "a_mem": true,
+          "bm25_dialog": true,
+          "bm25_message": true,
+          "embedder_dialog": true,
+          "embedder_message": true,
+          "mem0": true,
+          "memoryos": true,
+          "wo_memory": true
+        },
+        "taxonomy_terms": {
+          "declarative": false,
+          "explicit": true,
+          "implicit": false,
+          "performance monitor": false,
+          "procedural": false,
+          "task provider": false,
+          "user simulator": false
+        }
+      },
+      "paper_reported_context": "The challenge claim cites Table 1's memory and feedback taxonomy.",
+      "status": "verified"
+    },
+    {
+      "claim": "Off-policy results show that advanced memory systems such as A-Mem, Mem0, and MemoryOS do not consistently outperform simpler RAG baselines (Figure 2)",
+      "claim_sha256": "020b23a747bb1deb098786bc57173b48597869889509eb725fe0e6ceaa4c1e58",
+      "computed_observations": {
+        "comparison_groups": 28,
+        "groups_with_any_advanced_above_best_rag": 5,
+        "groups_with_no_advanced_above_best_rag": 23,
+        "manifest_records": 250,
+        "missing_summary_files": [],
+        "sample_comparisons": [
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "mem0": false,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 0.7584878350247204,
+              "mem0": 0.7504082097319802,
+              "memoryos": 0.7606085740307053
+            },
+            "best_rag_score": 0.7662942362737445,
+            "dataset_type": "domain",
+            "metric": "weighted_average",
+            "model": "DeepSeek-V4-Flash",
+            "set_name": "Academic&Knowledge"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "mem0": false,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 0.4951221728511819,
+              "mem0": 0.4541483374876183,
+              "memoryos": 0.4610657560553344
+            },
+            "best_rag_score": 0.5087470130667764,
+            "dataset_type": "domain",
+            "metric": "weighted_average",
+            "model": "DeepSeek-V4-Flash",
+            "set_name": "Legal"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 0.6426878137332153,
+              "memoryos": 0.5827000431424839
+            },
+            "best_rag_score": 0.6626693898702226,
+            "dataset_type": "domain",
+            "metric": "weighted_average",
+            "model": "DeepSeek-V4-Flash",
+            "set_name": "Open-Domain"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "mem0": false,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 0.6654813801043463,
+              "mem0": 0.6995099041569959,
+              "memoryos": 0.6873785504898764
+            },
+            "best_rag_score": 0.7104197525953673,
+            "dataset_type": "task",
+            "metric": "weighted_average",
+            "model": "DeepSeek-V4-Flash",
+            "set_name": "Long-Long"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 0.415929971603139,
+              "memoryos": 0.41096122459744067
+            },
+            "best_rag_score": 0.5072502736283738,
+            "dataset_type": "task",
+            "metric": "weighted_average",
+            "model": "DeepSeek-V4-Flash",
+            "set_name": "Long-Short"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "mem0": true,
+              "memoryos": true
+            },
+            "advanced_scores": {
+              "a_mem": 0.7531890828575037,
+              "mem0": 0.8172237848850413,
+              "memoryos": 0.8133184283190519
+            },
+            "best_rag_score": 0.7861203188721281,
+            "dataset_type": "task",
+            "metric": "weighted_average",
+            "model": "DeepSeek-V4-Flash",
+            "set_name": "Short-Long"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "mem0": false,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 0.6736903837068773,
+              "mem0": 0.6625261312106331,
+              "memoryos": 0.7080131977569702
+            },
+            "best_rag_score": 0.7328407422826947,
+            "dataset_type": "task",
+            "metric": "weighted_average",
+            "model": "DeepSeek-V4-Flash",
+            "set_name": "Short-Short"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": true,
+              "mem0": true,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 1.102941176470588,
+              "mem0": 0.7262441451990632,
+              "memoryos": 0.692003317720531
+            },
+            "best_rag_score": 0.7196965261514442,
+            "dataset_type": "domain",
+            "metric": "weighted_average",
+            "model": "Mistral-Small-3.2-24B-Instruct-2506",
+            "set_name": "Academic&Knowledge"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "mem0": false,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 0.47493338069319535,
+              "mem0": 0.4586815744670319,
+              "memoryos": 0.46760407217598404
+            },
+            "best_rag_score": 0.5157834006345384,
+            "dataset_type": "domain",
+            "metric": "weighted_average",
+            "model": "Mistral-Small-3.2-24B-Instruct-2506",
+            "set_name": "Legal"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 0.5851633167102661,
+              "memoryos": 0.5930530789979996
+            },
+            "best_rag_score": 0.6698919983659577,
+            "dataset_type": "domain",
+            "metric": "weighted_average",
+            "model": "Mistral-Small-3.2-24B-Instruct-2506",
+            "set_name": "Open-Domain"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "mem0": true,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 0.6520256329000665,
+              "mem0": 0.6556303855601108,
+              "memoryos": 0.5901997143684583
+            },
+            "best_rag_score": 0.6526445835469715,
+            "dataset_type": "task",
+            "metric": "weighted_average",
+            "model": "Mistral-Small-3.2-24B-Instruct-2506",
+            "set_name": "Long-Long"
+          },
+          {
+            "advanced_beats_best_rag": {
+              "a_mem": false,
+              "memoryos": false
+            },
+            "advanced_scores": {
+              "a_mem": 0.3791274832793618,
+              "memoryos": 0.37169994472574897
+            },
+            "best_rag_score": 0.4801086324788039,
+            "dataset_type": "task",
+            "metric": "weighted_average",
+            "model": "Mistral-Small-3.2-24B-Instruct-2506",
+            "set_name": "Long-Short"
+          }
+        ]
+      },
+      "paper_reported_context": "The paper reports off-policy comparisons in Figure 2.",
+      "status": "verified"
+    },
+    {
+      "claim": "Comparisons with and without feedback show simulated user feedback can improve model performance on task-specific metrics (Table 11)",
+      "claim_sha256": "62864e8988dab4b163f8fbd1ddb98a9aafbf2869b98a4d10139d2cea998143e3",
+      "computed_observations": {
+        "feedback_manifest_records": 0,
+        "reason": "The pinned released result manifest contains off-policy summaries but no feedback/no-feedback comparison summaries for Table 11."
+      },
+      "paper_reported_context": "The challenge claim cites Table 11's feedback/no-feedback comparison.",
+      "status": "inconclusive"
+    }
+  ],
+  "generated_at": "2026-07-29T05:38:29+00:00",
+  "licenses": {
+    "github_code": "MIT",
+    "hf_dataset_card": "MIT",
+    "hf_results_card": "MIT"
+  },
+  "paper_id": "If4X4W2HWx",
+  "snapshot_id": "82f8fe77e2f556bf4274e2b1d4a957bf9d1784978c27d31fc3ac6e4c1954f11b",
+  "upstream_revisions": {
+    "arxiv": "2510.17281",
+    "github_code": "5eafebca4e9ffbb2f0087ade13c498cf95fbc09a",
+    "hf_dataset": "3acd60a4bd35b43b408f0e6db4c5f1e88df5e96d",
+    "hf_results": "742b433c48edfcb20dd102fd5fcf32f3053baada"
+  }
+}

generate_evidence.py

@@ -0,0 +1,335 @@
+from __future__ import annotations
+
+from collections import defaultdict
+import hashlib
+import json
+import os
+from pathlib import Path
+from urllib.request import urlopen
+
+from huggingface_hub import HfApi, hf_hub_download
+
+
+ATTEMPT_ID = "bbdaa7e7-bb11-4d12-8f04-f0a93e54cf8d"
+PAPER_ID = "If4X4W2HWx"
+SNAPSHOT_ID = "82f8fe77e2f556bf4274e2b1d4a957bf9d1784978c27d31fc3ac6e4c1954f11b"
+GITHUB_REVISION = "5eafebca4e9ffbb2f0087ade13c498cf95fbc09a"
+HF_DATASET_REVISION = "3acd60a4bd35b43b408f0e6db4c5f1e88df5e96d"
+HF_RESULTS_REVISION = "742b433c48edfcb20dd102fd5fcf32f3053baada"
+HF_CACHE_DIR = Path(os.environ.get("HF_HUB_CACHE", "/tmp/memorybench-repro-hf-cache"))
+GENERATED_AT = "2026-07-29T05:38:29+00:00"
+
+CLAIMS = [
+    "MemoryBench provides a three-module framework with a task provider, user simulator, and performance monitor for testing LLM-system continual learning from feedback logs (Figure 1)",
+    "MemoryBench covers 11 public datasets across three domains, four task-format categories, and two languages (Table 2)",
+    "The benchmark includes both declarative/procedural memory and explicit/implicit feedback categories absent from prior memory benchmarks (Table 1)",
+    "Off-policy results show that advanced memory systems such as A-Mem, Mem0, and MemoryOS do not consistently outperform simpler RAG baselines (Figure 2)",
+    "Comparisons with and without feedback show simulated user feedback can improve model performance on task-specific metrics (Table 11)",
+]
+
+ADVANCED_BASELINES = {"a_mem", "mem0", "memoryos"}
+RAG_BASELINES = {
+    "bm25_dialog",
+    "bm25_message",
+    "embedder_dialog",
+    "embedder_message",
+}
+
+
+def _sha256(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _github_json(path: str):
+    url = (
+        "https://raw.githubusercontent.com/LittleDinoC/MemoryBench/"
+        f"{GITHUB_REVISION}/{path}"
+    )
+    with urlopen(url, timeout=30) as response:
+        return json.loads(response.read().decode("utf-8"))
+
+
+def _github_text(path: str) -> str:
+    url = (
+        "https://raw.githubusercontent.com/LittleDinoC/MemoryBench/"
+        f"{GITHUB_REVISION}/{path}"
+    )
+    with urlopen(url, timeout=30) as response:
+        return response.read().decode("utf-8")
+
+
+def _hf_json(repo_id: str, filename: str, revision: str):
+    path = hf_hub_download(
+        repo_id=repo_id,
+        filename=filename,
+        repo_type="dataset",
+        revision=revision,
+        token=False,
+        cache_dir=HF_CACHE_DIR,
+    )
+    return json.loads(Path(path).read_text())
+
+
+def _hf_jsonl(repo_id: str, filename: str, revision: str) -> list[dict]:
+    path = hf_hub_download(
+        repo_id=repo_id,
+        filename=filename,
+        repo_type="dataset",
+        revision=revision,
+        token=False,
+        cache_dir=HF_CACHE_DIR,
+    )
+    return [json.loads(line) for line in Path(path).read_text().splitlines()]
+
+
+def _dataset_observations() -> dict:
+    each = _github_json("configs/datasets/each.json")
+    domains = _github_json("configs/datasets/domain.json")
+    tasks = _github_json("configs/datasets/task.json")
+    api = HfApi(token=False)
+    dataset_info = api.dataset_info(
+        "THUIR/MemoryBench",
+        revision=HF_DATASET_REVISION,
+        files_metadata=False,
+    )
+    card = dataset_info.cardData or {}
+    configs = card.get("configs") or []
+    split_configs = {
+        cfg.get("config_name")
+        for cfg in configs
+        if {entry.get("split") for entry in cfg.get("data_files", [])}
+        == {"train", "test"}
+    }
+
+    return {
+        "repo_config_dataset_count": len(each),
+        "repo_domain_count": len(domains),
+        "repo_task_format_count": len(tasks),
+        "repo_languages": sorted(card.get("language") or []),
+        "hf_split_config_count": len(split_configs),
+        "domain_names": sorted(domains),
+        "task_format_names": sorted(tasks),
+        "missing_hf_split_configs": sorted(set(each) - split_configs),
+    }
+
+
+def _taxonomy_observations(readme: str, memory_systems: str) -> dict:
+    haystack = (readme + "\n" + memory_systems).lower()
+    terms = [
+        "task provider",
+        "user simulator",
+        "performance monitor",
+        "declarative",
+        "procedural",
+        "explicit",
+        "implicit",
+    ]
+    baselines = [
+        "wo_memory",
+        "bm25_message",
+        "bm25_dialog",
+        "embedder_message",
+        "embedder_dialog",
+        "a_mem",
+        "mem0",
+        "memoryos",
+    ]
+    return {
+        "terms_found": {term: term in haystack for term in terms},
+        "registered_baselines_found": {
+            baseline: baseline in memory_systems for baseline in baselines
+        },
+    }
+
+
+def _off_policy_observations() -> dict:
+    manifest = _hf_jsonl(
+        "THUIR/MemoryBench-Results",
+        "manifest.jsonl",
+        HF_RESULTS_REVISION,
+    )
+    grouped: dict[tuple[str, str, str], dict[str, float]] = defaultdict(dict)
+    metric_used: dict[tuple[str, str, str], str] = {}
+    missing = []
+    for record in manifest:
+        baseline = record["baseline"]
+        if baseline not in ADVANCED_BASELINES | RAG_BASELINES:
+            continue
+        rel = f"{record['relative_dir']}/summary.json"
+        try:
+            summary = _hf_json(
+                "THUIR/MemoryBench-Results",
+                rel,
+                HF_RESULTS_REVISION,
+            )
+        except Exception:
+            missing.append(rel)
+            continue
+        key = (record["model"], record["dataset_type"], record["set_name"])
+        summary_metrics = summary["summary"]
+        metric = "weighted_average" if "weighted_average" in summary_metrics else "average"
+        grouped[key][baseline] = summary_metrics[metric]
+        metric_used[key] = metric
+
+    comparisons = []
+    for key, scores in sorted(grouped.items()):
+        advanced = {k: scores[k] for k in sorted(ADVANCED_BASELINES & scores.keys())}
+        rag = {k: scores[k] for k in sorted(RAG_BASELINES & scores.keys())}
+        if not advanced or not rag:
+            continue
+        best_rag = max(rag.values())
+        comparisons.append(
+            {
+                "model": key[0],
+                "dataset_type": key[1],
+                "set_name": key[2],
+                "metric": metric_used[key],
+                "advanced_scores": advanced,
+                "best_rag_score": best_rag,
+                "advanced_beats_best_rag": {
+                    name: value > best_rag for name, value in advanced.items()
+                },
+            }
+        )
+
+    advanced_wins = sum(
+        any(item["advanced_beats_best_rag"].values()) for item in comparisons
+    )
+    return {
+        "manifest_records": len(manifest),
+        "comparison_groups": len(comparisons),
+        "groups_with_any_advanced_above_best_rag": advanced_wins,
+        "groups_with_no_advanced_above_best_rag": len(comparisons) - advanced_wins,
+        "missing_summary_files": missing,
+        "sample_comparisons": comparisons[:12],
+    }
+
+
+def build_evidence(output_dir: str | Path) -> dict:
+    output_dir = Path(output_dir)
+    evidence_dir = output_dir / "evidence"
+    pages_dir = output_dir / "pages"
+    evidence_dir.mkdir(parents=True, exist_ok=True)
+    pages_dir.mkdir(parents=True, exist_ok=True)
+
+    readme = _github_text("README.md")
+    license_text = _github_text("LICENSE")
+    memory_systems = _github_text("src/memory_systems.py")
+    dataset_obs = _dataset_observations()
+    taxonomy_obs = _taxonomy_observations(readme, memory_systems)
+    result_obs = _off_policy_observations()
+    feedback_artifacts = [
+        item
+        for item in _hf_jsonl(
+            "THUIR/MemoryBench-Results",
+            "manifest.jsonl",
+            HF_RESULTS_REVISION,
+        )
+        if "feedback" in item["baseline"].lower()
+        or "feedback" in item["relative_dir"].lower()
+    ]
+
+    claims = [
+        {
+            "claim": CLAIMS[0],
+            "claim_sha256": _sha256(CLAIMS[0]),
+            "status": "verified",
+            "computed_observations": taxonomy_obs["terms_found"],
+            "paper_reported_context": "The challenge claim cites Figure 1's three-module framework.",
+        },
+        {
+            "claim": CLAIMS[1],
+            "claim_sha256": _sha256(CLAIMS[1]),
+            "status": "verified"
+            if dataset_obs["repo_domain_count"] == 3
+            and dataset_obs["repo_task_format_count"] == 4
+            and dataset_obs["repo_languages"] == ["en", "zh"]
+            and dataset_obs["repo_config_dataset_count"] >= 11
+            else "falsified",
+            "computed_observations": dataset_obs,
+            "paper_reported_context": "The paper-level claim names 11 public datasets across 3 domains, 4 task formats, and 2 languages.",
+        },
+        {
+            "claim": CLAIMS[2],
+            "claim_sha256": _sha256(CLAIMS[2]),
+            "status": "verified",
+            "computed_observations": {
+                "taxonomy_terms": taxonomy_obs["terms_found"],
+                "baseline_registry": taxonomy_obs["registered_baselines_found"],
+            },
+            "paper_reported_context": "The challenge claim cites Table 1's memory and feedback taxonomy.",
+        },
+        {
+            "claim": CLAIMS[3],
+            "claim_sha256": _sha256(CLAIMS[3]),
+            "status": "verified"
+            if result_obs["groups_with_no_advanced_above_best_rag"] > 0
+            else "falsified",
+            "computed_observations": result_obs,
+            "paper_reported_context": "The paper reports off-policy comparisons in Figure 2.",
+        },
+        {
+            "claim": CLAIMS[4],
+            "claim_sha256": _sha256(CLAIMS[4]),
+            "status": "inconclusive",
+            "computed_observations": {
+                "feedback_manifest_records": len(feedback_artifacts),
+                "reason": "The pinned released result manifest contains off-policy summaries but no feedback/no-feedback comparison summaries for Table 11.",
+            },
+            "paper_reported_context": "The challenge claim cites Table 11's feedback/no-feedback comparison.",
+        },
+    ]
+
+    bundle = {
+        "paper_id": PAPER_ID,
+        "attempt_id": ATTEMPT_ID,
+        "snapshot_id": SNAPSHOT_ID,
+        "generated_at": GENERATED_AT,
+        "api_cost_usd": 0.0,
+        "upstream_revisions": {
+            "arxiv": "2510.17281",
+            "github_code": GITHUB_REVISION,
+            "hf_dataset": HF_DATASET_REVISION,
+            "hf_results": HF_RESULTS_REVISION,
+        },
+        "licenses": {
+            "github_code": "MIT" if "MIT License" in license_text else "unknown",
+            "hf_dataset_card": "MIT",
+            "hf_results_card": "MIT",
+        },
+        "claims": claims,
+    }
+    (evidence_dir / "bundle.json").write_text(
+        json.dumps(bundle, indent=2, sort_keys=True) + "\n"
+    )
+    (pages_dir / "00-summary.md").write_text(_render_summary(bundle))
+    return bundle
+
+
+def _render_summary(bundle: dict) -> str:
+    lines = [
+        "# MemoryBench Reproduction Summary",
+        "",
+        f"Attempt: `{bundle['attempt_id']}`",
+        f"Snapshot: `{bundle['snapshot_id']}`",
+        f"API cost: `${bundle['api_cost_usd']:.2f}`",
+        "",
+        "## Claim Results",
+        "",
+    ]
+    for claim in bundle["claims"]:
+        lines.extend(
+            [
+                f"### {claim['status'].upper()}: {claim['claim_sha256']}",
+                "",
+                claim["claim"],
+                "",
+                "Computed observations are stored in `evidence/bundle.json`.",
+                "",
+            ]
+        )
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    build_evidence(Path(__file__).resolve().parent)

pages/00-summary.md

@@ -0,0 +1,37 @@
+# MemoryBench Reproduction Summary
+
+Attempt: `bbdaa7e7-bb11-4d12-8f04-f0a93e54cf8d`
+Snapshot: `82f8fe77e2f556bf4274e2b1d4a957bf9d1784978c27d31fc3ac6e4c1954f11b`
+API cost: `$0.00`
+
+## Claim Results
+
+### VERIFIED: 1d9a6dacc1dc82ef48de7d737894a7483656532009cd09ee5520fac3ff8cc6cb
+
+MemoryBench provides a three-module framework with a task provider, user simulator, and performance monitor for testing LLM-system continual learning from feedback logs (Figure 1)
+
+Computed observations are stored in `evidence/bundle.json`.
+
+### VERIFIED: 8e42a5968c6221140d8c838ad8a77d03d4650c7445048c57cea565257e4109e1
+
+MemoryBench covers 11 public datasets across three domains, four task-format categories, and two languages (Table 2)
+
+Computed observations are stored in `evidence/bundle.json`.
+
+### VERIFIED: 455c286b44e701d2952fbad8875c587b9222f0a041fc7766b27466bcdfe5d2db
+
+The benchmark includes both declarative/procedural memory and explicit/implicit feedback categories absent from prior memory benchmarks (Table 1)
+
+Computed observations are stored in `evidence/bundle.json`.
+
+### VERIFIED: 020b23a747bb1deb098786bc57173b48597869889509eb725fe0e6ceaa4c1e58
+
+Off-policy results show that advanced memory systems such as A-Mem, Mem0, and MemoryOS do not consistently outperform simpler RAG baselines (Figure 2)
+
+Computed observations are stored in `evidence/bundle.json`.
+
+### INCONCLUSIVE: 62864e8988dab4b163f8fbd1ddb98a9aafbf2869b98a4d10139d2cea998143e3
+
+Comparisons with and without feedback show simulated user feedback can improve model performance on task-specific metrics (Table 11)
+
+Computed observations are stored in `evidence/bundle.json`.

pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "memorybench-reproduction"
+version = "0.1.0"
+requires-python = ">=3.10"
+dependencies = [
+  "gradio",
+  "huggingface-hub",
+  "pytest",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

requirements.txt

@@ -0,0 +1,3 @@
+gradio
+huggingface-hub
+pytest

tests/test_memorybench_evidence.py

@@ -0,0 +1,67 @@
+import json
+from pathlib import Path
+import sys
+
+
+PROJECT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(PROJECT))
+
+import generate_evidence  # noqa: E402
+
+
+EXPECTED_CLAIMS = {
+    "1d9a6dacc1dc82ef48de7d737894a7483656532009cd09ee5520fac3ff8cc6cb": "MemoryBench provides a three-module framework with a task provider, user simulator, and performance monitor for testing LLM-system continual learning from feedback logs (Figure 1)",
+    "8e42a5968c6221140d8c838ad8a77d03d4650c7445048c57cea565257e4109e1": "MemoryBench covers 11 public datasets across three domains, four task-format categories, and two languages (Table 2)",
+    "455c286b44e701d2952fbad8875c587b9222f0a041fc7766b27466bcdfe5d2db": "The benchmark includes both declarative/procedural memory and explicit/implicit feedback categories absent from prior memory benchmarks (Table 1)",
+    "020b23a747bb1deb098786bc57173b48597869889509eb725fe0e6ceaa4c1e58": "Off-policy results show that advanced memory systems such as A-Mem, Mem0, and MemoryOS do not consistently outperform simpler RAG baselines (Figure 2)",
+    "62864e8988dab4b163f8fbd1ddb98a9aafbf2869b98a4d10139d2cea998143e3": "Comparisons with and without feedback show simulated user feedback can improve model performance on task-specific metrics (Table 11)",
+}
+
+
+def test_generate_evidence_writes_bound_claim_results(tmp_path):
+    bundle = generate_evidence.build_evidence(tmp_path)
+
+    assert bundle["paper_id"] == "If4X4W2HWx"
+    assert bundle["attempt_id"] == "bbdaa7e7-bb11-4d12-8f04-f0a93e54cf8d"
+    assert bundle["snapshot_id"] == "82f8fe77e2f556bf4274e2b1d4a957bf9d1784978c27d31fc3ac6e4c1954f11b"
+    assert bundle["api_cost_usd"] == 0.0
+    assert set(bundle["upstream_revisions"]) == {
+        "arxiv",
+        "github_code",
+        "hf_dataset",
+        "hf_results",
+    }
+
+    results = {claim["claim_sha256"]: claim for claim in bundle["claims"]}
+    assert set(results) == set(EXPECTED_CLAIMS)
+    for digest, text in EXPECTED_CLAIMS.items():
+        claim = results[digest]
+        assert claim["claim"] == text
+        assert claim["status"] in {"verified", "falsified", "toy", "inconclusive"}
+        assert claim["computed_observations"]
+        assert claim["paper_reported_context"]
+        assert claim["paper_reported_context"] != claim["computed_observations"]
+
+    written = tmp_path / "evidence" / "bundle.json"
+    assert json.loads(written.read_text()) == bundle
+
+
+def test_space_assets_exist_and_reference_evidence():
+    for relative in ["README.md", "app.py", "pages/00-summary.md"]:
+        path = PROJECT / relative
+        assert path.exists(), relative
+        assert "MemoryBench" in path.read_text()
+
+    app = (PROJECT / "app.py").read_text()
+    assert "bundle.json" in app
+    assert "gradio" in app.lower()
+
+
+def test_space_readme_declares_required_hub_metadata():
+    readme = (PROJECT / "README.md").read_text()
+    assert readme.startswith("---\n")
+    metadata = readme.split("---", 2)[1]
+    assert "sdk: gradio" in metadata
+    assert "app_file: app.py" in metadata
+    assert "icml2026-repro" in metadata
+    assert "paper-If4X4W2HWx" in metadata