{
 "repo": "hematteo/sparse-readout-prism",
 "generated": "2026-09-03",
 "paper": {
  "title": "Sparse Readout Prism: Explaining Logit-Lens Scores in Features Instead of Tokens",
  "arxiv": "2609.01936",
  "url": "https://arxiv.org/abs/2609.01936"
 },
 "code": {
  "repo": "https://github.com/hematteo/sparse-readout-prism",
  "version": "0.2.2",
  "registry": "configs/registries/exp2_selected_sae_checkpoints.yaml",
  "loader": "sparse_readout_prism.load_factorizer"
 },
 "file_layout": "<model>/<operating_point>/checkpoint.pt",
 "metrics": {
  "rowEV": "row-centered explained variance of the reconstructed unembedding rows (held-out)",
  "top1": "argmax agreement of the reconstructed readout with the original on held-out hidden states",
  "KL_bits": "readout KL in bits; Gemma values computed through the final-logit softcap"
 },
 "layouts": {
  "config": {
   "keys": [
    "model_state_dict",
    "config"
   ],
   "factorizer_block": "config.factorizer",
   "embeds_row_stats": false,
   "embeds_metrics": false,
   "note": "config holds the full training config (data, factorizer, training, run)"
  },
  "flat": {
   "keys": [
    "model_state_dict",
    "factorizer",
    "evaluation",
    "row_mean",
    "row_norms",
    "row_token_ids",
    "metrics"
   ],
   "factorizer_block": "factorizer",
   "embeds_row_stats": true,
   "embeds_metrics": true,
   "note": "row_mean/row_norms are the preprocessing pinned at training time; metrics is the held-out eval"
  }
 },
 "preprocessing": "rows centered then L2-normalized; the paper's analysis runs center on the live full-vocabulary mean of W_U (preprocess_rows(W_U); --centering live), training centered on the text-token rows",
 "checkpoints": [
  {
   "id": "gemma_e2b_k128",
   "path": "gemma-4-e2b/k128_16x/checkpoint.pt",
   "model": "Gemma-4-E2B",
   "base_model": "google/gemma-4-E2B-it",
   "width": "16x",
   "d_features": 24576,
   "d_model": 1536,
   "k": 128,
   "rowEV": 0.714,
   "top1": 0.623,
   "KL_bits": 1.94,
   "size_bytes": 302091735,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table), softcap-correct top-1/KL"
  },
  {
   "id": "gemma_e2b_k256",
   "path": "gemma-4-e2b/k256_32x/checkpoint.pt",
   "model": "Gemma-4-E2B",
   "base_model": "google/gemma-4-E2B-it",
   "width": "32x",
   "d_features": 49152,
   "d_model": 1536,
   "k": 256,
   "rowEV": 0.834,
   "top1": 0.333,
   "KL_bits": 6.37,
   "size_bytes": 604179927,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table), softcap-correct top-1/KL"
  },
  {
   "id": "gemma_e4b_k128",
   "path": "gemma-4-e4b/k128_16x/checkpoint.pt",
   "model": "Gemma-4-E4B",
   "base_model": "google/gemma-4-E4B-it",
   "width": "16x",
   "d_features": 40960,
   "d_model": 2560,
   "k": 128,
   "rowEV": 0.693,
   "top1": 0.669,
   "KL_bits": 1.82,
   "size_bytes": 839028183,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table), softcap-correct top-1/KL"
  },
  {
   "id": "gemma_e4b_k256",
   "path": "gemma-4-e4b/k256_32x/checkpoint.pt",
   "model": "Gemma-4-E4B",
   "base_model": "google/gemma-4-E4B-it",
   "width": "32x",
   "d_features": 81920,
   "d_model": 2560,
   "k": 256,
   "rowEV": 0.827,
   "top1": 0.736,
   "KL_bits": 1.22,
   "size_bytes": 1678052823,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table), softcap-correct top-1/KL"
  },
  {
   "id": "ministral8b_k128",
   "path": "ministral-3-8b/k128_16x/checkpoint.pt",
   "model": "Ministral-3-8B-Base",
   "base_model": "mistralai/Ministral-3-8B-Base-2512",
   "width": "16x",
   "d_features": 65536,
   "d_model": 4096,
   "k": 128,
   "rowEV": 0.806,
   "top1": 0.885,
   "KL_bits": 0.13,
   "size_bytes": 2149339577,
   "layout": "flat",
   "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E"
  },
  {
   "id": "ministral8b_k256",
   "path": "ministral-3-8b/k256_32x/checkpoint.pt",
   "model": "Ministral-3-8B-Base",
   "base_model": "mistralai/Ministral-3-8B-Base-2512",
   "width": "32x",
   "d_features": 131072,
   "d_model": 4096,
   "k": 256,
   "rowEV": 0.888,
   "top1": 0.904,
   "KL_bits": 0.087,
   "size_bytes": 4297085449,
   "layout": "flat",
   "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E"
  },
  {
   "id": "qwen0p8b_k128",
   "path": "qwen3.5-0.8b/k128_16x/checkpoint.pt",
   "model": "Qwen3.5-0.8B",
   "base_model": "Qwen/Qwen3.5-0.8B",
   "width": "16x",
   "d_features": 16384,
   "d_model": 1024,
   "k": 128,
   "rowEV": 0.76,
   "top1": 0.844,
   "KL_bits": 0.277,
   "size_bytes": 134286807,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F"
  },
  {
   "id": "qwen0p8b_k256",
   "path": "qwen3.5-0.8b/k256_32x/checkpoint.pt",
   "model": "Qwen3.5-0.8B",
   "base_model": "Qwen/Qwen3.5-0.8B",
   "width": "32x",
   "d_features": 32768,
   "d_model": 1024,
   "k": 256,
   "rowEV": 0.877,
   "top1": 0.891,
   "KL_bits": 0.135,
   "size_bytes": 268570071,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F"
  },
  {
   "id": "qwen2b_k128",
   "path": "qwen3.5-2b/k128_16x/checkpoint.pt",
   "model": "Qwen3.5-2B",
   "base_model": "Qwen/Qwen3.5-2B",
   "width": "16x",
   "d_features": 32768,
   "d_model": 2048,
   "k": 128,
   "rowEV": 0.712,
   "top1": 0.858,
   "KL_bits": 0.261,
   "size_bytes": 537005527,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F"
  },
  {
   "id": "qwen2b_k256",
   "path": "qwen3.5-2b/k256_32x/checkpoint.pt",
   "model": "Qwen3.5-2B",
   "base_model": "Qwen/Qwen3.5-2B",
   "width": "32x",
   "d_features": 65536,
   "d_model": 2048,
   "k": 256,
   "rowEV": 0.847,
   "top1": 0.887,
   "KL_bits": 0.136,
   "size_bytes": 1074007511,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F"
  },
  {
   "id": "qwen9b_k128_fixed",
   "path": "qwen3.5-9b/k128_8x/checkpoint.pt",
   "model": "Qwen3.5-9B",
   "base_model": "Qwen/Qwen3.5-9B",
   "width": "8x",
   "d_features": 32768,
   "d_model": 4096,
   "k": 128,
   "rowEV": 0.621,
   "top1": 0.846,
   "KL_bits": 0.296,
   "size_bytes": 1073876183,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F"
  },
  {
   "id": "qwen9b_k256_16x",
   "path": "qwen3.5-9b/k256_16x/checkpoint.pt",
   "model": "Qwen3.5-9B",
   "base_model": "Qwen/Qwen3.5-9B",
   "width": "16x",
   "d_features": 65536,
   "d_model": 4096,
   "k": 256,
   "rowEV": 0.761,
   "top1": 0.874,
   "KL_bits": 0.167,
   "size_bytes": 2147749079,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F"
  },
  {
   "id": "qwen9b_k256_32x",
   "path": "qwen3.5-9b/k256_32x/checkpoint.pt",
   "model": "Qwen3.5-9B",
   "base_model": "Qwen/Qwen3.5-9B",
   "width": "32x",
   "d_features": 131072,
   "d_model": 4096,
   "k": 256,
   "rowEV": 0.857,
   "top1": 0.9,
   "KL_bits": 0.105,
   "size_bytes": 4295495171,
   "layout": "config",
   "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F"
  },
  {
   "id": "r1dist_llama8b_k128",
   "path": "r1-distill-llama-8b/k128_16x/checkpoint.pt",
   "model": "R1-Distill-Llama-8B",
   "base_model": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B",
   "width": "16x",
   "d_features": 65536,
   "d_model": 4096,
   "k": 128,
   "rowEV": 0.796,
   "top1": 0.725,
   "KL_bits": 0.536,
   "size_bytes": 2149305849,
   "layout": "flat",
   "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E"
  },
  {
   "id": "r1dist_llama8b_k256",
   "path": "r1-distill-llama-8b/k256_32x/checkpoint.pt",
   "model": "R1-Distill-Llama-8B",
   "base_model": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B",
   "width": "32x",
   "d_features": 131072,
   "d_model": 4096,
   "k": 256,
   "rowEV": 0.888,
   "top1": 0.754,
   "KL_bits": 0.434,
   "size_bytes": 4297051721,
   "layout": "flat",
   "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E"
  },
  {
   "id": "r1dist_qwen7b_k128",
   "path": "r1-distill-qwen-7b/k128_16x/checkpoint.pt",
   "model": "R1-Distill-Qwen-7B",
   "base_model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",
   "width": "16x",
   "d_features": 57344,
   "d_model": 3584,
   "k": 128,
   "rowEV": 0.709,
   "top1": 0.695,
   "KL_bits": 0.777,
   "size_bytes": 1646240185,
   "layout": "flat",
   "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E"
  },
  {
   "id": "r1dist_qwen7b_k256",
   "path": "r1-distill-qwen-7b/k256_32x/checkpoint.pt",
   "model": "R1-Distill-Qwen-7B",
   "base_model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",
   "width": "32x",
   "d_features": 114688,
   "d_model": 3584,
   "k": 256,
   "rowEV": 0.844,
   "top1": 0.76,
   "KL_bits": 0.489,
   "size_bytes": 3290636793,
   "layout": "flat",
   "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E"
  }
 ],
 "replicates": {
  "note": "Seed-variation family used end to end by the paper's basis-stability appendix (cross-seed stability, feature-group matching, leave-one-out core recovery). Three independently seeded dictionaries per width, trained on Qwen3.5-2B under the seed-variation preprocessing recipe. The paper says this recipe differs from the one behind the released qwen3.5-2b/k128_16x and k256_32x dictionaries, so seed 0 here is NOT the released checkpoint; no table in that appendix mixes the two families.",
  "checkpoints": [
   {
    "id": "qwen2b_k128_16x_seedvar_s0",
    "path": "qwen3.5-2b/k128_16x_seedvar_s0/checkpoint.pt",
    "model": "Qwen3.5-2B",
    "base_model": "Qwen/Qwen3.5-2B",
    "family": "seed_variation",
    "seed": 0,
    "width": "16x",
    "d_features": 32768,
    "d_model": 2048,
    "k": 128,
    "rowEV": 0.744,
    "top1": 0.805,
    "KL_bits": 0.286,
    "steps": 15000,
    "size_bytes": 539994553,
    "sha256": "cd5462a450f2696c8353a3b03e2b222ee57f8e0c912473b8850f71a6eada99ed",
    "layout": "config",
    "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)",
    "sidecars": [
     "qwen3.5-2b/k128_16x_seedvar_s0/config.yaml",
     "qwen3.5-2b/k128_16x_seedvar_s0/metrics.json"
    ]
   },
   {
    "id": "qwen2b_k128_16x_seedvar_s1",
    "path": "qwen3.5-2b/k128_16x_seedvar_s1/checkpoint.pt",
    "model": "Qwen3.5-2B",
    "base_model": "Qwen/Qwen3.5-2B",
    "family": "seed_variation",
    "seed": 1,
    "width": "16x",
    "d_features": 32768,
    "d_model": 2048,
    "k": 128,
    "rowEV": 0.745,
    "top1": 0.805,
    "KL_bits": 0.291,
    "steps": 20000,
    "size_bytes": 539994553,
    "sha256": "450c99237a3252084f43d12c6eda1f41a4eaf56ac2391ccbd9d8c9c34d95b584",
    "layout": "config",
    "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)",
    "sidecars": [
     "qwen3.5-2b/k128_16x_seedvar_s1/config.yaml",
     "qwen3.5-2b/k128_16x_seedvar_s1/metrics.json"
    ]
   },
   {
    "id": "qwen2b_k128_16x_seedvar_s2",
    "path": "qwen3.5-2b/k128_16x_seedvar_s2/checkpoint.pt",
    "model": "Qwen3.5-2B",
    "base_model": "Qwen/Qwen3.5-2B",
    "family": "seed_variation",
    "seed": 2,
    "width": "16x",
    "d_features": 32768,
    "d_model": 2048,
    "k": 128,
    "rowEV": 0.744,
    "top1": 0.816,
    "KL_bits": 0.293,
    "steps": 20000,
    "size_bytes": 539994553,
    "sha256": "485432a34dd5b9a135487d5358cedd319197489dc1d493bcf801ce0e55ec80aa",
    "layout": "config",
    "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)",
    "sidecars": [
     "qwen3.5-2b/k128_16x_seedvar_s2/config.yaml",
     "qwen3.5-2b/k128_16x_seedvar_s2/metrics.json"
    ]
   },
   {
    "id": "qwen2b_k256_32x_seedvar_s0",
    "path": "qwen3.5-2b/k256_32x_seedvar_s0/checkpoint.pt",
    "model": "Qwen3.5-2B",
    "base_model": "Qwen/Qwen3.5-2B",
    "family": "seed_variation",
    "seed": 0,
    "width": "32x",
    "d_features": 65536,
    "d_model": 2048,
    "k": 256,
    "rowEV": 0.824,
    "top1": 0.807,
    "KL_bits": 0.202,
    "steps": 17500,
    "size_bytes": 1076996537,
    "sha256": "81cad1148e0ed0d9065691b5fec2dc033f51a5374f8b23d0c1c01be830d653e7",
    "layout": "config",
    "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)",
    "sidecars": [
     "qwen3.5-2b/k256_32x_seedvar_s0/config.yaml",
     "qwen3.5-2b/k256_32x_seedvar_s0/metrics.json"
    ]
   },
   {
    "id": "qwen2b_k256_32x_seedvar_s1",
    "path": "qwen3.5-2b/k256_32x_seedvar_s1/checkpoint.pt",
    "model": "Qwen3.5-2B",
    "base_model": "Qwen/Qwen3.5-2B",
    "family": "seed_variation",
    "seed": 1,
    "width": "32x",
    "d_features": 65536,
    "d_model": 2048,
    "k": 256,
    "rowEV": 0.823,
    "top1": 0.822,
    "KL_bits": 0.222,
    "steps": 17500,
    "size_bytes": 1076996537,
    "sha256": "f0ce4816b260a25e23834472ae76b3c4a01b1bcb18a681a21f16349aa6cd9acf",
    "layout": "config",
    "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)",
    "sidecars": [
     "qwen3.5-2b/k256_32x_seedvar_s1/config.yaml",
     "qwen3.5-2b/k256_32x_seedvar_s1/metrics.json"
    ]
   },
   {
    "id": "qwen2b_k256_32x_seedvar_s2",
    "path": "qwen3.5-2b/k256_32x_seedvar_s2/checkpoint.pt",
    "model": "Qwen3.5-2B",
    "base_model": "Qwen/Qwen3.5-2B",
    "family": "seed_variation",
    "seed": 2,
    "width": "32x",
    "d_features": 65536,
    "d_model": 2048,
    "k": 256,
    "rowEV": 0.826,
    "top1": 0.848,
    "KL_bits": 0.194,
    "steps": 17500,
    "size_bytes": 1076996537,
    "sha256": "c30f537c387c66f5a92fa5321aef38278d69ede42b9f78254e288160735d7a57",
    "layout": "config",
    "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)",
    "sidecars": [
     "qwen3.5-2b/k256_32x_seedvar_s2/config.yaml",
     "qwen3.5-2b/k256_32x_seedvar_s2/metrics.json"
    ]
   }
  ]
 }
}