| { |
| "repo": "hematteo/sparse-readout-prism", |
| "generated": "2026-09-03", |
| "paper": { |
| "title": "Sparse Readout Prism: Explaining Logit-Lens Scores in Features Instead of Tokens", |
| "arxiv": "2609.01936", |
| "url": "https://arxiv.org/abs/2609.01936" |
| }, |
| "code": { |
| "repo": "https://github.com/hematteo/sparse-readout-prism", |
| "version": "0.2.2", |
| "registry": "configs/registries/exp2_selected_sae_checkpoints.yaml", |
| "loader": "sparse_readout_prism.load_factorizer" |
| }, |
| "file_layout": "<model>/<operating_point>/checkpoint.pt", |
| "metrics": { |
| "rowEV": "row-centered explained variance of the reconstructed unembedding rows (held-out)", |
| "top1": "argmax agreement of the reconstructed readout with the original on held-out hidden states", |
| "KL_bits": "readout KL in bits; Gemma values computed through the final-logit softcap" |
| }, |
| "layouts": { |
| "config": { |
| "keys": [ |
| "model_state_dict", |
| "config" |
| ], |
| "factorizer_block": "config.factorizer", |
| "embeds_row_stats": false, |
| "embeds_metrics": false, |
| "note": "config holds the full training config (data, factorizer, training, run)" |
| }, |
| "flat": { |
| "keys": [ |
| "model_state_dict", |
| "factorizer", |
| "evaluation", |
| "row_mean", |
| "row_norms", |
| "row_token_ids", |
| "metrics" |
| ], |
| "factorizer_block": "factorizer", |
| "embeds_row_stats": true, |
| "embeds_metrics": true, |
| "note": "row_mean/row_norms are the preprocessing pinned at training time; metrics is the held-out eval" |
| } |
| }, |
| "preprocessing": "rows centered then L2-normalized; the paper's analysis runs center on the live full-vocabulary mean of W_U (preprocess_rows(W_U); --centering live), training centered on the text-token rows", |
| "checkpoints": [ |
| { |
| "id": "gemma_e2b_k128", |
| "path": "gemma-4-e2b/k128_16x/checkpoint.pt", |
| "model": "Gemma-4-E2B", |
| "base_model": "google/gemma-4-E2B-it", |
| "width": "16x", |
| "d_features": 24576, |
| "d_model": 1536, |
| "k": 128, |
| "rowEV": 0.714, |
| "top1": 0.623, |
| "KL_bits": 1.94, |
| "size_bytes": 302091735, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table), softcap-correct top-1/KL" |
| }, |
| { |
| "id": "gemma_e2b_k256", |
| "path": "gemma-4-e2b/k256_32x/checkpoint.pt", |
| "model": "Gemma-4-E2B", |
| "base_model": "google/gemma-4-E2B-it", |
| "width": "32x", |
| "d_features": 49152, |
| "d_model": 1536, |
| "k": 256, |
| "rowEV": 0.834, |
| "top1": 0.333, |
| "KL_bits": 6.37, |
| "size_bytes": 604179927, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table), softcap-correct top-1/KL" |
| }, |
| { |
| "id": "gemma_e4b_k128", |
| "path": "gemma-4-e4b/k128_16x/checkpoint.pt", |
| "model": "Gemma-4-E4B", |
| "base_model": "google/gemma-4-E4B-it", |
| "width": "16x", |
| "d_features": 40960, |
| "d_model": 2560, |
| "k": 128, |
| "rowEV": 0.693, |
| "top1": 0.669, |
| "KL_bits": 1.82, |
| "size_bytes": 839028183, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table), softcap-correct top-1/KL" |
| }, |
| { |
| "id": "gemma_e4b_k256", |
| "path": "gemma-4-e4b/k256_32x/checkpoint.pt", |
| "model": "Gemma-4-E4B", |
| "base_model": "google/gemma-4-E4B-it", |
| "width": "32x", |
| "d_features": 81920, |
| "d_model": 2560, |
| "k": 256, |
| "rowEV": 0.827, |
| "top1": 0.736, |
| "KL_bits": 1.22, |
| "size_bytes": 1678052823, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table), softcap-correct top-1/KL" |
| }, |
| { |
| "id": "ministral8b_k128", |
| "path": "ministral-3-8b/k128_16x/checkpoint.pt", |
| "model": "Ministral-3-8B-Base", |
| "base_model": "mistralai/Ministral-3-8B-Base-2512", |
| "width": "16x", |
| "d_features": 65536, |
| "d_model": 4096, |
| "k": 128, |
| "rowEV": 0.806, |
| "top1": 0.885, |
| "KL_bits": 0.13, |
| "size_bytes": 2149339577, |
| "layout": "flat", |
| "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E" |
| }, |
| { |
| "id": "ministral8b_k256", |
| "path": "ministral-3-8b/k256_32x/checkpoint.pt", |
| "model": "Ministral-3-8B-Base", |
| "base_model": "mistralai/Ministral-3-8B-Base-2512", |
| "width": "32x", |
| "d_features": 131072, |
| "d_model": 4096, |
| "k": 256, |
| "rowEV": 0.888, |
| "top1": 0.904, |
| "KL_bits": 0.087, |
| "size_bytes": 4297085449, |
| "layout": "flat", |
| "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E" |
| }, |
| { |
| "id": "qwen0p8b_k128", |
| "path": "qwen3.5-0.8b/k128_16x/checkpoint.pt", |
| "model": "Qwen3.5-0.8B", |
| "base_model": "Qwen/Qwen3.5-0.8B", |
| "width": "16x", |
| "d_features": 16384, |
| "d_model": 1024, |
| "k": 128, |
| "rowEV": 0.76, |
| "top1": 0.844, |
| "KL_bits": 0.277, |
| "size_bytes": 134286807, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F" |
| }, |
| { |
| "id": "qwen0p8b_k256", |
| "path": "qwen3.5-0.8b/k256_32x/checkpoint.pt", |
| "model": "Qwen3.5-0.8B", |
| "base_model": "Qwen/Qwen3.5-0.8B", |
| "width": "32x", |
| "d_features": 32768, |
| "d_model": 1024, |
| "k": 256, |
| "rowEV": 0.877, |
| "top1": 0.891, |
| "KL_bits": 0.135, |
| "size_bytes": 268570071, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F" |
| }, |
| { |
| "id": "qwen2b_k128", |
| "path": "qwen3.5-2b/k128_16x/checkpoint.pt", |
| "model": "Qwen3.5-2B", |
| "base_model": "Qwen/Qwen3.5-2B", |
| "width": "16x", |
| "d_features": 32768, |
| "d_model": 2048, |
| "k": 128, |
| "rowEV": 0.712, |
| "top1": 0.858, |
| "KL_bits": 0.261, |
| "size_bytes": 537005527, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F" |
| }, |
| { |
| "id": "qwen2b_k256", |
| "path": "qwen3.5-2b/k256_32x/checkpoint.pt", |
| "model": "Qwen3.5-2B", |
| "base_model": "Qwen/Qwen3.5-2B", |
| "width": "32x", |
| "d_features": 65536, |
| "d_model": 2048, |
| "k": 256, |
| "rowEV": 0.847, |
| "top1": 0.887, |
| "KL_bits": 0.136, |
| "size_bytes": 1074007511, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F" |
| }, |
| { |
| "id": "qwen9b_k128_fixed", |
| "path": "qwen3.5-9b/k128_8x/checkpoint.pt", |
| "model": "Qwen3.5-9B", |
| "base_model": "Qwen/Qwen3.5-9B", |
| "width": "8x", |
| "d_features": 32768, |
| "d_model": 4096, |
| "k": 128, |
| "rowEV": 0.621, |
| "top1": 0.846, |
| "KL_bits": 0.296, |
| "size_bytes": 1073876183, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F" |
| }, |
| { |
| "id": "qwen9b_k256_16x", |
| "path": "qwen3.5-9b/k256_16x/checkpoint.pt", |
| "model": "Qwen3.5-9B", |
| "base_model": "Qwen/Qwen3.5-9B", |
| "width": "16x", |
| "d_features": 65536, |
| "d_model": 4096, |
| "k": 256, |
| "rowEV": 0.761, |
| "top1": 0.874, |
| "KL_bits": 0.167, |
| "size_bytes": 2147749079, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F" |
| }, |
| { |
| "id": "qwen9b_k256_32x", |
| "path": "qwen3.5-9b/k256_32x/checkpoint.pt", |
| "model": "Qwen3.5-9B", |
| "base_model": "Qwen/Qwen3.5-9B", |
| "width": "32x", |
| "d_features": 131072, |
| "d_model": 4096, |
| "k": 256, |
| "rowEV": 0.857, |
| "top1": 0.9, |
| "KL_bits": 0.105, |
| "size_bytes": 4295495171, |
| "layout": "config", |
| "metrics_source": "paper Appendix C (selection table); the 9B 8x point is in Appendices D and F" |
| }, |
| { |
| "id": "r1dist_llama8b_k128", |
| "path": "r1-distill-llama-8b/k128_16x/checkpoint.pt", |
| "model": "R1-Distill-Llama-8B", |
| "base_model": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", |
| "width": "16x", |
| "d_features": 65536, |
| "d_model": 4096, |
| "k": 128, |
| "rowEV": 0.796, |
| "top1": 0.725, |
| "KL_bits": 0.536, |
| "size_bytes": 2149305849, |
| "layout": "flat", |
| "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E" |
| }, |
| { |
| "id": "r1dist_llama8b_k256", |
| "path": "r1-distill-llama-8b/k256_32x/checkpoint.pt", |
| "model": "R1-Distill-Llama-8B", |
| "base_model": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", |
| "width": "32x", |
| "d_features": 131072, |
| "d_model": 4096, |
| "k": 256, |
| "rowEV": 0.888, |
| "top1": 0.754, |
| "KL_bits": 0.434, |
| "size_bytes": 4297051721, |
| "layout": "flat", |
| "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E" |
| }, |
| { |
| "id": "r1dist_qwen7b_k128", |
| "path": "r1-distill-qwen-7b/k128_16x/checkpoint.pt", |
| "model": "R1-Distill-Qwen-7B", |
| "base_model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", |
| "width": "16x", |
| "d_features": 57344, |
| "d_model": 3584, |
| "k": 128, |
| "rowEV": 0.709, |
| "top1": 0.695, |
| "KL_bits": 0.777, |
| "size_bytes": 1646240185, |
| "layout": "flat", |
| "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E" |
| }, |
| { |
| "id": "r1dist_qwen7b_k256", |
| "path": "r1-distill-qwen-7b/k256_32x/checkpoint.pt", |
| "model": "R1-Distill-Qwen-7B", |
| "base_model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", |
| "width": "32x", |
| "d_features": 114688, |
| "d_model": 3584, |
| "k": 256, |
| "rowEV": 0.844, |
| "top1": 0.76, |
| "KL_bits": 0.489, |
| "size_bytes": 3290636793, |
| "layout": "flat", |
| "metrics_source": "checkpoint['metrics'] (held-out eval); paper Appendix E" |
| } |
| ], |
| "replicates": { |
| "note": "Seed-variation family used end to end by the paper's basis-stability appendix (cross-seed stability, feature-group matching, leave-one-out core recovery). Three independently seeded dictionaries per width, trained on Qwen3.5-2B under the seed-variation preprocessing recipe. The paper says this recipe differs from the one behind the released qwen3.5-2b/k128_16x and k256_32x dictionaries, so seed 0 here is NOT the released checkpoint; no table in that appendix mixes the two families.", |
| "checkpoints": [ |
| { |
| "id": "qwen2b_k128_16x_seedvar_s0", |
| "path": "qwen3.5-2b/k128_16x_seedvar_s0/checkpoint.pt", |
| "model": "Qwen3.5-2B", |
| "base_model": "Qwen/Qwen3.5-2B", |
| "family": "seed_variation", |
| "seed": 0, |
| "width": "16x", |
| "d_features": 32768, |
| "d_model": 2048, |
| "k": 128, |
| "rowEV": 0.744, |
| "top1": 0.805, |
| "KL_bits": 0.286, |
| "steps": 15000, |
| "size_bytes": 539994553, |
| "sha256": "cd5462a450f2696c8353a3b03e2b222ee57f8e0c912473b8850f71a6eada99ed", |
| "layout": "config", |
| "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)", |
| "sidecars": [ |
| "qwen3.5-2b/k128_16x_seedvar_s0/config.yaml", |
| "qwen3.5-2b/k128_16x_seedvar_s0/metrics.json" |
| ] |
| }, |
| { |
| "id": "qwen2b_k128_16x_seedvar_s1", |
| "path": "qwen3.5-2b/k128_16x_seedvar_s1/checkpoint.pt", |
| "model": "Qwen3.5-2B", |
| "base_model": "Qwen/Qwen3.5-2B", |
| "family": "seed_variation", |
| "seed": 1, |
| "width": "16x", |
| "d_features": 32768, |
| "d_model": 2048, |
| "k": 128, |
| "rowEV": 0.745, |
| "top1": 0.805, |
| "KL_bits": 0.291, |
| "steps": 20000, |
| "size_bytes": 539994553, |
| "sha256": "450c99237a3252084f43d12c6eda1f41a4eaf56ac2391ccbd9d8c9c34d95b584", |
| "layout": "config", |
| "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)", |
| "sidecars": [ |
| "qwen3.5-2b/k128_16x_seedvar_s1/config.yaml", |
| "qwen3.5-2b/k128_16x_seedvar_s1/metrics.json" |
| ] |
| }, |
| { |
| "id": "qwen2b_k128_16x_seedvar_s2", |
| "path": "qwen3.5-2b/k128_16x_seedvar_s2/checkpoint.pt", |
| "model": "Qwen3.5-2B", |
| "base_model": "Qwen/Qwen3.5-2B", |
| "family": "seed_variation", |
| "seed": 2, |
| "width": "16x", |
| "d_features": 32768, |
| "d_model": 2048, |
| "k": 128, |
| "rowEV": 0.744, |
| "top1": 0.816, |
| "KL_bits": 0.293, |
| "steps": 20000, |
| "size_bytes": 539994553, |
| "sha256": "485432a34dd5b9a135487d5358cedd319197489dc1d493bcf801ce0e55ec80aa", |
| "layout": "config", |
| "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)", |
| "sidecars": [ |
| "qwen3.5-2b/k128_16x_seedvar_s2/config.yaml", |
| "qwen3.5-2b/k128_16x_seedvar_s2/metrics.json" |
| ] |
| }, |
| { |
| "id": "qwen2b_k256_32x_seedvar_s0", |
| "path": "qwen3.5-2b/k256_32x_seedvar_s0/checkpoint.pt", |
| "model": "Qwen3.5-2B", |
| "base_model": "Qwen/Qwen3.5-2B", |
| "family": "seed_variation", |
| "seed": 0, |
| "width": "32x", |
| "d_features": 65536, |
| "d_model": 2048, |
| "k": 256, |
| "rowEV": 0.824, |
| "top1": 0.807, |
| "KL_bits": 0.202, |
| "steps": 17500, |
| "size_bytes": 1076996537, |
| "sha256": "81cad1148e0ed0d9065691b5fec2dc033f51a5374f8b23d0c1c01be830d653e7", |
| "layout": "config", |
| "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)", |
| "sidecars": [ |
| "qwen3.5-2b/k256_32x_seedvar_s0/config.yaml", |
| "qwen3.5-2b/k256_32x_seedvar_s0/metrics.json" |
| ] |
| }, |
| { |
| "id": "qwen2b_k256_32x_seedvar_s1", |
| "path": "qwen3.5-2b/k256_32x_seedvar_s1/checkpoint.pt", |
| "model": "Qwen3.5-2B", |
| "base_model": "Qwen/Qwen3.5-2B", |
| "family": "seed_variation", |
| "seed": 1, |
| "width": "32x", |
| "d_features": 65536, |
| "d_model": 2048, |
| "k": 256, |
| "rowEV": 0.823, |
| "top1": 0.822, |
| "KL_bits": 0.222, |
| "steps": 17500, |
| "size_bytes": 1076996537, |
| "sha256": "f0ce4816b260a25e23834472ae76b3c4a01b1bcb18a681a21f16349aa6cd9acf", |
| "layout": "config", |
| "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)", |
| "sidecars": [ |
| "qwen3.5-2b/k256_32x_seedvar_s1/config.yaml", |
| "qwen3.5-2b/k256_32x_seedvar_s1/metrics.json" |
| ] |
| }, |
| { |
| "id": "qwen2b_k256_32x_seedvar_s2", |
| "path": "qwen3.5-2b/k256_32x_seedvar_s2/checkpoint.pt", |
| "model": "Qwen3.5-2B", |
| "base_model": "Qwen/Qwen3.5-2B", |
| "family": "seed_variation", |
| "seed": 2, |
| "width": "32x", |
| "d_features": 65536, |
| "d_model": 2048, |
| "k": 256, |
| "rowEV": 0.826, |
| "top1": 0.848, |
| "KL_bits": 0.194, |
| "steps": 17500, |
| "size_bytes": 1076996537, |
| "sha256": "c30f537c387c66f5a92fa5321aef38278d69ede42b9f78254e288160735d7a57", |
| "layout": "config", |
| "metrics_source": "metrics.json alongside the checkpoint (held-out eval at training time)", |
| "sidecars": [ |
| "qwen3.5-2b/k256_32x_seedvar_s2/config.yaml", |
| "qwen3.5-2b/k256_32x_seedvar_s2/metrics.json" |
| ] |
| } |
| ] |
| } |
| } |
