| { |
| "rgcn_link_prediction": { |
| "validation": { |
| "mean_reciprocal_rank": 0.0734197945466392, |
| "hits_at_10": 0.14442630250853494, |
| "hits_at_1": 0.03933501558557221 |
| }, |
| "testing": { |
| "mean_reciprocal_rank": 0.07925546916450926, |
| "hits_at_10": 0.15405164737310775, |
| "hits_at_1": 0.04407836153161175 |
| }, |
| "split_counts": { |
| "train": 121266, |
| "validation": 6737, |
| "test": 6738 |
| }, |
| "transe_init": true |
| }, |
| "rollup": { |
| "recorded_at": "2026-06-05", |
| "phase2_targets": { |
| "link_pred_mrr": 0.2, |
| "source": "TODO.md Phase 2 readiness criterion" |
| }, |
| "link_prediction_split": { |
| "train": 121266, |
| "validation": 6737, |
| "test": 6738, |
| "ratios": [ |
| 0.9, |
| 0.05, |
| 0.05 |
| ], |
| "seed": 42 |
| }, |
| "transe": { |
| "config": "data/processed/transe_config.json", |
| "artifacts": "data/processed/transe_emb.pt", |
| "hyperparams": { |
| "embedding_dim": 128, |
| "learning_rate": 0.001, |
| "batch_size": 256, |
| "num_epochs": 200, |
| "num_negs_per_pos": 10 |
| }, |
| "validation": { |
| "mean_reciprocal_rank": 0.0688, |
| "hits_at_10": 0.1718, |
| "hits_at_1": 0.0056 |
| }, |
| "testing": { |
| "mean_reciprocal_rank": 0.0684, |
| "hits_at_10": 0.1702, |
| "hits_at_1": 0.0045 |
| }, |
| "meets_mrr_target": false |
| }, |
| "rgcn": { |
| "config": "data/processed/rgcn_config.json", |
| "artifacts": "checkpoints/wikimovies_rgcn.pt", |
| "hyperparams": { |
| "hidden_dim": 128, |
| "num_layers": 2, |
| "num_bases": 4, |
| "learning_rate": 0.001, |
| "batch_size": 1024, |
| "num_epochs": 50, |
| "num_negs_per_pos": 10, |
| "transe_init": true, |
| "frozen_after_train": true |
| }, |
| "validation": { |
| "mean_reciprocal_rank": 0.0734, |
| "hits_at_10": 0.1444, |
| "hits_at_1": 0.0393 |
| }, |
| "testing": { |
| "mean_reciprocal_rank": 0.0793, |
| "hits_at_10": 0.1541, |
| "hits_at_1": 0.0441 |
| }, |
| "meets_mrr_target": false, |
| "vs_transe": { |
| "test_mrr_delta": 0.0109, |
| "test_hits10_delta": -0.0161, |
| "note": "MRR slightly up; Hits@10 down vs TransE" |
| } |
| }, |
| "hf_publish": { |
| "model_name": "wikimovies-transe-rgcn", |
| "version": "v1-experimental", |
| "status": "staged_locally", |
| "staging_dir": "models/wikimovies-transe-rgcn", |
| "upload_script": "scripts/publish_encoder_hf.py", |
| "note": "Experimental baseline; expect v2 after remediation. Upload requires: hf auth login && python scripts/publish_encoder_hf.py --repo-id USER/wikimovies-transe-rgcn --upload" |
| }, |
| "encode_api": { |
| "module": "graph_align/encoders/encoder.py", |
| "checkpoint": "checkpoints/wikimovies_rgcn.pt", |
| "benchmark": "results/encode_benchmark.json", |
| "num_subgraphs": 1000, |
| "split": "poc_train", |
| "per_subgraph_ms": { |
| "cpu_node": 2.98, |
| "cpu_graph": 2.91, |
| "mps_node": 40.32, |
| "mps_graph": 30.86 |
| }, |
| "note": "Sequential per-subgraph forward; MPS slower than CPU on small subgraphs (kernel launch overhead). Use CPU or batched encode_subgraphs for inference." |
| }, |
| "retrieval": { |
| "config": "data/processed/config.json", |
| "hop_max": 3, |
| "n_max": 500, |
| "train_samples": 114196, |
| "answer_in_khop_rate": 1.0, |
| "answer_in_retrieved_rate": 0.5623, |
| "meets_retrieval_target": false, |
| "note": "No formal MRR target; 56% cap coverage is QA bottleneck separate from link-pred MRR" |
| }, |
| "gaps": [ |
| { |
| "id": "link_pred_mrr", |
| "current": 0.079, |
| "target": 0.2, |
| "metric": "rgcn test MRR", |
| "blocks_mvp": false, |
| "blocks_phase2_criterion": true |
| }, |
| { |
| "id": "retrieval_coverage", |
| "current": 0.562, |
| "target": null, |
| "metric": "answer_in_retrieved_rate at N_max=500", |
| "blocks_mvp": false, |
| "note": "44% train QA: gold answer outside capped subgraph" |
| } |
| ], |
| "remediation": { |
| "link_pred_mrr": [ |
| { |
| "priority": 1, |
| "action": "Stronger KGE init: train ComplEx or RotatE (PyKEEN) on same split; use as R-GCN node init", |
| "expected": "Often +0.05\u20130.15 MRR on multi-rel KGs vs TransE L1", |
| "effort": "medium" |
| }, |
| { |
| "priority": 2, |
| "action": "R-GCN hyperparam sweep: 100\u2013200 epochs, lr 5e-4, dropout 0.1, num_bases=9, unfreeze/finetune init", |
| "expected": "Modest MRR gain; current 50-epoch run under-trained", |
| "effort": "low" |
| }, |
| { |
| "priority": 3, |
| "action": "Better negative sampling: type-constrained / degree-aware (PyKEEN Bernoulli or LCWA+filtered)", |
| "expected": "Better ranking tail prediction", |
| "effort": "medium" |
| }, |
| { |
| "priority": 4, |
| "action": "Add inverse relations (+9 rels) and self-loops in message-passing graph", |
| "expected": "Improves undirected reachability in conv layers", |
| "effort": "medium" |
| }, |
| { |
| "priority": 5, |
| "action": "Calibrate eval: run PyKEEN TransE/ComplEx baseline on identical split; verify our eval matches", |
| "expected": "Rule out eval implementation gap", |
| "effort": "low" |
| } |
| ], |
| "retrieval_coverage": [ |
| { |
| "priority": 1, |
| "action": "Raise N_max budget (1000 / 2000) and rebuild subgraphs; re-measure answer_in_retrieved_rate", |
| "expected": "Direct lift toward 80%+ retrieved; linear cost in subgraph size", |
| "effort": "low", |
| "command": "python scripts/build_subgraphs.py --n-max-budget 2000" |
| }, |
| { |
| "priority": 2, |
| "action": "Smarter truncation: hop-first then degree (already); add answer-aware cap ablation for upper bound", |
| "expected": "Diagnostic only \u2014 sets ceiling for retrieval", |
| "effort": "low" |
| } |
| ], |
| "mvp_path": "Proceed with frozen R-GCN (MRR 0.08) + N_max=500 (56% retrieval): link-pred MRR is init quality; QA accuracy driven mainly by retrieval + projection alignment, not global MRR>0.2." |
| } |
| } |
| } |
