File size: 10,746 Bytes
d12b611 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | """learn_report.json → AUBIN_LEARN.md (HF'ye giden rapor). Dürüst: seçim dev'de, test bir kez; Kev referansları yanında.
python make_learn_report.py runs/learn/learn_report.json release/AUBIN_LEARN.md
"""
import json, sys
def pct(x):
return "—" if x is None else f"{100 * x:.1f}"
def glance(online_path, R):
"""Sonuçlar bir bakışta — dürüst: işe yarayan ve yaramayan birlikte."""
out = ["## Results at a glance", ""]
if online_path:
O = json.load(open(online_path, encoding="utf-8"))["runs"]
gains = [(rn, rr["kev_test"]["model"], rr["kev_test"]["aubin_learn"]) for rn, rr in O.items() if "kev_test" in rr]
tg = [(rn, rr["kev_transfer_test"]["model"], rr["kev_transfer_test"]["aubin_learn"]) for rn, rr in O.items() if "kev_transfer_test" in rr]
best = max(gains, key=lambda g: g[2])
out += [f"* **Learning from feedback works:** on the kev_test stream every one of {len(gains)} AUBIN variants improved "
f"(+{100 * min(b - a for _, a, b in gains):.1f} to +{100 * max(b - a for _, a, b in gains):.1f} points); best "
f"`{best[0]}` {pct(best[1])} → **{pct(best[2])}**. Online protocol (labels revealed after each answer) — not a static test score.",
f"* **Never-seen sources (transfer, memory starts empty):** change "
f"{100 * min(b - a for _, a, b in tg):+.1f} to {100 * max(b - a for _, a, b in tg):+.1f} points — the self-calibrator keeps "
"trust on the model when memory is not yet useful, so it does not hurt; a few hundred feedbacks per source are not enough to help."]
st = []
for emb, mems in R.get("embeds", {}).items():
for mname, res in mems.items():
for rn, rr in res["runs"].items():
if "kev_test" in rr:
st.append(rr["kev_test"]["fused"] - rr["kev_test"]["model"])
if st:
out.append(f"* **Static locked test (no feedback):** fusing a frozen memory into an already-fine-tuned AUBIN changes kev_test by "
f"{100 * min(st):+.1f} to {100 * max(st):+.1f} points — no reliable gain; AUBIN already learned these sources.")
out += ["* Latency: writing a case to memory takes well under a millisecond with the hashing embedder (tens of ms with a "
"sentence embedder); recall is a matrix product over the partition.", ""]
return out
def main():
R = json.load(open(sys.argv[1], encoding="utf-8")); out = sys.argv[2]
ref = R.get("reference", {})
L = ["# AUBIN-Learn — instant self-learning memory for AUBIN (Norovox core)", "",
"AUBIN-Learn plugs the Norovox self-learning core into AUBIN's fast decision loop. Knowledge is not baked into "
"weights: verified cases live in an external **decision memory**. When AUBIN is unsure, it recalls the most similar "
"solved cases of the same question type and fuses their vote with its own probabilities; when it is told the right "
"answer, `learn()` writes it to memory and the very next decision uses it — no gradient step, no retraining.", "",
"```python", "from aubin.learn import DecisionMemory, AubinLearner",
"mem = DecisionMemory() # or DecisionMemory(embed_fn=<sentence embedder>)",
"agent = AubinLearner(score_fn=scorer.score, memory=mem, weights=dev_tuned_weights, margin=4.0,",
" research_fn=Retrieval().fetch) # Norovox Retrieval: local KB -> web, when memory has nothing",
"choice, logprobs, evidence = agent.decide(item)", "agent.feedback(item, correct_index) # learned instantly", "```", "",
*glance(sys.argv[3] if len(sys.argv) > 3 else "", R),
"## Protocol", "",
"* Memory = Kev decision-v7 **train** (+ kev_augment extra rows). Test / development / calibration / transfer items never enter memory.",
"* Model log-probabilities come from previously measured AUBIN runs (same item order; labels verified to match).",
"* Fusion weight (per source, by log-loss), neighbour count and temperature are selected on development data only "
"(kev_dev; cal-300 when a run has no dev); kev_test and kev_transfer_test are reported once.",
"* Also measured: a learning curve (memory 0% → 100%), latency, and an online-feedback stream (separate protocol).", "",
f"Memory: {R['memory']['kev_train_items']} Kev decision-v7 train items + {R['memory']['extra_items']} leak-free extra items "
"(kev_augment.py: same HF source datasets, rows not used anywhere in Kev's suites).", ""]
for emb, mems in R.get("embeds", {}).items():
for mname, res in mems.items():
L += [f"## Embedding `{emb}`, memory `{mname}` ({res.get('memory_items', '?')} items)", "",
f"Latency: learn {res['latency_ms']['learn_median']} ms (median, incl. embedding) · recall "
f"{res['latency_ms']['recall_mean_incl_embed']} ms/query (incl. embedding).", "",
"| model predictions | tuned on | kev_test model | kev_test + memory | memory alone | kev_transfer_test model | + memory |",
"|---|---|---|---|---|---|---|"]
for rn, rr in res["runs"].items():
t, x = rr.get("kev_test", {}), rr.get("kev_transfer_test", {})
L.append(f"| {rn} | {rr['dev_used']} | {pct(t.get('model'))} | {pct(t.get('fused'))} | {pct(t.get('memory_only'))} | "
f"{pct(x.get('model'))} | {pct(x.get('fused'))} |")
best = max(res["runs"].items(), key=lambda kv: kv[1].get("kev_test", {}).get("fused", 0))
rn, rr = best
L += ["", f"Reference on kev_test (Kev's own training sources): Kev-9B {pct(ref.get('Kev-9B kev_test'))}, "
f"Kev-27B {pct(ref.get('Kev-27B kev_test'))}. Best row above: `{rn}` {pct(rr['kev_test']['fused'])}.", ""]
if "kev_test" in rr:
L += ["Per source (kev_test), model → model + memory:", "", "| source | n | model | + memory |", "|---|---|---|---|"]
for s, (a, n) in rr["kev_test"]["by_source_model"].items():
L.append(f"| {s} | {n} | {pct(a)} | {pct(rr['kev_test']['by_source_fused'][s][0])} |")
L.append("")
if "learning_curve" in rr:
L += ["Learning curve — the same model, memory grown from 0% to 100% (no retraining):", "",
"| memory fraction | items | kev_test |", "|---|---|---|"]
for f, c in rr["learning_curve"].items():
L.append(f"| {f} | {c['memory_items']} | {pct(c['kev_test_fused'])} |")
L.append("")
if "online_feedback_kev_test" in rr:
o = rr["online_feedback_kev_test"]
L += [f"Online feedback (separate protocol, not a static test score): answering kev_test as a stream and writing "
f"each correct label to memory after answering gives {pct(o['accuracy'])} overall, "
f"{pct(o['second_half'])} on the second half.", ""]
if len(sys.argv) > 4 and sys.argv[4]: # hızlı beceri (bellekten saniyede öğrenilen doğrusal sınıflandırıcı)
S = json.load(open(sys.argv[4], encoding="utf-8"))
L += ["## Fast skills (learned from memory in seconds)", "",
"The Norovox skill-library idea applied to decisions: for every source, a small softmax classifier is fitted on the "
"memory's embeddings (labels by name) in seconds on a CPU, and fused with AUBIN's probabilities. Regularisation and "
"per-source fusion weights were selected on kev_dev only; test is reported once. A skill is switched on for a source "
"only if, on kev_dev, it raises accuracy by at least 2 questions **and** lowers log-loss; everywhere else it stays off. "
"(An ungated variant that picked weights by log-loss alone gave no gain — `reports/learn_skill_ungated.json`.)", "",
"| predictions | kev_test model | kev_test + skills | kev_transfer_test model | + skills |", "|---|---|---|---|---|"]
for rn, rr in S["runs"].items():
t, x = rr.get("kev_test", {}), rr.get("kev_transfer_test", {})
L.append(f"| {rn} | {pct(t.get('model'))} | {pct(t.get('fused'))} | {pct(x.get('model'))} | {pct(x.get('fused'))} |")
best = max(S["runs"].items(), key=lambda kv: kv[1].get("kev_test", {}).get("fused", 0))
L += ["", f"Per source for `{best[0]}` (kev_test): n · model · + skills", ""]
for s, (n, m, f) in best[1]["kev_test"]["by_source"].items():
L.append(f"* {s}: {n} · {pct(m)} · {pct(f)}")
L.append("")
if len(sys.argv) > 3 and sys.argv[3]: # çevrimiçi (geri bildirimli) akış deneyi
O = json.load(open(sys.argv[3], encoding="utf-8"))
L += ["## Online self-learning (feedback stream)", "",
"Questions arrive one by one; after each answer the correct label is revealed. AUBIN-Learn writes it to memory "
"(instant) and its **self-calibrator** (Hedge / multiplicative weights over three experts — model, memory, fused) "
"shifts trust per source towards whichever has been more accurate. On kev_transfer_test the memory starts **empty**: "
"these sources were never seen in training by AUBIN or Kev. Hyper-parameters (η, initial trust) were selected on the "
"development streams only. This is a separate protocol, not a static locked-test score.", "",
"| predictions | stream | model | AUBIN-Learn | model, 2nd half | AUBIN-Learn, 2nd half |", "|---|---|---|---|---|---|"]
for rn, rr in O["runs"].items():
for s in ("kev_transfer_test", "kev_test"):
if s in rr:
x = rr[s]
L.append(f"| {rn} | {s} | {pct(x['model'])} | {pct(x['aubin_learn'])} | {pct(x['model_2nd_half'])} | {pct(x['aubin_learn_2nd_half'])} |")
L.append("")
L += ["## Honest notes", "",
"* Fusion weights, neighbour count and temperature were chosen on development data only; test numbers are reported once.",
"* Memory never contains test, development, calibration or transfer items.",
"* Sources unseen by the memory (all of transfer-v4) fall back to the model unchanged — memory cannot hurt them.",
"* Any comparison with Kev uses the same public Kev suites; Kev models are not part of AUBIN-Learn."]
open(out, "w", encoding="utf-8").write("\n".join(L) + "\n")
print(out)
if __name__ == "__main__":
main()
|