| { |
| "data_sha256": "8442fc4dbeb740baab4603a0ebc079f5fa53474e7b5bba9b4f82a206d5a65dc7", |
| "sample_sha256": "25555efe29519e4c3e230b8cc7f7fd5aa0d667a5085028e2fe71735a43fb9872", |
| "seed": 42, |
| "per_source": 0, |
| "count": 17476, |
| "gpu": "NVIDIA GeForce RTX 4090", |
| "torch": "2.11.0+cu130", |
| "laya_version": "0.3.20", |
| "laya_source_expected": "970dc8c5f63d7b886a68409493f37d569424f933", |
| "laya_revision": "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851", |
| "kev_model": "/workspace/kev-run/runs/train-20260924-185636/final", |
| "notes": [ |
| "Paired test of Kev's dataset mixture, not an independent generalization benchmark; Laya training overlap unknown.", |
| "Native shipped calibration and precision; no temperature fitted on test data. Kev had in-domain calibration.", |
| "Laya uses its native input limits. shared_untruncated excludes Laya clipping/mask replacement; Kev rejects overlength inputs.", |
| "Laya API probabilities rounded to 4 decimals and renormalized. Log loss floors zero at 1e-300.", |
| "ECE uses target mass, accuracy accepts any positive target; ordinal MAE is in level indices.", |
| "Latency: one decision/request, three warmups, synchronized GPU, excludes downloads/load/filtering; native runtimes, not equal precision." |
| ], |
| "kev_temperature": 1.3734692667114208, |
| "kev_config_sha256": "53a9b5365795cd4515dce2ac28827f0bbffbf2660fc1903ae60dbeea1bbf0b90", |
| "kev_weights_sha256": "8e2c6d85fb9ded84dc59e9d9c047721dade6962dc007db9f085209996eda0609", |
| "kev": { |
| "latency_median_ms": 17.827488132752478, |
| "latency_p95_ms": 22.653514053672552, |
| "decisions_per_second": 52.454379622202104, |
| "peak_allocated_gib": 1.799546718597412, |
| "metrics": { |
| "overall": { |
| "count": 17476, |
| "accuracy": 0.7993247882810712, |
| "log_loss": 0.47473593158579847, |
| "brier_score": 0.276944144606486, |
| "ece_15": 0.01499784008965703, |
| "ordinal_mae": 0.39187796655282964, |
| "ordinal_count": 997 |
| }, |
| "source/google/boolq": { |
| "count": 828, |
| "accuracy": 0.8623188405797102, |
| "log_loss": 0.32017628056957004, |
| "brier_score": 0.19625255264609132, |
| "ece_15": 0.03878371236679417, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "kind/noul": { |
| "count": 828, |
| "accuracy": 0.8623188405797102, |
| "log_loss": 0.32017628056957004, |
| "brier_score": 0.19625255264609132, |
| "ece_15": 0.03878371236679417, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "shared_untruncated": { |
| "count": 15879, |
| "accuracy": 0.7944454940487437, |
| "log_loss": 0.4848516415026143, |
| "brier_score": 0.2833957148988989, |
| "ece_15": 0.01351641796079561, |
| "ordinal_mae": 0.39300533843821434, |
| "ordinal_count": 969 |
| }, |
| "source/tasksource/defeasible-nli": { |
| "count": 2000, |
| "accuracy": 0.8755, |
| "log_loss": 0.29917000923023196, |
| "brier_score": 0.18311932099491463, |
| "ece_15": 0.030246663003452377, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "kind/choice": { |
| "count": 15651, |
| "accuracy": 0.8015462270781419, |
| "log_loss": 0.47080995086420485, |
| "brier_score": 0.2736998187648513, |
| "ece_15": 0.015109753374155774, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/FOL-nli": { |
| "count": 2000, |
| "accuracy": 0.8315, |
| "log_loss": 0.4253180067613509, |
| "brier_score": 0.24239928794092896, |
| "ece_15": 0.04729400868572198, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/zero-shot-label-nli": { |
| "count": 1979, |
| "accuracy": 0.729156139464376, |
| "log_loss": 0.5152899138960492, |
| "brier_score": 0.3483716637653681, |
| "ece_15": 0.031776498083021916, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/nyu-mll/multi_nli": { |
| "count": 2000, |
| "accuracy": 0.907, |
| "log_loss": 0.26788311256993835, |
| "brier_score": 0.14492574189565635, |
| "ece_15": 0.04858235192126606, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/doc-nli": { |
| "count": 1776, |
| "accuracy": 0.8682432432432432, |
| "log_loss": 0.3774372164976728, |
| "brier_score": 0.20400877870993306, |
| "ece_15": 0.07071918120546522, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/Rowan/hellaswag": { |
| "count": 2000, |
| "accuracy": 0.8635, |
| "log_loss": 0.3611164615832854, |
| "brier_score": 0.19575059958417662, |
| "ece_15": 0.024213224725977266, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/facebook/anli": { |
| "count": 2000, |
| "accuracy": 0.579, |
| "log_loss": 0.9135263254515801, |
| "brier_score": 0.5611907148330435, |
| "ece_15": 0.12435406994979702, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/bigbench": { |
| "count": 1896, |
| "accuracy": 0.7631856540084389, |
| "log_loss": 0.6036558405500002, |
| "brier_score": 0.3044079216301446, |
| "ece_15": 0.05166735277454641, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/tasksource-instruct-v0": { |
| "count": 997, |
| "accuracy": 0.7121364092276831, |
| "log_loss": 0.6647268195648457, |
| "brier_score": 0.3948875567334653, |
| "ece_15": 0.06109424008639756, |
| "ordinal_mae": 0.39187796655282964, |
| "ordinal_count": 997 |
| }, |
| "kind/score": { |
| "count": 997, |
| "accuracy": 0.7121364092276831, |
| "log_loss": 0.6647268195648457, |
| "brier_score": 0.3948875567334653, |
| "ece_15": 0.06109424008639756, |
| "ordinal_mae": 0.39187796655282964, |
| "ordinal_count": 997 |
| } |
| } |
| }, |
| "laya_config": { |
| "encoder": "answerdotai/ModernBERT-large", |
| "head_layers": 2, |
| "max_len": 512, |
| "head_max_len": 192, |
| "max_prefixes": 6, |
| "act_costs": { |
| "escalate": 0.5 |
| }, |
| "cost_wrong_act": 3.0, |
| "amp_dtype": "bf16", |
| "model_name": "rl-agent", |
| "temperature": [ |
| 1.6369030475616455, |
| 1.2514300346374512, |
| 1.983399510383606 |
| ], |
| "temperature_by_options": { |
| "choice:3-5": 1.7601518630981445, |
| "choice:6-10": 1.0000158548355103, |
| "score:3-5": 1.2514300346374512, |
| "noul:2": 1.983399510383606, |
| "choice:11+": 0.10058280825614929, |
| "choice:2": 1.9063563346862793 |
| }, |
| "training": { |
| "updates": 7313, |
| "epochs_completed": 1, |
| "hours": 1.96, |
| "world_size": 1, |
| "fine_tuned_from_checkpoint": true |
| } |
| }, |
| "laya": { |
| "latency_median_ms": 20.587651059031487, |
| "latency_p95_ms": 22.047329926863313, |
| "decisions_per_second": 48.54963820832984, |
| "peak_allocated_gib": 2.2828011512756348, |
| "metrics": { |
| "overall": { |
| "count": 17476, |
| "accuracy": 0.5163080796520944, |
| "log_loss": 1.246543205431566, |
| "brier_score": 0.6572239964545026, |
| "ece_15": 0.2051937618147546, |
| "ordinal_mae": 0.8264893538631242, |
| "ordinal_count": 997 |
| }, |
| "source/google/boolq": { |
| "count": 828, |
| "accuracy": 0.8466183574879227, |
| "log_loss": 0.3902746100231246, |
| "brier_score": 0.23805258207729438, |
| "ece_15": 0.07251304347826099, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "kind/noul": { |
| "count": 828, |
| "accuracy": 0.8466183574879227, |
| "log_loss": 0.3902746100231246, |
| "brier_score": 0.23805258207729438, |
| "ece_15": 0.07251304347826099, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "shared_untruncated": { |
| "count": 15879, |
| "accuracy": 0.5293154480760753, |
| "log_loss": 1.2492520084248007, |
| "brier_score": 0.6527813164520726, |
| "ece_15": 0.2196989991270484, |
| "ordinal_mae": 0.8282910369726099, |
| "ordinal_count": 969 |
| }, |
| "source/tasksource/defeasible-nli": { |
| "count": 2000, |
| "accuracy": 0.5825, |
| "log_loss": 0.952491156760722, |
| "brier_score": 0.6141003590499987, |
| "ece_15": 0.21883975000000003, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "kind/choice": { |
| "count": 15651, |
| "accuracy": 0.5009903520541819, |
| "log_loss": 1.2676000656054562, |
| "brier_score": 0.6753007087251631, |
| "ece_15": 0.21164316804299338, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/FOL-nli": { |
| "count": 2000, |
| "accuracy": 0.389, |
| "log_loss": 1.4171403527363373, |
| "brier_score": 0.823208540406562, |
| "ece_15": 0.3076837893706574, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/zero-shot-label-nli": { |
| "count": 1979, |
| "accuracy": 0.2769075290550783, |
| "log_loss": 1.85790882550375, |
| "brier_score": 1.0321144313757578, |
| "ece_15": 0.46389314898652, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/nyu-mll/multi_nli": { |
| "count": 2000, |
| "accuracy": 0.884, |
| "log_loss": 0.33853652283624674, |
| "brier_score": 0.18216245952339305, |
| "ece_15": 0.03418428408961287, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/doc-nli": { |
| "count": 1776, |
| "accuracy": 0.6548423423423423, |
| "log_loss": 0.7602912050384334, |
| "brier_score": 0.49615899890765763, |
| "ece_15": 0.1476695382882884, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/Rowan/hellaswag": { |
| "count": 2000, |
| "accuracy": 0.322, |
| "log_loss": 1.3616799927239023, |
| "brier_score": 0.7380228883572477, |
| "ece_15": 0.018641407260095413, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/facebook/anli": { |
| "count": 2000, |
| "accuracy": 0.5025, |
| "log_loss": 1.3613812182618001, |
| "brier_score": 0.7585082783178929, |
| "ece_15": 0.3206912302510386, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/bigbench": { |
| "count": 1896, |
| "accuracy": 0.4061181434599156, |
| "log_loss": 2.0831600181080487, |
| "brier_score": 0.7454607063263152, |
| "ece_15": 0.21542547758310301, |
| "ordinal_mae": null, |
| "ordinal_count": 0 |
| }, |
| "source/tasksource/tasksource-instruct-v0": { |
| "count": 997, |
| "accuracy": 0.48244734202607825, |
| "log_loss": 1.627114397424211, |
| "brier_score": 0.721572348867965, |
| "ece_15": 0.22077087104690687, |
| "ordinal_mae": 0.8264893538631242, |
| "ordinal_count": 997 |
| }, |
| "kind/score": { |
| "count": 997, |
| "accuracy": 0.48244734202607825, |
| "log_loss": 1.627114397424211, |
| "brier_score": 0.721572348867965, |
| "ece_15": 0.22077087104690687, |
| "ordinal_mae": 0.8264893538631242, |
| "ordinal_count": 997 |
| } |
| } |
| }, |
| "laya_altered_input_count": 1597, |
| "laya_altered_input_by_source": { |
| "tasksource/doc-nli": 562, |
| "Rowan/hellaswag": 772, |
| "tasksource/FOL-nli": 155, |
| "tasksource/tasksource-instruct-v0": 28, |
| "tasksource/bigbench": 58, |
| "tasksource/zero-shot-label-nli": 19, |
| "google/boolq": 3 |
| } |
| } |
|
|
