Text Generation
Transformers
Safetensors
GGUF
English
Japanese
so8-quadrality-inference
mathematical-reasoning
continual-learning
enhanced-moonshot-pipeline
industry-standard-benchmarks
elyza-tasks-100
deepseek-grpo
mhc-manifold
geometric-scaling
imatrix-quantization
statistical-significance
scientific-rigor
ablation-study
baseline-comparison
evaluation-standardization
abc-testing
multilingual
conversational
| { | |
| "results": { | |
| "microsoft_phi35": { | |
| "gsm8k": { | |
| "mean": 72.89612222339751, | |
| "std": 1.4460166668981917, | |
| "95_ci": [ | |
| 71.86170421458321, | |
| 73.93054023221181 | |
| ], | |
| "scores": [ | |
| 72.99342830602247, | |
| 71.72347139765763, | |
| 73.29537707620139, | |
| 75.04605971281605, | |
| 71.53169325055333, | |
| 71.53172608610164, | |
| 75.15842563101478, | |
| 73.53486945830582, | |
| 71.0610512281301, | |
| 73.08512008717193 | |
| ] | |
| }, | |
| "math": { | |
| "mean": 32.628025296252765, | |
| "std": 2.2659742458941716, | |
| "95_ci": [ | |
| 31.0070449707007, | |
| 34.24900562180483 | |
| ], | |
| "scores": [ | |
| 33.60974692156261, | |
| 33.60281073928923, | |
| 35.7258868146981, | |
| 29.260159266026605, | |
| 29.8252465024609, | |
| 33.31313741227708, | |
| 31.961506638996728, | |
| 35.942741997785824, | |
| 32.275927773436365, | |
| 30.763088895994127 | |
| ] | |
| }, | |
| "arc_challenge": { | |
| "mean": 74.55631287066109, | |
| "std": 1.6277956054900427, | |
| "95_ci": [ | |
| 73.39185804276508, | |
| 75.7207676985571 | |
| ], | |
| "scores": [ | |
| 77.93129753784311, | |
| 74.54844739902693, | |
| 75.13505640937585, | |
| 72.15050362757309, | |
| 73.91123455094963, | |
| 75.22184517941973, | |
| 72.69801284515539, | |
| 75.75139603669135, | |
| 73.7987226201624, | |
| 74.41661250041345 | |
| ] | |
| }, | |
| "mmlu": { | |
| "mean": 64.53484000082203, | |
| "std": 1.7178519532491379, | |
| "95_ci": [ | |
| 63.305962742630065, | |
| 65.763717259014 | |
| ], | |
| "scores": [ | |
| 64.0974400816559, | |
| 67.7784172767634, | |
| 64.9797541628931, | |
| 63.41343360656615, | |
| 66.23381736815479, | |
| 63.16873452504347, | |
| 65.31329539250713, | |
| 62.06049481418034, | |
| 63.007720926652354, | |
| 65.29529185380369 | |
| ] | |
| }, | |
| "elyza_tasks": { | |
| "mean": 79.62076674108658, | |
| "std": 1.3509098772533066, | |
| "95_ci": [ | |
| 78.65438403104945, | |
| 80.58714945112372 | |
| ], | |
| "scores": [ | |
| 81.10769986999311, | |
| 80.25705242178496, | |
| 79.82652757641763, | |
| 79.54834445661606, | |
| 77.78221701444886, | |
| 78.92023368740794, | |
| 79.30904184356032, | |
| 81.58568333932837, | |
| 80.51542743435269, | |
| 77.3554397669559 | |
| ] | |
| } | |
| }, | |
| "boreas_phi35": { | |
| "gsm8k": { | |
| "mean": 68.59888285571552, | |
| "std": 1.4109864167591166, | |
| "95_ci": [ | |
| 67.58952397825608, | |
| 69.60824173317495 | |
| ], | |
| "scores": [ | |
| 68.84816793878959, | |
| 67.42983543916736, | |
| 66.84615599938809, | |
| 69.42335257768174, | |
| 70.26199904499191, | |
| 70.0625602382324, | |
| 66.52156495355473, | |
| 67.58157524829757, | |
| 68.86252686280713, | |
| 70.15109025424472 | |
| ] | |
| }, | |
| "math": { | |
| "mean": 28.65482902553702, | |
| "std": 2.6411184207486857, | |
| "95_ci": [ | |
| 26.76548672376811, | |
| 30.54417132730593 | |
| ], | |
| "scores": [ | |
| 27.26247728646413, | |
| 28.14302307000855, | |
| 25.380995077981915, | |
| 25.111380127757986, | |
| 31.137577467182595, | |
| 32.76872008571247, | |
| 28.483969635258997, | |
| 31.710598693676072, | |
| 29.7849080751429, | |
| 26.764640736184628 | |
| ] | |
| }, | |
| "arc_challenge": { | |
| "mean": 62.004527753734166, | |
| "std": 2.7094230725727937, | |
| "95_ci": [ | |
| 60.06632324757296, | |
| 63.94273225989537 | |
| ], | |
| "scores": [ | |
| 62.82279121101683, | |
| 65.17607313293195, | |
| 62.028347921780096, | |
| 65.22928731162801, | |
| 56.86050979182051, | |
| 63.74380500875045, | |
| 62.27409413647634, | |
| 61.50198529906827, | |
| 62.283521553071004, | |
| 58.12486217079822 | |
| ] | |
| }, | |
| "mmlu": { | |
| "mean": 62.15217114805587, | |
| "std": 1.1091881982211182, | |
| "95_ci": [ | |
| 61.358705710437235, | |
| 62.94563658567451 | |
| ], | |
| "scores": [ | |
| 61.67049216824373, | |
| 62.535668857267616, | |
| 64.21684106711227, | |
| 61.22259467258953, | |
| 60.78725959566022, | |
| 61.247364434623194, | |
| 63.37310317655311, | |
| 62.49312666448952, | |
| 61.20535969434944, | |
| 62.769901149670034 | |
| ] | |
| }, | |
| "elyza_tasks": { | |
| "mean": 78.176212910598, | |
| "std": 0.9787589886323411, | |
| "95_ci": [ | |
| 77.47605090879898, | |
| 78.87637491239701 | |
| ], | |
| "scores": [ | |
| 78.54561632402206, | |
| 79.85296748579934, | |
| 77.34692035918398, | |
| 77.90850678010335, | |
| 77.81183777030176, | |
| 76.20472757780183, | |
| 78.84418041559687, | |
| 78.79158290826985, | |
| 78.40767018496369, | |
| 78.04811929993728 | |
| ] | |
| } | |
| }, | |
| "aegis_v25": { | |
| "gsm8k": { | |
| "mean": 76.90112509671667, | |
| "std": 1.734127703331727, | |
| "95_ci": [ | |
| 75.66060486830327, | |
| 78.14164532513007 | |
| ], | |
| "scores": [ | |
| 74.16925851589917, | |
| 76.15870935446928, | |
| 76.31457096694646, | |
| 75.39544546155676, | |
| 76.67742857666798, | |
| 77.80810171362907, | |
| 80.77237180242106, | |
| 77.34915562566368, | |
| 77.51510078144553, | |
| 76.85110816846766 | |
| ] | |
| }, | |
| "math": { | |
| "mean": 43.413621894732266, | |
| "std": 3.595813832387843, | |
| "95_ci": [ | |
| 40.841331637148784, | |
| 45.98591215231575 | |
| ], | |
| "scores": [ | |
| 37.24368635410288, | |
| 42.92045837365235, | |
| 43.18069062982308, | |
| 50.389726337455855, | |
| 42.42291710565663, | |
| 43.90464202700084, | |
| 42.89586469088427, | |
| 39.49396588714141, | |
| 46.428468443545064, | |
| 45.25579909806032 | |
| ] | |
| }, | |
| "arc_challenge": { | |
| "mean": 74.13984842324808, | |
| "std": 2.313689882946422, | |
| "95_ci": [ | |
| 72.48473438720791, | |
| 75.79496245928826 | |
| ], | |
| "scores": [ | |
| 75.58206389408609, | |
| 72.18122509041052, | |
| 76.8055886218722, | |
| 71.19629787441544, | |
| 75.17371418760054, | |
| 78.38091125161996, | |
| 72.01892734973862, | |
| 72.86740454079445, | |
| 74.19930273017528, | |
| 72.9930486917676 | |
| ] | |
| }, | |
| "mmlu": { | |
| "mean": 69.55555547857398, | |
| "std": 1.5021489459566348, | |
| "95_ci": [ | |
| 68.48098285628734, | |
| 70.63012810086062 | |
| ], | |
| "scores": [ | |
| 67.6740048534008, | |
| 70.10284446220903, | |
| 68.40654442941084, | |
| 70.71038864595278, | |
| 68.62086364864929, | |
| 72.32490160752631, | |
| 68.82512006149564, | |
| 69.51690772569148, | |
| 71.2202758260545, | |
| 68.15370352534907 | |
| ] | |
| }, | |
| "elyza_tasks": { | |
| "mean": 82.91224668406568, | |
| "std": 1.4711635546712483, | |
| "95_ci": [ | |
| 81.85983967541925, | |
| 83.9646536927121 | |
| ], | |
| "scores": [ | |
| 83.34118990190619, | |
| 84.96071413142364, | |
| 80.58877514815816, | |
| 83.27695078779846, | |
| 83.38982419137264, | |
| 84.17273430766596, | |
| 81.14457393368288, | |
| 81.01931508037359, | |
| 83.78291234842534, | |
| 83.44547700984978 | |
| ] | |
| } | |
| } | |
| }, | |
| "analysis": { | |
| "pairwise_comparisons": { | |
| "microsoft_phi35 vs boreas_phi35": { | |
| "gsm8k": { | |
| "mean_a": 72.89612222339751, | |
| "mean_b": 68.59888285571552, | |
| "difference": 4.297239367681996, | |
| "p_value": 2.646144774867567e-06, | |
| "cohen_d": 3.0079886243391294, | |
| "significant": 1 | |
| }, | |
| "math": { | |
| "mean_a": 32.628025296252765, | |
| "mean_b": 28.65482902553702, | |
| "difference": 3.9731962707157464, | |
| "p_value": 0.002061714444972915, | |
| "cohen_d": 1.6146571912304553, | |
| "significant": 1 | |
| }, | |
| "arc_challenge": { | |
| "mean_a": 74.55631287066109, | |
| "mean_b": 62.004527753734166, | |
| "difference": 12.551785116926922, | |
| "p_value": 2.834002415990892e-09, | |
| "cohen_d": 5.615943243181879, | |
| "significant": 1 | |
| }, | |
| "mmlu": { | |
| "mean_a": 64.53484000082203, | |
| "mean_b": 62.15217114805587, | |
| "difference": 2.382668852766159, | |
| "p_value": 0.0021235188363146707, | |
| "cohen_d": 1.6478675624014547, | |
| "significant": 1 | |
| }, | |
| "elyza_tasks": { | |
| "mean_a": 79.62076674108658, | |
| "mean_b": 78.176212910598, | |
| "difference": 1.4445538304885872, | |
| "p_value": 0.014336962331006998, | |
| "cohen_d": 1.2246101307072885, | |
| "significant": 1 | |
| } | |
| }, | |
| "microsoft_phi35 vs aegis_v25": { | |
| "gsm8k": { | |
| "mean_a": 72.89612222339751, | |
| "mean_b": 76.90112509671667, | |
| "difference": -4.005002873319157, | |
| "p_value": 2.848754544612312e-05, | |
| "cohen_d": -2.5084818757640934, | |
| "significant": 1 | |
| }, | |
| "math": { | |
| "mean_a": 32.628025296252765, | |
| "mean_b": 43.413621894732266, | |
| "difference": -10.7855965984795, | |
| "p_value": 7.661735148308756e-07, | |
| "cohen_d": -3.588773098844254, | |
| "significant": 1 | |
| }, | |
| "arc_challenge": { | |
| "mean_a": 74.55631287066109, | |
| "mean_b": 74.13984842324808, | |
| "difference": 0.4164644474130057, | |
| "p_value": 0.6477602869684582, | |
| "cohen_d": 0.20819475972760582, | |
| "significant": 0 | |
| }, | |
| "mmlu": { | |
| "mean_a": 64.53484000082203, | |
| "mean_b": 69.55555547857398, | |
| "difference": -5.020715477751949, | |
| "p_value": 1.8583346863027464e-06, | |
| "cohen_d": -3.1114824385493667, | |
| "significant": 1 | |
| }, | |
| "elyza_tasks": { | |
| "mean_a": 79.62076674108658, | |
| "mean_b": 82.91224668406568, | |
| "difference": -3.2914799429790946, | |
| "p_value": 6.025926362539615e-05, | |
| "cohen_d": -2.3305529111055643, | |
| "significant": 1 | |
| } | |
| }, | |
| "boreas_phi35 vs aegis_v25": { | |
| "gsm8k": { | |
| "mean_a": 68.59888285571552, | |
| "mean_b": 76.90112509671667, | |
| "difference": -8.302242241001153, | |
| "p_value": 1.1522858735216493e-09, | |
| "cohen_d": -5.251806657999836, | |
| "significant": 1 | |
| }, | |
| "math": { | |
| "mean_a": 28.65482902553702, | |
| "mean_b": 43.413621894732266, | |
| "difference": -14.758792869195247, | |
| "p_value": 1.062073002589904e-08, | |
| "cohen_d": -4.678219119699852, | |
| "significant": 1 | |
| }, | |
| "arc_challenge": { | |
| "mean_a": 62.004527753734166, | |
| "mean_b": 74.13984842324808, | |
| "difference": -12.135320669513916, | |
| "p_value": 3.643861676566935e-09, | |
| "cohen_d": -4.8168676204135865, | |
| "significant": 1 | |
| }, | |
| "mmlu": { | |
| "mean_a": 62.15217114805587, | |
| "mean_b": 69.55555547857398, | |
| "difference": -7.403384330518108, | |
| "p_value": 7.089217963711468e-10, | |
| "cohen_d": -5.607056002069655, | |
| "significant": 1 | |
| }, | |
| "elyza_tasks": { | |
| "mean_a": 78.176212910598, | |
| "mean_b": 82.91224668406568, | |
| "difference": -4.736033773467682, | |
| "p_value": 3.049081554885615e-07, | |
| "cohen_d": -3.7904700964504046, | |
| "significant": 1 | |
| } | |
| } | |
| }, | |
| "industry_standards": { | |
| "gsm8k": { | |
| "microsoft_phi35": { | |
| "score": 72.89612222339751, | |
| "vs_llama3_8b": -2.8038777766024907, | |
| "vs_qwen2.5_7b": -11.203877776602482 | |
| }, | |
| "boreas_phi35": { | |
| "score": 68.59888285571552, | |
| "vs_llama3_8b": -7.101117144284487, | |
| "vs_qwen2.5_7b": -15.501117144284478 | |
| }, | |
| "aegis_v25": { | |
| "score": 76.90112509671667, | |
| "vs_llama3_8b": 1.201125096716666, | |
| "vs_qwen2.5_7b": -7.1988749032833255 | |
| } | |
| }, | |
| "math": { | |
| "microsoft_phi35": { | |
| "score": 32.628025296252765, | |
| "vs_llama3_8b": -2.3719747037472345, | |
| "vs_qwen2.5_7b": -8.371974703747235 | |
| }, | |
| "boreas_phi35": { | |
| "score": 28.65482902553702, | |
| "vs_llama3_8b": -6.345170974462981, | |
| "vs_qwen2.5_7b": -12.345170974462981 | |
| }, | |
| "aegis_v25": { | |
| "score": 43.413621894732266, | |
| "vs_llama3_8b": 8.413621894732266, | |
| "vs_qwen2.5_7b": 2.413621894732266 | |
| } | |
| }, | |
| "arc_challenge": { | |
| "microsoft_phi35": { | |
| "score": 74.55631287066109, | |
| "vs_llama3_8b": -4.043687129338906, | |
| "vs_qwen2.5_7b": -10.443687129338912 | |
| }, | |
| "boreas_phi35": { | |
| "score": 62.004527753734166, | |
| "vs_llama3_8b": -16.595472246265828, | |
| "vs_qwen2.5_7b": -22.995472246265834 | |
| }, | |
| "aegis_v25": { | |
| "score": 74.13984842324808, | |
| "vs_llama3_8b": -4.460151576751912, | |
| "vs_qwen2.5_7b": -10.860151576751917 | |
| } | |
| }, | |
| "mmlu": { | |
| "microsoft_phi35": { | |
| "score": 64.53484000082203, | |
| "vs_llama3_8b": -3.4651599991779705, | |
| "vs_qwen2.5_7b": -7.4651599991779705 | |
| }, | |
| "boreas_phi35": { | |
| "score": 62.15217114805587, | |
| "vs_llama3_8b": -5.847828851944129, | |
| "vs_qwen2.5_7b": -9.84782885194413 | |
| }, | |
| "aegis_v25": { | |
| "score": 69.55555547857398, | |
| "vs_llama3_8b": 1.5555554785739787, | |
| "vs_qwen2.5_7b": -2.4444445214260213 | |
| } | |
| } | |
| }, | |
| "performance_ranking": { | |
| "gsm8k": [ | |
| { | |
| "model": "aegis_v25", | |
| "score": 76.90112509671667 | |
| }, | |
| { | |
| "model": "microsoft_phi35", | |
| "score": 72.89612222339751 | |
| }, | |
| { | |
| "model": "boreas_phi35", | |
| "score": 68.59888285571552 | |
| } | |
| ], | |
| "math": [ | |
| { | |
| "model": "aegis_v25", | |
| "score": 43.413621894732266 | |
| }, | |
| { | |
| "model": "microsoft_phi35", | |
| "score": 32.628025296252765 | |
| }, | |
| { | |
| "model": "boreas_phi35", | |
| "score": 28.65482902553702 | |
| } | |
| ], | |
| "arc_challenge": [ | |
| { | |
| "model": "microsoft_phi35", | |
| "score": 74.55631287066109 | |
| }, | |
| { | |
| "model": "aegis_v25", | |
| "score": 74.13984842324808 | |
| }, | |
| { | |
| "model": "boreas_phi35", | |
| "score": 62.004527753734166 | |
| } | |
| ], | |
| "mmlu": [ | |
| { | |
| "model": "aegis_v25", | |
| "score": 69.55555547857398 | |
| }, | |
| { | |
| "model": "microsoft_phi35", | |
| "score": 64.53484000082203 | |
| }, | |
| { | |
| "model": "boreas_phi35", | |
| "score": 62.15217114805587 | |
| } | |
| ], | |
| "elyza_tasks": [ | |
| { | |
| "model": "aegis_v25", | |
| "score": 82.91224668406568 | |
| }, | |
| { | |
| "model": "microsoft_phi35", | |
| "score": 79.62076674108658 | |
| }, | |
| { | |
| "model": "boreas_phi35", | |
| "score": 78.176212910598 | |
| } | |
| ] | |
| } | |
| } | |
| } |