{
"schema_version": 1,
"date": "2026-10-04",
"scope": "Native Swift/MLX CUDA checkpoint diagnostic; Apple full-checkpoint memory fit is unverified.",
"checkpoint": {
"repository": "Aleph-Alpha/Kolibri-1-BF16",
"revision": "7a8f290e7858825c3cf5e4c447ba68345de9f1d3"
},
"runtime": {
"schema_version": 1,
"runtime": "Native Swift/MLX on CUDA using the production Kolibri model, loader, and scoring code",
"harness": "Temporary executable target isolates production scoring from unrelated CLI targets; it does not validate all Linux CLI integrations.",
"swift_version": "6.2",
"mlx_swift_revision": "001d1aa3e5655d7efc073fb9632f952f57cdf528",
"mlx_core_version": "0.32.1",
"conversion_mlx_version": "0.32.2",
"gpu": "NVIDIA B200, 180 GB class",
"source_files": {
"Sources/KolibriQualificationHarness/Harness.swift": "4a815afa5866809e744adb0af7843a39de5d8b95d51876953289fe7d3cb085c9",
"Sources/KolibriQualificationHarness/KolibriLoader.swift": "3b3f40e05584efd120f37f32d2ea15b16e50ce931ad997b08dbcefdfadb6564e",
"Sources/KolibriQualificationHarness/KolibriLogprobBenchmark.swift": "7ad448b7c69ffe6490d50f18b67dcb50662391860cf74bca050e85c115dbcab3",
"Sources/KolibriQualificationHarness/KolibriResources.swift": "517ab9ea94faa23335fe4e769ca8ac5d7d2841743feac2eea08c63d1ba39b0f2",
"Sources/KolibriQualificationHarness/KolibriWeightVerification.swift": "d8ff24cae20ba20d91582b673a92bf3fd94100347320b190e0ab144122bf862b",
"Sources/MereRunKolibriModel/KolibriAttention.swift": "e82da4a0b035075f15e4746eecc27af32e5b943574b5946a0eb2d1dabb6bc08a",
"Sources/MereRunKolibriModel/KolibriConfiguration.swift": "efc75064a820119b54cb8a3686d09d07637e3edda56df7a3dbb37cf82b48b980",
"Sources/MereRunKolibriModel/KolibriLayers.swift": "3bd52b12f955880d141eebc97aee267a4fb0080e5e678e917e08a0e6b191f34b",
"Sources/MereRunKolibriModel/KolibriModel.swift": "3103e2be471908b6deb55b5d35f0e80cd8627d1a0d4a24913638d3e396797ede",
"Sources/MereRunTensor/DenseLayer.swift": "077dbdbb0d36ecd02c8150a3e0a666fdc5860b01ead2eb0f04b0682f8bff19d3",
"Sources/MereRunTensor/DynamicSparseAttention.swift": "37079e09500dcd5faf214389bf0d76b2a4f89187fd01fbc62c53bcb554a86873",
"Sources/MereRunTensor/FusedQuantizedProjection.swift": "aa35809d5c96e1d2b0bd10464075d0fa2947301465ab1e0b689e2874a279fcb4",
"Sources/MereRunTensor/HFSafetensorsIndex.swift": "4b848730a1cedb641314a72ea8258497901335cfc1532eef879f4ffb0257ebf3",
"Sources/MereRunTensor/HFSafetensorsWeightsLoader.swift": "85af8e44fd1bbab7c041418271ae409e32bf01339f86d70e2626e4afa7d23a77",
"Sources/MereRunTensor/MLXCheckpoint.swift": "2ea22a051cc1c312d4829471a955d1c5cfccc4ea47dbfa14caa34a5ed1410787",
"Sources/MereRunTensor/MLXRequestStreams.swift": "0b91988d7a59936af93e3882261fe6ef7116af911d7c16afcb338fc7ada35b1d",
"Sources/MereRunTensor/ModelWeightsLoader.swift": "027707af40c2eeae4ef0397c76761688ff0776d4082adc413470c8eb4b6a7d79",
"Sources/MereRunTensor/PortableQuantizedMatmul.swift": "7183309bd9f4bfc0bc56aafda28c346cdc2ee2b2dc967ffacaf13463be4f12c3",
"Sources/MereRunTensor/ResidualQuantizedLinear.swift": "0cc21cd10d95edf7fe4e649cfc9726131a149e55aa189f10cfddf92b692dac61",
"Sources/MereRunTensor/RoutedMoERouting+Gather.swift": "25ff1b1dee1e52a3fa787f95d607d6ad5ca8853162d03ae5568751a95bb2076f",
"Sources/MereRunTensor/RoutedMoERouting+Residual.swift": "ae8c44c84af05c6ff7d92ba1b799b9994f15737f62a8d34c7fce86a4523a3048",
"Sources/MereRunTensor/RoutedMoERouting+Sorted.swift": "1f9724ccc4032057803275e10b5f52a3f07f115363c437e4b411c5b6051de6f8",
"Sources/MereRunTensor/RoutedMoERouting+Staging.swift": "1cf2dd8df780585d8de123e0bab72e1e3b904cb5a3d26009a3ef9c118857b319",
"Sources/MereRunTensor/RoutedMoERouting.swift": "95fca53fbbf8938b053d6be750e2a7889b58e0059e62356eb1974048fa5851ef",
"Sources/MereRunTensor/SafetensorsStreamingLoader.swift": "dfe93d7e95ddbe64c22fa0969c066c49c2558efab84e11e6a7f516a896a0204c",
"Sources/MereRunTensor/SmallBatchAffineGatherQMV.swift": "2337941e65e5810f82b53678ca403d6e722ca9e76bd7e357a4a07ab81ccc7a1c",
"Sources/MereRunTensor/SmallBatchAffineQMV.swift": "7efd4cbd4e1d526e2f4a34a377c7ce6e35b30a1bba0eeb74094603c501e9af5b"
},
"reference_binary_sha256": "8ee839e1e8c5844ae49ec0199bfd37c6bffdcfe8407a260d899be28de2af4196",
"candidate_binary_sha256": "bd436db1f42cc945c4fe61d39218ab7671501fdedc1dceeec0f0c1e18fac3514",
"reference_build_configuration": "debug",
"candidate_build_configuration": "release",
"tiny_quant_release_debug_max_abs_logits_delta": 0.0,
"final_source_differences": {
"Sources/MereRunCore/Kolibri/KolibriResources.swift": {
"final_sha256": "e2dc9a0ca2849d398d631d3c911837a91cde82b06c18a2a032908d247cae912c",
"change": "Added local-directory identification; source revision and benchmark math unchanged. Scoring does not use the new path helper."
}
}
},
"harness_source": "import Foundation\nimport MLX\n\nstruct ChatRequestIssue: Error, LocalizedError {\n let option: String\n let message: String\n init(_ option: String, _ message: String) { self.option = option; self.message = message }\n var errorDescription: String? { \"\\(option): \\(message)\" }\n}\n\n@main struct Harness {\n static func main() throws {\n let args = CommandLine.arguments\n guard args.count == 4 || args.count == 5 else { throw ChatRequestIssue(\"arguments\", \"model-root suite output [calibration-file]\") }\n try Device.withDefaultDevice(Device(.gpu)) {\n Memory.cacheLimit = 256 * 1024 * 1024\n try KolibriLogprobBenchmark.run(modelRoot: URL(fileURLWithPath: args[1]),\n suite: URL(fileURLWithPath: args[2]), output: URL(fileURLWithPath: args[3]),\n calibrationOutput: args.count == 5 ? URL(fileURLWithPath: args[4]) : nil,\n progressHandler: { FileHandle.standardError.write(Data(($0 + \"\\n\").utf8)) })\n }\n }\n}\n",
"suite": {
"sha256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b",
"sequences": [
{
"id": "cal-en",
"language": "en",
"task": "reasoning",
"split": "calibration",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
17033,
17407,
1776,
1519,
32,
56,
29522,
46,
25241,
29522,
568,
7880,
46,
742,
900,
1750,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
49,
57,
29522,
1750,
58,
32,
51,
18092,
32,
56,
32099,
32,
53,
576,
32,
49,
57,
46,
127906
],
"scoreStart": 57
},
{
"id": "cal-de",
"language": "de",
"task": "reasoning",
"split": "calibration",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1113,
5958,
32206,
32,
57,
48,
15592,
622,
32,
49,
44,
53,
6838,
46,
2447,
4023,
2459,
2796,
89902,
15705,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
89902,
15705,
12165,
32,
54,
48,
14202,
31351,
58,
32,
57,
48,
49722,
32,
49,
44,
53,
576,
32,
54,
48,
46,
127906
],
"scoreStart": 61
},
{
"id": "cal-code",
"language": "en",
"task": "code",
"split": "calibration",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
7794,
2931,
41,
327,
26550,
14169,
26455,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
7794,
2931,
2488,
14141,
1778,
14169,
1921,
14169,
127906
],
"scoreStart": 53
},
{
"id": "held-en-fact",
"language": "en",
"task": "knowledge",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1798,
735,
262,
13134,
9996,
379,
14272,
63,
1741,
622,
2403,
14264,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
13134,
9996,
379,
14272,
735,
2859,
46,
127906
],
"scoreStart": 50
},
{
"id": "held-de-fact",
"language": "de",
"task": "knowledge",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
61701,
44187,
14107,
968,
16395,
63,
97011,
622,
1097,
16952,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
5396,
44187,
14107,
614,
16395,
2459,
3294,
46,
127906
],
"scoreStart": 48
},
{
"id": "held-en-arithmetic",
"language": "en",
"task": "reasoning",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
65,
4777,
63655,
32,
49,
55,
84038,
3660,
1688,
54,
1776,
286,
94448,
1688,
49,
50,
46,
611,
735,
3327,
15482,
7086,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
15482,
7086,
735,
1688,
57,
48,
58,
32,
49,
55,
18092,
1688,
54,
32099,
1688,
49,
50,
576,
1688,
57,
48,
46,
127906
],
"scoreStart": 63
},
{
"id": "held-de-arithmetic",
"language": "de",
"task": "reasoning",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
11717,
75971,
16611,
32,
50,
52,
10343,
775,
614,
3408,
32,
52,
2527,
420,
2941,
32,
56,
2527,
2340,
46,
2447,
4023,
2459,
948,
42842,
268,
8776,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
8590,
42842,
268,
8776,
12165,
32,
56,
56,
2527,
58,
32,
50,
52,
18092,
32,
52,
32099,
32,
56,
576,
32,
56,
56,
46,
127906
],
"scoreStart": 68
},
{
"id": "held-en-logic",
"language": "en",
"task": "reasoning",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4419,
35311,
568,
26933,
46,
3463,
26933,
476,
23134,
46,
7302,
3326,
3480,
327,
938,
35311,
476,
23134,
63,
20274,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
3913,
46,
543,
26933,
327,
476,
23134,
1026,
502,
2396,
1578,
35311,
44,
1409,
262,
5082,
830,
502,
3480,
46,
127906
],
"scoreStart": 59
},
{
"id": "held-de-logic",
"language": "de",
"task": "reasoning",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
13518,
25525,
1200,
10273,
46,
44871,
10273,
1200,
13752,
329,
46,
8470,
5871,
25412,
44,
1469,
46789,
25525,
13752,
329,
1200,
63,
840,
7166,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1170,
46,
1047,
13752,
1021,
10273,
4165,
3514,
25525,
2260,
59,
346,
90837,
11637,
14276,
939,
46,
127906
],
"scoreStart": 63
},
{
"id": "held-en-code",
"language": "en",
"task": "code",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
735,
105480,
14599,
41,
327,
26550,
2544,
14163,
735,
2051,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
735,
105480,
14599,
2488,
14141,
1778,
14163,
14628,
32,
50,
1339,
32,
48,
127906
],
"scoreStart": 56
},
{
"id": "held-de-code",
"language": "de",
"task": "code",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
22242,
699,
6797,
15836,
1427,
84780,
106925,
125942,
3003,
346,
518,
9678,
7307,
1117,
1233,
1151,
63064,
18996,
4330,
2340,
1673,
46,
32638,
2120,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
1427,
84780,
106925,
125942,
2488,
14141,
1778,
9468,
91,
48,
93,
1765,
9468,
4100,
4330,
127906
],
"scoreStart": 64
},
{
"id": "held-en-translate",
"language": "en",
"task": "translation",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
44761,
318,
4867,
58,
543,
14345,
52074,
3660,
14395,
286,
62712,
3660,
8445,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
30908,
56277,
7470,
24677,
5269,
420,
61556,
7470,
16363,
5269,
46,
127906
],
"scoreStart": 51
},
{
"id": "held-de-translate",
"language": "de",
"task": "translation",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1103,
57190,
120025,
98780,
58,
1047,
36235,
1030,
2361,
38749,
4118,
48837,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
8709,
378,
4191,
941,
5951,
1681,
11123,
46,
127906
],
"scoreStart": 50
},
{
"id": "held-en-window",
"language": "en",
"task": "window-crossing",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
5800,
1862,
12232,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
1798,
735,
262,
3973,
13709,
622,
11237,
32,
55,
63,
6442,
1444,
262,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
],
"scoreStart": 1327
},
{
"id": "held-de-window",
"language": "de",
"task": "window-crossing",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
76,
555,
2834,
94446,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
86170,
34360,
855,
12975,
622,
11237,
32,
55,
63,
32638,
2120,
1178,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
],
"scoreStart": 1328
},
{
"id": "held-en-tool",
"language": "en",
"task": "tools",
"split": "heldout",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
275,
35,
21955,
263,
652,
824,
5604,
2403,
3688,
828,
10852,
318,
9095,
310,
262,
3143,
15483,
275,
652,
568,
2349,
310,
1889,
53814,
2288,
3167,
32776,
1609,
32776,
62,
38357,
24727,
656,
60,
32776,
434,
1142,
676,
408,
4138,
8322,
408,
946,
2938,
262,
5995,
379,
941,
6547,
120260,
946,
653,
408,
946,
688,
95,
30279,
531,
946,
8785,
408,
4138,
15637,
408,
4138,
3547,
408,
4138,
805,
408,
946,
1383,
16506,
946,
22267,
408,
7311,
3547,
24148,
946,
805,
408,
946,
1300,
16506,
946,
805,
408,
946,
676,
100048,
376,
32776,
9215,
2674,
1776,
1889,
5604,
44,
1778,
941,
12597,
7438,
310,
1889,
5032,
286,
15477,
2288,
32,
127909,
127910,
38357,
24727,
656,
127909,
10,
1142,
653,
408,
3167,
676,
62679,
120073,
946,
7160,
408,
3167,
8510,
45,
6427,
123291,
62,
475,
127910,
127906,
10,
127904,
1646,
10,
5116,
262,
5995,
4324,
318,
2492,
2839,
262,
5995,
622,
67041,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
127909,
10,
1142,
653,
408,
946,
688,
95,
30279,
531,
946,
7160,
408,
4138,
3547,
408,
946,
50855,
67036,
34,
31257,
127910,
127906
],
"scoreStart": 183
}
],
"texts": [
{
"id": "cal-en",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nThree boxes each contain 8 marbles. Five marbles are removed. How many remain? Answer briefly.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "19 marbles remain: 3 \u00d7 8 \u2212 5 = 19."
},
{
"id": "cal-de",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nEin Zug f\u00e4hrt 90 Kilometer in 1,5 Stunden. Wie hoch ist seine Durchschnittsgeschwindigkeit? Antworte kurz.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "Die Durchschnittsgeschwindigkeit betr\u00e4gt 60 km/h: 90 \u00f7 1,5 = 60."
},
{
"id": "cal-code",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nWrite a Python function square(x) that returns x squared. Return only code.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "def square(x):\n return x * x"
},
{
"id": "held-en-fact",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nWhat is the chemical symbol for gold? Answer in one sentence.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "The chemical symbol for gold is Au."
},
{
"id": "held-de-fact",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nWelches chemische Symbol hat Eisen? Antworte in einem Satz.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "Das chemische Symbol f\u00fcr Eisen ist Fe."
},
{
"id": "held-en-arithmetic",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nA shop sells 17 notebooks at $6 each and refunds $12. What is its net revenue? Answer briefly.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "The net revenue is $90: 17 \u00d7 $6 \u2212 $12 = $90."
},
{
"id": "held-de-arithmetic",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nEine B\u00e4ckerei verkauft 24 Brote f\u00fcr je 4 Euro und gibt 8 Euro zur\u00fcck. Wie hoch ist der Nettoerl\u00f6s? Antworte kurz.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "Der Nettoerl\u00f6s betr\u00e4gt 88 Euro: 24 \u00d7 4 \u2212 8 = 88."
},
{
"id": "held-en-logic",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nAll whales are mammals. Some mammals can fly. Does it follow that some whales can fly? Explain briefly.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "No. The mammals that can fly need not include any whales, so the conclusion does not follow."
},
{
"id": "held-de-logic",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nAlle Rosen sind Pflanzen. Manche Pflanzen sind giftig. Folgt daraus, dass manche Rosen giftig sind? Begr\u00fcnde kurz.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "Nein. Die giftigen Pflanzen m\u00fcssen keine Rosen sein; die Schlussfolgerung folgt daher nicht."
},
{
"id": "held-en-code",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nWrite a Python function is_even(n) that returns whether n is even. Return only code.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "def is_even(n):\n return n % 2 == 0"
},
{
"id": "held-de-code",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nSchreibe eine Python-Funktion first_or_none(items), die das erste Element oder bei einer leeren Liste None zur\u00fcckgibt. Gib nur Code aus.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "def first_or_none(items):\n return items[0] if items else None"
},
{
"id": "held-en-translate",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nTranslate to German: The library opens at nine and closes at six.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "Die Bibliothek \u00f6ffnet um neun Uhr und schlie\u00dft um sechs Uhr."
},
{
"id": "held-de-translate",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\n\u00dcbersetze ins Englische: Die Br\u00fccke wurde vor hundert Jahren gebaut.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "The bridge was built a hundred years ago."
},
{
"id": "held-en-window",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nRead these records.\nRecord 0: reference code R0000, status checked.\nRecord 1: reference code R0001, status checked.\nRecord 2: reference code R0002, status checked.\nRecord 3: reference code R0003, status checked.\nRecord 4: reference code R0004, status checked.\nRecord 5: reference code R0005, status checked.\nRecord 6: reference code R0006, status checked.\nRecord 7: reference code AZURE-731, status verified.\nRecord 8: reference code R0008, status checked.\nRecord 9: reference code R0009, status checked.\nRecord 10: reference code R0010, status checked.\nRecord 11: reference code R0011, status checked.\nRecord 12: reference code R0012, status checked.\nRecord 13: reference code R0013, status checked.\nRecord 14: reference code R0014, status checked.\nRecord 15: reference code R0015, status checked.\nRecord 16: reference code R0016, status checked.\nRecord 17: reference code R0017, status checked.\nRecord 18: reference code R0018, status checked.\nRecord 19: reference code R0019, status checked.\nRecord 20: reference code R0020, status checked.\nRecord 21: reference code R0021, status checked.\nRecord 22: reference code R0022, status checked.\nRecord 23: reference code R0023, status checked.\nRecord 24: reference code R0024, status checked.\nRecord 25: reference code R0025, status checked.\nRecord 26: reference code R0026, status checked.\nRecord 27: reference code R0027, status checked.\nRecord 28: reference code R0028, status checked.\nRecord 29: reference code R0029, status checked.\nRecord 30: reference code R0030, status checked.\nRecord 31: reference code R0031, status checked.\nRecord 32: reference code R0032, status checked.\nRecord 33: reference code R0033, status checked.\nRecord 34: reference code R0034, status checked.\nRecord 35: reference code R0035, status checked.\nRecord 36: reference code R0036, status checked.\nRecord 37: reference code R0037, status checked.\nRecord 38: reference code R0038, status checked.\nRecord 39: reference code R0039, status checked.\nRecord 40: reference code R0040, status checked.\nRecord 41: reference code R0041, status checked.\nRecord 42: reference code R0042, status checked.\nRecord 43: reference code R0043, status checked.\nRecord 44: reference code R0044, status checked.\nRecord 45: reference code R0045, status checked.\nRecord 46: reference code R0046, status checked.\nRecord 47: reference code R0047, status checked.\nRecord 48: reference code R0048, status checked.\nRecord 49: reference code R0049, status checked.\nRecord 50: reference code R0050, status checked.\nRecord 51: reference code R0051, status checked.\nRecord 52: reference code R0052, status checked.\nRecord 53: reference code R0053, status checked.\nRecord 54: reference code R0054, status checked.\nRecord 55: reference code R0055, status checked.\nRecord 56: reference code R0056, status checked.\nRecord 57: reference code R0057, status checked.\nRecord 58: reference code R0058, status checked.\nRecord 59: reference code R0059, status checked.\nRecord 60: reference code R0060, status checked.\nRecord 61: reference code R0061, status checked.\nRecord 62: reference code R0062, status checked.\nRecord 63: reference code R0063, status checked.\nRecord 64: reference code R0064, status checked.\nRecord 65: reference code R0065, status checked.\nRecord 66: reference code R0066, status checked.\nRecord 67: reference code R0067, status checked.\nRecord 68: reference code R0068, status checked.\nRecord 69: reference code R0069, status checked.\nRecord 70: reference code R0070, status checked.\nRecord 71: reference code R0071, status checked.\nRecord 72: reference code R0072, status checked.\nRecord 73: reference code R0073, status checked.\nRecord 74: reference code R0074, status checked.\nRecord 75: reference code R0075, status checked.\nRecord 76: reference code R0076, status checked.\nRecord 77: reference code R0077, status checked.\nRecord 78: reference code R0078, status checked.\nRecord 79: reference code R0079, status checked.\nWhat is the reference code in Record 7? Return only the code.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "AZURE-731"
},
{
"id": "held-de-window",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.<|im_end|>\n<|im_start|>user\nLies diese Datens\u00e4tze.\nRecord 0: reference code R0000, status checked.\nRecord 1: reference code R0001, status checked.\nRecord 2: reference code R0002, status checked.\nRecord 3: reference code R0003, status checked.\nRecord 4: reference code R0004, status checked.\nRecord 5: reference code R0005, status checked.\nRecord 6: reference code R0006, status checked.\nRecord 7: reference code AZURE-731, status verified.\nRecord 8: reference code R0008, status checked.\nRecord 9: reference code R0009, status checked.\nRecord 10: reference code R0010, status checked.\nRecord 11: reference code R0011, status checked.\nRecord 12: reference code R0012, status checked.\nRecord 13: reference code R0013, status checked.\nRecord 14: reference code R0014, status checked.\nRecord 15: reference code R0015, status checked.\nRecord 16: reference code R0016, status checked.\nRecord 17: reference code R0017, status checked.\nRecord 18: reference code R0018, status checked.\nRecord 19: reference code R0019, status checked.\nRecord 20: reference code R0020, status checked.\nRecord 21: reference code R0021, status checked.\nRecord 22: reference code R0022, status checked.\nRecord 23: reference code R0023, status checked.\nRecord 24: reference code R0024, status checked.\nRecord 25: reference code R0025, status checked.\nRecord 26: reference code R0026, status checked.\nRecord 27: reference code R0027, status checked.\nRecord 28: reference code R0028, status checked.\nRecord 29: reference code R0029, status checked.\nRecord 30: reference code R0030, status checked.\nRecord 31: reference code R0031, status checked.\nRecord 32: reference code R0032, status checked.\nRecord 33: reference code R0033, status checked.\nRecord 34: reference code R0034, status checked.\nRecord 35: reference code R0035, status checked.\nRecord 36: reference code R0036, status checked.\nRecord 37: reference code R0037, status checked.\nRecord 38: reference code R0038, status checked.\nRecord 39: reference code R0039, status checked.\nRecord 40: reference code R0040, status checked.\nRecord 41: reference code R0041, status checked.\nRecord 42: reference code R0042, status checked.\nRecord 43: reference code R0043, status checked.\nRecord 44: reference code R0044, status checked.\nRecord 45: reference code R0045, status checked.\nRecord 46: reference code R0046, status checked.\nRecord 47: reference code R0047, status checked.\nRecord 48: reference code R0048, status checked.\nRecord 49: reference code R0049, status checked.\nRecord 50: reference code R0050, status checked.\nRecord 51: reference code R0051, status checked.\nRecord 52: reference code R0052, status checked.\nRecord 53: reference code R0053, status checked.\nRecord 54: reference code R0054, status checked.\nRecord 55: reference code R0055, status checked.\nRecord 56: reference code R0056, status checked.\nRecord 57: reference code R0057, status checked.\nRecord 58: reference code R0058, status checked.\nRecord 59: reference code R0059, status checked.\nRecord 60: reference code R0060, status checked.\nRecord 61: reference code R0061, status checked.\nRecord 62: reference code R0062, status checked.\nRecord 63: reference code R0063, status checked.\nRecord 64: reference code R0064, status checked.\nRecord 65: reference code R0065, status checked.\nRecord 66: reference code R0066, status checked.\nRecord 67: reference code R0067, status checked.\nRecord 68: reference code R0068, status checked.\nRecord 69: reference code R0069, status checked.\nRecord 70: reference code R0070, status checked.\nRecord 71: reference code R0071, status checked.\nRecord 72: reference code R0072, status checked.\nRecord 73: reference code R0073, status checked.\nRecord 74: reference code R0074, status checked.\nRecord 75: reference code R0075, status checked.\nRecord 76: reference code R0076, status checked.\nRecord 77: reference code R0077, status checked.\nRecord 78: reference code R0078, status checked.\nRecord 79: reference code R0079, status checked.\nWelcher Referenzcode steht in Record 7? Gib nur den Code aus.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "AZURE-731"
},
{
"id": "held-en-tool",
"prompt": "<|im_start|>system\n# Reasoning effort\n\nReasoning is disabled. Proceed straight to answering according to the user's instructions.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"function\": {\"description\": \"Get the weather for a city.\", \"name\": \"get_weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"], \"type\": \"object\"}}, \"type\": \"function\"}\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n<|im_start|>user\nUse the weather tool to look up the weather in Halifax.<|im_end|>\n<|im_start|>assistant\n\n\n\n\n",
"continuation": "\n{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Halifax\"}}\n"
}
]
},
"artifacts": {
"reference": {
"conversion_sha256": "00017e76438afff556792421c677bed6299324b13e311179a5e288c9e3b13476",
"conversion": {
"architecture_revision": "049a6a7bd2405b27d6d280d256bd3d585191c7ae",
"logical_weight_bytes": 156206187520,
"native_expert_order": "numeric",
"output_files": {
"global.safetensors": {
"bytes": 1310725428,
"sha256": "20bdf1aafd0ad0307547d1aa5ae4b785fa81ae942d42de14180eb9715cc28aed"
},
"layer-000.safetensors": {
"bytes": 3097911388,
"sha256": "b6cc7af0a8051623cbe68fdf62c26da471c278427ac18f9d3531bf86739c027d"
},
"layer-001.safetensors": {
"bytes": 3097911388,
"sha256": "4de15ab0569ae1f1d643b75dcaf8f7219fea50670f8f00e9256f86a4a45af78a"
},
"layer-002.safetensors": {
"bytes": 3097911388,
"sha256": "9e0c89d27090659b1079c5fa49c9534b33a3f63f8cb5aa7470868e6bef9a3ec8"
},
"layer-003.safetensors": {
"bytes": 3097911388,
"sha256": "989532a52a2e4986cefc388f3277c50db2ab2a2addbdcaa4b8989a47dc32911a"
},
"layer-004.safetensors": {
"bytes": 3097911388,
"sha256": "8c8928755675f82e053272e1d5d691b15a706b8a03cbf435938355559cd06841"
},
"layer-005.safetensors": {
"bytes": 3097911362,
"sha256": "acc70d202ad553de00ac835e025fcc3148384441ac895ca79549d87fb9d3043f"
},
"layer-006.safetensors": {
"bytes": 3097911388,
"sha256": "218f91ab4439823e0412b51c6dde72c888199ecc8308a970bf585994cea3f6e7"
},
"layer-007.safetensors": {
"bytes": 3097911388,
"sha256": "7b40882e4cfd345f9b89315a112d97a4f2d93b9cf83ae1ec181349d9f7ce7faf"
},
"layer-008.safetensors": {
"bytes": 3097911388,
"sha256": "a3f65144214d0cf3253b639b646b61a39db74e580fa97ed728f3eece372e92c8"
},
"layer-009.safetensors": {
"bytes": 3097911388,
"sha256": "187aaeaebb35554cb2217e2c51f572e14c5f86acb1a2d2085d2a435dbf1dcce9"
},
"layer-010.safetensors": {
"bytes": 3097911406,
"sha256": "2534ee9c9bd1f97a6c71a3bee687801a510f89db092abb7a95ff21ad004afaad"
},
"layer-011.safetensors": {
"bytes": 3097911406,
"sha256": "ce4e341e5b1de941a0c595cf8c9ee2bc6a6591a6104890f80c86a32463597a66"
},
"layer-012.safetensors": {
"bytes": 3097911406,
"sha256": "ab4d8c699b92a9d816e8e1d74a33fbf5f5c2c3c81dea8d8d8767c5f7fa0b2171"
},
"layer-013.safetensors": {
"bytes": 3097911406,
"sha256": "1fcd53cc50887533da4b055a908c8c6c06bc2839e3306ac5e9e5ff33ddf9f671"
},
"layer-014.safetensors": {
"bytes": 3097911406,
"sha256": "84738c21d0372eb40b6307a005103e95006108a3e590436c4c68ae7511b38f5b"
},
"layer-015.safetensors": {
"bytes": 3097911406,
"sha256": "818d0c27e26b264b18e4446451cb52b7a5e9f18f17bdc6d9cdfbf79a59daa6b0"
},
"layer-016.safetensors": {
"bytes": 3097911406,
"sha256": "b0efcd066453998d62a9cb20ed1b530f80849e41db21d320bd1d591bdb5b526a"
},
"layer-017.safetensors": {
"bytes": 3097911406,
"sha256": "157fa687f33b8a13864b150e25b21f12ab94e5c4d76343a6cb938dd0a4c6dd52"
},
"layer-018.safetensors": {
"bytes": 3097911406,
"sha256": "526d147278794e9c1a5f449a489bf0191f23cf5414acbf8e2828ce0ba733b9b9"
},
"layer-019.safetensors": {
"bytes": 3097911406,
"sha256": "43651255839ddfc3867bfaea594e5325d4c1bc7f43915c6fbdb3930d6d72550b"
},
"layer-020.safetensors": {
"bytes": 3097911406,
"sha256": "3f357c72d4eb40fa7c9090b23d15ba61fa7f7bc11dbd631327fbb2ea0a4e03e0"
},
"layer-021.safetensors": {
"bytes": 3097911406,
"sha256": "bc17c79d66d3f595ed156633dd2add30cb67a82b6c714ec47d4e496ed194e718"
},
"layer-022.safetensors": {
"bytes": 3097911406,
"sha256": "8c60494711a8cb2d10e82b134ef8fb42733e79d823d967b8d9caee27e2666611"
},
"layer-023.safetensors": {
"bytes": 3097911406,
"sha256": "15a326f5d0161cbabd4b58f9bff71cada2783d798a0e49f9239c3d3aadbd024b"
},
"layer-024.safetensors": {
"bytes": 3097911406,
"sha256": "947656d82d9491226e658bf84dd12bb054c489f1a2aeee3792107b0996d7143d"
},
"layer-025.safetensors": {
"bytes": 3097911406,
"sha256": "66edc0317823d6af130b0ae912be5bf642fce5806524a2198f0eab395a3f05c0"
},
"layer-026.safetensors": {
"bytes": 3097911406,
"sha256": "ab1f35cfc6140792e2cb2ef2efbd195fabfdd0ac9ffa286c0d0b3036487a89ad"
},
"layer-027.safetensors": {
"bytes": 3097911406,
"sha256": "e89587f376f60e0a597c58159739da145a2e1ace4d6b8bbe49ce21f580398814"
},
"layer-028.safetensors": {
"bytes": 3097911406,
"sha256": "20ff9ad475095508e87f4103e376e09b37d65c35f11b77a09329b48d496e3d8b"
},
"layer-029.safetensors": {
"bytes": 3097911406,
"sha256": "d8e8ab395c8e7302cc052b6cb2545d5192781c2d691ff1b9d32731fbdde987e5"
},
"layer-030.safetensors": {
"bytes": 3097911406,
"sha256": "aea2670d66126e67caf9cf910b6c2c3f22fcc57a0f58f342f8b592ce17b3a053"
},
"layer-031.safetensors": {
"bytes": 3097911406,
"sha256": "b081b769a18eb09a853b2465c842b593c97dddfc760125e5e7a83e0f7bbee44c"
},
"layer-032.safetensors": {
"bytes": 3097911382,
"sha256": "fd13623a6713783c220cb984a94261ababe403e116c1ac88cc472b8e93b87214"
},
"layer-033.safetensors": {
"bytes": 3097911406,
"sha256": "e80617f60f8938c917fca10d1fd7883c7c0882989bfc5b9323a2f8891913f2f0"
},
"layer-034.safetensors": {
"bytes": 3097911406,
"sha256": "28b73b6f83d8d38fa68ea5f3078cac2ab0771be52772db1924be2b09dbf59b24"
},
"layer-035.safetensors": {
"bytes": 3097911406,
"sha256": "fa6f48c269dc11bea968f5cef12a7eafc3675be8d5bf2ceae102532e6fa149fa"
},
"layer-036.safetensors": {
"bytes": 3097911406,
"sha256": "d83d140e69812be878332dbb318956c909613a45a51eeb9b41ca26efa805e7cb"
},
"layer-037.safetensors": {
"bytes": 3097911406,
"sha256": "669f89e10f95db88bb1e9d2ccb2c3e397fb4c43cfdfe19c5abca75f6f65608f4"
},
"layer-038.safetensors": {
"bytes": 3097911406,
"sha256": "8334c2a89c29efa207801abc234e6a28b6d23b4587d3b9fa4e88065ab6eb4ca4"
},
"layer-039.safetensors": {
"bytes": 3097911406,
"sha256": "2b3421b4bb91d9b1ff9d89127f8c5d707e579e3f1a4fcaa532587b59f072326a"
},
"layer-040.safetensors": {
"bytes": 3097911406,
"sha256": "5e22241b062d7366546a957190c028af621aceaca3fe7a7b054537589adacf11"
},
"layer-041.safetensors": {
"bytes": 3097911406,
"sha256": "41dc023aa05ae039f31505cdcb8c03423a5345c07f10fde998ec3de42f5ce39e"
},
"layer-042.safetensors": {
"bytes": 3097911406,
"sha256": "163c47f591dd93002e4c287f53883b7912ed8ec652cceea7269e4e8e00a02917"
},
"layer-043.safetensors": {
"bytes": 3097911406,
"sha256": "ddc389f7e20ff0d168125aac52c40e3ce9889573c64388d3b13613b043bd1270"
},
"layer-044.safetensors": {
"bytes": 3097911406,
"sha256": "862be6dc06cf96e312ebb7abe1190ccc9d35c89d6175274b851ae7775d922e95"
},
"layer-045.safetensors": {
"bytes": 3097911406,
"sha256": "3a010d64c101c90ffabfb5de8168ad6681e60fb24e2a6cb003cabd4db1b1ec98"
},
"layer-046.safetensors": {
"bytes": 3097911406,
"sha256": "e4dcb1a0cec4bd8c177679838be09584114c0e5a62af24a71f9e212aa6d665b8"
},
"layer-047.safetensors": {
"bytes": 3097911406,
"sha256": "85bd4ea301afff5dd3bf649b2888a58f56b8bdebf2b983a87580b6db2bb36362"
},
"layer-048.safetensors": {
"bytes": 3097911406,
"sha256": "23e83bd2c305fcbf10c72db403fba2866c8c916060273f49155f5837b3bb46ff"
},
"layer-049.safetensors": {
"bytes": 3097911406,
"sha256": "4c733bb0d2e12122c5bbfd0ba114e9c5163759a725d6251763d25c51371601d2"
}
},
"profile": "reference",
"quality_qualified": false,
"schema_version": 1,
"source_files": {
"model-00001-of-00032.safetensors": {
"bytes": 4998628528,
"sha256": "e626660074fd9ed3e3644f1d62f23be6adad9437e876f5f9b18260d8382d260b"
},
"model-00002-of-00032.safetensors": {
"bytes": 4998686392,
"sha256": "77259681f2b864d0c0634225d5b31cc15befa1405d062dd2089b41efeb5e6a8b"
},
"model-00003-of-00032.safetensors": {
"bytes": 4998050640,
"sha256": "ba03f0cbe6e48bed0140e5c52fc540cc44f4854887550db5ff6fac91a866142a"
},
"model-00004-of-00032.safetensors": {
"bytes": 4998050400,
"sha256": "68cc07ab7a3614104ab0ee8ce186737638eac3e8f32eb8d2c56742a0437d1680"
},
"model-00005-of-00032.safetensors": {
"bytes": 4998686720,
"sha256": "80ab29ca3d0d917cd83da0205d05b976d8e06100e2ca95ed66175e316550e3ae"
},
"model-00006-of-00032.safetensors": {
"bytes": 4998050320,
"sha256": "24e268797a3e13be638f9edf1f55c872346cf506786322afc6582e4cfa8c4373"
},
"model-00007-of-00032.safetensors": {
"bytes": 4998687256,
"sha256": "f3c8fcdf3bbc5b3af5c12bf8395d091b14c29957afa061f48c4c2685b5b8c8db"
},
"model-00008-of-00032.safetensors": {
"bytes": 4998052664,
"sha256": "648bbb1d3637a96df3f75cb6865846322a4af13bf38fd7817c3e40077d651cc7"
},
"model-00009-of-00032.safetensors": {
"bytes": 4998052208,
"sha256": "481958b40e88a712ea468da50b8f501bf2fe329330881fc3b1b1c3be1ec90689"
},
"model-00010-of-00032.safetensors": {
"bytes": 4998688544,
"sha256": "55d415d65584784fc523a26c9b328e56aec0ddea9fe42a9b48988bc800e67ae1"
},
"model-00011-of-00032.safetensors": {
"bytes": 4998052336,
"sha256": "1d36e5b599317d08d0232bb5e3bdb79f53bd6008294888a66b54d202c5263fea"
},
"model-00012-of-00032.safetensors": {
"bytes": 4998687968,
"sha256": "02be77ce110c6afc379c6ef23fad928205cddf26ddea8155aa274bed1cfd0efe"
},
"model-00013-of-00032.safetensors": {
"bytes": 4998052816,
"sha256": "94c607a19b8254ad78648d91d8808b2ffee5733e27739683225175e050c465c3"
},
"model-00014-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "021de7d92c76f23e31cfaf7f0053e1ce9359efeb93fd90728e658cceb9fe2b2a"
},
"model-00015-of-00032.safetensors": {
"bytes": 4998688344,
"sha256": "1383a4d989beac327c464bd3dd0a8008a199fca3d8c93d2e05dacc76d1f49163"
},
"model-00016-of-00032.safetensors": {
"bytes": 4998052488,
"sha256": "caf525ce7d499e4fbb7e3c6388346a1e8d602baa35f1946387aabb0143b7b696"
},
"model-00017-of-00032.safetensors": {
"bytes": 4984946504,
"sha256": "23af12d4220b2ebd42fcc030c5f98651adb73e96f0819e7dfbfed9454f98c2bd"
},
"model-00018-of-00032.safetensors": {
"bytes": 4998686528,
"sha256": "84e7b32097760e1fbe4380d424ab2c723d101af2283f3aea956a7f8dd30625da"
},
"model-00019-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "a95e33a7082c63f590d5f1893425f17bb87adf8814039dbf76adbfd64fd0e639"
},
"model-00020-of-00032.safetensors": {
"bytes": 4998688200,
"sha256": "55e35b6f1d394cd024d3f4819c63a66bbd5c135168823b02fdce2c736e79e330"
},
"model-00021-of-00032.safetensors": {
"bytes": 4998052592,
"sha256": "e2c07bd50fdec4fabdac18cf7a24db21561bccc33c73dd038a409db2e9632c8f"
},
"model-00022-of-00032.safetensors": {
"bytes": 4998052248,
"sha256": "41e87862dc6209d187842822e7f075763b9eb0dcc439b5be71d82e8162fb8765"
},
"model-00023-of-00032.safetensors": {
"bytes": 4998688616,
"sha256": "e0d663b7503a40dff70cf78b7497a06688efbf07d584f9cbb3b82cc70267c422"
},
"model-00024-of-00032.safetensors": {
"bytes": 4998052232,
"sha256": "8fae68d67883f561d205a9dde47af5815e35b472ff43452e9599cd26eeec2060"
},
"model-00025-of-00032.safetensors": {
"bytes": 4998688040,
"sha256": "0b46e002887b466bc2c9ae63709afaa1a2ad088498da76dd0919b70e5a6126b8"
},
"model-00026-of-00032.safetensors": {
"bytes": 4998052744,
"sha256": "b25cdfb7c0a656ac63b506310ff9db4e1383a76a9455722eb0938de0f697e998"
},
"model-00027-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "765f2b2e020262d9c17a8a9a20ffd711f14ea3014b222f26d3ec1aaa11959aa2"
},
"model-00028-of-00032.safetensors": {
"bytes": 4998688432,
"sha256": "c811b837b63d0cb79116f76a856dca1de6e3915db5df65ca62388c4484903a0d"
},
"model-00029-of-00032.safetensors": {
"bytes": 4998052440,
"sha256": "42c9771487382501d2aa2a5942fbb15062bf18bdbade5aed7b8b4ca180d7e1ff"
},
"model-00030-of-00032.safetensors": {
"bytes": 4998687904,
"sha256": "d6b1e31e529b6472a2da690e879ed9ae70719030b4932e560b1a952aae3748e5"
},
"model-00031-of-00032.safetensors": {
"bytes": 4998052896,
"sha256": "9a7c95968efd33a0bc6c15210a8df021f342ebd4bf3dd442975422a006238418"
},
"model-00032-of-00032.safetensors": {
"bytes": 1278688952,
"sha256": "8aa64fa468f49b55e5fd80da2d6830a01635a1ef53fd5ae0e573a1a6c657d2b3"
}
},
"source_repository": "Aleph-Alpha/Kolibri-1-BF16",
"source_revision": "7a8f290e7858825c3cf5e4c447ba68345de9f1d3",
"source_tensor_count": 58353,
"tensor_count": 903
},
"scoring_receipt": {
"calibrationCaseIDs": [
"cal-en",
"cal-de",
"cal-code"
],
"calibrationSHA256": "b4b18a667a326a9b6cf27f32c61c93c7009081226c403fc644a6f449bc3d3949",
"cases": [
{
"elapsedSeconds": 26.816213369369507,
"logitsFile": "cal-en.safetensors",
"logitsSHA256": "58421f32fa07c62e338926ce3a98646b28a221ba8825401732e8fd6ce9d0755e",
"meanNegativeLogLikelihood": 0.6549556255340576,
"sequence": {
"id": "cal-en",
"language": "en",
"scoreStart": 57,
"split": "calibration",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
17033,
17407,
1776,
1519,
32,
56,
29522,
46,
25241,
29522,
568,
7880,
46,
742,
900,
1750,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
49,
57,
29522,
1750,
58,
32,
51,
18092,
32,
56,
32099,
32,
53,
576,
32,
49,
57,
46,
127906
]
},
"tokenCount": 19
},
{
"elapsedSeconds": 9.448264479637146,
"logitsFile": "cal-de.safetensors",
"logitsSHA256": "fcd39636ddf83d05acc7633e7049f55257609987c24a292a567a59550ae37072",
"meanNegativeLogLikelihood": 0.7154402732849121,
"sequence": {
"id": "cal-de",
"language": "de",
"scoreStart": 61,
"split": "calibration",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1113,
5958,
32206,
32,
57,
48,
15592,
622,
32,
49,
44,
53,
6838,
46,
2447,
4023,
2459,
2796,
89902,
15705,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
89902,
15705,
12165,
32,
54,
48,
14202,
31351,
58,
32,
57,
48,
49722,
32,
49,
44,
53,
576,
32,
54,
48,
46,
127906
]
},
"tokenCount": 24
},
{
"elapsedSeconds": 8.602163434028625,
"logitsFile": "cal-code.safetensors",
"logitsSHA256": "08494db971d43b0244969a7a39780715a8542741d60e8b5830645ed1463fcaa0",
"meanNegativeLogLikelihood": 0.16979160904884338,
"sequence": {
"id": "cal-code",
"language": "en",
"scoreStart": 53,
"split": "calibration",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
7794,
2931,
41,
327,
26550,
14169,
26455,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
7794,
2931,
2488,
14141,
1778,
14169,
1921,
14169,
127906
]
},
"tokenCount": 10
},
{
"elapsedSeconds": 5.188398718833923,
"logitsFile": "held-en-fact.safetensors",
"logitsSHA256": "a13e3354469cc5c0487f1f0a3df8768476e6c38ba06e4508491f36be11312579",
"meanNegativeLogLikelihood": 0.05097516253590584,
"sequence": {
"id": "held-en-fact",
"language": "en",
"scoreStart": 50,
"split": "heldout",
"task": "knowledge",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1798,
735,
262,
13134,
9996,
379,
14272,
63,
1741,
622,
2403,
14264,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
13134,
9996,
379,
14272,
735,
2859,
46,
127906
]
},
"tokenCount": 9
},
{
"elapsedSeconds": 3.2529808282852173,
"logitsFile": "held-de-fact.safetensors",
"logitsSHA256": "c6a52f545603ad555f2f5d8e90b39df0d801670296e3ad06806ad6d813a541b2",
"meanNegativeLogLikelihood": 0.12108675390481949,
"sequence": {
"id": "held-de-fact",
"language": "de",
"scoreStart": 48,
"split": "heldout",
"task": "knowledge",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
61701,
44187,
14107,
968,
16395,
63,
97011,
622,
1097,
16952,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
5396,
44187,
14107,
614,
16395,
2459,
3294,
46,
127906
]
},
"tokenCount": 9
},
{
"elapsedSeconds": 6.185012340545654,
"logitsFile": "held-en-arithmetic.safetensors",
"logitsSHA256": "ed3e6010370c2979c05c77b24c4faa7beb3cd8843621a781cad9f2d2a05cefcc",
"meanNegativeLogLikelihood": 1.0540345907211304,
"sequence": {
"id": "held-en-arithmetic",
"language": "en",
"scoreStart": 63,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
65,
4777,
63655,
32,
49,
55,
84038,
3660,
1688,
54,
1776,
286,
94448,
1688,
49,
50,
46,
611,
735,
3327,
15482,
7086,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
15482,
7086,
735,
1688,
57,
48,
58,
32,
49,
55,
18092,
1688,
54,
32099,
1688,
49,
50,
576,
1688,
57,
48,
46,
127906
]
},
"tokenCount": 24
},
{
"elapsedSeconds": 4.494347929954529,
"logitsFile": "held-de-arithmetic.safetensors",
"logitsSHA256": "c44870e97c70320c525ca581755563c29e4381a4b60a005cff515a8cb95f5bde",
"meanNegativeLogLikelihood": 0.6239409446716309,
"sequence": {
"id": "held-de-arithmetic",
"language": "de",
"scoreStart": 68,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
11717,
75971,
16611,
32,
50,
52,
10343,
775,
614,
3408,
32,
52,
2527,
420,
2941,
32,
56,
2527,
2340,
46,
2447,
4023,
2459,
948,
42842,
268,
8776,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
8590,
42842,
268,
8776,
12165,
32,
56,
56,
2527,
58,
32,
50,
52,
18092,
32,
52,
32099,
32,
56,
576,
32,
56,
56,
46,
127906
]
},
"tokenCount": 25
},
{
"elapsedSeconds": 7.470496892929077,
"logitsFile": "held-en-logic.safetensors",
"logitsSHA256": "d16a855ffc15e0e3e2ea8c95a34ccb26b50dec64489b16695989cf1cab539fe4",
"meanNegativeLogLikelihood": 1.0951416492462158,
"sequence": {
"id": "held-en-logic",
"language": "en",
"scoreStart": 59,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4419,
35311,
568,
26933,
46,
3463,
26933,
476,
23134,
46,
7302,
3326,
3480,
327,
938,
35311,
476,
23134,
63,
20274,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
3913,
46,
543,
26933,
327,
476,
23134,
1026,
502,
2396,
1578,
35311,
44,
1409,
262,
5082,
830,
502,
3480,
46,
127906
]
},
"tokenCount": 21
},
{
"elapsedSeconds": 4.444067001342773,
"logitsFile": "held-de-logic.safetensors",
"logitsSHA256": "9a3d4985ccabde3762e8732020368c0aecf6d891d32684b10fddfdf0e179cf56",
"meanNegativeLogLikelihood": 1.6221963167190552,
"sequence": {
"id": "held-de-logic",
"language": "de",
"scoreStart": 63,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
13518,
25525,
1200,
10273,
46,
44871,
10273,
1200,
13752,
329,
46,
8470,
5871,
25412,
44,
1469,
46789,
25525,
13752,
329,
1200,
63,
840,
7166,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1170,
46,
1047,
13752,
1021,
10273,
4165,
3514,
25525,
2260,
59,
346,
90837,
11637,
14276,
939,
46,
127906
]
},
"tokenCount": 18
},
{
"elapsedSeconds": 4.418267846107483,
"logitsFile": "held-en-code.safetensors",
"logitsSHA256": "c239585d12bb57f37d8b9a69fb8f45df605c54984c93bc59a44ed954e8ba2b2d",
"meanNegativeLogLikelihood": 0.05543060600757599,
"sequence": {
"id": "held-en-code",
"language": "en",
"scoreStart": 56,
"split": "heldout",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
735,
105480,
14599,
41,
327,
26550,
2544,
14163,
735,
2051,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
735,
105480,
14599,
2488,
14141,
1778,
14163,
14628,
32,
50,
1339,
32,
48,
127906
]
},
"tokenCount": 15
},
{
"elapsedSeconds": 3.419432520866394,
"logitsFile": "held-de-code.safetensors",
"logitsSHA256": "a6a0df917875c9f7146db93eb6989f698509bb3ef7f0b17ced41b5a65c49acd1",
"meanNegativeLogLikelihood": 0.1295350044965744,
"sequence": {
"id": "held-de-code",
"language": "de",
"scoreStart": 64,
"split": "heldout",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
22242,
699,
6797,
15836,
1427,
84780,
106925,
125942,
3003,
346,
518,
9678,
7307,
1117,
1233,
1151,
63064,
18996,
4330,
2340,
1673,
46,
32638,
2120,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
1427,
84780,
106925,
125942,
2488,
14141,
1778,
9468,
91,
48,
93,
1765,
9468,
4100,
4330,
127906
]
},
"tokenCount": 17
},
{
"elapsedSeconds": 4.418137073516846,
"logitsFile": "held-en-translate.safetensors",
"logitsSHA256": "cfc4e6eab647a9c7b42dd9877ea87252483981d846a8099fd75c144bccd171e0",
"meanNegativeLogLikelihood": 0.1965249925851822,
"sequence": {
"id": "held-en-translate",
"language": "en",
"scoreStart": 51,
"split": "heldout",
"task": "translation",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
44761,
318,
4867,
58,
543,
14345,
52074,
3660,
14395,
286,
62712,
3660,
8445,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
30908,
56277,
7470,
24677,
5269,
420,
61556,
7470,
16363,
5269,
46,
127906
]
},
"tokenCount": 13
},
{
"elapsedSeconds": 4.087977409362793,
"logitsFile": "held-de-translate.safetensors",
"logitsSHA256": "ffb0f655ba0b856e0f64519d94be5b8fc15d331148ef44ea9c770fa5ebc524ba",
"meanNegativeLogLikelihood": 0.08180637657642365,
"sequence": {
"id": "held-de-translate",
"language": "de",
"scoreStart": 50,
"split": "heldout",
"task": "translation",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1103,
57190,
120025,
98780,
58,
1047,
36235,
1030,
2361,
38749,
4118,
48837,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
8709,
378,
4191,
941,
5951,
1681,
11123,
46,
127906
]
},
"tokenCount": 10
},
{
"elapsedSeconds": 85.96948957443237,
"logitsFile": "held-en-window.safetensors",
"logitsSHA256": "7b08206320b52687b4951ef7edce82641754f77ab548eaff0085907103552992",
"meanNegativeLogLikelihood": 0.0007907322724349797,
"sequence": {
"id": "held-en-window",
"language": "en",
"scoreStart": 1327,
"split": "heldout",
"task": "window-crossing",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
5800,
1862,
12232,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
1798,
735,
262,
3973,
13709,
622,
11237,
32,
55,
63,
6442,
1444,
262,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
]
},
"tokenCount": 7
},
{
"elapsedSeconds": 35.08260667324066,
"logitsFile": "held-de-window.safetensors",
"logitsSHA256": "ebee3ae6aa5e466c49338351abc06452045d9d5cdb91b94c5f7df7cba8c0ffef",
"meanNegativeLogLikelihood": 0.0006425040191970766,
"sequence": {
"id": "held-de-window",
"language": "de",
"scoreStart": 1328,
"split": "heldout",
"task": "window-crossing",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
76,
555,
2834,
94446,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
86170,
34360,
855,
12975,
622,
11237,
32,
55,
63,
32638,
2120,
1178,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
]
},
"tokenCount": 7
},
{
"elapsedSeconds": 9.346731424331665,
"logitsFile": "held-en-tool.safetensors",
"logitsSHA256": "240a12a42689641a43f1d689e0064b4d222f12109522852fbc3d959ae9ae4c41",
"meanNegativeLogLikelihood": 0.00042243627831339836,
"sequence": {
"id": "held-en-tool",
"language": "en",
"scoreStart": 183,
"split": "heldout",
"task": "tools",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
275,
35,
21955,
263,
652,
824,
5604,
2403,
3688,
828,
10852,
318,
9095,
310,
262,
3143,
15483,
275,
652,
568,
2349,
310,
1889,
53814,
2288,
3167,
32776,
1609,
32776,
62,
38357,
24727,
656,
60,
32776,
434,
1142,
676,
408,
4138,
8322,
408,
946,
2938,
262,
5995,
379,
941,
6547,
120260,
946,
653,
408,
946,
688,
95,
30279,
531,
946,
8785,
408,
4138,
15637,
408,
4138,
3547,
408,
4138,
805,
408,
946,
1383,
16506,
946,
22267,
408,
7311,
3547,
24148,
946,
805,
408,
946,
1300,
16506,
946,
805,
408,
946,
676,
100048,
376,
32776,
9215,
2674,
1776,
1889,
5604,
44,
1778,
941,
12597,
7438,
310,
1889,
5032,
286,
15477,
2288,
32,
127909,
127910,
38357,
24727,
656,
127909,
10,
1142,
653,
408,
3167,
676,
62679,
120073,
946,
7160,
408,
3167,
8510,
45,
6427,
123291,
62,
475,
127910,
127906,
10,
127904,
1646,
10,
5116,
262,
5995,
4324,
318,
2492,
2839,
262,
5995,
622,
67041,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
127909,
10,
1142,
653,
408,
946,
688,
95,
30279,
531,
946,
7160,
408,
4138,
3547,
408,
946,
50855,
67036,
34,
31257,
127910,
127906
]
},
"tokenCount": 23
}
],
"checkpointPath": "/workspace/kolibri/artifacts/Kolibri-1-MLX-reference",
"chunkSize": 32,
"configSHA256": "81142ed9372166d9639731fbaee8b6b308281d119f02bcfd9ed96900114a0a7a",
"conversionSHA256": "00017e76438afff556792421c677bed6299324b13e311179a5e288c9e3b13476",
"device": "Device(gpu, 0)",
"indexSHA256": "a96aacb1e3f575ff395a7778e85823fce63e8871bd1145afbd364113c7cf8174",
"peakMemoryBytes": 157802842400,
"schemaVersion": 1,
"suiteSHA256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b"
}
},
"q8": {
"conversion_sha256": "bd32f040715df2a0ee2f39a3ee2530026ef74e0bd1e46967ac44bc72276eaa83",
"conversion": {
"architecture_revision": "049a6a7bd2405b27d6d280d256bd3d585191c7ae",
"logical_weight_bytes": 83645547520,
"native_expert_order": "numeric",
"output_files": {
"global.safetensors": {
"bytes": 1310725428,
"sha256": "20bdf1aafd0ad0307547d1aa5ae4b785fa81ae942d42de14180eb9715cc28aed"
},
"layer-000.safetensors": {
"bytes": 1646700957,
"sha256": "57fbad0263b3a8b965d20e8509ef1cfd2ecbd3f1a271cbaccb82fb12a8e13496"
},
"layer-001.safetensors": {
"bytes": 1646700957,
"sha256": "492511b31fad92f636770e03a48179b63b84056b9eec4221037275c5ad1a1e71"
},
"layer-002.safetensors": {
"bytes": 1646700929,
"sha256": "1ff4e4d5af6ab530868f5be9009c697e1b15320e2bc15a1538b7ea4a1bd97fbb"
},
"layer-003.safetensors": {
"bytes": 1646700967,
"sha256": "cbec9201b3e6aaaad8e3212deaa1fd28080438e610587b348fc4f3c18661f12d"
},
"layer-004.safetensors": {
"bytes": 1646700961,
"sha256": "5c5c39f78515471a5d7c4b99d08782f30d797edb550da5cb99708747eafefae7"
},
"layer-005.safetensors": {
"bytes": 1646700959,
"sha256": "a770c8d1a00c28bb086919a92681640a0572cd0242240f261fb38bd743e6690e"
},
"layer-006.safetensors": {
"bytes": 1646700967,
"sha256": "2b0db727c0840c386d1d6f02f7835482345778bcc051d96b5399333938c222c0"
},
"layer-007.safetensors": {
"bytes": 1646700943,
"sha256": "c8fc83827633f35a01ca0a1fb63c040a61d5dea3f70fb4ce92fb3d6d9a0d2e28"
},
"layer-008.safetensors": {
"bytes": 1646700949,
"sha256": "e0173c00930d5f0fef8426b50945497782830d289c1ac1f43b55a4dd1aea7333"
},
"layer-009.safetensors": {
"bytes": 1646700963,
"sha256": "d8c632e20ec9bb0199eea9e869e1efdd0c17c4b34d361a82f715e87ce834662a"
},
"layer-010.safetensors": {
"bytes": 1646700981,
"sha256": "3c940e2ba423c8e046ddda4bcf0c75bcdb10638e159785137406a0a2e9b5f4ee"
},
"layer-011.safetensors": {
"bytes": 1646700961,
"sha256": "9c3d61246822171748f45d070fde4ca9f6ec17cc8347606d79248e7b122d3c30"
},
"layer-012.safetensors": {
"bytes": 1646700945,
"sha256": "28ea458cafd8037550da724b935e7a22a9639e18d9dfc7980426adfc226c0bda"
},
"layer-013.safetensors": {
"bytes": 1646700997,
"sha256": "3f60394d531d8e9a8799df2070492b99534a27d7ddc350bd2577b7ef30cd4e2c"
},
"layer-014.safetensors": {
"bytes": 1646701005,
"sha256": "3caeeb5dfc97059ef3d0b979d856b16432c3598c80ad3bd1b739fb6c86ca099e"
},
"layer-015.safetensors": {
"bytes": 1646701001,
"sha256": "3422984ffdf5bab11d569a9d354aaf2ac2f1b626d7830ffe928dea16e871e605"
},
"layer-016.safetensors": {
"bytes": 1646700999,
"sha256": "4ef9ca340938474099e54b6a20beda815a806cb5a9f3f304f25cfe76853262a4"
},
"layer-017.safetensors": {
"bytes": 1646700971,
"sha256": "fd56deaf288a4f527595d24a6d866fff09c31fdf99b5649584dcdcc23d42d2de"
},
"layer-018.safetensors": {
"bytes": 1646701001,
"sha256": "d1042e51185d121d5586babbad50c6711641f37230b1a26f60668a841b9f72e8"
},
"layer-019.safetensors": {
"bytes": 1646701003,
"sha256": "4e50e35da90525b0d733564e004c25bf214d344a40eb19c32eae538c9937461a"
},
"layer-020.safetensors": {
"bytes": 1646700999,
"sha256": "c2e9189c7b06ddd0cc5d8f4841a6a63235efbbf56590f1f0291b86a99713789f"
},
"layer-021.safetensors": {
"bytes": 1646701001,
"sha256": "e7c6d9376cfb9281e09207a18cd8a4068052eec7b9c2d8f46a056a23b5fdf218"
},
"layer-022.safetensors": {
"bytes": 1646700995,
"sha256": "3bfb365ff60756932cc982716b597db58749897276cfaa3ffd79a9a7d2c18da5"
},
"layer-023.safetensors": {
"bytes": 1646700989,
"sha256": "a1e73235d59c7c820570981387ec1d6e521e5e70290626c8c515c3a5d9afc078"
},
"layer-024.safetensors": {
"bytes": 1646700999,
"sha256": "83041e586b5922cb931bef0b37678a3f8edb7ec2857d9f924e302d0697bedd61"
},
"layer-025.safetensors": {
"bytes": 1646701005,
"sha256": "9647fb584b5453128d45ebafbb3e399e1cedd0954107e432425845a8e2823fae"
},
"layer-026.safetensors": {
"bytes": 1646700991,
"sha256": "500bf6953a31094ce76b5e6af1dd28cfe35ae39d4bfb6652a7f718c0399b07f8"
},
"layer-027.safetensors": {
"bytes": 1646701001,
"sha256": "1b1976f8783bca4c42100307b18508ee4089063001dd8630e912de880c5b193a"
},
"layer-028.safetensors": {
"bytes": 1646700999,
"sha256": "67fb813f623d5d52749a5b2fb6a7dac4209300bada728f16ab09195f40e0ed98"
},
"layer-029.safetensors": {
"bytes": 1646700999,
"sha256": "c6a663276fe81f36194b33b60bc32217df0d070d49a21e3968b5012ff61fd34f"
},
"layer-030.safetensors": {
"bytes": 1646701003,
"sha256": "1686fab1fe10da882a1b023da28601b9e5abad89612b60e300448d5f031be669"
},
"layer-031.safetensors": {
"bytes": 1646700999,
"sha256": "39555d81e6db1eb57535cca3d917f3eca8c7c7a26ac403af1ee6e68819619180"
},
"layer-032.safetensors": {
"bytes": 1646701007,
"sha256": "025493713bd0b2ba467b170bc221738140d626e907e10a42fd14a86abc84943c"
},
"layer-033.safetensors": {
"bytes": 1646701009,
"sha256": "c4fc2a29533e7ac48d9e82100f6e4ed1fd55eba872677c78304d7542d7b38026"
},
"layer-034.safetensors": {
"bytes": 1646700999,
"sha256": "6049aa2a65ac059705d14250cd97d7e13d4401f802a414b651368a7e878ff188"
},
"layer-035.safetensors": {
"bytes": 1646700999,
"sha256": "da23a126716c2fc1b7d705f5bcfa0668aa1a5fc4b6380b51ef66bb7a0c34bb57"
},
"layer-036.safetensors": {
"bytes": 1646700965,
"sha256": "3ff2ee380475e4e145bfaa398ab434418276d1867136cd6bc16c2d5e1321aaf2"
},
"layer-037.safetensors": {
"bytes": 1646701003,
"sha256": "e68b72e9d3ec0fde7528fb5aeaaaa4807089949428952ba903dacf5750a15a01"
},
"layer-038.safetensors": {
"bytes": 1646701009,
"sha256": "315289ac78c567225174d2bba751058ec3a31f33f8155f55eaeb33a77e37cc6b"
},
"layer-039.safetensors": {
"bytes": 1646701003,
"sha256": "02a6edfd12ccab348c7ce72d0a70f156785b8f41fda39c87bde0aead7ce7aed4"
},
"layer-040.safetensors": {
"bytes": 1646701003,
"sha256": "b7c08b2d7010bfc39842d8cfb7fd92136ba434a1e6b13b7cb98e9d2009152690"
},
"layer-041.safetensors": {
"bytes": 1646700945,
"sha256": "8bc8e6cfbfe7077996cb92d905d0458880c28bd4457a3030c7d54608e5b01393"
},
"layer-042.safetensors": {
"bytes": 1646701001,
"sha256": "1ae206f860a05e5d5a91f1d3d9f1f82ce1fa5b82989b55831e791d31a93a82d9"
},
"layer-043.safetensors": {
"bytes": 1646701005,
"sha256": "11b8ed36b17252714cb8e58c5d686e2ecd04e38abb27881d4d17279115d422f0"
},
"layer-044.safetensors": {
"bytes": 1646700977,
"sha256": "ed71ce0dede05fe93945c39013fda3c3eb53f25bbcfed49bbfa3600aea399a31"
},
"layer-045.safetensors": {
"bytes": 1646700995,
"sha256": "71b675f82958edf63c348bbd9904e41bfc3382d0ebc5bbf64a09cf0af0487fc5"
},
"layer-046.safetensors": {
"bytes": 1646700999,
"sha256": "03acd12fe57cd55dec767823051cd9bad112c84789b402f2aef8ea3c9b805d0c"
},
"layer-047.safetensors": {
"bytes": 1646700991,
"sha256": "59c3124bb2576b819a8c189a73bdf6879b71c6fd04defcf942e96eeab4b5dbfd"
},
"layer-048.safetensors": {
"bytes": 1646700989,
"sha256": "cd4a67790b5f9f9765b0aacbbf1e13e8e33bf5848b2dffbb6c8caeaec60aa7e3"
},
"layer-049.safetensors": {
"bytes": 1646701001,
"sha256": "dc5b32c4141834ba5e2f6d927b87f90bec11305de10add0a8ac21db5b066ebfc"
}
},
"profile": "q8",
"quality_qualified": false,
"schema_version": 1,
"source_files": {
"model-00001-of-00032.safetensors": {
"bytes": 4998628528,
"sha256": "e626660074fd9ed3e3644f1d62f23be6adad9437e876f5f9b18260d8382d260b"
},
"model-00002-of-00032.safetensors": {
"bytes": 4998686392,
"sha256": "77259681f2b864d0c0634225d5b31cc15befa1405d062dd2089b41efeb5e6a8b"
},
"model-00003-of-00032.safetensors": {
"bytes": 4998050640,
"sha256": "ba03f0cbe6e48bed0140e5c52fc540cc44f4854887550db5ff6fac91a866142a"
},
"model-00004-of-00032.safetensors": {
"bytes": 4998050400,
"sha256": "68cc07ab7a3614104ab0ee8ce186737638eac3e8f32eb8d2c56742a0437d1680"
},
"model-00005-of-00032.safetensors": {
"bytes": 4998686720,
"sha256": "80ab29ca3d0d917cd83da0205d05b976d8e06100e2ca95ed66175e316550e3ae"
},
"model-00006-of-00032.safetensors": {
"bytes": 4998050320,
"sha256": "24e268797a3e13be638f9edf1f55c872346cf506786322afc6582e4cfa8c4373"
},
"model-00007-of-00032.safetensors": {
"bytes": 4998687256,
"sha256": "f3c8fcdf3bbc5b3af5c12bf8395d091b14c29957afa061f48c4c2685b5b8c8db"
},
"model-00008-of-00032.safetensors": {
"bytes": 4998052664,
"sha256": "648bbb1d3637a96df3f75cb6865846322a4af13bf38fd7817c3e40077d651cc7"
},
"model-00009-of-00032.safetensors": {
"bytes": 4998052208,
"sha256": "481958b40e88a712ea468da50b8f501bf2fe329330881fc3b1b1c3be1ec90689"
},
"model-00010-of-00032.safetensors": {
"bytes": 4998688544,
"sha256": "55d415d65584784fc523a26c9b328e56aec0ddea9fe42a9b48988bc800e67ae1"
},
"model-00011-of-00032.safetensors": {
"bytes": 4998052336,
"sha256": "1d36e5b599317d08d0232bb5e3bdb79f53bd6008294888a66b54d202c5263fea"
},
"model-00012-of-00032.safetensors": {
"bytes": 4998687968,
"sha256": "02be77ce110c6afc379c6ef23fad928205cddf26ddea8155aa274bed1cfd0efe"
},
"model-00013-of-00032.safetensors": {
"bytes": 4998052816,
"sha256": "94c607a19b8254ad78648d91d8808b2ffee5733e27739683225175e050c465c3"
},
"model-00014-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "021de7d92c76f23e31cfaf7f0053e1ce9359efeb93fd90728e658cceb9fe2b2a"
},
"model-00015-of-00032.safetensors": {
"bytes": 4998688344,
"sha256": "1383a4d989beac327c464bd3dd0a8008a199fca3d8c93d2e05dacc76d1f49163"
},
"model-00016-of-00032.safetensors": {
"bytes": 4998052488,
"sha256": "caf525ce7d499e4fbb7e3c6388346a1e8d602baa35f1946387aabb0143b7b696"
},
"model-00017-of-00032.safetensors": {
"bytes": 4984946504,
"sha256": "23af12d4220b2ebd42fcc030c5f98651adb73e96f0819e7dfbfed9454f98c2bd"
},
"model-00018-of-00032.safetensors": {
"bytes": 4998686528,
"sha256": "84e7b32097760e1fbe4380d424ab2c723d101af2283f3aea956a7f8dd30625da"
},
"model-00019-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "a95e33a7082c63f590d5f1893425f17bb87adf8814039dbf76adbfd64fd0e639"
},
"model-00020-of-00032.safetensors": {
"bytes": 4998688200,
"sha256": "55e35b6f1d394cd024d3f4819c63a66bbd5c135168823b02fdce2c736e79e330"
},
"model-00021-of-00032.safetensors": {
"bytes": 4998052592,
"sha256": "e2c07bd50fdec4fabdac18cf7a24db21561bccc33c73dd038a409db2e9632c8f"
},
"model-00022-of-00032.safetensors": {
"bytes": 4998052248,
"sha256": "41e87862dc6209d187842822e7f075763b9eb0dcc439b5be71d82e8162fb8765"
},
"model-00023-of-00032.safetensors": {
"bytes": 4998688616,
"sha256": "e0d663b7503a40dff70cf78b7497a06688efbf07d584f9cbb3b82cc70267c422"
},
"model-00024-of-00032.safetensors": {
"bytes": 4998052232,
"sha256": "8fae68d67883f561d205a9dde47af5815e35b472ff43452e9599cd26eeec2060"
},
"model-00025-of-00032.safetensors": {
"bytes": 4998688040,
"sha256": "0b46e002887b466bc2c9ae63709afaa1a2ad088498da76dd0919b70e5a6126b8"
},
"model-00026-of-00032.safetensors": {
"bytes": 4998052744,
"sha256": "b25cdfb7c0a656ac63b506310ff9db4e1383a76a9455722eb0938de0f697e998"
},
"model-00027-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "765f2b2e020262d9c17a8a9a20ffd711f14ea3014b222f26d3ec1aaa11959aa2"
},
"model-00028-of-00032.safetensors": {
"bytes": 4998688432,
"sha256": "c811b837b63d0cb79116f76a856dca1de6e3915db5df65ca62388c4484903a0d"
},
"model-00029-of-00032.safetensors": {
"bytes": 4998052440,
"sha256": "42c9771487382501d2aa2a5942fbb15062bf18bdbade5aed7b8b4ca180d7e1ff"
},
"model-00030-of-00032.safetensors": {
"bytes": 4998687904,
"sha256": "d6b1e31e529b6472a2da690e879ed9ae70719030b4932e560b1a952aae3748e5"
},
"model-00031-of-00032.safetensors": {
"bytes": 4998052896,
"sha256": "9a7c95968efd33a0bc6c15210a8df021f342ebd4bf3dd442975422a006238418"
},
"model-00032-of-00032.safetensors": {
"bytes": 1278688952,
"sha256": "8aa64fa468f49b55e5fd80da2d6830a01635a1ef53fd5ae0e573a1a6c657d2b3"
}
},
"source_repository": "Aleph-Alpha/Kolibri-1-BF16",
"source_revision": "7a8f290e7858825c3cf5e4c447ba68345de9f1d3",
"source_tensor_count": 58353,
"tensor_count": 1903
},
"scoring_receipt": {
"calibrationCaseIDs": [],
"cases": [
{
"elapsedSeconds": 22.145086765289307,
"logitsFile": "cal-en.safetensors",
"logitsSHA256": "756e5b920fd9211c76b50d9fd0d3a301126514ecb39153c414c4a765884bca64",
"meanNegativeLogLikelihood": 0.6767817139625549,
"sequence": {
"id": "cal-en",
"language": "en",
"scoreStart": 57,
"split": "calibration",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
17033,
17407,
1776,
1519,
32,
56,
29522,
46,
25241,
29522,
568,
7880,
46,
742,
900,
1750,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
49,
57,
29522,
1750,
58,
32,
51,
18092,
32,
56,
32099,
32,
53,
576,
32,
49,
57,
46,
127906
]
},
"tokenCount": 19
},
{
"elapsedSeconds": 6.327370882034302,
"logitsFile": "cal-de.safetensors",
"logitsSHA256": "bbf6eccdf4e7247db73b23fe294eaabd8fb735c527b4717c20cf22401b2cad90",
"meanNegativeLogLikelihood": 0.7239995002746582,
"sequence": {
"id": "cal-de",
"language": "de",
"scoreStart": 61,
"split": "calibration",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1113,
5958,
32206,
32,
57,
48,
15592,
622,
32,
49,
44,
53,
6838,
46,
2447,
4023,
2459,
2796,
89902,
15705,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
89902,
15705,
12165,
32,
54,
48,
14202,
31351,
58,
32,
57,
48,
49722,
32,
49,
44,
53,
576,
32,
54,
48,
46,
127906
]
},
"tokenCount": 24
},
{
"elapsedSeconds": 5.679817795753479,
"logitsFile": "cal-code.safetensors",
"logitsSHA256": "99e7507ece649d3ef1af871d63ea023c6aed8308f2d5401fa15ea4fcaecfaf6a",
"meanNegativeLogLikelihood": 0.17525406181812286,
"sequence": {
"id": "cal-code",
"language": "en",
"scoreStart": 53,
"split": "calibration",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
7794,
2931,
41,
327,
26550,
14169,
26455,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
7794,
2931,
2488,
14141,
1778,
14169,
1921,
14169,
127906
]
},
"tokenCount": 10
},
{
"elapsedSeconds": 4.819172143936157,
"logitsFile": "held-en-fact.safetensors",
"logitsSHA256": "5c250795ef8d8654a98ca9d48e90ce1082a4f18742dfc30a7bce6af49bdfdf1a",
"meanNegativeLogLikelihood": 0.0533599853515625,
"sequence": {
"id": "held-en-fact",
"language": "en",
"scoreStart": 50,
"split": "heldout",
"task": "knowledge",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1798,
735,
262,
13134,
9996,
379,
14272,
63,
1741,
622,
2403,
14264,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
13134,
9996,
379,
14272,
735,
2859,
46,
127906
]
},
"tokenCount": 9
},
{
"elapsedSeconds": 3.419869065284729,
"logitsFile": "held-de-fact.safetensors",
"logitsSHA256": "f64bd03c962c16b0cbe2129397ea3ba34d5a475b34f6e8d146bcbbbc69adf18e",
"meanNegativeLogLikelihood": 0.12277720123529434,
"sequence": {
"id": "held-de-fact",
"language": "de",
"scoreStart": 48,
"split": "heldout",
"task": "knowledge",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
61701,
44187,
14107,
968,
16395,
63,
97011,
622,
1097,
16952,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
5396,
44187,
14107,
614,
16395,
2459,
3294,
46,
127906
]
},
"tokenCount": 9
},
{
"elapsedSeconds": 5.549734354019165,
"logitsFile": "held-en-arithmetic.safetensors",
"logitsSHA256": "45298e294d092faf2d3fead0bb5c832a07bfcb7db41ba72141ff20d368e85932",
"meanNegativeLogLikelihood": 1.0762231349945068,
"sequence": {
"id": "held-en-arithmetic",
"language": "en",
"scoreStart": 63,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
65,
4777,
63655,
32,
49,
55,
84038,
3660,
1688,
54,
1776,
286,
94448,
1688,
49,
50,
46,
611,
735,
3327,
15482,
7086,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
15482,
7086,
735,
1688,
57,
48,
58,
32,
49,
55,
18092,
1688,
54,
32099,
1688,
49,
50,
576,
1688,
57,
48,
46,
127906
]
},
"tokenCount": 24
},
{
"elapsedSeconds": 4.200287103652954,
"logitsFile": "held-de-arithmetic.safetensors",
"logitsSHA256": "d1cdccf3e42f2e6887c14290d592bfeeee5753d0fce39eb08b50541d05cbaf33",
"meanNegativeLogLikelihood": 0.625777006149292,
"sequence": {
"id": "held-de-arithmetic",
"language": "de",
"scoreStart": 68,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
11717,
75971,
16611,
32,
50,
52,
10343,
775,
614,
3408,
32,
52,
2527,
420,
2941,
32,
56,
2527,
2340,
46,
2447,
4023,
2459,
948,
42842,
268,
8776,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
8590,
42842,
268,
8776,
12165,
32,
56,
56,
2527,
58,
32,
50,
52,
18092,
32,
52,
32099,
32,
56,
576,
32,
56,
56,
46,
127906
]
},
"tokenCount": 25
},
{
"elapsedSeconds": 5.937287092208862,
"logitsFile": "held-en-logic.safetensors",
"logitsSHA256": "bc6110b08f28bc1dac2922e3c83309c37bfda363cc4d07c88cb12416de9dbcaf",
"meanNegativeLogLikelihood": 1.1384713649749756,
"sequence": {
"id": "held-en-logic",
"language": "en",
"scoreStart": 59,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4419,
35311,
568,
26933,
46,
3463,
26933,
476,
23134,
46,
7302,
3326,
3480,
327,
938,
35311,
476,
23134,
63,
20274,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
3913,
46,
543,
26933,
327,
476,
23134,
1026,
502,
2396,
1578,
35311,
44,
1409,
262,
5082,
830,
502,
3480,
46,
127906
]
},
"tokenCount": 21
},
{
"elapsedSeconds": 4.192874073982239,
"logitsFile": "held-de-logic.safetensors",
"logitsSHA256": "e9f9bfaf6941e47f2c754417e212a56d823253e12cdc45477ae633d7684c4b02",
"meanNegativeLogLikelihood": 1.6359659433364868,
"sequence": {
"id": "held-de-logic",
"language": "de",
"scoreStart": 63,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
13518,
25525,
1200,
10273,
46,
44871,
10273,
1200,
13752,
329,
46,
8470,
5871,
25412,
44,
1469,
46789,
25525,
13752,
329,
1200,
63,
840,
7166,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1170,
46,
1047,
13752,
1021,
10273,
4165,
3514,
25525,
2260,
59,
346,
90837,
11637,
14276,
939,
46,
127906
]
},
"tokenCount": 18
},
{
"elapsedSeconds": 4.276385188102722,
"logitsFile": "held-en-code.safetensors",
"logitsSHA256": "7d86399fefadf29769997cb284cfeddece93e67445d2d90f47de0792a01ed74c",
"meanNegativeLogLikelihood": 0.050343070179224014,
"sequence": {
"id": "held-en-code",
"language": "en",
"scoreStart": 56,
"split": "heldout",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
735,
105480,
14599,
41,
327,
26550,
2544,
14163,
735,
2051,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
735,
105480,
14599,
2488,
14141,
1778,
14163,
14628,
32,
50,
1339,
32,
48,
127906
]
},
"tokenCount": 15
},
{
"elapsedSeconds": 3.077330231666565,
"logitsFile": "held-de-code.safetensors",
"logitsSHA256": "2680385a0c3638371331a5cf0c724dd853ed9be56c588624ff7e6a873d70a293",
"meanNegativeLogLikelihood": 0.11565365642309189,
"sequence": {
"id": "held-de-code",
"language": "de",
"scoreStart": 64,
"split": "heldout",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
22242,
699,
6797,
15836,
1427,
84780,
106925,
125942,
3003,
346,
518,
9678,
7307,
1117,
1233,
1151,
63064,
18996,
4330,
2340,
1673,
46,
32638,
2120,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
1427,
84780,
106925,
125942,
2488,
14141,
1778,
9468,
91,
48,
93,
1765,
9468,
4100,
4330,
127906
]
},
"tokenCount": 17
},
{
"elapsedSeconds": 4.099236726760864,
"logitsFile": "held-en-translate.safetensors",
"logitsSHA256": "4be499da54e23c6d8772bc1adc162298216906357b82142d25afcf0f27665599",
"meanNegativeLogLikelihood": 0.19412298500537872,
"sequence": {
"id": "held-en-translate",
"language": "en",
"scoreStart": 51,
"split": "heldout",
"task": "translation",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
44761,
318,
4867,
58,
543,
14345,
52074,
3660,
14395,
286,
62712,
3660,
8445,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
30908,
56277,
7470,
24677,
5269,
420,
61556,
7470,
16363,
5269,
46,
127906
]
},
"tokenCount": 13
},
{
"elapsedSeconds": 3.963321566581726,
"logitsFile": "held-de-translate.safetensors",
"logitsSHA256": "ac0ee6bae302ac508fba7576ac74d1774b5ad93366c45e27204346039a913221",
"meanNegativeLogLikelihood": 0.08650493621826172,
"sequence": {
"id": "held-de-translate",
"language": "de",
"scoreStart": 50,
"split": "heldout",
"task": "translation",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1103,
57190,
120025,
98780,
58,
1047,
36235,
1030,
2361,
38749,
4118,
48837,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
8709,
378,
4191,
941,
5951,
1681,
11123,
46,
127906
]
},
"tokenCount": 10
},
{
"elapsedSeconds": 92.78882348537445,
"logitsFile": "held-en-window.safetensors",
"logitsSHA256": "9dd31f928b09788dd9dc44388a3a6613637848b5cb2483c6423130f309eb111c",
"meanNegativeLogLikelihood": 0.0007073538727127016,
"sequence": {
"id": "held-en-window",
"language": "en",
"scoreStart": 1327,
"split": "heldout",
"task": "window-crossing",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
5800,
1862,
12232,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
1798,
735,
262,
3973,
13709,
622,
11237,
32,
55,
63,
6442,
1444,
262,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
]
},
"tokenCount": 7
},
{
"elapsedSeconds": 37.23163187503815,
"logitsFile": "held-de-window.safetensors",
"logitsSHA256": "f6ebe3c65ee2b4306d8f5ec31545046f6c95c3430ed540ba86887ada83a79639",
"meanNegativeLogLikelihood": 0.0005811963928863406,
"sequence": {
"id": "held-de-window",
"language": "de",
"scoreStart": 1328,
"split": "heldout",
"task": "window-crossing",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
76,
555,
2834,
94446,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
86170,
34360,
855,
12975,
622,
11237,
32,
55,
63,
32638,
2120,
1178,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
]
},
"tokenCount": 7
},
{
"elapsedSeconds": 8.452101707458496,
"logitsFile": "held-en-tool.safetensors",
"logitsSHA256": "797ded988fc3303f10a65551f445d1ae96ba70b1d820cf67235b99733d81b19d",
"meanNegativeLogLikelihood": 0.0001628709869692102,
"sequence": {
"id": "held-en-tool",
"language": "en",
"scoreStart": 183,
"split": "heldout",
"task": "tools",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
275,
35,
21955,
263,
652,
824,
5604,
2403,
3688,
828,
10852,
318,
9095,
310,
262,
3143,
15483,
275,
652,
568,
2349,
310,
1889,
53814,
2288,
3167,
32776,
1609,
32776,
62,
38357,
24727,
656,
60,
32776,
434,
1142,
676,
408,
4138,
8322,
408,
946,
2938,
262,
5995,
379,
941,
6547,
120260,
946,
653,
408,
946,
688,
95,
30279,
531,
946,
8785,
408,
4138,
15637,
408,
4138,
3547,
408,
4138,
805,
408,
946,
1383,
16506,
946,
22267,
408,
7311,
3547,
24148,
946,
805,
408,
946,
1300,
16506,
946,
805,
408,
946,
676,
100048,
376,
32776,
9215,
2674,
1776,
1889,
5604,
44,
1778,
941,
12597,
7438,
310,
1889,
5032,
286,
15477,
2288,
32,
127909,
127910,
38357,
24727,
656,
127909,
10,
1142,
653,
408,
3167,
676,
62679,
120073,
946,
7160,
408,
3167,
8510,
45,
6427,
123291,
62,
475,
127910,
127906,
10,
127904,
1646,
10,
5116,
262,
5995,
4324,
318,
2492,
2839,
262,
5995,
622,
67041,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
127909,
10,
1142,
653,
408,
946,
688,
95,
30279,
531,
946,
7160,
408,
4138,
3547,
408,
946,
50855,
67036,
34,
31257,
127910,
127906
]
},
"tokenCount": 23
}
],
"checkpointPath": "/workspace/kolibri/artifacts/Kolibri-1-MLX-q8",
"chunkSize": 32,
"configSHA256": "eba459d925b4847d3fd7c71d3cd5ff0625e1c58e67fed1f42ddb0aa2b46855aa",
"conversionSHA256": "bd32f040715df2a0ee2f39a3ee2530026ef74e0bd1e46967ac44bc72276eaa83",
"device": "Device(gpu, 0)",
"indexSHA256": "c6476bd195953e3cee56fbad36b6c11f56f596eb995a5023f094067ba9568c01",
"peakMemoryBytes": 86122756384,
"schemaVersion": 1,
"suiteSHA256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b"
}
},
"mixed2": {
"conversion_sha256": "ec78555322e1267973be8a64d5302d476eacf4021c7124af8fa94c8ceb1735cd",
"conversion": {
"architecture_revision": "049a6a7bd2405b27d6d280d256bd3d585191c7ae",
"logical_weight_bytes": 24663147520,
"native_expert_order": "numeric",
"output_files": {
"global.safetensors": {
"bytes": 1310725428,
"sha256": "20bdf1aafd0ad0307547d1aa5ae4b785fa81ae942d42de14180eb9715cc28aed"
},
"layer-000.safetensors": {
"bytes": 467052891,
"sha256": "77f630528501efddd1118b7dac63c98fe80c31960cbe0ad6dc50ff38bb5decfc"
},
"layer-001.safetensors": {
"bytes": 467052889,
"sha256": "6900091ef403540dc99c12251fad9866c8175b5906487369db2f4f66a0b6d5e2"
},
"layer-002.safetensors": {
"bytes": 467052879,
"sha256": "fbcacc650d3fed46d3555fac42c093912c6e7d48bff9a3e73b95041d9f0a8768"
},
"layer-003.safetensors": {
"bytes": 467052893,
"sha256": "6a51c3c4d9ddfc51c82010c01cafe1fe0bfb3bacd01cd57de60ec947c133b019"
},
"layer-004.safetensors": {
"bytes": 467052893,
"sha256": "2562a5e85e254e04db52335ee9ecbd74625e73046515c81504ac7c79781d483c"
},
"layer-005.safetensors": {
"bytes": 467052891,
"sha256": "13ed28fe4409b955371c70963527d0fd98bd3c282b3c3a8478ae530ab9430629"
},
"layer-006.safetensors": {
"bytes": 467052893,
"sha256": "5e6f02163d5548213dd7bda57bfb49477b9eaa93291d666b12a4ddde217df589"
},
"layer-007.safetensors": {
"bytes": 467052893,
"sha256": "327b02026877916131e10dea2d2d41950f7f9272399670f0205b12be77efa1f2"
},
"layer-008.safetensors": {
"bytes": 467052889,
"sha256": "13649c87c4c76fe793acc24ab98e9d8612fbf2b820012c60ffc5ac73395b5981"
},
"layer-009.safetensors": {
"bytes": 467052893,
"sha256": "150393e796980d2f57364020520bd7d664d5a198a1763fc715e47597135b4ba3"
},
"layer-010.safetensors": {
"bytes": 467052925,
"sha256": "56318848efbc70fe508792384a395d671c23e30123aa496a9c8a1f1ec39f74f1"
},
"layer-011.safetensors": {
"bytes": 467052929,
"sha256": "8f8a09f6254511b9d8b9cb92a2d6e56e8fd551f671f4c9767737f30fc1e931bc"
},
"layer-012.safetensors": {
"bytes": 467052927,
"sha256": "5e4a9aafcf705a3319fee00a47c81091b47c9cc337ad96d1cf9a1f9dbc7dc1c2"
},
"layer-013.safetensors": {
"bytes": 467052929,
"sha256": "f22464cfe094607457ea0c6f9608a7bda8fb24027fdd7c6a46e3ef0aba91bdcf"
},
"layer-014.safetensors": {
"bytes": 467052935,
"sha256": "3bf9767744d267de0577086ddcfd9a1d9d6cd197cf1f71b2535e708bea471bf3"
},
"layer-015.safetensors": {
"bytes": 467052931,
"sha256": "552ba19f479129444399ef38c454cb98b9d628327265dcc1c4a5ed673d8cfe8b"
},
"layer-016.safetensors": {
"bytes": 467052929,
"sha256": "ab8a259b692d70dc604b78a2f585484cacfeba0fb01f126adf2e76ace75839b0"
},
"layer-017.safetensors": {
"bytes": 467052923,
"sha256": "b0954befa990b6e5f1bb90c1178a91930f0a81055fd1ffd15c264d0a8ffab9fd"
},
"layer-018.safetensors": {
"bytes": 467052929,
"sha256": "4691d8e32fef697a81815e67ee2e9474c3a482e3d515ed9515a61f15800e9833"
},
"layer-019.safetensors": {
"bytes": 467052931,
"sha256": "7e6381ae1584539bc00f53cbd3eed55ac962e1d2d90d1f22dcad2b8f23b09bb2"
},
"layer-020.safetensors": {
"bytes": 467052929,
"sha256": "812c2adf657754c943c9144eb74da20506d43ca989a1731883125c4d5a4f23a7"
},
"layer-021.safetensors": {
"bytes": 467052931,
"sha256": "334db23be56bec4a43bba8787c531f146b466408faa94c496c3a48b618dd85e5"
},
"layer-022.safetensors": {
"bytes": 467052927,
"sha256": "70f95bed5333c466a36d493d26607023f42f0ae994b8a7ed7ddd09f51a3f50f1"
},
"layer-023.safetensors": {
"bytes": 467052927,
"sha256": "ce30753d4f9af37586f41cf3b52921b8dd9bed9d6f2bdd9d1663987b75b02fab"
},
"layer-024.safetensors": {
"bytes": 467052931,
"sha256": "f6820f85a3d64a2e6983b6b4a338fbdd50f645e1c60c9474535f5cbfccf8f8ce"
},
"layer-025.safetensors": {
"bytes": 467052935,
"sha256": "3bbaae4a7d3244df5c20bfb35c4121c8b8a16a73691dbf70ad9792de9690e7ee"
},
"layer-026.safetensors": {
"bytes": 467052925,
"sha256": "a08f68eead642048f26403cb5442fe803d90cd8fc1c70bab8c43722d48d16074"
},
"layer-027.safetensors": {
"bytes": 467052931,
"sha256": "9b721b9c00970bfa16c584ab86b3160bcd778291c4447e8b1a6178a6e1714eec"
},
"layer-028.safetensors": {
"bytes": 467052929,
"sha256": "256d81810989551cf4e6963507184b8ef7da9bb7fc37524e564da3a8ec8d1cd8"
},
"layer-029.safetensors": {
"bytes": 467052929,
"sha256": "1682797c8388ace7fa767c11249c5e51710163ea0c7d4b814dddb7995f143ee8"
},
"layer-030.safetensors": {
"bytes": 467052931,
"sha256": "cc4e20a193821ecc98df0db83460a5b6345d935bcaac02bfcdc66bdf93dc17c5"
},
"layer-031.safetensors": {
"bytes": 467052931,
"sha256": "1b58682408a22b340188e7a8ef6a3726f21e981cc62f489a33c91b86052846fe"
},
"layer-032.safetensors": {
"bytes": 467052935,
"sha256": "4542115edf92784c1c46bd527efc01ba7d1d6003bec965099e99570e24fe290c"
},
"layer-033.safetensors": {
"bytes": 467052935,
"sha256": "517a827330ad93e8ccae679e54553fe921eba9724da7e97b424c45af10bba6fa"
},
"layer-034.safetensors": {
"bytes": 467052931,
"sha256": "747c887c659a1d3a781b4964ee1c5832ced94881098cc41c7c8880d06762d7a3"
},
"layer-035.safetensors": {
"bytes": 467052931,
"sha256": "7b15669e6239b5c02d9792325088fcd8430548f2ac401c8a8f65faaa6ede1d22"
},
"layer-036.safetensors": {
"bytes": 467052927,
"sha256": "26d32efcdf440917d56091e4f8a0a4d0c189c30f4a79dcc4bf4684486da4f06a"
},
"layer-037.safetensors": {
"bytes": 467052931,
"sha256": "b3a138d185888da8118317de406cbd4e1d59d4c2751d06b7539f383421329391"
},
"layer-038.safetensors": {
"bytes": 467052935,
"sha256": "d3a19767c10093df15299483562aef79255c7dd01b72519b21e47e5e466cfbd0"
},
"layer-039.safetensors": {
"bytes": 467052931,
"sha256": "59a99c2876acf7bb8bbb4a5ee4d46c4346d8b1f2f2baafac2d4d5c11075312ae"
},
"layer-040.safetensors": {
"bytes": 467052935,
"sha256": "d143055b57e97aad55b2ba004dda9b925237a79c9329a8feb3fef6bc78ee9f59"
},
"layer-041.safetensors": {
"bytes": 467052907,
"sha256": "1b3572d28a27ed3aff60a2aa598d8c69a4a74bf0ad815b20be8fb374b458af4d"
},
"layer-042.safetensors": {
"bytes": 467052931,
"sha256": "bf2eae9eb03965c12641286641b606ea920212d9888d60cea1bc9faf75954e54"
},
"layer-043.safetensors": {
"bytes": 467052931,
"sha256": "c0e42538d8fdb7f4ae76edd6e0a57a33b2cf91616dfd31e14cf2dd341a93a3d8"
},
"layer-044.safetensors": {
"bytes": 467052931,
"sha256": "f67d23e5c2be25e9d8947f3cb36ac8f85116fc67de5b7f2e32e7ac7a53bca7f7"
},
"layer-045.safetensors": {
"bytes": 467052927,
"sha256": "c145ef196e49f38460fe14580a6cb2526abf96daec55c12c216da2c8130dd9bc"
},
"layer-046.safetensors": {
"bytes": 467052929,
"sha256": "e80f38b3ac35edbcf0ee33f4527a8fe6cf8e626c030bc33430f4dd5e45ec18bc"
},
"layer-047.safetensors": {
"bytes": 467052925,
"sha256": "13989b69d3d4622a673c010567d3b959da4a1cfab193631fe9ccb6e4096dcccc"
},
"layer-048.safetensors": {
"bytes": 467052929,
"sha256": "2657d43704d8cf9b9dd13215c90ae94618d441aa501e83e766caa6692b74e0f3"
},
"layer-049.safetensors": {
"bytes": 467052931,
"sha256": "545b63e29579cc4d7f4569838e611721baacf96484aa5c38607de1c219c0e8a8"
}
},
"profile": "mixed2",
"quality_qualified": false,
"schema_version": 1,
"source_files": {
"model-00001-of-00032.safetensors": {
"bytes": 4998628528,
"sha256": "e626660074fd9ed3e3644f1d62f23be6adad9437e876f5f9b18260d8382d260b"
},
"model-00002-of-00032.safetensors": {
"bytes": 4998686392,
"sha256": "77259681f2b864d0c0634225d5b31cc15befa1405d062dd2089b41efeb5e6a8b"
},
"model-00003-of-00032.safetensors": {
"bytes": 4998050640,
"sha256": "ba03f0cbe6e48bed0140e5c52fc540cc44f4854887550db5ff6fac91a866142a"
},
"model-00004-of-00032.safetensors": {
"bytes": 4998050400,
"sha256": "68cc07ab7a3614104ab0ee8ce186737638eac3e8f32eb8d2c56742a0437d1680"
},
"model-00005-of-00032.safetensors": {
"bytes": 4998686720,
"sha256": "80ab29ca3d0d917cd83da0205d05b976d8e06100e2ca95ed66175e316550e3ae"
},
"model-00006-of-00032.safetensors": {
"bytes": 4998050320,
"sha256": "24e268797a3e13be638f9edf1f55c872346cf506786322afc6582e4cfa8c4373"
},
"model-00007-of-00032.safetensors": {
"bytes": 4998687256,
"sha256": "f3c8fcdf3bbc5b3af5c12bf8395d091b14c29957afa061f48c4c2685b5b8c8db"
},
"model-00008-of-00032.safetensors": {
"bytes": 4998052664,
"sha256": "648bbb1d3637a96df3f75cb6865846322a4af13bf38fd7817c3e40077d651cc7"
},
"model-00009-of-00032.safetensors": {
"bytes": 4998052208,
"sha256": "481958b40e88a712ea468da50b8f501bf2fe329330881fc3b1b1c3be1ec90689"
},
"model-00010-of-00032.safetensors": {
"bytes": 4998688544,
"sha256": "55d415d65584784fc523a26c9b328e56aec0ddea9fe42a9b48988bc800e67ae1"
},
"model-00011-of-00032.safetensors": {
"bytes": 4998052336,
"sha256": "1d36e5b599317d08d0232bb5e3bdb79f53bd6008294888a66b54d202c5263fea"
},
"model-00012-of-00032.safetensors": {
"bytes": 4998687968,
"sha256": "02be77ce110c6afc379c6ef23fad928205cddf26ddea8155aa274bed1cfd0efe"
},
"model-00013-of-00032.safetensors": {
"bytes": 4998052816,
"sha256": "94c607a19b8254ad78648d91d8808b2ffee5733e27739683225175e050c465c3"
},
"model-00014-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "021de7d92c76f23e31cfaf7f0053e1ce9359efeb93fd90728e658cceb9fe2b2a"
},
"model-00015-of-00032.safetensors": {
"bytes": 4998688344,
"sha256": "1383a4d989beac327c464bd3dd0a8008a199fca3d8c93d2e05dacc76d1f49163"
},
"model-00016-of-00032.safetensors": {
"bytes": 4998052488,
"sha256": "caf525ce7d499e4fbb7e3c6388346a1e8d602baa35f1946387aabb0143b7b696"
},
"model-00017-of-00032.safetensors": {
"bytes": 4984946504,
"sha256": "23af12d4220b2ebd42fcc030c5f98651adb73e96f0819e7dfbfed9454f98c2bd"
},
"model-00018-of-00032.safetensors": {
"bytes": 4998686528,
"sha256": "84e7b32097760e1fbe4380d424ab2c723d101af2283f3aea956a7f8dd30625da"
},
"model-00019-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "a95e33a7082c63f590d5f1893425f17bb87adf8814039dbf76adbfd64fd0e639"
},
"model-00020-of-00032.safetensors": {
"bytes": 4998688200,
"sha256": "55e35b6f1d394cd024d3f4819c63a66bbd5c135168823b02fdce2c736e79e330"
},
"model-00021-of-00032.safetensors": {
"bytes": 4998052592,
"sha256": "e2c07bd50fdec4fabdac18cf7a24db21561bccc33c73dd038a409db2e9632c8f"
},
"model-00022-of-00032.safetensors": {
"bytes": 4998052248,
"sha256": "41e87862dc6209d187842822e7f075763b9eb0dcc439b5be71d82e8162fb8765"
},
"model-00023-of-00032.safetensors": {
"bytes": 4998688616,
"sha256": "e0d663b7503a40dff70cf78b7497a06688efbf07d584f9cbb3b82cc70267c422"
},
"model-00024-of-00032.safetensors": {
"bytes": 4998052232,
"sha256": "8fae68d67883f561d205a9dde47af5815e35b472ff43452e9599cd26eeec2060"
},
"model-00025-of-00032.safetensors": {
"bytes": 4998688040,
"sha256": "0b46e002887b466bc2c9ae63709afaa1a2ad088498da76dd0919b70e5a6126b8"
},
"model-00026-of-00032.safetensors": {
"bytes": 4998052744,
"sha256": "b25cdfb7c0a656ac63b506310ff9db4e1383a76a9455722eb0938de0f697e998"
},
"model-00027-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "765f2b2e020262d9c17a8a9a20ffd711f14ea3014b222f26d3ec1aaa11959aa2"
},
"model-00028-of-00032.safetensors": {
"bytes": 4998688432,
"sha256": "c811b837b63d0cb79116f76a856dca1de6e3915db5df65ca62388c4484903a0d"
},
"model-00029-of-00032.safetensors": {
"bytes": 4998052440,
"sha256": "42c9771487382501d2aa2a5942fbb15062bf18bdbade5aed7b8b4ca180d7e1ff"
},
"model-00030-of-00032.safetensors": {
"bytes": 4998687904,
"sha256": "d6b1e31e529b6472a2da690e879ed9ae70719030b4932e560b1a952aae3748e5"
},
"model-00031-of-00032.safetensors": {
"bytes": 4998052896,
"sha256": "9a7c95968efd33a0bc6c15210a8df021f342ebd4bf3dd442975422a006238418"
},
"model-00032-of-00032.safetensors": {
"bytes": 1278688952,
"sha256": "8aa64fa468f49b55e5fd80da2d6830a01635a1ef53fd5ae0e573a1a6c657d2b3"
}
},
"source_repository": "Aleph-Alpha/Kolibri-1-BF16",
"source_revision": "7a8f290e7858825c3cf5e4c447ba68345de9f1d3",
"source_tensor_count": 58353,
"tensor_count": 1903
},
"scoring_receipt": {
"calibrationCaseIDs": [],
"cases": [
{
"elapsedSeconds": 15.263413786888123,
"logitsFile": "cal-en.safetensors",
"logitsSHA256": "a915bd7bae6f27ff5725f4527fe9ff0a9324c8d02e3e0576f4d39d29e44afc32",
"meanNegativeLogLikelihood": 0.6718395948410034,
"sequence": {
"id": "cal-en",
"language": "en",
"scoreStart": 57,
"split": "calibration",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
17033,
17407,
1776,
1519,
32,
56,
29522,
46,
25241,
29522,
568,
7880,
46,
742,
900,
1750,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
49,
57,
29522,
1750,
58,
32,
51,
18092,
32,
56,
32099,
32,
53,
576,
32,
49,
57,
46,
127906
]
},
"tokenCount": 19
},
{
"elapsedSeconds": 4.814086675643921,
"logitsFile": "cal-de.safetensors",
"logitsSHA256": "5c0ff2bc79561dcdc78e3cef9f7954bff0ccb0ca91149e46033fe150e2db3b0e",
"meanNegativeLogLikelihood": 0.6605978012084961,
"sequence": {
"id": "cal-de",
"language": "de",
"scoreStart": 61,
"split": "calibration",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1113,
5958,
32206,
32,
57,
48,
15592,
622,
32,
49,
44,
53,
6838,
46,
2447,
4023,
2459,
2796,
89902,
15705,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
89902,
15705,
12165,
32,
54,
48,
14202,
31351,
58,
32,
57,
48,
49722,
32,
49,
44,
53,
576,
32,
54,
48,
46,
127906
]
},
"tokenCount": 24
},
{
"elapsedSeconds": 3.929967522621155,
"logitsFile": "cal-code.safetensors",
"logitsSHA256": "92d23ef65c2c7f3c3ac1e2f8c93e0021f9a98c5e844045a9db9ac3f0385f095f",
"meanNegativeLogLikelihood": 0.18414612114429474,
"sequence": {
"id": "cal-code",
"language": "en",
"scoreStart": 53,
"split": "calibration",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
7794,
2931,
41,
327,
26550,
14169,
26455,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
7794,
2931,
2488,
14141,
1778,
14169,
1921,
14169,
127906
]
},
"tokenCount": 10
},
{
"elapsedSeconds": 3.6074379682540894,
"logitsFile": "held-en-fact.safetensors",
"logitsSHA256": "b051919b94ba64a12e7ede482a89373aaf4efaa839ac8075d78020dac6c6c484",
"meanNegativeLogLikelihood": 0.03158802539110184,
"sequence": {
"id": "held-en-fact",
"language": "en",
"scoreStart": 50,
"split": "heldout",
"task": "knowledge",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1798,
735,
262,
13134,
9996,
379,
14272,
63,
1741,
622,
2403,
14264,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
13134,
9996,
379,
14272,
735,
2859,
46,
127906
]
},
"tokenCount": 9
},
{
"elapsedSeconds": 3.0807080268859863,
"logitsFile": "held-de-fact.safetensors",
"logitsSHA256": "f257e634d736cfe0ace91fad49e4da6498d717f959b5ac81a2545ec4a7678191",
"meanNegativeLogLikelihood": 0.08544593304395676,
"sequence": {
"id": "held-de-fact",
"language": "de",
"scoreStart": 48,
"split": "heldout",
"task": "knowledge",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
61701,
44187,
14107,
968,
16395,
63,
97011,
622,
1097,
16952,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
5396,
44187,
14107,
614,
16395,
2459,
3294,
46,
127906
]
},
"tokenCount": 9
},
{
"elapsedSeconds": 4.291423201560974,
"logitsFile": "held-en-arithmetic.safetensors",
"logitsSHA256": "9e568f9bf67e37a95512e18768ed7c2bcd11f39daf15da4e3cd5a60b91dfafb3",
"meanNegativeLogLikelihood": 1.059257984161377,
"sequence": {
"id": "held-en-arithmetic",
"language": "en",
"scoreStart": 63,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
65,
4777,
63655,
32,
49,
55,
84038,
3660,
1688,
54,
1776,
286,
94448,
1688,
49,
50,
46,
611,
735,
3327,
15482,
7086,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
15482,
7086,
735,
1688,
57,
48,
58,
32,
49,
55,
18092,
1688,
54,
32099,
1688,
49,
50,
576,
1688,
57,
48,
46,
127906
]
},
"tokenCount": 24
},
{
"elapsedSeconds": 4.270632743835449,
"logitsFile": "held-de-arithmetic.safetensors",
"logitsSHA256": "c148216e0ddb47602a270554096f8eea36ef70309607763686ed5b983af7a48e",
"meanNegativeLogLikelihood": 0.5934154987335205,
"sequence": {
"id": "held-de-arithmetic",
"language": "de",
"scoreStart": 68,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
11717,
75971,
16611,
32,
50,
52,
10343,
775,
614,
3408,
32,
52,
2527,
420,
2941,
32,
56,
2527,
2340,
46,
2447,
4023,
2459,
948,
42842,
268,
8776,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
8590,
42842,
268,
8776,
12165,
32,
56,
56,
2527,
58,
32,
50,
52,
18092,
32,
52,
32099,
32,
56,
576,
32,
56,
56,
46,
127906
]
},
"tokenCount": 25
},
{
"elapsedSeconds": 5.073943376541138,
"logitsFile": "held-en-logic.safetensors",
"logitsSHA256": "bebefce57310198e2d12120df01884ed4424fa869980ea78a8512c00125c6e24",
"meanNegativeLogLikelihood": 1.2498408555984497,
"sequence": {
"id": "held-en-logic",
"language": "en",
"scoreStart": 59,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4419,
35311,
568,
26933,
46,
3463,
26933,
476,
23134,
46,
7302,
3326,
3480,
327,
938,
35311,
476,
23134,
63,
20274,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
3913,
46,
543,
26933,
327,
476,
23134,
1026,
502,
2396,
1578,
35311,
44,
1409,
262,
5082,
830,
502,
3480,
46,
127906
]
},
"tokenCount": 21
},
{
"elapsedSeconds": 3.815176486968994,
"logitsFile": "held-de-logic.safetensors",
"logitsSHA256": "771176ba10bdc2004991dee8e418ea762635ada4dd38c32f1026d03269babb43",
"meanNegativeLogLikelihood": 1.8250385522842407,
"sequence": {
"id": "held-de-logic",
"language": "de",
"scoreStart": 63,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
13518,
25525,
1200,
10273,
46,
44871,
10273,
1200,
13752,
329,
46,
8470,
5871,
25412,
44,
1469,
46789,
25525,
13752,
329,
1200,
63,
840,
7166,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1170,
46,
1047,
13752,
1021,
10273,
4165,
3514,
25525,
2260,
59,
346,
90837,
11637,
14276,
939,
46,
127906
]
},
"tokenCount": 18
},
{
"elapsedSeconds": 3.9693586826324463,
"logitsFile": "held-en-code.safetensors",
"logitsSHA256": "3cef87d6c7a12ef990d84e857210e8c5200c257ed590b12a045bb1831ed89f46",
"meanNegativeLogLikelihood": 0.07828141003847122,
"sequence": {
"id": "held-en-code",
"language": "en",
"scoreStart": 56,
"split": "heldout",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
735,
105480,
14599,
41,
327,
26550,
2544,
14163,
735,
2051,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
735,
105480,
14599,
2488,
14141,
1778,
14163,
14628,
32,
50,
1339,
32,
48,
127906
]
},
"tokenCount": 15
},
{
"elapsedSeconds": 2.4145448207855225,
"logitsFile": "held-de-code.safetensors",
"logitsSHA256": "e475c0cf81e158f669de9554475c4cc06e9350e04094a93c41b654a92278f710",
"meanNegativeLogLikelihood": 0.1638011932373047,
"sequence": {
"id": "held-de-code",
"language": "de",
"scoreStart": 64,
"split": "heldout",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
22242,
699,
6797,
15836,
1427,
84780,
106925,
125942,
3003,
346,
518,
9678,
7307,
1117,
1233,
1151,
63064,
18996,
4330,
2340,
1673,
46,
32638,
2120,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
1427,
84780,
106925,
125942,
2488,
14141,
1778,
9468,
91,
48,
93,
1765,
9468,
4100,
4330,
127906
]
},
"tokenCount": 17
},
{
"elapsedSeconds": 4.184002757072449,
"logitsFile": "held-en-translate.safetensors",
"logitsSHA256": "c2c4a5084713fe1a30d4832229c5652c951c0a1b36b16fbba23e1918f016f069",
"meanNegativeLogLikelihood": 0.19104181230068207,
"sequence": {
"id": "held-en-translate",
"language": "en",
"scoreStart": 51,
"split": "heldout",
"task": "translation",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
44761,
318,
4867,
58,
543,
14345,
52074,
3660,
14395,
286,
62712,
3660,
8445,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
30908,
56277,
7470,
24677,
5269,
420,
61556,
7470,
16363,
5269,
46,
127906
]
},
"tokenCount": 13
},
{
"elapsedSeconds": 4.078127980232239,
"logitsFile": "held-de-translate.safetensors",
"logitsSHA256": "ea66cb5288bdec956bd1a9ddb022c955bd59973a2acf873e1a401292962a6a53",
"meanNegativeLogLikelihood": 0.08230161666870117,
"sequence": {
"id": "held-de-translate",
"language": "de",
"scoreStart": 50,
"split": "heldout",
"task": "translation",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1103,
57190,
120025,
98780,
58,
1047,
36235,
1030,
2361,
38749,
4118,
48837,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
8709,
378,
4191,
941,
5951,
1681,
11123,
46,
127906
]
},
"tokenCount": 10
},
{
"elapsedSeconds": 83.12866699695587,
"logitsFile": "held-en-window.safetensors",
"logitsSHA256": "688f3bffa06ce28d66fed14a1e8e928e4beb0df71c95d9a215d6c76c140657d9",
"meanNegativeLogLikelihood": 0.0010858264286071062,
"sequence": {
"id": "held-en-window",
"language": "en",
"scoreStart": 1327,
"split": "heldout",
"task": "window-crossing",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
5800,
1862,
12232,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
1798,
735,
262,
3973,
13709,
622,
11237,
32,
55,
63,
6442,
1444,
262,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
]
},
"tokenCount": 7
},
{
"elapsedSeconds": 33.63522005081177,
"logitsFile": "held-de-window.safetensors",
"logitsSHA256": "4e3d58e199452b92d5a9ac142a1418c8ff33e0da12b9b3474e74dc59b6842b26",
"meanNegativeLogLikelihood": 0.001338141388259828,
"sequence": {
"id": "held-de-window",
"language": "de",
"scoreStart": 1328,
"split": "heldout",
"task": "window-crossing",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
76,
555,
2834,
94446,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
86170,
34360,
855,
12975,
622,
11237,
32,
55,
63,
32638,
2120,
1178,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
]
},
"tokenCount": 7
},
{
"elapsedSeconds": 7.10475492477417,
"logitsFile": "held-en-tool.safetensors",
"logitsSHA256": "9615469f3c425898c47af696ef0d0ace71c0a147a13cb3265c1838c3d675bb25",
"meanNegativeLogLikelihood": 0.0049936044961214066,
"sequence": {
"id": "held-en-tool",
"language": "en",
"scoreStart": 183,
"split": "heldout",
"task": "tools",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
275,
35,
21955,
263,
652,
824,
5604,
2403,
3688,
828,
10852,
318,
9095,
310,
262,
3143,
15483,
275,
652,
568,
2349,
310,
1889,
53814,
2288,
3167,
32776,
1609,
32776,
62,
38357,
24727,
656,
60,
32776,
434,
1142,
676,
408,
4138,
8322,
408,
946,
2938,
262,
5995,
379,
941,
6547,
120260,
946,
653,
408,
946,
688,
95,
30279,
531,
946,
8785,
408,
4138,
15637,
408,
4138,
3547,
408,
4138,
805,
408,
946,
1383,
16506,
946,
22267,
408,
7311,
3547,
24148,
946,
805,
408,
946,
1300,
16506,
946,
805,
408,
946,
676,
100048,
376,
32776,
9215,
2674,
1776,
1889,
5604,
44,
1778,
941,
12597,
7438,
310,
1889,
5032,
286,
15477,
2288,
32,
127909,
127910,
38357,
24727,
656,
127909,
10,
1142,
653,
408,
3167,
676,
62679,
120073,
946,
7160,
408,
3167,
8510,
45,
6427,
123291,
62,
475,
127910,
127906,
10,
127904,
1646,
10,
5116,
262,
5995,
4324,
318,
2492,
2839,
262,
5995,
622,
67041,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
127909,
10,
1142,
653,
408,
946,
688,
95,
30279,
531,
946,
7160,
408,
4138,
3547,
408,
946,
50855,
67036,
34,
31257,
127910,
127906
]
},
"tokenCount": 23
}
],
"checkpointPath": "/workspace/kolibri/artifacts/Kolibri-1-MLX-mixed2",
"chunkSize": 32,
"configSHA256": "cd3d15e8f44b3dc24937f9f9730d4eb7d621759f28cc7ce5f33b633a37aa6d31",
"conversionSHA256": "ec78555322e1267973be8a64d5302d476eacf4021c7124af8fa94c8ceb1735cd",
"device": "Device(gpu, 0)",
"indexSHA256": "036de221d5da648136f71b4b5d078004193101e2ab16d3764854ad66d0628da9",
"peakMemoryBytes": 26570516768,
"schemaVersion": 1,
"suiteSHA256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b"
}
},
"mixed2-down3": {
"conversion_sha256": "5a540860411e5b9afe2d57a5add44d0614d55a2697cd9e586a46d4867b30bd39",
"conversion": {
"architecture_revision": "049a6a7bd2405b27d6d280d256bd3d585191c7ae",
"logical_weight_bytes": 28595307520,
"native_expert_order": "numeric",
"output_files": {
"global.safetensors": {
"bytes": 1310725428,
"sha256": "20bdf1aafd0ad0307547d1aa5ae4b785fa81ae942d42de14180eb9715cc28aed"
},
"layer-000.safetensors": {
"bytes": 545696093,
"sha256": "c10c2f935b2dc8e7baec5b3147f13e34062e72f2802389bf520dd7f3e0fb84fd"
},
"layer-001.safetensors": {
"bytes": 545696091,
"sha256": "ef9f7da42b1966452d66f6833c4776686e39784150e6277a76c9547b48ff9bf7"
},
"layer-002.safetensors": {
"bytes": 545696081,
"sha256": "615496ac8f3a5c0424fc97d15f5906e3c76f205ca53f7bc0e196978456696bc6"
},
"layer-003.safetensors": {
"bytes": 545696095,
"sha256": "34b86365c3287a3acf0699af929dba068cbd9f455a0e07103f89c8066f563065"
},
"layer-004.safetensors": {
"bytes": 545696095,
"sha256": "1f7fce0acef0b5717f400bbecf43001222bbb625a935606622752875aa19d485"
},
"layer-005.safetensors": {
"bytes": 545696093,
"sha256": "b765e702edac7cb2ad6bf1bb744eb4c56a5e2bedbce5c77822c23d98ff64e6e3"
},
"layer-006.safetensors": {
"bytes": 545696095,
"sha256": "34f3528fcd67bfc76a99919f81b83afe811be65e7ed040d9cad590949e0ae787"
},
"layer-007.safetensors": {
"bytes": 545696095,
"sha256": "20e7b11e9c90896ec9e76d33b4e8bd0ebc26d6437cc6f098b922e52789c9912d"
},
"layer-008.safetensors": {
"bytes": 545696091,
"sha256": "2db2322c1172f881e7019c98011581cec56b912891cb70be33c5cc61b16cb1ba"
},
"layer-009.safetensors": {
"bytes": 545696095,
"sha256": "97af0e76ac00e36a9b4357035105d88e30d46d5e0ef18235a7d380724bafa8c7"
},
"layer-010.safetensors": {
"bytes": 545696127,
"sha256": "3ab19eed33ec258ea38e186921048afee5ddb93b0ed1c7def33735177482e27a"
},
"layer-011.safetensors": {
"bytes": 545696131,
"sha256": "4813a1a4280d63510ecdfef225e01df47b19785f402890af6b3a82fe8fa8027a"
},
"layer-012.safetensors": {
"bytes": 545696129,
"sha256": "cc872691b892e25501bcce8c8375cd814f9f7b064aa9d6f21f070829ce07fbf9"
},
"layer-013.safetensors": {
"bytes": 545696131,
"sha256": "595c9f82fbf8c51a3eb245badbba0db57cc821a6147df6e1951257dad572cb2f"
},
"layer-014.safetensors": {
"bytes": 545696135,
"sha256": "973a25cab1b665fe3af791e38790a7e388ca10d4fc09299f34021913a47da571"
},
"layer-015.safetensors": {
"bytes": 545696133,
"sha256": "c291d6e077316ba7cc23d20ad17b4453b6962f8a2f5135d8bac32f5804e2bc78"
},
"layer-016.safetensors": {
"bytes": 545696131,
"sha256": "6bc1c1e60a9129ce5ed62b101f4981dc9e9a4b0af23e86e365fe9adce2e53136"
},
"layer-017.safetensors": {
"bytes": 545696125,
"sha256": "cc804ecba32b0f065ae93d1053b98c82bede05667df5e5a24a6f69ab4a800785"
},
"layer-018.safetensors": {
"bytes": 545696131,
"sha256": "b0c5afd1dc4b1601efcb395591c3f026a52367a20c9096565339a7cd8d0e6635"
},
"layer-019.safetensors": {
"bytes": 545696133,
"sha256": "6c74d5986b5444001fb4c623e6ba4514319a313a563756412408c3bfe67e8d79"
},
"layer-020.safetensors": {
"bytes": 545696131,
"sha256": "d62e1c5b1214df5284660df2d677c7b5d3736fdc6a06d6e2cfe2c9df49929972"
},
"layer-021.safetensors": {
"bytes": 545696133,
"sha256": "3c257dced1eb187164b95d90820af30a12463af06ec7a4c5c5f1928cd7dbdf72"
},
"layer-022.safetensors": {
"bytes": 545696129,
"sha256": "7f20934cf05334e4a35dfc721ef564f87cbd09b54f67f4e63882c4ef4a35f979"
},
"layer-023.safetensors": {
"bytes": 545696129,
"sha256": "a28557f37c71e62d64b60a7ab8204e538fe2388a995279c40dc784b75b0f558d"
},
"layer-024.safetensors": {
"bytes": 545696133,
"sha256": "aef6f8c30ab9732a9b0c09c0bda34f785cdffa34c1ce386dd0103d17ae1ff655"
},
"layer-025.safetensors": {
"bytes": 545696135,
"sha256": "fcf5e4f17a97d31245afd340d497a9cdc2a1da8e0d7fbd2db6438f2a1f96e4a4"
},
"layer-026.safetensors": {
"bytes": 545696127,
"sha256": "78b2a6e20614004465af629af4193d1d41f154af3be5f11f9a65201301578351"
},
"layer-027.safetensors": {
"bytes": 545696133,
"sha256": "3e0bd3e604735c8c85c4b760358b50dd86e2fca722f265794d9dc3b5d0d6a525"
},
"layer-028.safetensors": {
"bytes": 545696131,
"sha256": "6c465dffe4fa179c125480ea2d392012ec1b9b3f0feb914047bd80059fb869cf"
},
"layer-029.safetensors": {
"bytes": 545696131,
"sha256": "5963924f93778db1e781794f94cc9214e9c724711cf0f80dc3e0fd2115ded809"
},
"layer-030.safetensors": {
"bytes": 545696133,
"sha256": "3fcf92ea34c9dd0d6e524fee1d6eecb0f62e53e7b09348b5421df0b75f7510b2"
},
"layer-031.safetensors": {
"bytes": 545696133,
"sha256": "1127a549d521c40b228bd56b090bba9d67a8669ea9d5e944ad504819f6e53dd7"
},
"layer-032.safetensors": {
"bytes": 545696135,
"sha256": "d9a41abb12f70b78b3cb88edba82e79ea8158f9d0416a74579991208b8ff2912"
},
"layer-033.safetensors": {
"bytes": 545696135,
"sha256": "a2ecc3361da91fb3edcf682aa58acac2fc8c12ecffbc10529a5e7fb67e294f1d"
},
"layer-034.safetensors": {
"bytes": 545696133,
"sha256": "f2baf3fafa985c36b051731882621234a1fe8517fa5b8198faa7d2df4d250127"
},
"layer-035.safetensors": {
"bytes": 545696133,
"sha256": "d707dbc6c8438cf56edb6e029ab18a57ddebadc22c886f5857b1781d601c339a"
},
"layer-036.safetensors": {
"bytes": 545696129,
"sha256": "1ab720cf74993c1b748137e91d8f4da100583a0c3bd4ec93e5d594577224523a"
},
"layer-037.safetensors": {
"bytes": 545696133,
"sha256": "359c2d2cb54dad7c70f77ecf94563e541a13cf1d158899c88dc49c8f8dbd75ca"
},
"layer-038.safetensors": {
"bytes": 545696135,
"sha256": "dceffe948c52b00858d1078a00bea6d06713d46ed878f85ac09fa754c6204611"
},
"layer-039.safetensors": {
"bytes": 545696133,
"sha256": "5d05a36879b548f034f79058286b8e8cd79d85c848074e896ff75a7ee263e520"
},
"layer-040.safetensors": {
"bytes": 545696135,
"sha256": "889fe26fa92971be1cf1f12ae10bb8bf045a484d96b85d3abaa13d16caafe336"
},
"layer-041.safetensors": {
"bytes": 545696107,
"sha256": "281768bfecdcf9df4fa09be5bd843b8cd7a2e2fd84d54b170175321e479da0d0"
},
"layer-042.safetensors": {
"bytes": 545696133,
"sha256": "9709ce18c5843d55a849ec224f3c363889a30178f4f211453fe1c746b1fc7f7a"
},
"layer-043.safetensors": {
"bytes": 545696131,
"sha256": "d1b413cd9bf34644cbc82e8a6e516415886911533e1857a740484ecad48f5502"
},
"layer-044.safetensors": {
"bytes": 545696133,
"sha256": "da85b9bbf71ce3221103d58e135fe869a895022a2d25fcc10be13671ad6e881a"
},
"layer-045.safetensors": {
"bytes": 545696129,
"sha256": "324eb92525abd6342ea502ea4837211467b13db8cfff612aec88edd91bcd24ec"
},
"layer-046.safetensors": {
"bytes": 545696131,
"sha256": "89f51a9fabd70dc57d4b441c304d2d47a6e760a644ddebd5686a8124037da011"
},
"layer-047.safetensors": {
"bytes": 545696127,
"sha256": "2339b6e531de36ea3f605fb603ae63eb2555149df84579535c834fc4ef4dc904"
},
"layer-048.safetensors": {
"bytes": 545696131,
"sha256": "37dc2772004d588955ae3f77e3160eeb315fa1417936dbc4053defbedaebf952"
},
"layer-049.safetensors": {
"bytes": 545696133,
"sha256": "4b07e59df6c1bb4703dcd762128248a29808471478cb2d0747b76a8868cd4ead"
}
},
"profile": "mixed2-down3",
"quality_qualified": false,
"schema_version": 1,
"source_files": {
"model-00001-of-00032.safetensors": {
"bytes": 4998628528,
"sha256": "e626660074fd9ed3e3644f1d62f23be6adad9437e876f5f9b18260d8382d260b"
},
"model-00002-of-00032.safetensors": {
"bytes": 4998686392,
"sha256": "77259681f2b864d0c0634225d5b31cc15befa1405d062dd2089b41efeb5e6a8b"
},
"model-00003-of-00032.safetensors": {
"bytes": 4998050640,
"sha256": "ba03f0cbe6e48bed0140e5c52fc540cc44f4854887550db5ff6fac91a866142a"
},
"model-00004-of-00032.safetensors": {
"bytes": 4998050400,
"sha256": "68cc07ab7a3614104ab0ee8ce186737638eac3e8f32eb8d2c56742a0437d1680"
},
"model-00005-of-00032.safetensors": {
"bytes": 4998686720,
"sha256": "80ab29ca3d0d917cd83da0205d05b976d8e06100e2ca95ed66175e316550e3ae"
},
"model-00006-of-00032.safetensors": {
"bytes": 4998050320,
"sha256": "24e268797a3e13be638f9edf1f55c872346cf506786322afc6582e4cfa8c4373"
},
"model-00007-of-00032.safetensors": {
"bytes": 4998687256,
"sha256": "f3c8fcdf3bbc5b3af5c12bf8395d091b14c29957afa061f48c4c2685b5b8c8db"
},
"model-00008-of-00032.safetensors": {
"bytes": 4998052664,
"sha256": "648bbb1d3637a96df3f75cb6865846322a4af13bf38fd7817c3e40077d651cc7"
},
"model-00009-of-00032.safetensors": {
"bytes": 4998052208,
"sha256": "481958b40e88a712ea468da50b8f501bf2fe329330881fc3b1b1c3be1ec90689"
},
"model-00010-of-00032.safetensors": {
"bytes": 4998688544,
"sha256": "55d415d65584784fc523a26c9b328e56aec0ddea9fe42a9b48988bc800e67ae1"
},
"model-00011-of-00032.safetensors": {
"bytes": 4998052336,
"sha256": "1d36e5b599317d08d0232bb5e3bdb79f53bd6008294888a66b54d202c5263fea"
},
"model-00012-of-00032.safetensors": {
"bytes": 4998687968,
"sha256": "02be77ce110c6afc379c6ef23fad928205cddf26ddea8155aa274bed1cfd0efe"
},
"model-00013-of-00032.safetensors": {
"bytes": 4998052816,
"sha256": "94c607a19b8254ad78648d91d8808b2ffee5733e27739683225175e050c465c3"
},
"model-00014-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "021de7d92c76f23e31cfaf7f0053e1ce9359efeb93fd90728e658cceb9fe2b2a"
},
"model-00015-of-00032.safetensors": {
"bytes": 4998688344,
"sha256": "1383a4d989beac327c464bd3dd0a8008a199fca3d8c93d2e05dacc76d1f49163"
},
"model-00016-of-00032.safetensors": {
"bytes": 4998052488,
"sha256": "caf525ce7d499e4fbb7e3c6388346a1e8d602baa35f1946387aabb0143b7b696"
},
"model-00017-of-00032.safetensors": {
"bytes": 4984946504,
"sha256": "23af12d4220b2ebd42fcc030c5f98651adb73e96f0819e7dfbfed9454f98c2bd"
},
"model-00018-of-00032.safetensors": {
"bytes": 4998686528,
"sha256": "84e7b32097760e1fbe4380d424ab2c723d101af2283f3aea956a7f8dd30625da"
},
"model-00019-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "a95e33a7082c63f590d5f1893425f17bb87adf8814039dbf76adbfd64fd0e639"
},
"model-00020-of-00032.safetensors": {
"bytes": 4998688200,
"sha256": "55e35b6f1d394cd024d3f4819c63a66bbd5c135168823b02fdce2c736e79e330"
},
"model-00021-of-00032.safetensors": {
"bytes": 4998052592,
"sha256": "e2c07bd50fdec4fabdac18cf7a24db21561bccc33c73dd038a409db2e9632c8f"
},
"model-00022-of-00032.safetensors": {
"bytes": 4998052248,
"sha256": "41e87862dc6209d187842822e7f075763b9eb0dcc439b5be71d82e8162fb8765"
},
"model-00023-of-00032.safetensors": {
"bytes": 4998688616,
"sha256": "e0d663b7503a40dff70cf78b7497a06688efbf07d584f9cbb3b82cc70267c422"
},
"model-00024-of-00032.safetensors": {
"bytes": 4998052232,
"sha256": "8fae68d67883f561d205a9dde47af5815e35b472ff43452e9599cd26eeec2060"
},
"model-00025-of-00032.safetensors": {
"bytes": 4998688040,
"sha256": "0b46e002887b466bc2c9ae63709afaa1a2ad088498da76dd0919b70e5a6126b8"
},
"model-00026-of-00032.safetensors": {
"bytes": 4998052744,
"sha256": "b25cdfb7c0a656ac63b506310ff9db4e1383a76a9455722eb0938de0f697e998"
},
"model-00027-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "765f2b2e020262d9c17a8a9a20ffd711f14ea3014b222f26d3ec1aaa11959aa2"
},
"model-00028-of-00032.safetensors": {
"bytes": 4998688432,
"sha256": "c811b837b63d0cb79116f76a856dca1de6e3915db5df65ca62388c4484903a0d"
},
"model-00029-of-00032.safetensors": {
"bytes": 4998052440,
"sha256": "42c9771487382501d2aa2a5942fbb15062bf18bdbade5aed7b8b4ca180d7e1ff"
},
"model-00030-of-00032.safetensors": {
"bytes": 4998687904,
"sha256": "d6b1e31e529b6472a2da690e879ed9ae70719030b4932e560b1a952aae3748e5"
},
"model-00031-of-00032.safetensors": {
"bytes": 4998052896,
"sha256": "9a7c95968efd33a0bc6c15210a8df021f342ebd4bf3dd442975422a006238418"
},
"model-00032-of-00032.safetensors": {
"bytes": 1278688952,
"sha256": "8aa64fa468f49b55e5fd80da2d6830a01635a1ef53fd5ae0e573a1a6c657d2b3"
}
},
"source_repository": "Aleph-Alpha/Kolibri-1-BF16",
"source_revision": "7a8f290e7858825c3cf5e4c447ba68345de9f1d3",
"source_tensor_count": 58353,
"tensor_count": 1903
},
"scoring_receipt": {
"calibrationCaseIDs": [],
"cases": [
{
"elapsedSeconds": 16.09254252910614,
"logitsFile": "cal-en.safetensors",
"logitsSHA256": "beb21bd2d2c021a73ca781d704d4eeaa939af05c818dbdafc0d63a520b9521ab",
"meanNegativeLogLikelihood": 0.6593186855316162,
"sequence": {
"id": "cal-en",
"language": "en",
"scoreStart": 57,
"split": "calibration",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
17033,
17407,
1776,
1519,
32,
56,
29522,
46,
25241,
29522,
568,
7880,
46,
742,
900,
1750,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
49,
57,
29522,
1750,
58,
32,
51,
18092,
32,
56,
32099,
32,
53,
576,
32,
49,
57,
46,
127906
]
},
"tokenCount": 19
},
{
"elapsedSeconds": 4.987935900688171,
"logitsFile": "cal-de.safetensors",
"logitsSHA256": "fc33a651e387d2bb50ce8851c2ec40e4931338ae10d628be304958766f575c71",
"meanNegativeLogLikelihood": 0.6791297197341919,
"sequence": {
"id": "cal-de",
"language": "de",
"scoreStart": 61,
"split": "calibration",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1113,
5958,
32206,
32,
57,
48,
15592,
622,
32,
49,
44,
53,
6838,
46,
2447,
4023,
2459,
2796,
89902,
15705,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
89902,
15705,
12165,
32,
54,
48,
14202,
31351,
58,
32,
57,
48,
49722,
32,
49,
44,
53,
576,
32,
54,
48,
46,
127906
]
},
"tokenCount": 24
},
{
"elapsedSeconds": 3.962475061416626,
"logitsFile": "cal-code.safetensors",
"logitsSHA256": "224462e46bca2619b5016bb48bce212d34ca5afa78222d49b3ce4249df8134b0",
"meanNegativeLogLikelihood": 0.1894523650407791,
"sequence": {
"id": "cal-code",
"language": "en",
"scoreStart": 53,
"split": "calibration",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
7794,
2931,
41,
327,
26550,
14169,
26455,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
7794,
2931,
2488,
14141,
1778,
14169,
1921,
14169,
127906
]
},
"tokenCount": 10
},
{
"elapsedSeconds": 3.493422746658325,
"logitsFile": "held-en-fact.safetensors",
"logitsSHA256": "06a66e55aac6a8f698c31d427672f88db6e7df0396fc9544fd73e0f64c11012a",
"meanNegativeLogLikelihood": 0.0261400006711483,
"sequence": {
"id": "held-en-fact",
"language": "en",
"scoreStart": 50,
"split": "heldout",
"task": "knowledge",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1798,
735,
262,
13134,
9996,
379,
14272,
63,
1741,
622,
2403,
14264,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
13134,
9996,
379,
14272,
735,
2859,
46,
127906
]
},
"tokenCount": 9
},
{
"elapsedSeconds": 3.043730139732361,
"logitsFile": "held-de-fact.safetensors",
"logitsSHA256": "0c45f9ab4f4427b324ab0c32f62edfe784a4b41b5bdeb2a1aaff152b5a532a1b",
"meanNegativeLogLikelihood": 0.12533971667289734,
"sequence": {
"id": "held-de-fact",
"language": "de",
"scoreStart": 48,
"split": "heldout",
"task": "knowledge",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
61701,
44187,
14107,
968,
16395,
63,
97011,
622,
1097,
16952,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
5396,
44187,
14107,
614,
16395,
2459,
3294,
46,
127906
]
},
"tokenCount": 9
},
{
"elapsedSeconds": 4.496983170509338,
"logitsFile": "held-en-arithmetic.safetensors",
"logitsSHA256": "f255da3c1799a1d071fde300fbc08e3d910e9c8192818a34811ca2d109120b33",
"meanNegativeLogLikelihood": 1.070218801498413,
"sequence": {
"id": "held-en-arithmetic",
"language": "en",
"scoreStart": 63,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
65,
4777,
63655,
32,
49,
55,
84038,
3660,
1688,
54,
1776,
286,
94448,
1688,
49,
50,
46,
611,
735,
3327,
15482,
7086,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
15482,
7086,
735,
1688,
57,
48,
58,
32,
49,
55,
18092,
1688,
54,
32099,
1688,
49,
50,
576,
1688,
57,
48,
46,
127906
]
},
"tokenCount": 24
},
{
"elapsedSeconds": 4.025557637214661,
"logitsFile": "held-de-arithmetic.safetensors",
"logitsSHA256": "8c235a9b70de1708aa6f7982fa74b1cb39b8c51491bf4b42594d20130ae415b5",
"meanNegativeLogLikelihood": 0.5806379318237305,
"sequence": {
"id": "held-de-arithmetic",
"language": "de",
"scoreStart": 68,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
11717,
75971,
16611,
32,
50,
52,
10343,
775,
614,
3408,
32,
52,
2527,
420,
2941,
32,
56,
2527,
2340,
46,
2447,
4023,
2459,
948,
42842,
268,
8776,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
8590,
42842,
268,
8776,
12165,
32,
56,
56,
2527,
58,
32,
50,
52,
18092,
32,
52,
32099,
32,
56,
576,
32,
56,
56,
46,
127906
]
},
"tokenCount": 25
},
{
"elapsedSeconds": 4.995562672615051,
"logitsFile": "held-en-logic.safetensors",
"logitsSHA256": "e5ce95e40952166c4422feef28f0e83ec6c0f159c3a16b662e795994b0c246de",
"meanNegativeLogLikelihood": 1.1815303564071655,
"sequence": {
"id": "held-en-logic",
"language": "en",
"scoreStart": 59,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4419,
35311,
568,
26933,
46,
3463,
26933,
476,
23134,
46,
7302,
3326,
3480,
327,
938,
35311,
476,
23134,
63,
20274,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
3913,
46,
543,
26933,
327,
476,
23134,
1026,
502,
2396,
1578,
35311,
44,
1409,
262,
5082,
830,
502,
3480,
46,
127906
]
},
"tokenCount": 21
},
{
"elapsedSeconds": 3.7372764348983765,
"logitsFile": "held-de-logic.safetensors",
"logitsSHA256": "52112c7725a4880172550b96a8f0507a6f49f821416806a9b7fa48abad685b94",
"meanNegativeLogLikelihood": 1.6653589010238647,
"sequence": {
"id": "held-de-logic",
"language": "de",
"scoreStart": 63,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
13518,
25525,
1200,
10273,
46,
44871,
10273,
1200,
13752,
329,
46,
8470,
5871,
25412,
44,
1469,
46789,
25525,
13752,
329,
1200,
63,
840,
7166,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1170,
46,
1047,
13752,
1021,
10273,
4165,
3514,
25525,
2260,
59,
346,
90837,
11637,
14276,
939,
46,
127906
]
},
"tokenCount": 18
},
{
"elapsedSeconds": 3.8253928422927856,
"logitsFile": "held-en-code.safetensors",
"logitsSHA256": "bfa92544e9c1e6f581290dba0838b3649f91170d67734df3c53fef700d08a673",
"meanNegativeLogLikelihood": 0.07841593772172928,
"sequence": {
"id": "held-en-code",
"language": "en",
"scoreStart": 56,
"split": "heldout",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
735,
105480,
14599,
41,
327,
26550,
2544,
14163,
735,
2051,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
735,
105480,
14599,
2488,
14141,
1778,
14163,
14628,
32,
50,
1339,
32,
48,
127906
]
},
"tokenCount": 15
},
{
"elapsedSeconds": 2.5872596502304077,
"logitsFile": "held-de-code.safetensors",
"logitsSHA256": "cd871af25e4b5c7d71dc39caf63e472f52fe79eb7431f84486ef7f15cd917ecf",
"meanNegativeLogLikelihood": 0.16967223584651947,
"sequence": {
"id": "held-de-code",
"language": "de",
"scoreStart": 64,
"split": "heldout",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
22242,
699,
6797,
15836,
1427,
84780,
106925,
125942,
3003,
346,
518,
9678,
7307,
1117,
1233,
1151,
63064,
18996,
4330,
2340,
1673,
46,
32638,
2120,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
1427,
84780,
106925,
125942,
2488,
14141,
1778,
9468,
91,
48,
93,
1765,
9468,
4100,
4330,
127906
]
},
"tokenCount": 17
},
{
"elapsedSeconds": 3.8054038286209106,
"logitsFile": "held-en-translate.safetensors",
"logitsSHA256": "69e5e8ece226477c7d93441d1dce4a211851ca218ed262892baecc817d7d908d",
"meanNegativeLogLikelihood": 0.1986902803182602,
"sequence": {
"id": "held-en-translate",
"language": "en",
"scoreStart": 51,
"split": "heldout",
"task": "translation",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
44761,
318,
4867,
58,
543,
14345,
52074,
3660,
14395,
286,
62712,
3660,
8445,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
30908,
56277,
7470,
24677,
5269,
420,
61556,
7470,
16363,
5269,
46,
127906
]
},
"tokenCount": 13
},
{
"elapsedSeconds": 3.6340898275375366,
"logitsFile": "held-de-translate.safetensors",
"logitsSHA256": "a5c430c626f374ba895808452d67adb15c61e6946afada92e77ee12fd16c1aca",
"meanNegativeLogLikelihood": 0.0933631882071495,
"sequence": {
"id": "held-de-translate",
"language": "de",
"scoreStart": 50,
"split": "heldout",
"task": "translation",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1103,
57190,
120025,
98780,
58,
1047,
36235,
1030,
2361,
38749,
4118,
48837,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
8709,
378,
4191,
941,
5951,
1681,
11123,
46,
127906
]
},
"tokenCount": 10
},
{
"elapsedSeconds": 88.07801222801208,
"logitsFile": "held-en-window.safetensors",
"logitsSHA256": "5f6711ea6b82cefac18d4118017858841a5c3ffa92d40e1d54ac512a2572e622",
"meanNegativeLogLikelihood": 0.0008247920777648687,
"sequence": {
"id": "held-en-window",
"language": "en",
"scoreStart": 1327,
"split": "heldout",
"task": "window-crossing",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
5800,
1862,
12232,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
1798,
735,
262,
3973,
13709,
622,
11237,
32,
55,
63,
6442,
1444,
262,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
]
},
"tokenCount": 7
},
{
"elapsedSeconds": 38.96505916118622,
"logitsFile": "held-de-window.safetensors",
"logitsSHA256": "0decd7dbcac93d8090522942ea2815172c39a4e88e86b3e9534e6f3aca365d2a",
"meanNegativeLogLikelihood": 0.0007692064973525703,
"sequence": {
"id": "held-de-window",
"language": "de",
"scoreStart": 1328,
"split": "heldout",
"task": "window-crossing",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
76,
555,
2834,
94446,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
86170,
34360,
855,
12975,
622,
11237,
32,
55,
63,
32638,
2120,
1178,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
]
},
"tokenCount": 7
},
{
"elapsedSeconds": 7.147426247596741,
"logitsFile": "held-en-tool.safetensors",
"logitsSHA256": "dfe8deba5c5b03c137671526ac44f7a7b0e3ee104e0c13eb8387d5e137c641ac",
"meanNegativeLogLikelihood": 0.00022191587777342647,
"sequence": {
"id": "held-en-tool",
"language": "en",
"scoreStart": 183,
"split": "heldout",
"task": "tools",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
275,
35,
21955,
263,
652,
824,
5604,
2403,
3688,
828,
10852,
318,
9095,
310,
262,
3143,
15483,
275,
652,
568,
2349,
310,
1889,
53814,
2288,
3167,
32776,
1609,
32776,
62,
38357,
24727,
656,
60,
32776,
434,
1142,
676,
408,
4138,
8322,
408,
946,
2938,
262,
5995,
379,
941,
6547,
120260,
946,
653,
408,
946,
688,
95,
30279,
531,
946,
8785,
408,
4138,
15637,
408,
4138,
3547,
408,
4138,
805,
408,
946,
1383,
16506,
946,
22267,
408,
7311,
3547,
24148,
946,
805,
408,
946,
1300,
16506,
946,
805,
408,
946,
676,
100048,
376,
32776,
9215,
2674,
1776,
1889,
5604,
44,
1778,
941,
12597,
7438,
310,
1889,
5032,
286,
15477,
2288,
32,
127909,
127910,
38357,
24727,
656,
127909,
10,
1142,
653,
408,
3167,
676,
62679,
120073,
946,
7160,
408,
3167,
8510,
45,
6427,
123291,
62,
475,
127910,
127906,
10,
127904,
1646,
10,
5116,
262,
5995,
4324,
318,
2492,
2839,
262,
5995,
622,
67041,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
127909,
10,
1142,
653,
408,
946,
688,
95,
30279,
531,
946,
7160,
408,
4138,
3547,
408,
946,
50855,
67036,
34,
31257,
127910,
127906
]
},
"tokenCount": 23
}
],
"checkpointPath": "/workspace/kolibri/artifacts/Kolibri-1-MLX-mixed2-down3",
"chunkSize": 32,
"configSHA256": "50c585a969b4ab3b753f31484deca930f2d50ac722efe00e18a87fcd342e157f",
"conversionSHA256": "5a540860411e5b9afe2d57a5add44d0614d55a2697cd9e586a46d4867b30bd39",
"device": "Device(gpu, 0)",
"indexSHA256": "ead0095100a38a2db625f27c261c7f60cc008a9eb1576c5e82f1a54cb71f524a",
"peakMemoryBytes": 30521997600,
"schemaVersion": 1,
"suiteSHA256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b"
}
},
"mixed2-calibrated": {
"conversion_sha256": "39b678a298301b867dcbbf93ad529fe982fa647a99a37b4c49a9961f9a56ec33",
"conversion": {
"architecture_revision": "049a6a7bd2405b27d6d280d256bd3d585191c7ae",
"calibration_case_ids": [
"cal-en",
"cal-de",
"cal-code"
],
"calibration_sha256": "b4b18a667a326a9b6cf27f32c61c93c7009081226c403fc644a6f449bc3d3949",
"calibration_suite_sha256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b",
"fit": "diagonal input second moments; BF16 affine levels; eight Lloyd iterations; groupwise weighted-error acceptance",
"fitting": {
"model.layers.0.mlp.experts.down_proj": {
"improved_group_fraction": 0.9975138902664185,
"weighted_error_ratio": 0.7698951959609985
},
"model.layers.0.mlp.experts.gate_proj": {
"improved_group_fraction": 0.46531423926353455,
"weighted_error_ratio": 0.4639914929866791
},
"model.layers.0.mlp.experts.up_proj": {
"improved_group_fraction": 0.46540936827659607,
"weighted_error_ratio": 0.44145655632019043
},
"model.layers.1.mlp.experts.down_proj": {
"improved_group_fraction": 0.9977730512619019,
"weighted_error_ratio": 0.7438682913780212
},
"model.layers.1.mlp.experts.gate_proj": {
"improved_group_fraction": 0.4861798882484436,
"weighted_error_ratio": 0.30408596992492676
},
"model.layers.1.mlp.experts.up_proj": {
"improved_group_fraction": 0.4859863519668579,
"weighted_error_ratio": 0.3023447096347809
},
"model.layers.10.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996386766433716,
"weighted_error_ratio": 0.5361248850822449
},
"model.layers.10.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9969963431358337,
"weighted_error_ratio": 0.6691825985908508
},
"model.layers.10.mlp.experts.up_proj": {
"improved_group_fraction": 0.9970105886459351,
"weighted_error_ratio": 0.6702075600624084
},
"model.layers.11.mlp.experts.down_proj": {
"improved_group_fraction": 0.9995847344398499,
"weighted_error_ratio": 0.6346104145050049
},
"model.layers.11.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9994911551475525,
"weighted_error_ratio": 0.663370668888092
},
"model.layers.11.mlp.experts.up_proj": {
"improved_group_fraction": 0.9994911551475525,
"weighted_error_ratio": 0.6638196706771851
},
"model.layers.12.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996742606163025,
"weighted_error_ratio": 0.6409187912940979
},
"model.layers.12.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9988048076629639,
"weighted_error_ratio": 0.6625343561172485
},
"model.layers.12.mlp.experts.up_proj": {
"improved_group_fraction": 0.9988284111022949,
"weighted_error_ratio": 0.6629210710525513
},
"model.layers.13.mlp.experts.down_proj": {
"improved_group_fraction": 0.999610424041748,
"weighted_error_ratio": 0.603240430355072
},
"model.layers.13.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9975337386131287,
"weighted_error_ratio": 0.6669063568115234
},
"model.layers.13.mlp.experts.up_proj": {
"improved_group_fraction": 0.9975438714027405,
"weighted_error_ratio": 0.6676194071769714
},
"model.layers.14.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996902942657471,
"weighted_error_ratio": 0.597445547580719
},
"model.layers.14.mlp.experts.gate_proj": {
"improved_group_fraction": 0.999293327331543,
"weighted_error_ratio": 0.6576365828514099
},
"model.layers.14.mlp.experts.up_proj": {
"improved_group_fraction": 0.9992943406105042,
"weighted_error_ratio": 0.6581912040710449
},
"model.layers.15.mlp.experts.down_proj": {
"improved_group_fraction": 0.9995730519294739,
"weighted_error_ratio": 0.6087181568145752
},
"model.layers.15.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9974602460861206,
"weighted_error_ratio": 0.6594696640968323
},
"model.layers.15.mlp.experts.up_proj": {
"improved_group_fraction": 0.9975622296333313,
"weighted_error_ratio": 0.6601839065551758
},
"model.layers.16.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996928572654724,
"weighted_error_ratio": 0.6334097385406494
},
"model.layers.16.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998660087585449,
"weighted_error_ratio": 0.6622950434684753
},
"model.layers.16.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998756647109985,
"weighted_error_ratio": 0.663432776927948
},
"model.layers.17.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996440410614014,
"weighted_error_ratio": 0.594506561756134
},
"model.layers.17.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9991092085838318,
"weighted_error_ratio": 0.6636205911636353
},
"model.layers.17.mlp.experts.up_proj": {
"improved_group_fraction": 0.9991216659545898,
"weighted_error_ratio": 0.6642159819602966
},
"model.layers.18.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996330738067627,
"weighted_error_ratio": 0.5585047006607056
},
"model.layers.18.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9988256096839905,
"weighted_error_ratio": 0.6605659127235413
},
"model.layers.18.mlp.experts.up_proj": {
"improved_group_fraction": 0.9988548755645752,
"weighted_error_ratio": 0.660987377166748
},
"model.layers.19.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997925162315369,
"weighted_error_ratio": 0.65301114320755
},
"model.layers.19.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998571276664734,
"weighted_error_ratio": 0.661312460899353
},
"model.layers.19.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998670220375061,
"weighted_error_ratio": 0.6616130471229553
},
"model.layers.2.mlp.experts.down_proj": {
"improved_group_fraction": 0.9984421133995056,
"weighted_error_ratio": 0.38732653856277466
},
"model.layers.2.mlp.experts.gate_proj": {
"improved_group_fraction": 0.7343750596046448,
"weighted_error_ratio": 0.7475119829177856
},
"model.layers.2.mlp.experts.up_proj": {
"improved_group_fraction": 0.7349548935890198,
"weighted_error_ratio": 0.7472378611564636
},
"model.layers.20.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997902512550354,
"weighted_error_ratio": 0.6558893918991089
},
"model.layers.20.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9982829093933105,
"weighted_error_ratio": 0.661159336566925
},
"model.layers.20.mlp.experts.up_proj": {
"improved_group_fraction": 0.9982963800430298,
"weighted_error_ratio": 0.6611374020576477
},
"model.layers.21.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997808337211609,
"weighted_error_ratio": 0.6550851464271545
},
"model.layers.21.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998103380203247,
"weighted_error_ratio": 0.6597232222557068
},
"model.layers.21.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998133778572083,
"weighted_error_ratio": 0.659915566444397
},
"model.layers.22.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997648000717163,
"weighted_error_ratio": 0.5699103474617004
},
"model.layers.22.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9992724657058716,
"weighted_error_ratio": 0.6573889255523682
},
"model.layers.22.mlp.experts.up_proj": {
"improved_group_fraction": 0.9992765188217163,
"weighted_error_ratio": 0.6579917073249817
},
"model.layers.23.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996742606163025,
"weighted_error_ratio": 0.6332929730415344
},
"model.layers.23.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9990077018737793,
"weighted_error_ratio": 0.6563268303871155
},
"model.layers.23.mlp.experts.up_proj": {
"improved_group_fraction": 0.9990249872207642,
"weighted_error_ratio": 0.6570913195610046
},
"model.layers.24.mlp.experts.down_proj": {
"improved_group_fraction": 0.9998487234115601,
"weighted_error_ratio": 0.6581493616104126
},
"model.layers.24.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998334646224976,
"weighted_error_ratio": 0.6573989987373352
},
"model.layers.24.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998596906661987,
"weighted_error_ratio": 0.6578843593597412
},
"model.layers.25.mlp.experts.down_proj": {
"improved_group_fraction": 0.9998438954353333,
"weighted_error_ratio": 0.6579828858375549
},
"model.layers.25.mlp.experts.gate_proj": {
"improved_group_fraction": 0.99986732006073,
"weighted_error_ratio": 0.6533841490745544
},
"model.layers.25.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998749494552612,
"weighted_error_ratio": 0.6551067233085632
},
"model.layers.26.mlp.experts.down_proj": {
"improved_group_fraction": 0.9998868703842163,
"weighted_error_ratio": 0.6596474051475525
},
"model.layers.26.mlp.experts.gate_proj": {
"improved_group_fraction": 0.999878466129303,
"weighted_error_ratio": 0.6527830958366394
},
"model.layers.26.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998881816864014,
"weighted_error_ratio": 0.6545938849449158
},
"model.layers.27.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996699690818787,
"weighted_error_ratio": 0.614827573299408
},
"model.layers.27.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998677968978882,
"weighted_error_ratio": 0.6547960638999939
},
"model.layers.27.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998774528503418,
"weighted_error_ratio": 0.6566148400306702
},
"model.layers.28.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997012615203857,
"weighted_error_ratio": 0.6058957576751709
},
"model.layers.28.mlp.experts.gate_proj": {
"improved_group_fraction": 0.999756395816803,
"weighted_error_ratio": 0.6543420553207397
},
"model.layers.28.mlp.experts.up_proj": {
"improved_group_fraction": 0.9997714161872864,
"weighted_error_ratio": 0.6564900279045105
},
"model.layers.29.mlp.experts.down_proj": {
"improved_group_fraction": 0.9998090863227844,
"weighted_error_ratio": 0.6575601100921631
},
"model.layers.29.mlp.experts.gate_proj": {
"improved_group_fraction": 0.99986732006073,
"weighted_error_ratio": 0.653963029384613
},
"model.layers.29.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998744130134583,
"weighted_error_ratio": 0.6561363339424133
},
"model.layers.3.mlp.experts.down_proj": {
"improved_group_fraction": 0.9994321465492249,
"weighted_error_ratio": 0.36661526560783386
},
"model.layers.3.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9629674553871155,
"weighted_error_ratio": 0.7011687755584717
},
"model.layers.3.mlp.experts.up_proj": {
"improved_group_fraction": 0.9631834030151367,
"weighted_error_ratio": 0.7012413144111633
},
"model.layers.30.mlp.experts.down_proj": {
"improved_group_fraction": 0.9998652935028076,
"weighted_error_ratio": 0.6612080335617065
},
"model.layers.30.mlp.experts.gate_proj": {
"improved_group_fraction": 0.999878466129303,
"weighted_error_ratio": 0.6527523398399353
},
"model.layers.30.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998772144317627,
"weighted_error_ratio": 0.6547605395317078
},
"model.layers.31.mlp.experts.down_proj": {
"improved_group_fraction": 0.9998195171356201,
"weighted_error_ratio": 0.6018158793449402
},
"model.layers.31.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998822808265686,
"weighted_error_ratio": 0.6554486751556396
},
"model.layers.31.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998924732208252,
"weighted_error_ratio": 0.6573114395141602
},
"model.layers.32.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997462630271912,
"weighted_error_ratio": 0.6488978862762451
},
"model.layers.32.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998746514320374,
"weighted_error_ratio": 0.6547339558601379
},
"model.layers.32.mlp.experts.up_proj": {
"improved_group_fraction": 0.999879777431488,
"weighted_error_ratio": 0.6563697457313538
},
"model.layers.33.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997966289520264,
"weighted_error_ratio": 0.6586382389068604
},
"model.layers.33.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998680353164673,
"weighted_error_ratio": 0.6547629237174988
},
"model.layers.33.mlp.experts.up_proj": {
"improved_group_fraction": 0.999879777431488,
"weighted_error_ratio": 0.6563228964805603
},
"model.layers.34.mlp.experts.down_proj": {
"improved_group_fraction": 0.999756395816803,
"weighted_error_ratio": 0.6574217677116394
},
"model.layers.34.mlp.experts.gate_proj": {
"improved_group_fraction": 0.999873161315918,
"weighted_error_ratio": 0.6541325449943542
},
"model.layers.34.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998777508735657,
"weighted_error_ratio": 0.6559650301933289
},
"model.layers.35.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997320175170898,
"weighted_error_ratio": 0.5903333425521851
},
"model.layers.35.mlp.experts.gate_proj": {
"improved_group_fraction": 0.999872624874115,
"weighted_error_ratio": 0.6543296575546265
},
"model.layers.35.mlp.experts.up_proj": {
"improved_group_fraction": 0.999866783618927,
"weighted_error_ratio": 0.6558845043182373
},
"model.layers.36.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997419118881226,
"weighted_error_ratio": 0.6572784185409546
},
"model.layers.36.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998670220375061,
"weighted_error_ratio": 0.6543537378311157
},
"model.layers.36.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998685717582703,
"weighted_error_ratio": 0.6565313339233398
},
"model.layers.37.mlp.experts.down_proj": {
"improved_group_fraction": 0.9998136162757874,
"weighted_error_ratio": 0.6601080894470215
},
"model.layers.37.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998675584793091,
"weighted_error_ratio": 0.6532811522483826
},
"model.layers.37.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998701214790344,
"weighted_error_ratio": 0.6555066704750061
},
"model.layers.38.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996646046638489,
"weighted_error_ratio": 0.5803616642951965
},
"model.layers.38.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998568892478943,
"weighted_error_ratio": 0.6529878973960876
},
"model.layers.38.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998800158500671,
"weighted_error_ratio": 0.655458927154541
},
"model.layers.39.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997477531433105,
"weighted_error_ratio": 0.6283152103424072
},
"model.layers.39.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998670220375061,
"weighted_error_ratio": 0.6530816555023193
},
"model.layers.39.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998665452003479,
"weighted_error_ratio": 0.6555950045585632
},
"model.layers.4.mlp.experts.down_proj": {
"improved_group_fraction": 0.9995361566543579,
"weighted_error_ratio": 0.4446414113044739
},
"model.layers.4.mlp.experts.gate_proj": {
"improved_group_fraction": 0.962312638759613,
"weighted_error_ratio": 0.7124764919281006
},
"model.layers.4.mlp.experts.up_proj": {
"improved_group_fraction": 0.9625381827354431,
"weighted_error_ratio": 0.7117166519165039
},
"model.layers.40.mlp.experts.down_proj": {
"improved_group_fraction": 0.99980628490448,
"weighted_error_ratio": 0.6579837799072266
},
"model.layers.40.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998464584350586,
"weighted_error_ratio": 0.6530960202217102
},
"model.layers.40.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998357892036438,
"weighted_error_ratio": 0.6558077931404114
},
"model.layers.41.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997109174728394,
"weighted_error_ratio": 0.6286647319793701
},
"model.layers.41.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998436570167542,
"weighted_error_ratio": 0.6509476900100708
},
"model.layers.41.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998515248298645,
"weighted_error_ratio": 0.6541886925697327
},
"model.layers.42.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997900128364563,
"weighted_error_ratio": 0.6519435048103333
},
"model.layers.42.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998072981834412,
"weighted_error_ratio": 0.6504195928573608
},
"model.layers.42.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998215436935425,
"weighted_error_ratio": 0.6540864109992981
},
"model.layers.43.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996681809425354,
"weighted_error_ratio": 0.6415640115737915
},
"model.layers.43.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9997762441635132,
"weighted_error_ratio": 0.648736298084259
},
"model.layers.43.mlp.experts.up_proj": {
"improved_group_fraction": 0.9997983574867249,
"weighted_error_ratio": 0.6523913741111755
},
"model.layers.44.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997958540916443,
"weighted_error_ratio": 0.6546920537948608
},
"model.layers.44.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998065233230591,
"weighted_error_ratio": 0.6491928696632385
},
"model.layers.44.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998250603675842,
"weighted_error_ratio": 0.6534722447395325
},
"model.layers.45.mlp.experts.down_proj": {
"improved_group_fraction": 0.9998502731323242,
"weighted_error_ratio": 0.6582046747207642
},
"model.layers.45.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9997861981391907,
"weighted_error_ratio": 0.6447946429252625
},
"model.layers.45.mlp.experts.up_proj": {
"improved_group_fraction": 0.999818742275238,
"weighted_error_ratio": 0.6496479511260986
},
"model.layers.46.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997541308403015,
"weighted_error_ratio": 0.47684627771377563
},
"model.layers.46.mlp.experts.gate_proj": {
"improved_group_fraction": 0.999781608581543,
"weighted_error_ratio": 0.6401781439781189
},
"model.layers.46.mlp.experts.up_proj": {
"improved_group_fraction": 0.9997958540916443,
"weighted_error_ratio": 0.6448078751564026
},
"model.layers.47.mlp.experts.down_proj": {
"improved_group_fraction": 0.9997134208679199,
"weighted_error_ratio": 0.5086941719055176
},
"model.layers.47.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9997701644897461,
"weighted_error_ratio": 0.632804811000824
},
"model.layers.47.mlp.experts.up_proj": {
"improved_group_fraction": 0.9997836351394653,
"weighted_error_ratio": 0.6364538669586182
},
"model.layers.48.mlp.experts.down_proj": {
"improved_group_fraction": 0.999733030796051,
"weighted_error_ratio": 0.6196533441543579
},
"model.layers.48.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9997587203979492,
"weighted_error_ratio": 0.6246802806854248
},
"model.layers.48.mlp.experts.up_proj": {
"improved_group_fraction": 0.9997653365135193,
"weighted_error_ratio": 0.6259937286376953
},
"model.layers.49.mlp.experts.down_proj": {
"improved_group_fraction": 0.9998019337654114,
"weighted_error_ratio": 0.6527794599533081
},
"model.layers.49.mlp.experts.gate_proj": {
"improved_group_fraction": 0.999690055847168,
"weighted_error_ratio": 0.5713881254196167
},
"model.layers.49.mlp.experts.up_proj": {
"improved_group_fraction": 0.9997029900550842,
"weighted_error_ratio": 0.5698012113571167
},
"model.layers.5.mlp.experts.down_proj": {
"improved_group_fraction": 0.9995267987251282,
"weighted_error_ratio": 0.5512955188751221
},
"model.layers.5.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9890663623809814,
"weighted_error_ratio": 0.6902539134025574
},
"model.layers.5.mlp.experts.up_proj": {
"improved_group_fraction": 0.9893702864646912,
"weighted_error_ratio": 0.6904846429824829
},
"model.layers.6.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996076226234436,
"weighted_error_ratio": 0.5002883672714233
},
"model.layers.6.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9965111017227173,
"weighted_error_ratio": 0.7103538513183594
},
"model.layers.6.mlp.experts.up_proj": {
"improved_group_fraction": 0.9966515302658081,
"weighted_error_ratio": 0.7123474478721619
},
"model.layers.7.mlp.experts.down_proj": {
"improved_group_fraction": 0.9996328353881836,
"weighted_error_ratio": 0.6247726678848267
},
"model.layers.7.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9947327375411987,
"weighted_error_ratio": 0.6741361021995544
},
"model.layers.7.mlp.experts.up_proj": {
"improved_group_fraction": 0.9947502613067627,
"weighted_error_ratio": 0.6756290793418884
},
"model.layers.8.mlp.experts.down_proj": {
"improved_group_fraction": 0.9995837211608887,
"weighted_error_ratio": 0.6263865232467651
},
"model.layers.8.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9998378157615662,
"weighted_error_ratio": 0.6677034497261047
},
"model.layers.8.mlp.experts.up_proj": {
"improved_group_fraction": 0.9998370409011841,
"weighted_error_ratio": 0.669352114200592
},
"model.layers.9.mlp.experts.down_proj": {
"improved_group_fraction": 0.9995588064193726,
"weighted_error_ratio": 0.23276858031749725
},
"model.layers.9.mlp.experts.gate_proj": {
"improved_group_fraction": 0.9982033371925354,
"weighted_error_ratio": 0.6682053208351135
},
"model.layers.9.mlp.experts.up_proj": {
"improved_group_fraction": 0.9982224106788635,
"weighted_error_ratio": 0.6698547005653381
}
},
"logical_weight_bytes": 24663147520,
"native_expert_order": "numeric",
"output_files": {
"global.safetensors": {
"bytes": 1310725430,
"sha256": "01cbadf86e75907cda0e5e517ec7d610381f29e905d07601a4d0cb14326af44f"
},
"layer-000.safetensors": {
"bytes": 467052849,
"sha256": "e925a3b7af18cbc6967b9ca489d74df755122b8b15287f0c683124354fd05923"
},
"layer-001.safetensors": {
"bytes": 467052867,
"sha256": "49728742a29b5039be28c971e2a0caec440458b5b1776a773596f7c369cae135"
},
"layer-002.safetensors": {
"bytes": 467052835,
"sha256": "f82c16c2bba82db77413881062366356377b13533ab7241dcae94e7af72e8a3a"
},
"layer-003.safetensors": {
"bytes": 467052851,
"sha256": "61c9e3a6a41faf9604d0f91d8b39adfc9c71c4d1f4a331d332402f9b4ff7e100"
},
"layer-004.safetensors": {
"bytes": 467052843,
"sha256": "1dd7d7e9eb0d20d32cc1a498cce0e51f9105729444df64fa5e85ba3cfd7e2ccc"
},
"layer-005.safetensors": {
"bytes": 467052861,
"sha256": "57ef3b36fbd8697e22c3b1304faa08c91240a30ca216a416ea27aaefd70fce2b"
},
"layer-006.safetensors": {
"bytes": 467052869,
"sha256": "2d50a445a769d6e8300e02ec798c3350fc34e9ea11506f68441877194fd08f7d"
},
"layer-007.safetensors": {
"bytes": 467052891,
"sha256": "b26e44512cf2a63aa6ffc04df8deeedb449df576fc1679c78654ce38aac48f3c"
},
"layer-008.safetensors": {
"bytes": 467052845,
"sha256": "f7fdd2fdfd92995bd3c9c58abb76b079b6948cf7eb0727e4d11bb95f2e47169c"
},
"layer-009.safetensors": {
"bytes": 467052851,
"sha256": "53394e86c048320232953c14f60fb8108eba336d13f986dc0103a37a70865364"
},
"layer-010.safetensors": {
"bytes": 467052905,
"sha256": "172296121653573411579a8ce99d5804e3f04ea73f7611479820cf7c788d8656"
},
"layer-011.safetensors": {
"bytes": 467052865,
"sha256": "d225f87e63a397d72283495cf5ed71110ab5dce8a5e5af7738f72d0200763a26"
},
"layer-012.safetensors": {
"bytes": 467052863,
"sha256": "2ae7ce4c2f397166bae28f92641e5e29ccd958480078daba17b2af84a5511d81"
},
"layer-013.safetensors": {
"bytes": 467052893,
"sha256": "a5bf855183e52751eab0957e9726dfd77c71322526260ab9994bc9a6c2a221cb"
},
"layer-014.safetensors": {
"bytes": 467052851,
"sha256": "080f83dbe1b25bada230a914d2067b28a4968ff1e13c7f7b42a77c8f036b2ca9"
},
"layer-015.safetensors": {
"bytes": 467052899,
"sha256": "4c4e72f3a3edff79c8900293b8b8ac9012ada16dffe4f6b5c8d21d8704f18232"
},
"layer-016.safetensors": {
"bytes": 467052909,
"sha256": "9819169a1622b283c4a4234d6f804d51e0734b5189595ae8d0ff0e36b8113395"
},
"layer-017.safetensors": {
"bytes": 467052887,
"sha256": "21eeda8e9c83d21f2a532acd52f3fd85c2ff6998891e721d0856bf23234af053"
},
"layer-018.safetensors": {
"bytes": 467052905,
"sha256": "aed7b97282ed10cd08ddc90314c3cec7954179b0d449657cf341fb60bc6ad6e9"
},
"layer-019.safetensors": {
"bytes": 467052871,
"sha256": "18c92656852b8c7d2b3bca41f6c99bb693f6d5143bc2d2a6e13b65f8d35a9a47"
},
"layer-020.safetensors": {
"bytes": 467052931,
"sha256": "98bc38cd6cd9320eb2f46c2f0208d42b9f79ff8c94d0631c02ad6789db8de792"
},
"layer-021.safetensors": {
"bytes": 467052883,
"sha256": "fab36ac1da8064716ed8d46a602ba04f5008c1cb353e2e0e5c193b93da4f1f03"
},
"layer-022.safetensors": {
"bytes": 467052889,
"sha256": "f97fe3814d7ede568e421ad9db9bb51aaf209d4df3b3b447a5cefe811ac54e4b"
},
"layer-023.safetensors": {
"bytes": 467052883,
"sha256": "4fc255782acc1cf79775ffd3820dc145f06424423cc7dfdc782d15a0e190f2d7"
},
"layer-024.safetensors": {
"bytes": 467052911,
"sha256": "67b429910ada5f83581ea90ec18226f4c3dfcede169e05cdffeb4f0486b5af80"
},
"layer-025.safetensors": {
"bytes": 467052931,
"sha256": "f500f4a885ac7cd50749833573cbc4541c084aaf094bf5a5dac43990a93cfc03"
},
"layer-026.safetensors": {
"bytes": 467052919,
"sha256": "6f4af76a103c0660d4645442618408311185e59edcf2cb0e6fbb8217ebc80af5"
},
"layer-027.safetensors": {
"bytes": 467052887,
"sha256": "c48333fce653b7837399f57d73bfa91a766a6a4782ea50a0bf28d4daac5e7531"
},
"layer-028.safetensors": {
"bytes": 467052905,
"sha256": "5399e39014275ad8a29de514620db742d17cad29c2429601350ce843cccbf85d"
},
"layer-029.safetensors": {
"bytes": 467052877,
"sha256": "ef9b1506e60b0f83e5c459284707ff57e4e975b57e4b1db4e67edc0d1272e028"
},
"layer-030.safetensors": {
"bytes": 467052909,
"sha256": "b676d39a34c48eeb4c667d9c6ebb3436bd56764fbfd779329f24d164d15281a2"
},
"layer-031.safetensors": {
"bytes": 467052863,
"sha256": "3fc7c387c804604bc14790aaa2c009dd825d42ff2f8d0dc40f4951e4825bee84"
},
"layer-032.safetensors": {
"bytes": 467052875,
"sha256": "ff63586e55f7c49267da9a9330ae69397536525a03f30865938ed2e909d49e8e"
},
"layer-033.safetensors": {
"bytes": 467052889,
"sha256": "c859d390315f69b67eb3089dede8163074c6d341bb4ed5fbd5df5e644fa017be"
},
"layer-034.safetensors": {
"bytes": 467052907,
"sha256": "19eefdeb520fbd33ec9cb018078a31db89e23759ee583d4021cf1f94b8441f6c"
},
"layer-035.safetensors": {
"bytes": 467052899,
"sha256": "883f4fd0d07cbf0097d0e32d81b99a2bc29af43b90bb8151cf7218a2937b53fc"
},
"layer-036.safetensors": {
"bytes": 467052921,
"sha256": "2a0f939248bb758e51c850d5e3f662c8047fac4e23afced596154f693c543ecd"
},
"layer-037.safetensors": {
"bytes": 467052875,
"sha256": "e64bd677b00ba8ff5c90c207a7c2c0186e25ca11eef003e8e0d5e206f441dce3"
},
"layer-038.safetensors": {
"bytes": 467052879,
"sha256": "a964d760fca85381d62caf49825aa062f4accbfe76af8afb362f66decc1126da"
},
"layer-039.safetensors": {
"bytes": 467052909,
"sha256": "109b9f1ec5faab58d1333038d8f89cfbe70b88329f7a47666219deedc52ca547"
},
"layer-040.safetensors": {
"bytes": 467052893,
"sha256": "bb6a76a583a25913a23d32cb2d6d9cc306a3a5c3194e1c9f7e15cedf0e44b3cf"
},
"layer-041.safetensors": {
"bytes": 467052913,
"sha256": "b9263ef6cde2f353f86d4b51954643f6c1c687ba6a2bbf98301e46efb31e76d6"
},
"layer-042.safetensors": {
"bytes": 467052893,
"sha256": "89f429a464f5c2678973e4a1678f7f7dfb21245a4bc9f461627461cd2a888954"
},
"layer-043.safetensors": {
"bytes": 467052899,
"sha256": "19d3cb265371ab7fa75501ebf0021e1383011bb5a1940fdc9176f2bcae8110ed"
},
"layer-044.safetensors": {
"bytes": 467052895,
"sha256": "c3978c9f17b5693c1cd4161cbae92d93678de63383b49da94f49e2f9fc2c63e0"
},
"layer-045.safetensors": {
"bytes": 467052859,
"sha256": "bf4cbf83ab49043c6ecf90ccd5773697ffbe9f4d18f1421647b586c9253dbbcf"
},
"layer-046.safetensors": {
"bytes": 467052905,
"sha256": "b2daa1adf4199f4710cda103b60dfd17123532861f6839296fa2dc5fdebd1b71"
},
"layer-047.safetensors": {
"bytes": 467052893,
"sha256": "97b205fc44586ed813bc3993d9db345dcbd0f06b27973fb89583568189c8e03b"
},
"layer-048.safetensors": {
"bytes": 467052903,
"sha256": "8de4e474a610165c05ca3bb0ff7a67192970122ddd578adba1b1cab971bf8264"
},
"layer-049.safetensors": {
"bytes": 467052901,
"sha256": "453daeb12164e945f4924f2f9fa1e39050b1d144e1f13da8a5cdff7148e327e1"
}
},
"profile": "mixed2-calibrated",
"quality_qualified": false,
"schema_version": 1,
"source_files": {
"model-00001-of-00032.safetensors": {
"bytes": 4998628528,
"sha256": "e626660074fd9ed3e3644f1d62f23be6adad9437e876f5f9b18260d8382d260b"
},
"model-00002-of-00032.safetensors": {
"bytes": 4998686392,
"sha256": "77259681f2b864d0c0634225d5b31cc15befa1405d062dd2089b41efeb5e6a8b"
},
"model-00003-of-00032.safetensors": {
"bytes": 4998050640,
"sha256": "ba03f0cbe6e48bed0140e5c52fc540cc44f4854887550db5ff6fac91a866142a"
},
"model-00004-of-00032.safetensors": {
"bytes": 4998050400,
"sha256": "68cc07ab7a3614104ab0ee8ce186737638eac3e8f32eb8d2c56742a0437d1680"
},
"model-00005-of-00032.safetensors": {
"bytes": 4998686720,
"sha256": "80ab29ca3d0d917cd83da0205d05b976d8e06100e2ca95ed66175e316550e3ae"
},
"model-00006-of-00032.safetensors": {
"bytes": 4998050320,
"sha256": "24e268797a3e13be638f9edf1f55c872346cf506786322afc6582e4cfa8c4373"
},
"model-00007-of-00032.safetensors": {
"bytes": 4998687256,
"sha256": "f3c8fcdf3bbc5b3af5c12bf8395d091b14c29957afa061f48c4c2685b5b8c8db"
},
"model-00008-of-00032.safetensors": {
"bytes": 4998052664,
"sha256": "648bbb1d3637a96df3f75cb6865846322a4af13bf38fd7817c3e40077d651cc7"
},
"model-00009-of-00032.safetensors": {
"bytes": 4998052208,
"sha256": "481958b40e88a712ea468da50b8f501bf2fe329330881fc3b1b1c3be1ec90689"
},
"model-00010-of-00032.safetensors": {
"bytes": 4998688544,
"sha256": "55d415d65584784fc523a26c9b328e56aec0ddea9fe42a9b48988bc800e67ae1"
},
"model-00011-of-00032.safetensors": {
"bytes": 4998052336,
"sha256": "1d36e5b599317d08d0232bb5e3bdb79f53bd6008294888a66b54d202c5263fea"
},
"model-00012-of-00032.safetensors": {
"bytes": 4998687968,
"sha256": "02be77ce110c6afc379c6ef23fad928205cddf26ddea8155aa274bed1cfd0efe"
},
"model-00013-of-00032.safetensors": {
"bytes": 4998052816,
"sha256": "94c607a19b8254ad78648d91d8808b2ffee5733e27739683225175e050c465c3"
},
"model-00014-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "021de7d92c76f23e31cfaf7f0053e1ce9359efeb93fd90728e658cceb9fe2b2a"
},
"model-00015-of-00032.safetensors": {
"bytes": 4998688344,
"sha256": "1383a4d989beac327c464bd3dd0a8008a199fca3d8c93d2e05dacc76d1f49163"
},
"model-00016-of-00032.safetensors": {
"bytes": 4998052488,
"sha256": "caf525ce7d499e4fbb7e3c6388346a1e8d602baa35f1946387aabb0143b7b696"
},
"model-00017-of-00032.safetensors": {
"bytes": 4984946504,
"sha256": "23af12d4220b2ebd42fcc030c5f98651adb73e96f0819e7dfbfed9454f98c2bd"
},
"model-00018-of-00032.safetensors": {
"bytes": 4998686528,
"sha256": "84e7b32097760e1fbe4380d424ab2c723d101af2283f3aea956a7f8dd30625da"
},
"model-00019-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "a95e33a7082c63f590d5f1893425f17bb87adf8814039dbf76adbfd64fd0e639"
},
"model-00020-of-00032.safetensors": {
"bytes": 4998688200,
"sha256": "55e35b6f1d394cd024d3f4819c63a66bbd5c135168823b02fdce2c736e79e330"
},
"model-00021-of-00032.safetensors": {
"bytes": 4998052592,
"sha256": "e2c07bd50fdec4fabdac18cf7a24db21561bccc33c73dd038a409db2e9632c8f"
},
"model-00022-of-00032.safetensors": {
"bytes": 4998052248,
"sha256": "41e87862dc6209d187842822e7f075763b9eb0dcc439b5be71d82e8162fb8765"
},
"model-00023-of-00032.safetensors": {
"bytes": 4998688616,
"sha256": "e0d663b7503a40dff70cf78b7497a06688efbf07d584f9cbb3b82cc70267c422"
},
"model-00024-of-00032.safetensors": {
"bytes": 4998052232,
"sha256": "8fae68d67883f561d205a9dde47af5815e35b472ff43452e9599cd26eeec2060"
},
"model-00025-of-00032.safetensors": {
"bytes": 4998688040,
"sha256": "0b46e002887b466bc2c9ae63709afaa1a2ad088498da76dd0919b70e5a6126b8"
},
"model-00026-of-00032.safetensors": {
"bytes": 4998052744,
"sha256": "b25cdfb7c0a656ac63b506310ff9db4e1383a76a9455722eb0938de0f697e998"
},
"model-00027-of-00032.safetensors": {
"bytes": 4998052200,
"sha256": "765f2b2e020262d9c17a8a9a20ffd711f14ea3014b222f26d3ec1aaa11959aa2"
},
"model-00028-of-00032.safetensors": {
"bytes": 4998688432,
"sha256": "c811b837b63d0cb79116f76a856dca1de6e3915db5df65ca62388c4484903a0d"
},
"model-00029-of-00032.safetensors": {
"bytes": 4998052440,
"sha256": "42c9771487382501d2aa2a5942fbb15062bf18bdbade5aed7b8b4ca180d7e1ff"
},
"model-00030-of-00032.safetensors": {
"bytes": 4998687904,
"sha256": "d6b1e31e529b6472a2da690e879ed9ae70719030b4932e560b1a952aae3748e5"
},
"model-00031-of-00032.safetensors": {
"bytes": 4998052896,
"sha256": "9a7c95968efd33a0bc6c15210a8df021f342ebd4bf3dd442975422a006238418"
},
"model-00032-of-00032.safetensors": {
"bytes": 1278688952,
"sha256": "8aa64fa468f49b55e5fd80da2d6830a01635a1ef53fd5ae0e573a1a6c657d2b3"
}
},
"source_repository": "Aleph-Alpha/Kolibri-1-BF16",
"source_revision": "7a8f290e7858825c3cf5e4c447ba68345de9f1d3",
"source_tensor_count": 58353,
"tensor_count": 1903
},
"scoring_receipt": {
"calibrationCaseIDs": [],
"cases": [
{
"elapsedSeconds": 15.18481969833374,
"logitsFile": "cal-en.safetensors",
"logitsSHA256": "d10a7afbe27ea21fee9c0cb03b2ff2fdf13d3bd3d5be9599cf9965d21ba55e8a",
"meanNegativeLogLikelihood": 0.6938142776489258,
"sequence": {
"id": "cal-en",
"language": "en",
"scoreStart": 57,
"split": "calibration",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
17033,
17407,
1776,
1519,
32,
56,
29522,
46,
25241,
29522,
568,
7880,
46,
742,
900,
1750,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
49,
57,
29522,
1750,
58,
32,
51,
18092,
32,
56,
32099,
32,
53,
576,
32,
49,
57,
46,
127906
]
},
"tokenCount": 19
},
{
"elapsedSeconds": 4.667045950889587,
"logitsFile": "cal-de.safetensors",
"logitsSHA256": "77e6bd4a7df47c2efd18a7ece8c5bf61acdc574eacef9cae1bf2747de2cfc8cb",
"meanNegativeLogLikelihood": 0.582632303237915,
"sequence": {
"id": "cal-de",
"language": "de",
"scoreStart": 61,
"split": "calibration",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1113,
5958,
32206,
32,
57,
48,
15592,
622,
32,
49,
44,
53,
6838,
46,
2447,
4023,
2459,
2796,
89902,
15705,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
89902,
15705,
12165,
32,
54,
48,
14202,
31351,
58,
32,
57,
48,
49722,
32,
49,
44,
53,
576,
32,
54,
48,
46,
127906
]
},
"tokenCount": 24
},
{
"elapsedSeconds": 3.9933032989501953,
"logitsFile": "cal-code.safetensors",
"logitsSHA256": "9b3040c88e50b6f3acdd228372816aa9e75d49307468f687faa142678d073961",
"meanNegativeLogLikelihood": 0.19747400283813477,
"sequence": {
"id": "cal-code",
"language": "en",
"scoreStart": 53,
"split": "calibration",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
7794,
2931,
41,
327,
26550,
14169,
26455,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
7794,
2931,
2488,
14141,
1778,
14169,
1921,
14169,
127906
]
},
"tokenCount": 10
},
{
"elapsedSeconds": 3.3992522954940796,
"logitsFile": "held-en-fact.safetensors",
"logitsSHA256": "bbcfc6837117104d947cf2addc1939ff959e1771163ec3b0be3f0a7174a984b1",
"meanNegativeLogLikelihood": 0.06728151440620422,
"sequence": {
"id": "held-en-fact",
"language": "en",
"scoreStart": 50,
"split": "heldout",
"task": "knowledge",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1798,
735,
262,
13134,
9996,
379,
14272,
63,
1741,
622,
2403,
14264,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
13134,
9996,
379,
14272,
735,
2859,
46,
127906
]
},
"tokenCount": 9
},
{
"elapsedSeconds": 2.8967736959457397,
"logitsFile": "held-de-fact.safetensors",
"logitsSHA256": "74341a66cc892b1765a481508c22f6a83f876571838d11521ff19883d10b329c",
"meanNegativeLogLikelihood": 0.12728945910930634,
"sequence": {
"id": "held-de-fact",
"language": "de",
"scoreStart": 48,
"split": "heldout",
"task": "knowledge",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
61701,
44187,
14107,
968,
16395,
63,
97011,
622,
1097,
16952,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
5396,
44187,
14107,
614,
16395,
2459,
3294,
46,
127906
]
},
"tokenCount": 9
},
{
"elapsedSeconds": 3.861657500267029,
"logitsFile": "held-en-arithmetic.safetensors",
"logitsSHA256": "79f169ea32aabf26aff28da7839522e826c9bbbcf63612247dc84053b7eb5783",
"meanNegativeLogLikelihood": 1.1074368953704834,
"sequence": {
"id": "held-en-arithmetic",
"language": "en",
"scoreStart": 63,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
65,
4777,
63655,
32,
49,
55,
84038,
3660,
1688,
54,
1776,
286,
94448,
1688,
49,
50,
46,
611,
735,
3327,
15482,
7086,
63,
1741,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
15482,
7086,
735,
1688,
57,
48,
58,
32,
49,
55,
18092,
1688,
54,
32099,
1688,
49,
50,
576,
1688,
57,
48,
46,
127906
]
},
"tokenCount": 24
},
{
"elapsedSeconds": 3.627297878265381,
"logitsFile": "held-de-arithmetic.safetensors",
"logitsSHA256": "0d1de455a1b316209986745fdf85baaab04918712bd5d760ac46505a35b27b54",
"meanNegativeLogLikelihood": 0.5787030458450317,
"sequence": {
"id": "held-de-arithmetic",
"language": "de",
"scoreStart": 68,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
11717,
75971,
16611,
32,
50,
52,
10343,
775,
614,
3408,
32,
52,
2527,
420,
2941,
32,
56,
2527,
2340,
46,
2447,
4023,
2459,
948,
42842,
268,
8776,
63,
97011,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
8590,
42842,
268,
8776,
12165,
32,
56,
56,
2527,
58,
32,
50,
52,
18092,
32,
52,
32099,
32,
56,
576,
32,
56,
56,
46,
127906
]
},
"tokenCount": 25
},
{
"elapsedSeconds": 4.649676203727722,
"logitsFile": "held-en-logic.safetensors",
"logitsSHA256": "da3f439af6b1dc45430f5116bb2a30ccb1c7263076172ea4df324abb3d77564d",
"meanNegativeLogLikelihood": 1.108681082725525,
"sequence": {
"id": "held-en-logic",
"language": "en",
"scoreStart": 59,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4419,
35311,
568,
26933,
46,
3463,
26933,
476,
23134,
46,
7302,
3326,
3480,
327,
938,
35311,
476,
23134,
63,
20274,
23559,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
3913,
46,
543,
26933,
327,
476,
23134,
1026,
502,
2396,
1578,
35311,
44,
1409,
262,
5082,
830,
502,
3480,
46,
127906
]
},
"tokenCount": 21
},
{
"elapsedSeconds": 3.408438801765442,
"logitsFile": "held-de-logic.safetensors",
"logitsSHA256": "09d3e4f9d60053c8265d06a42b3212b3027fc745ee316cfe71544170686651a2",
"meanNegativeLogLikelihood": 1.829041600227356,
"sequence": {
"id": "held-de-logic",
"language": "de",
"scoreStart": 63,
"split": "heldout",
"task": "reasoning",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
13518,
25525,
1200,
10273,
46,
44871,
10273,
1200,
13752,
329,
46,
8470,
5871,
25412,
44,
1469,
46789,
25525,
13752,
329,
1200,
63,
840,
7166,
14460,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1170,
46,
1047,
13752,
1021,
10273,
4165,
3514,
25525,
2260,
59,
346,
90837,
11637,
14276,
939,
46,
127906
]
},
"tokenCount": 18
},
{
"elapsedSeconds": 3.567003846168518,
"logitsFile": "held-en-code.safetensors",
"logitsSHA256": "639b99a9b9ed6a20851c16b1a1455ad1aa8562d480d3e832b4109bf08e27d4b3",
"meanNegativeLogLikelihood": 0.07403717190027237,
"sequence": {
"id": "held-en-code",
"language": "en",
"scoreStart": 56,
"split": "heldout",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
4999,
941,
6797,
1889,
735,
105480,
14599,
41,
327,
26550,
2544,
14163,
735,
2051,
46,
6442,
1444,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
735,
105480,
14599,
2488,
14141,
1778,
14163,
14628,
32,
50,
1339,
32,
48,
127906
]
},
"tokenCount": 15
},
{
"elapsedSeconds": 2.269721508026123,
"logitsFile": "held-de-code.safetensors",
"logitsSHA256": "ca03560a2e5927714e38d32d762c6dd4404512ee88724fc0c9bd6fbd55af0eba",
"meanNegativeLogLikelihood": 0.13556255400180817,
"sequence": {
"id": "held-de-code",
"language": "de",
"scoreStart": 64,
"split": "heldout",
"task": "code",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
22242,
699,
6797,
15836,
1427,
84780,
106925,
125942,
3003,
346,
518,
9678,
7307,
1117,
1233,
1151,
63064,
18996,
4330,
2340,
1673,
46,
32638,
2120,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
1363,
1427,
84780,
106925,
125942,
2488,
14141,
1778,
9468,
91,
48,
93,
1765,
9468,
4100,
4330,
127906
]
},
"tokenCount": 17
},
{
"elapsedSeconds": 3.87636399269104,
"logitsFile": "held-en-translate.safetensors",
"logitsSHA256": "e5837a25ba66d3fbf1407ac44f9a77022e6e135bd00fd4beeb60cf0695f6f90f",
"meanNegativeLogLikelihood": 0.1646750569343567,
"sequence": {
"id": "held-en-translate",
"language": "en",
"scoreStart": 51,
"split": "heldout",
"task": "translation",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
44761,
318,
4867,
58,
543,
14345,
52074,
3660,
14395,
286,
62712,
3660,
8445,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
452,
30908,
56277,
7470,
24677,
5269,
420,
61556,
7470,
16363,
5269,
46,
127906
]
},
"tokenCount": 13
},
{
"elapsedSeconds": 3.6765007972717285,
"logitsFile": "held-de-translate.safetensors",
"logitsSHA256": "7d8824717792a45ee2050bbb985ebe4820c70052fcd31755248ede8250662cf9",
"meanNegativeLogLikelihood": 0.08966350555419922,
"sequence": {
"id": "held-de-translate",
"language": "de",
"scoreStart": 50,
"split": "heldout",
"task": "translation",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
1103,
57190,
120025,
98780,
58,
1047,
36235,
1030,
2361,
38749,
4118,
48837,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
325,
8709,
378,
4191,
941,
5951,
1681,
11123,
46,
127906
]
},
"tokenCount": 10
},
{
"elapsedSeconds": 80.61282813549042,
"logitsFile": "held-en-window.safetensors",
"logitsSHA256": "9feb623c206791636d6fe129fc503b4a30333dac159df30e377cb13f06e3d88a",
"meanNegativeLogLikelihood": 0.005408423487097025,
"sequence": {
"id": "held-en-window",
"language": "en",
"scoreStart": 1327,
"split": "heldout",
"task": "window-crossing",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
5800,
1862,
12232,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
1798,
735,
262,
3973,
13709,
622,
11237,
32,
55,
63,
6442,
1444,
262,
13709,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
]
},
"tokenCount": 7
},
{
"elapsedSeconds": 33.705241680145264,
"logitsFile": "held-de-window.safetensors",
"logitsSHA256": "c1f9bfd71e847dba7699672548751229ebca6f61ba24af037175e480abb8263a",
"meanNegativeLogLikelihood": 0.0031283244024962187,
"sequence": {
"id": "held-de-window",
"language": "de",
"scoreStart": 1328,
"split": "heldout",
"task": "window-crossing",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
46,
127906,
10,
127904,
1646,
10,
76,
555,
2834,
94446,
314,
10571,
32,
48,
58,
3973,
13709,
1321,
48,
48,
48,
48,
44,
12284,
28964,
314,
10571,
32,
49,
58,
3973,
13709,
1321,
48,
48,
48,
49,
44,
12284,
28964,
314,
10571,
32,
50,
58,
3973,
13709,
1321,
48,
48,
48,
50,
44,
12284,
28964,
314,
10571,
32,
51,
58,
3973,
13709,
1321,
48,
48,
48,
51,
44,
12284,
28964,
314,
10571,
32,
52,
58,
3973,
13709,
1321,
48,
48,
48,
52,
44,
12284,
28964,
314,
10571,
32,
53,
58,
3973,
13709,
1321,
48,
48,
48,
53,
44,
12284,
28964,
314,
10571,
32,
54,
58,
3973,
13709,
1321,
48,
48,
48,
54,
44,
12284,
28964,
314,
10571,
32,
55,
58,
3973,
13709,
79000,
9953,
45,
55,
51,
49,
44,
12284,
25082,
314,
10571,
32,
56,
58,
3973,
13709,
1321,
48,
48,
48,
56,
44,
12284,
28964,
314,
10571,
32,
57,
58,
3973,
13709,
1321,
48,
48,
48,
57,
44,
12284,
28964,
314,
10571,
32,
49,
48,
58,
3973,
13709,
1321,
48,
48,
49,
48,
44,
12284,
28964,
314,
10571,
32,
49,
49,
58,
3973,
13709,
1321,
48,
48,
49,
49,
44,
12284,
28964,
314,
10571,
32,
49,
50,
58,
3973,
13709,
1321,
48,
48,
49,
50,
44,
12284,
28964,
314,
10571,
32,
49,
51,
58,
3973,
13709,
1321,
48,
48,
49,
51,
44,
12284,
28964,
314,
10571,
32,
49,
52,
58,
3973,
13709,
1321,
48,
48,
49,
52,
44,
12284,
28964,
314,
10571,
32,
49,
53,
58,
3973,
13709,
1321,
48,
48,
49,
53,
44,
12284,
28964,
314,
10571,
32,
49,
54,
58,
3973,
13709,
1321,
48,
48,
49,
54,
44,
12284,
28964,
314,
10571,
32,
49,
55,
58,
3973,
13709,
1321,
48,
48,
49,
55,
44,
12284,
28964,
314,
10571,
32,
49,
56,
58,
3973,
13709,
1321,
48,
48,
49,
56,
44,
12284,
28964,
314,
10571,
32,
49,
57,
58,
3973,
13709,
1321,
48,
48,
49,
57,
44,
12284,
28964,
314,
10571,
32,
50,
48,
58,
3973,
13709,
1321,
48,
48,
50,
48,
44,
12284,
28964,
314,
10571,
32,
50,
49,
58,
3973,
13709,
1321,
48,
48,
50,
49,
44,
12284,
28964,
314,
10571,
32,
50,
50,
58,
3973,
13709,
1321,
48,
48,
50,
50,
44,
12284,
28964,
314,
10571,
32,
50,
51,
58,
3973,
13709,
1321,
48,
48,
50,
51,
44,
12284,
28964,
314,
10571,
32,
50,
52,
58,
3973,
13709,
1321,
48,
48,
50,
52,
44,
12284,
28964,
314,
10571,
32,
50,
53,
58,
3973,
13709,
1321,
48,
48,
50,
53,
44,
12284,
28964,
314,
10571,
32,
50,
54,
58,
3973,
13709,
1321,
48,
48,
50,
54,
44,
12284,
28964,
314,
10571,
32,
50,
55,
58,
3973,
13709,
1321,
48,
48,
50,
55,
44,
12284,
28964,
314,
10571,
32,
50,
56,
58,
3973,
13709,
1321,
48,
48,
50,
56,
44,
12284,
28964,
314,
10571,
32,
50,
57,
58,
3973,
13709,
1321,
48,
48,
50,
57,
44,
12284,
28964,
314,
10571,
32,
51,
48,
58,
3973,
13709,
1321,
48,
48,
51,
48,
44,
12284,
28964,
314,
10571,
32,
51,
49,
58,
3973,
13709,
1321,
48,
48,
51,
49,
44,
12284,
28964,
314,
10571,
32,
51,
50,
58,
3973,
13709,
1321,
48,
48,
51,
50,
44,
12284,
28964,
314,
10571,
32,
51,
51,
58,
3973,
13709,
1321,
48,
48,
51,
51,
44,
12284,
28964,
314,
10571,
32,
51,
52,
58,
3973,
13709,
1321,
48,
48,
51,
52,
44,
12284,
28964,
314,
10571,
32,
51,
53,
58,
3973,
13709,
1321,
48,
48,
51,
53,
44,
12284,
28964,
314,
10571,
32,
51,
54,
58,
3973,
13709,
1321,
48,
48,
51,
54,
44,
12284,
28964,
314,
10571,
32,
51,
55,
58,
3973,
13709,
1321,
48,
48,
51,
55,
44,
12284,
28964,
314,
10571,
32,
51,
56,
58,
3973,
13709,
1321,
48,
48,
51,
56,
44,
12284,
28964,
314,
10571,
32,
51,
57,
58,
3973,
13709,
1321,
48,
48,
51,
57,
44,
12284,
28964,
314,
10571,
32,
52,
48,
58,
3973,
13709,
1321,
48,
48,
52,
48,
44,
12284,
28964,
314,
10571,
32,
52,
49,
58,
3973,
13709,
1321,
48,
48,
52,
49,
44,
12284,
28964,
314,
10571,
32,
52,
50,
58,
3973,
13709,
1321,
48,
48,
52,
50,
44,
12284,
28964,
314,
10571,
32,
52,
51,
58,
3973,
13709,
1321,
48,
48,
52,
51,
44,
12284,
28964,
314,
10571,
32,
52,
52,
58,
3973,
13709,
1321,
48,
48,
52,
52,
44,
12284,
28964,
314,
10571,
32,
52,
53,
58,
3973,
13709,
1321,
48,
48,
52,
53,
44,
12284,
28964,
314,
10571,
32,
52,
54,
58,
3973,
13709,
1321,
48,
48,
52,
54,
44,
12284,
28964,
314,
10571,
32,
52,
55,
58,
3973,
13709,
1321,
48,
48,
52,
55,
44,
12284,
28964,
314,
10571,
32,
52,
56,
58,
3973,
13709,
1321,
48,
48,
52,
56,
44,
12284,
28964,
314,
10571,
32,
52,
57,
58,
3973,
13709,
1321,
48,
48,
52,
57,
44,
12284,
28964,
314,
10571,
32,
53,
48,
58,
3973,
13709,
1321,
48,
48,
53,
48,
44,
12284,
28964,
314,
10571,
32,
53,
49,
58,
3973,
13709,
1321,
48,
48,
53,
49,
44,
12284,
28964,
314,
10571,
32,
53,
50,
58,
3973,
13709,
1321,
48,
48,
53,
50,
44,
12284,
28964,
314,
10571,
32,
53,
51,
58,
3973,
13709,
1321,
48,
48,
53,
51,
44,
12284,
28964,
314,
10571,
32,
53,
52,
58,
3973,
13709,
1321,
48,
48,
53,
52,
44,
12284,
28964,
314,
10571,
32,
53,
53,
58,
3973,
13709,
1321,
48,
48,
53,
53,
44,
12284,
28964,
314,
10571,
32,
53,
54,
58,
3973,
13709,
1321,
48,
48,
53,
54,
44,
12284,
28964,
314,
10571,
32,
53,
55,
58,
3973,
13709,
1321,
48,
48,
53,
55,
44,
12284,
28964,
314,
10571,
32,
53,
56,
58,
3973,
13709,
1321,
48,
48,
53,
56,
44,
12284,
28964,
314,
10571,
32,
53,
57,
58,
3973,
13709,
1321,
48,
48,
53,
57,
44,
12284,
28964,
314,
10571,
32,
54,
48,
58,
3973,
13709,
1321,
48,
48,
54,
48,
44,
12284,
28964,
314,
10571,
32,
54,
49,
58,
3973,
13709,
1321,
48,
48,
54,
49,
44,
12284,
28964,
314,
10571,
32,
54,
50,
58,
3973,
13709,
1321,
48,
48,
54,
50,
44,
12284,
28964,
314,
10571,
32,
54,
51,
58,
3973,
13709,
1321,
48,
48,
54,
51,
44,
12284,
28964,
314,
10571,
32,
54,
52,
58,
3973,
13709,
1321,
48,
48,
54,
52,
44,
12284,
28964,
314,
10571,
32,
54,
53,
58,
3973,
13709,
1321,
48,
48,
54,
53,
44,
12284,
28964,
314,
10571,
32,
54,
54,
58,
3973,
13709,
1321,
48,
48,
54,
54,
44,
12284,
28964,
314,
10571,
32,
54,
55,
58,
3973,
13709,
1321,
48,
48,
54,
55,
44,
12284,
28964,
314,
10571,
32,
54,
56,
58,
3973,
13709,
1321,
48,
48,
54,
56,
44,
12284,
28964,
314,
10571,
32,
54,
57,
58,
3973,
13709,
1321,
48,
48,
54,
57,
44,
12284,
28964,
314,
10571,
32,
55,
48,
58,
3973,
13709,
1321,
48,
48,
55,
48,
44,
12284,
28964,
314,
10571,
32,
55,
49,
58,
3973,
13709,
1321,
48,
48,
55,
49,
44,
12284,
28964,
314,
10571,
32,
55,
50,
58,
3973,
13709,
1321,
48,
48,
55,
50,
44,
12284,
28964,
314,
10571,
32,
55,
51,
58,
3973,
13709,
1321,
48,
48,
55,
51,
44,
12284,
28964,
314,
10571,
32,
55,
52,
58,
3973,
13709,
1321,
48,
48,
55,
52,
44,
12284,
28964,
314,
10571,
32,
55,
53,
58,
3973,
13709,
1321,
48,
48,
55,
53,
44,
12284,
28964,
314,
10571,
32,
55,
54,
58,
3973,
13709,
1321,
48,
48,
55,
54,
44,
12284,
28964,
314,
10571,
32,
55,
55,
58,
3973,
13709,
1321,
48,
48,
55,
55,
44,
12284,
28964,
314,
10571,
32,
55,
56,
58,
3973,
13709,
1321,
48,
48,
55,
56,
44,
12284,
28964,
314,
10571,
32,
55,
57,
58,
3973,
13709,
1321,
48,
48,
55,
57,
44,
12284,
28964,
314,
86170,
34360,
855,
12975,
622,
11237,
32,
55,
63,
32638,
2120,
1178,
6157,
1953,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
38666,
9953,
45,
55,
51,
49,
127906
]
},
"tokenCount": 7
},
{
"elapsedSeconds": 6.974622488021851,
"logitsFile": "held-en-tool.safetensors",
"logitsSHA256": "1df1e87e69f57dc9a2761359b18dbc09fd4a649020871ff73b29244119ec6505",
"meanNegativeLogLikelihood": 0.017238948494195938,
"sequence": {
"id": "held-en-tool",
"language": "en",
"scoreStart": 183,
"split": "heldout",
"task": "tools",
"tokens": [
127904,
940,
10,
35,
70894,
5303,
263,
29230,
259,
735,
12416,
46,
24254,
14354,
318,
56320,
3863,
318,
262,
3143,
589,
15436,
275,
35,
21955,
263,
652,
824,
5604,
2403,
3688,
828,
10852,
318,
9095,
310,
262,
3143,
15483,
275,
652,
568,
2349,
310,
1889,
53814,
2288,
3167,
32776,
1609,
32776,
62,
38357,
24727,
656,
60,
32776,
434,
1142,
676,
408,
4138,
8322,
408,
946,
2938,
262,
5995,
379,
941,
6547,
120260,
946,
653,
408,
946,
688,
95,
30279,
531,
946,
8785,
408,
4138,
15637,
408,
4138,
3547,
408,
4138,
805,
408,
946,
1383,
16506,
946,
22267,
408,
7311,
3547,
24148,
946,
805,
408,
946,
1300,
16506,
946,
805,
408,
946,
676,
100048,
376,
32776,
9215,
2674,
1776,
1889,
5604,
44,
1778,
941,
12597,
7438,
310,
1889,
5032,
286,
15477,
2288,
32,
127909,
127910,
38357,
24727,
656,
127909,
10,
1142,
653,
408,
3167,
676,
62679,
120073,
946,
7160,
408,
3167,
8510,
45,
6427,
123291,
62,
475,
127910,
127906,
10,
127904,
1646,
10,
5116,
262,
5995,
4324,
318,
2492,
2839,
262,
5995,
622,
67041,
46,
127906,
10,
127904,
64090,
10,
127907,
263,
127908,
263,
127909,
10,
1142,
653,
408,
946,
688,
95,
30279,
531,
946,
7160,
408,
4138,
3547,
408,
946,
50855,
67036,
34,
31257,
127910,
127906
]
},
"tokenCount": 23
}
],
"checkpointPath": "/workspace/kolibri/artifacts/Kolibri-1-MLX-mixed2-calibrated",
"chunkSize": 32,
"configSHA256": "cd3d15e8f44b3dc24937f9f9730d4eb7d621759f28cc7ce5f33b633a37aa6d31",
"conversionSHA256": "39b678a298301b867dcbbf93ad529fe982fa647a99a37b4c49a9961f9a56ec33",
"device": "Device(gpu, 0)",
"indexSHA256": "036de221d5da648136f71b4b5d078004193101e2ab16d3764854ad66d0628da9",
"peakMemoryBytes": 26550516000,
"schemaVersion": 1,
"suiteSHA256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b"
}
}
},
"q2_selection": {
"reports": [
{
"schema_version": 1,
"reference": "/workspace/kolibri/results/reference",
"candidate": "/workspace/kolibri/results/mixed2",
"suite_sha256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b",
"gates": {
"mean_kl_max": 0.15,
"case_mean_kl_max": 0.3,
"perplexity_ratio_max": 1.1,
"case_perplexity_ratio_max": 1.25,
"top1_agreement_min": 0.8
},
"cases": [
{
"id": "cal-en",
"language": "en",
"task": "reasoning",
"split": "calibration",
"tokens": 19,
"mean_kl": 0.01978035512636936,
"p95_kl": 0.07019272293615497,
"top1_agreement": 0.9473684210526315,
"delta_nll": 0.016883991853967422,
"reference_nll": 0.6549558352843168,
"candidate_nll": 0.6718398271382842,
"perplexity_ratio": 1.0170273320262022,
"token_logprob_rmse": 0.2031434871379188,
"p95_abs_token_logprob_delta": 0.43077514485836166
},
{
"id": "cal-de",
"language": "de",
"task": "reasoning",
"split": "calibration",
"tokens": 24,
"mean_kl": 0.005667014272128068,
"p95_kl": 0.017980519144710057,
"top1_agreement": 1.0,
"delta_nll": -0.05484244519971531,
"reference_nll": 0.7154404572220049,
"candidate_nll": 0.6605980120222895,
"perplexity_ratio": 0.9466342829801032,
"token_logprob_rmse": 0.1955187889650095,
"p95_abs_token_logprob_delta": 0.2572200537461546
},
{
"id": "cal-code",
"language": "en",
"task": "code",
"split": "calibration",
"tokens": 10,
"mean_kl": 0.0090121566551538,
"p95_kl": 0.04494164323497877,
"top1_agreement": 0.9,
"delta_nll": 0.014354575798796019,
"reference_nll": 0.16979148134074312,
"candidate_nll": 0.18414605713953916,
"perplexity_ratio": 1.0144580974654194,
"token_logprob_rmse": 0.14400732283730655,
"p95_abs_token_logprob_delta": 0.32123469005405736
},
{
"id": "held-en-fact",
"language": "en",
"task": "knowledge",
"split": "heldout",
"tokens": 9,
"mean_kl": 0.008532572412014376,
"p95_kl": 0.03399869121037098,
"top1_agreement": 1.0,
"delta_nll": -0.01938650548337346,
"reference_nll": 0.050975090498802944,
"candidate_nll": 0.031588585015429486,
"perplexity_ratio": 0.9808002043171551,
"token_logprob_rmse": 0.03415090424272494,
"p95_abs_token_logprob_delta": 0.06652148796448626
},
{
"id": "held-de-fact",
"language": "de",
"task": "knowledge",
"split": "heldout",
"tokens": 9,
"mean_kl": 0.014508015541622411,
"p95_kl": 0.046303629362566164,
"top1_agreement": 1.0,
"delta_nll": -0.03564093058530338,
"reference_nll": 0.121087029966858,
"candidate_nll": 0.08544609938155465,
"perplexity_ratio": 0.9649867285020843,
"token_logprob_rmse": 0.053836523572273254,
"p95_abs_token_logprob_delta": 0.10077339452351758
},
{
"id": "held-en-arithmetic",
"language": "en",
"task": "reasoning",
"split": "heldout",
"tokens": 24,
"mean_kl": 0.02731870599303889,
"p95_kl": 0.11100863036357914,
"top1_agreement": 0.9166666666666666,
"delta_nll": 0.005223143733265969,
"reference_nll": 1.0540350248188621,
"candidate_nll": 1.0592581685521283,
"perplexity_ratio": 1.0052368081285032,
"token_logprob_rmse": 0.3765455041140242,
"p95_abs_token_logprob_delta": 0.8446805415397292
},
{
"id": "held-de-arithmetic",
"language": "de",
"task": "reasoning",
"split": "heldout",
"tokens": 25,
"mean_kl": 0.038012473264846844,
"p95_kl": 0.1842930397861482,
"top1_agreement": 0.92,
"delta_nll": -0.03052520898120546,
"reference_nll": 0.6239411885152131,
"candidate_nll": 0.5934159795340076,
"perplexity_ratio": 0.9699359806611073,
"token_logprob_rmse": 0.1811096744080099,
"p95_abs_token_logprob_delta": 0.43715357180532766
},
{
"id": "held-en-logic",
"language": "en",
"task": "reasoning",
"split": "heldout",
"tokens": 21,
"mean_kl": 0.06703000742128305,
"p95_kl": 0.19975681968907602,
"top1_agreement": 1.0,
"delta_nll": 0.1546991239601873,
"reference_nll": 1.09514205294828,
"candidate_nll": 1.249841176908467,
"perplexity_ratio": 1.1673066936485625,
"token_logprob_rmse": 0.5531070306744612,
"p95_abs_token_logprob_delta": 1.0158225248812665
},
{
"id": "held-de-logic",
"language": "de",
"task": "reasoning",
"split": "heldout",
"tokens": 18,
"mean_kl": 0.099907261091536,
"p95_kl": 0.30538032502326773,
"top1_agreement": 0.9444444444444444,
"delta_nll": 0.20284248227687066,
"reference_nll": 1.6221964826714386,
"candidate_nll": 1.8250389649483099,
"perplexity_ratio": 1.2248795128189813,
"token_logprob_rmse": 0.946217730605902,
"p95_abs_token_logprob_delta": 1.8555619116560873
},
{
"id": "held-en-code",
"language": "en",
"task": "code",
"split": "heldout",
"tokens": 15,
"mean_kl": 0.002400378283496873,
"p95_kl": 0.010770139261838363,
"top1_agreement": 1.0,
"delta_nll": 0.022850559417013642,
"reference_nll": 0.05543094233111131,
"candidate_nll": 0.07828150174812494,
"perplexity_ratio": 1.0231136334244717,
"token_logprob_rmse": 0.08794100503269633,
"p95_abs_token_logprob_delta": 0.10309197261457706
},
{
"id": "held-de-code",
"language": "de",
"task": "code",
"split": "heldout",
"tokens": 17,
"mean_kl": 0.001824994084877997,
"p95_kl": 0.008106041428038294,
"top1_agreement": 1.0,
"delta_nll": 0.034265870417392236,
"reference_nll": 0.12953522427201575,
"candidate_nll": 0.16380109468940798,
"perplexity_ratio": 1.034859708738459,
"token_logprob_rmse": 0.14251232480811332,
"p95_abs_token_logprob_delta": 0.12690560307912951
},
{
"id": "held-en-translate",
"language": "en",
"task": "translation",
"split": "heldout",
"tokens": 13,
"mean_kl": 0.007539816885054277,
"p95_kl": 0.02404970845431429,
"top1_agreement": 1.0,
"delta_nll": -0.005483094382172056,
"reference_nll": 0.19652510627505373,
"candidate_nll": 0.19104201189288167,
"perplexity_ratio": 0.9945319103431952,
"token_logprob_rmse": 0.03406796843613715,
"p95_abs_token_logprob_delta": 0.07896416980130834
},
{
"id": "held-de-translate",
"language": "de",
"task": "translation",
"split": "heldout",
"tokens": 10,
"mean_kl": 0.003643223327261212,
"p95_kl": 0.012914849091083438,
"top1_agreement": 1.0,
"delta_nll": 0.0004951360818267412,
"reference_nll": 0.08180677501291742,
"candidate_nll": 0.08230191109474415,
"perplexity_ratio": 1.0004952586819302,
"token_logprob_rmse": 0.025559004479840934,
"p95_abs_token_logprob_delta": 0.056366833815666734
},
{
"id": "held-en-window",
"language": "en",
"task": "window-crossing",
"split": "heldout",
"tokens": 7,
"mean_kl": 0.00014587426416638974,
"p95_kl": 0.0005033452945314835,
"top1_agreement": 1.0,
"delta_nll": 0.00029546422358195785,
"reference_nll": 0.000790756349371949,
"candidate_nll": 0.001086220572953907,
"perplexity_ratio": 1.000295507877435,
"token_logprob_rmse": 0.000572263733238149,
"p95_abs_token_logprob_delta": 0.0011095665807016796
},
{
"id": "held-de-window",
"language": "de",
"task": "window-crossing",
"split": "heldout",
"tokens": 7,
"mean_kl": 0.00040019105023655655,
"p95_kl": 0.0014269024844973787,
"top1_agreement": 1.0,
"delta_nll": 0.0006952818340340136,
"reference_nll": 0.0006427372332391178,
"candidate_nll": 0.001338019067273131,
"perplexity_ratio": 1.0006955235984767,
"token_logprob_rmse": 0.0012725182394150018,
"p95_abs_token_logprob_delta": 0.002530889731930041
},
{
"id": "held-en-tool",
"language": "en",
"task": "tools",
"split": "heldout",
"tokens": 23,
"mean_kl": 0.0035667630644439677,
"p95_kl": 2.1634635573068513e-05,
"top1_agreement": 1.0,
"delta_nll": 0.004570928290456717,
"reference_nll": 0.0004227024073699562,
"candidate_nll": 0.004993630697826673,
"perplexity_ratio": 1.0045813909184083,
"token_logprob_rmse": 0.02189957766465678,
"p95_abs_token_logprob_delta": 6.111664522470433e-05
}
],
"calibration": {
"cases": 3,
"tokens": 53,
"mean_kl": 0.011357672763842067,
"top1_agreement": 0.9622641509433962,
"delta_nll": -0.016073152482638232,
"reference_nll": 0.5908046539082561,
"candidate_nll": 0.5747315014256177,
"perplexity_ratio": 0.9840553313317,
"worst_case_mean_kl": 0.01978035512636936,
"worst_case_perplexity_ratio": 1.0170273320262022,
"passes_diagnostic_gates": true
},
"heldout": {
"cases": 13,
"tokens": 198,
"mean_kl": 0.026801138165091706,
"top1_agreement": 0.9747474747474747,
"delta_nll": 0.034029497373143834,
"reference_nll": 0.5104427418306431,
"candidate_nll": 0.544472239203787,
"perplexity_ratio": 1.0346151247064093,
"worst_case_mean_kl": 0.099907261091536,
"worst_case_perplexity_ratio": 1.2248795128189813,
"passes_diagnostic_gates": true
},
"by_language": {
"de": {
"cases": 6,
"tokens": 86,
"mean_kl": 0.034296199316402494,
"top1_agreement": 0.9651162790697675,
"delta_nll": 0.036739583859743935,
"reference_nll": 0.5687501790006285,
"candidate_nll": 0.6054897628603726,
"perplexity_ratio": 1.0374228240102343,
"worst_case_mean_kl": 0.099907261091536,
"worst_case_perplexity_ratio": 1.2248795128189813,
"passes_diagnostic_gates": true
},
"en": {
"cases": 7,
"tokens": 112,
"mean_kl": 0.02104600192390662,
"top1_agreement": 0.9821428571428571,
"delta_nll": 0.031948538106647335,
"reference_nll": 0.4656709597179757,
"candidate_nll": 0.49761949782462295,
"perplexity_ratio": 1.032464371366574,
"worst_case_mean_kl": 0.06703000742128305,
"worst_case_perplexity_ratio": 1.1673066936485625,
"passes_diagnostic_gates": true
}
},
"passes_all_heldout_gates": true
},
{
"schema_version": 1,
"reference": "/workspace/kolibri/results/reference",
"candidate": "/workspace/kolibri/results/mixed2-calibrated",
"suite_sha256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b",
"gates": {
"mean_kl_max": 0.15,
"case_mean_kl_max": 0.3,
"perplexity_ratio_max": 1.1,
"case_perplexity_ratio_max": 1.25,
"top1_agreement_min": 0.8
},
"cases": [
{
"id": "cal-en",
"language": "en",
"task": "reasoning",
"split": "calibration",
"tokens": 19,
"mean_kl": 0.02211196501066192,
"p95_kl": 0.06998506304236718,
"top1_agreement": 1.0,
"delta_nll": 0.03885864389967459,
"reference_nll": 0.6549558352843168,
"candidate_nll": 0.6938144791839914,
"perplexity_ratio": 1.0396235161369776,
"token_logprob_rmse": 0.14516357869105817,
"p95_abs_token_logprob_delta": 0.27220308971295737
},
{
"id": "cal-de",
"language": "de",
"task": "reasoning",
"split": "calibration",
"tokens": 24,
"mean_kl": 0.006228458694438874,
"p95_kl": 0.021947641501769814,
"top1_agreement": 1.0,
"delta_nll": -0.13280798215165446,
"reference_nll": 0.7154404572220049,
"candidate_nll": 0.5826324750703503,
"perplexity_ratio": 0.8756332131726654,
"token_logprob_rmse": 0.42107263685174673,
"p95_abs_token_logprob_delta": 1.2468944483897746
},
{
"id": "cal-code",
"language": "en",
"task": "code",
"split": "calibration",
"tokens": 10,
"mean_kl": 0.0098743866565869,
"p95_kl": 0.04800079429876301,
"top1_agreement": 0.9,
"delta_nll": 0.027682419837785615,
"reference_nll": 0.16979148134074312,
"candidate_nll": 0.19747390117852875,
"perplexity_ratio": 1.0280691382081901,
"token_logprob_rmse": 0.15882306770863566,
"p95_abs_token_logprob_delta": 0.3426315440100572
},
{
"id": "held-en-fact",
"language": "en",
"task": "knowledge",
"split": "heldout",
"tokens": 9,
"mean_kl": 0.013860991016841292,
"p95_kl": 0.06008918738239512,
"top1_agreement": 1.0,
"delta_nll": 0.016306890673749158,
"reference_nll": 0.050975090498802944,
"candidate_nll": 0.0672819811725521,
"perplexity_ratio": 1.0164405736783282,
"token_logprob_rmse": 0.06387911869090385,
"p95_abs_token_logprob_delta": 0.13236386777259135
},
{
"id": "held-de-fact",
"language": "de",
"task": "knowledge",
"split": "heldout",
"tokens": 9,
"mean_kl": 0.02144369110853793,
"p95_kl": 0.08873728765804512,
"top1_agreement": 1.0,
"delta_nll": 0.006202692986858449,
"reference_nll": 0.121087029966858,
"candidate_nll": 0.12728972295371646,
"perplexity_ratio": 1.0062219695218697,
"token_logprob_rmse": 0.10011333658104801,
"p95_abs_token_logprob_delta": 0.21373884006218008
},
{
"id": "held-en-arithmetic",
"language": "en",
"task": "reasoning",
"split": "heldout",
"tokens": 24,
"mean_kl": 0.03080768638351052,
"p95_kl": 0.1200434432997904,
"top1_agreement": 0.9583333333333334,
"delta_nll": 0.05340228036189116,
"reference_nll": 1.0540350248188621,
"candidate_nll": 1.1074373051807533,
"perplexity_ratio": 1.0548539067889309,
"token_logprob_rmse": 0.21064076911800278,
"p95_abs_token_logprob_delta": 0.45580880094096016
},
{
"id": "held-de-arithmetic",
"language": "de",
"task": "reasoning",
"split": "heldout",
"tokens": 25,
"mean_kl": 0.05260390814872606,
"p95_kl": 0.24884708269381914,
"top1_agreement": 0.96,
"delta_nll": -0.045238003175085965,
"reference_nll": 0.6239411885152131,
"candidate_nll": 0.578703185340127,
"perplexity_ratio": 0.9557699784713846,
"token_logprob_rmse": 0.16336856515566492,
"p95_abs_token_logprob_delta": 0.3690056274915401
},
{
"id": "held-en-logic",
"language": "en",
"task": "reasoning",
"split": "heldout",
"tokens": 21,
"mean_kl": 0.08366261552269878,
"p95_kl": 0.2290998067222127,
"top1_agreement": 1.0,
"delta_nll": 0.013539426306791856,
"reference_nll": 1.09514205294828,
"candidate_nll": 1.1086814792550714,
"perplexity_ratio": 1.0136314994088744,
"token_logprob_rmse": 0.3936792438847804,
"p95_abs_token_logprob_delta": 0.7286462366951483
},
{
"id": "held-de-logic",
"language": "de",
"task": "reasoning",
"split": "heldout",
"tokens": 18,
"mean_kl": 0.09821001084494624,
"p95_kl": 0.44627087366421353,
"top1_agreement": 0.9444444444444444,
"delta_nll": 0.2068456879446453,
"reference_nll": 1.6221964826714386,
"candidate_nll": 1.8290421706160842,
"perplexity_ratio": 1.2297927852855994,
"token_logprob_rmse": 1.1028888501263066,
"p95_abs_token_logprob_delta": 2.326009618249263
},
{
"id": "held-en-code",
"language": "en",
"task": "code",
"split": "heldout",
"tokens": 15,
"mean_kl": 0.0020208906826010477,
"p95_kl": 0.010643171099672614,
"top1_agreement": 1.0,
"delta_nll": 0.018606391224066945,
"reference_nll": 0.05543094233111131,
"candidate_nll": 0.07403733355517825,
"perplexity_ratio": 1.018780568715705,
"token_logprob_rmse": 0.06903425695345042,
"p95_abs_token_logprob_delta": 0.08677259989155586
},
{
"id": "held-de-code",
"language": "de",
"task": "code",
"split": "heldout",
"tokens": 17,
"mean_kl": 0.004740728395334873,
"p95_kl": 0.016953669689652433,
"top1_agreement": 1.0,
"delta_nll": 0.00602756266674631,
"reference_nll": 0.12953522427201575,
"candidate_nll": 0.1355627869387621,
"perplexity_ratio": 1.0060457649760732,
"token_logprob_rmse": 0.028553592316214772,
"p95_abs_token_logprob_delta": 0.05007979207277108
},
{
"id": "held-en-translate",
"language": "en",
"task": "translation",
"split": "heldout",
"tokens": 13,
"mean_kl": 0.01907380808154005,
"p95_kl": 0.08411538577707617,
"top1_agreement": 1.0,
"delta_nll": -0.031849906075468386,
"reference_nll": 0.19652510627505373,
"candidate_nll": 0.16467520019958537,
"perplexity_ratio": 0.9686519599429306,
"token_logprob_rmse": 0.07828410366431433,
"p95_abs_token_logprob_delta": 0.1499787338874024
},
{
"id": "held-de-translate",
"language": "de",
"task": "translation",
"split": "heldout",
"tokens": 10,
"mean_kl": 0.008595051520903516,
"p95_kl": 0.04123864680645645,
"top1_agreement": 1.0,
"delta_nll": 0.007857006935255494,
"reference_nll": 0.08180677501291742,
"candidate_nll": 0.0896637819481729,
"perplexity_ratio": 1.0078879542121395,
"token_logprob_rmse": 0.06455211857627316,
"p95_abs_token_logprob_delta": 0.14345496760932136
},
{
"id": "held-en-window",
"language": "en",
"task": "window-crossing",
"split": "heldout",
"tokens": 7,
"mean_kl": 0.004304654982215195,
"p95_kl": 0.02033676101502991,
"top1_agreement": 1.0,
"delta_nll": 0.004617753317589899,
"reference_nll": 0.000790756349371949,
"candidate_nll": 0.0054085096669618474,
"perplexity_ratio": 1.0046284315706269,
"token_logprob_rmse": 0.011463303796780445,
"p95_abs_token_logprob_delta": 0.0217389328721245
},
{
"id": "held-de-window",
"language": "de",
"task": "window-crossing",
"split": "heldout",
"tokens": 7,
"mean_kl": 0.0015985640012336887,
"p95_kl": 0.006167841727117904,
"top1_agreement": 1.0,
"delta_nll": 0.0024853705928913716,
"reference_nll": 0.0006427372332391178,
"candidate_nll": 0.0031281078261304897,
"perplexity_ratio": 1.0024884616866907,
"token_logprob_rmse": 0.004931898894341712,
"p95_abs_token_logprob_delta": 0.009936099669039836
},
{
"id": "held-en-tool",
"language": "en",
"task": "tools",
"split": "heldout",
"tokens": 23,
"mean_kl": 0.015299446797203355,
"p95_kl": 0.00037061329993125187,
"top1_agreement": 1.0,
"delta_nll": 0.01681645772253505,
"reference_nll": 0.0004227024073699562,
"candidate_nll": 0.01723916012990501,
"perplexity_ratio": 1.0169586502878978,
"token_logprob_rmse": 0.08067722844033608,
"p95_abs_token_logprob_delta": 0.00024332947396456836
}
],
"calibration": {
"cases": 3,
"tokens": 53,
"mean_kl": 0.012610456800659971,
"top1_agreement": 0.9811320754716981,
"delta_nll": -0.04098590828618931,
"reference_nll": 0.5908046539082561,
"candidate_nll": 0.5498187456220667,
"perplexity_ratio": 0.9598426556805054,
"worst_case_mean_kl": 0.02211196501066192,
"worst_case_perplexity_ratio": 1.0396235161369776,
"passes_diagnostic_gates": true
},
"heldout": {
"cases": 13,
"tokens": 198,
"mean_kl": 0.034014873786288424,
"top1_agreement": 0.9848484848484849,
"delta_nll": 0.02446175432706457,
"reference_nll": 0.5104427418306431,
"candidate_nll": 0.5349044961577077,
"perplexity_ratio": 1.0247633975919717,
"worst_case_mean_kl": 0.09821001084494624,
"worst_case_perplexity_ratio": 1.2297927852855994,
"passes_diagnostic_gates": true
},
"by_language": {
"de": {
"cases": 6,
"tokens": 86,
"mean_kl": 0.04015818563770219,
"top1_agreement": 0.9767441860465116,
"delta_nll": 0.033099218248205504,
"reference_nll": 0.5687501790006285,
"candidate_nll": 0.6018493972488342,
"perplexity_ratio": 1.0336530914026796,
"worst_case_mean_kl": 0.09821001084494624,
"worst_case_perplexity_ratio": 1.2297927852855994,
"passes_diagnostic_gates": true
},
"en": {
"cases": 7,
"tokens": 112,
"mean_kl": 0.029297687900381415,
"top1_agreement": 0.9910714285714286,
"delta_nll": 0.017829415959045625,
"reference_nll": 0.4656709597179757,
"candidate_nll": 0.48350037567702125,
"perplexity_ratio": 1.0179893088478078,
"worst_case_mean_kl": 0.08366261552269878,
"worst_case_perplexity_ratio": 1.0548539067889309,
"passes_diagnostic_gates": true
}
},
"passes_all_heldout_gates": true
}
],
"selected_by_calibration": "/workspace/kolibri/results/mixed2",
"selected_passes_heldout": true
},
"controls": {
"q8": {
"schema_version": 1,
"reference": "/workspace/kolibri/results/reference",
"candidate": "/workspace/kolibri/results/q8",
"suite_sha256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b",
"gates": {
"mean_kl_max": 0.15,
"case_mean_kl_max": 0.3,
"perplexity_ratio_max": 1.1,
"case_perplexity_ratio_max": 1.25,
"top1_agreement_min": 0.8
},
"cases": [
{
"id": "cal-en",
"language": "en",
"task": "reasoning",
"split": "calibration",
"tokens": 19,
"mean_kl": 0.000831320104940008,
"p95_kl": 0.003566044949457418,
"top1_agreement": 1.0,
"delta_nll": 0.021826107363083186,
"reference_nll": 0.6549558352843168,
"candidate_nll": 0.6767819426473999,
"perplexity_ratio": 1.0220660392579246,
"token_logprob_rmse": 0.07582195593830164,
"p95_abs_token_logprob_delta": 0.1421862872241261
},
{
"id": "cal-de",
"language": "de",
"task": "reasoning",
"split": "calibration",
"tokens": 24,
"mean_kl": 0.001603534293301628,
"p95_kl": 0.008784063066375124,
"top1_agreement": 1.0,
"delta_nll": 0.008559198182151962,
"reference_nll": 0.7154404572220049,
"candidate_nll": 0.7239996554041569,
"perplexity_ratio": 1.0085959328505507,
"token_logprob_rmse": 0.14025923664807696,
"p95_abs_token_logprob_delta": 0.22186145759812326
},
{
"id": "cal-code",
"language": "en",
"task": "code",
"split": "calibration",
"tokens": 10,
"mean_kl": 6.357170376642824e-05,
"p95_kl": 0.0003159591968290493,
"top1_agreement": 1.0,
"delta_nll": 0.005462543200587691,
"reference_nll": 0.16979148134074312,
"candidate_nll": 0.17525402454133082,
"perplexity_ratio": 1.005477490093319,
"token_logprob_rmse": 0.012322163811100278,
"p95_abs_token_logprob_delta": 0.027687492222026037
},
{
"id": "held-en-fact",
"language": "en",
"task": "knowledge",
"split": "heldout",
"tokens": 9,
"mean_kl": 0.00034482231273267477,
"p95_kl": 0.0017243604907017165,
"top1_agreement": 1.0,
"delta_nll": 0.0023849090239729132,
"reference_nll": 0.050975090498802944,
"candidate_nll": 0.053359999522775856,
"perplexity_ratio": 1.0023877551816585,
"token_logprob_rmse": 0.009140663898531534,
"p95_abs_token_logprob_delta": 0.018648715096632212
},
{
"id": "held-de-fact",
"language": "de",
"task": "knowledge",
"split": "heldout",
"tokens": 9,
"mean_kl": 0.0005495791920162519,
"p95_kl": 0.0023680372240675812,
"top1_agreement": 1.0,
"delta_nll": 0.0016904004070122576,
"reference_nll": 0.121087029966858,
"candidate_nll": 0.12277743037387026,
"perplexity_ratio": 1.0016918299391606,
"token_logprob_rmse": 0.015197832860456789,
"p95_abs_token_logprob_delta": 0.029628444624761212
},
{
"id": "held-en-arithmetic",
"language": "en",
"task": "reasoning",
"split": "heldout",
"tokens": 24,
"mean_kl": 0.0010954928930793377,
"p95_kl": 0.005571998192872811,
"top1_agreement": 1.0,
"delta_nll": 0.02218834879075779,
"reference_nll": 1.0540350248188621,
"candidate_nll": 1.07622337360962,
"perplexity_ratio": 1.022436340984427,
"token_logprob_rmse": 0.11484926002334328,
"p95_abs_token_logprob_delta": 0.27475494599210337
},
{
"id": "held-de-arithmetic",
"language": "de",
"task": "reasoning",
"split": "heldout",
"tokens": 25,
"mean_kl": 0.002019731721887198,
"p95_kl": 0.002624062239918569,
"top1_agreement": 0.92,
"delta_nll": 0.0018361978977208585,
"reference_nll": 0.6239411885152131,
"candidate_nll": 0.6257773864129339,
"perplexity_ratio": 1.0018378847413822,
"token_logprob_rmse": 0.01882822111026853,
"p95_abs_token_logprob_delta": 0.0427697261119568
},
{
"id": "held-en-logic",
"language": "en",
"task": "reasoning",
"split": "heldout",
"tokens": 21,
"mean_kl": 0.004024918910447815,
"p95_kl": 0.013950529888978587,
"top1_agreement": 1.0,
"delta_nll": 0.043329627158220654,
"reference_nll": 1.09514205294828,
"candidate_nll": 1.1384716801065002,
"perplexity_ratio": 1.0442820618525301,
"token_logprob_rmse": 0.23321261386389014,
"p95_abs_token_logprob_delta": 0.1956837340293549
},
{
"id": "held-de-logic",
"language": "de",
"task": "reasoning",
"split": "heldout",
"tokens": 18,
"mean_kl": 0.007719610002441162,
"p95_kl": 0.030517342615799168,
"top1_agreement": 1.0,
"delta_nll": 0.01376963094001011,
"reference_nll": 1.6221964826714386,
"candidate_nll": 1.6359661136114487,
"perplexity_ratio": 1.013864868936757,
"token_logprob_rmse": 0.23384264800173754,
"p95_abs_token_logprob_delta": 0.4238459386839542
},
{
"id": "held-en-code",
"language": "en",
"task": "code",
"split": "heldout",
"tokens": 15,
"mean_kl": 0.0001617395942762165,
"p95_kl": 0.0007318381125298323,
"top1_agreement": 1.0,
"delta_nll": -0.005087732978421787,
"reference_nll": 0.05543094233111131,
"candidate_nll": 0.05034320935268952,
"perplexity_ratio": 0.994925187613547,
"token_logprob_rmse": 0.019635795534873525,
"p95_abs_token_logprob_delta": 0.023011587576001052
},
{
"id": "held-de-code",
"language": "de",
"task": "code",
"split": "heldout",
"tokens": 17,
"mean_kl": 0.00037997528762981884,
"p95_kl": 0.0017589766198376506,
"top1_agreement": 1.0,
"delta_nll": -0.01388152977240937,
"reference_nll": 0.12953522427201575,
"candidate_nll": 0.11565369449960639,
"perplexity_ratio": 0.9862143743836661,
"token_logprob_rmse": 0.05811108954992289,
"p95_abs_token_logprob_delta": 0.05713196952096862
},
{
"id": "held-en-translate",
"language": "en",
"task": "translation",
"split": "heldout",
"tokens": 13,
"mean_kl": 0.0008596627504175353,
"p95_kl": 0.004892968617108096,
"top1_agreement": 1.0,
"delta_nll": -0.002401907069333245,
"reference_nll": 0.19652510627505373,
"candidate_nll": 0.19412319920572046,
"perplexity_ratio": 0.9976009752013411,
"token_logprob_rmse": 0.012124488957275727,
"p95_abs_token_logprob_delta": 0.026367461032273245
},
{
"id": "held-de-translate",
"language": "de",
"task": "translation",
"split": "heldout",
"tokens": 10,
"mean_kl": 0.0003098095122019714,
"p95_kl": 0.0016094573251632105,
"top1_agreement": 1.0,
"delta_nll": 0.004698338021975006,
"reference_nll": 0.08180677501291742,
"candidate_nll": 0.08650511303489242,
"perplexity_ratio": 1.004709392517865,
"token_logprob_rmse": 0.015327589719428571,
"p95_abs_token_logprob_delta": 0.02739905572351341
},
{
"id": "held-en-window",
"language": "en",
"task": "window-crossing",
"split": "heldout",
"tokens": 7,
"mean_kl": 1.7140796708703047e-05,
"p95_kl": 7.785417717286156e-05,
"top1_agreement": 1.0,
"delta_nll": -8.289372083654321e-05,
"reference_nll": 0.000790756349371949,
"candidate_nll": 0.000707862628535406,
"perplexity_ratio": 0.999917109714753,
"token_logprob_rmse": 0.00021644695834224762,
"p95_abs_token_logprob_delta": 0.0004103308466218927
},
{
"id": "held-de-window",
"language": "de",
"task": "window-crossing",
"split": "heldout",
"tokens": 7,
"mean_kl": 1.327133543409924e-05,
"p95_kl": 4.869777090541761e-05,
"top1_agreement": 1.0,
"delta_nll": -6.126680745652733e-05,
"reference_nll": 0.0006427372332391178,
"candidate_nll": 0.0005814704257825904,
"perplexity_ratio": 0.999938735069316,
"token_logprob_rmse": 0.00014433799285058088,
"p95_abs_token_logprob_delta": 0.00028263960642897426
},
{
"id": "held-en-tool",
"language": "en",
"task": "tools",
"split": "heldout",
"tokens": 23,
"mean_kl": 0.0001650757965565867,
"p95_kl": 1.9025558629223244e-06,
"top1_agreement": 1.0,
"delta_nll": -0.0002596787279482234,
"reference_nll": 0.0004227024073699562,
"candidate_nll": 0.0001630236794217328,
"perplexity_ratio": 0.9997403549856544,
"token_logprob_rmse": 0.001241060191339088,
"p95_abs_token_logprob_delta": 3.0115593699186143e-05
}
],
"calibration": {
"cases": 3,
"tokens": 53,
"mean_kl": 0.0010361438126559152,
"top1_agreement": 1.0,
"delta_nll": 0.012730985439171784,
"reference_nll": 0.5908046539082561,
"candidate_nll": 0.6035356393474278,
"perplexity_ratio": 1.012812369433733,
"worst_case_mean_kl": 0.001603534293301628,
"worst_case_perplexity_ratio": 1.0220660392579246,
"passes_diagnostic_gates": true
},
"heldout": {
"cases": 13,
"tokens": 198,
"mean_kl": 0.0016943435662336783,
"top1_agreement": 0.98989898989899,
"delta_nll": 0.007420977189965464,
"reference_nll": 0.5104427418306431,
"candidate_nll": 0.5178637190206086,
"perplexity_ratio": 1.0074485808810656,
"worst_case_mean_kl": 0.007719610002441162,
"worst_case_perplexity_ratio": 1.0442820618525301,
"passes_diagnostic_gates": true
},
"by_language": {
"de": {
"cases": 6,
"tokens": 86,
"mean_kl": 0.0023725937230120054,
"top1_agreement": 0.9767441860465116,
"delta_nll": 0.0013900048193361493,
"reference_nll": 0.5687501790006285,
"candidate_nll": 0.5701401838199647,
"perplexity_ratio": 1.0013909713237985,
"worst_case_mean_kl": 0.007719610002441162,
"worst_case_perplexity_ratio": 1.013864868936757,
"passes_diagnostic_gates": true
},
"en": {
"cases": 7,
"tokens": 112,
"mean_kl": 0.0011735443387074634,
"top1_agreement": 1.0,
"delta_nll": 0.012051902403127259,
"reference_nll": 0.4656709597179757,
"candidate_nll": 0.47772286212110293,
"perplexity_ratio": 1.0121248192132222,
"worst_case_mean_kl": 0.004024918910447815,
"worst_case_perplexity_ratio": 1.0442820618525301,
"passes_diagnostic_gates": true
}
},
"passes_all_heldout_gates": true
},
"mixed2-down3": {
"schema_version": 1,
"reference": "/workspace/kolibri/results/reference",
"candidate": "/workspace/kolibri/results/mixed2-down3",
"suite_sha256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b",
"gates": {
"mean_kl_max": 0.15,
"case_mean_kl_max": 0.3,
"perplexity_ratio_max": 1.1,
"case_perplexity_ratio_max": 1.25,
"top1_agreement_min": 0.8
},
"cases": [
{
"id": "cal-en",
"language": "en",
"task": "reasoning",
"split": "calibration",
"tokens": 19,
"mean_kl": 0.022287621710548947,
"p95_kl": 0.1447904366487688,
"top1_agreement": 1.0,
"delta_nll": 0.004363044334306552,
"reference_nll": 0.6549558352843168,
"candidate_nll": 0.6593188796186235,
"perplexity_ratio": 1.004372576269949,
"token_logprob_rmse": 0.24286216850578204,
"p95_abs_token_logprob_delta": 0.5419241343555933
},
{
"id": "cal-de",
"language": "de",
"task": "reasoning",
"split": "calibration",
"tokens": 24,
"mean_kl": 0.007260988555919265,
"p95_kl": 0.029563822436890767,
"top1_agreement": 1.0,
"delta_nll": -0.03631071531037027,
"reference_nll": 0.7154404572220049,
"candidate_nll": 0.6791297419116346,
"perplexity_ratio": 0.9643406115352335,
"token_logprob_rmse": 0.21867267718165878,
"p95_abs_token_logprob_delta": 0.6251351139120079
},
{
"id": "cal-code",
"language": "en",
"task": "code",
"split": "calibration",
"tokens": 10,
"mean_kl": 0.007037385756701542,
"p95_kl": 0.03626246218563995,
"top1_agreement": 0.9,
"delta_nll": 0.019660886050099243,
"reference_nll": 0.16979148134074312,
"candidate_nll": 0.18945236739084237,
"perplexity_ratio": 1.0198554341747101,
"token_logprob_rmse": 0.14154199337545095,
"p95_abs_token_logprob_delta": 0.310981706679991
},
{
"id": "held-en-fact",
"language": "en",
"task": "knowledge",
"split": "heldout",
"tokens": 9,
"mean_kl": 0.01423419324148334,
"p95_kl": 0.06001400123658908,
"top1_agreement": 1.0,
"delta_nll": -0.02483512167025481,
"reference_nll": 0.050975090498802944,
"candidate_nll": 0.026139968828548125,
"perplexity_ratio": 0.9754707327551739,
"token_logprob_rmse": 0.043561690307823496,
"p95_abs_token_logprob_delta": 0.08128050235832697
},
{
"id": "held-de-fact",
"language": "de",
"task": "knowledge",
"split": "heldout",
"tokens": 9,
"mean_kl": 0.015779768106219696,
"p95_kl": 0.05441238955611876,
"top1_agreement": 1.0,
"delta_nll": 0.004252887056540055,
"reference_nll": 0.121087029966858,
"candidate_nll": 0.12533991702339808,
"perplexity_ratio": 1.0042619434147027,
"token_logprob_rmse": 0.10819031163989325,
"p95_abs_token_logprob_delta": 0.21885599685849833
},
{
"id": "held-en-arithmetic",
"language": "en",
"task": "reasoning",
"split": "heldout",
"tokens": 24,
"mean_kl": 0.016232595839573278,
"p95_kl": 0.11285045020375233,
"top1_agreement": 0.9166666666666666,
"delta_nll": 0.016184163172342326,
"reference_nll": 1.0540350248188621,
"candidate_nll": 1.0702191879912044,
"perplexity_ratio": 1.016315836120913,
"token_logprob_rmse": 0.31559493406355205,
"p95_abs_token_logprob_delta": 0.6290415628914359
},
{
"id": "held-de-arithmetic",
"language": "de",
"task": "reasoning",
"split": "heldout",
"tokens": 25,
"mean_kl": 0.01943047836619761,
"p95_kl": 0.07637523146023015,
"top1_agreement": 0.88,
"delta_nll": -0.04330305666654819,
"reference_nll": 0.6239411885152131,
"candidate_nll": 0.5806381318486648,
"perplexity_ratio": 0.9576211326187573,
"token_logprob_rmse": 0.17632810315918787,
"p95_abs_token_logprob_delta": 0.33836131232658245
},
{
"id": "held-en-logic",
"language": "en",
"task": "reasoning",
"split": "heldout",
"tokens": 21,
"mean_kl": 0.059990803453216,
"p95_kl": 0.23054326489766488,
"top1_agreement": 1.0,
"delta_nll": 0.08638857535374063,
"reference_nll": 1.09514205294828,
"candidate_nll": 1.1815306283020202,
"perplexity_ratio": 1.090229882470684,
"token_logprob_rmse": 0.29448011242288624,
"p95_abs_token_logprob_delta": 0.5057039679067198
},
{
"id": "held-de-logic",
"language": "de",
"task": "reasoning",
"split": "heldout",
"tokens": 18,
"mean_kl": 0.06567859401424665,
"p95_kl": 0.2074714477173848,
"top1_agreement": 0.8888888888888888,
"delta_nll": 0.043162703830147045,
"reference_nll": 1.6221964826714386,
"candidate_nll": 1.6653591865015855,
"perplexity_ratio": 1.0441077613631318,
"token_logprob_rmse": 0.6398019877607051,
"p95_abs_token_logprob_delta": 1.4899336172976605
},
{
"id": "held-en-code",
"language": "en",
"task": "code",
"split": "heldout",
"tokens": 15,
"mean_kl": 0.0024156101823623152,
"p95_kl": 0.011009395769025702,
"top1_agreement": 1.0,
"delta_nll": 0.022984849609144032,
"reference_nll": 0.05543094233111131,
"candidate_nll": 0.07841579194025534,
"perplexity_ratio": 1.0232510367766303,
"token_logprob_rmse": 0.08830685147470625,
"p95_abs_token_logprob_delta": 0.10407475418039441
},
{
"id": "held-de-code",
"language": "de",
"task": "code",
"split": "heldout",
"tokens": 17,
"mean_kl": 0.00216444305865071,
"p95_kl": 0.010116049034585475,
"top1_agreement": 1.0,
"delta_nll": 0.040136892596446226,
"reference_nll": 0.12953522427201575,
"candidate_nll": 0.169672116868462,
"perplexity_ratio": 1.0409532632343015,
"token_logprob_rmse": 0.15692891481356833,
"p95_abs_token_logprob_delta": 0.15947780996348082
},
{
"id": "held-en-translate",
"language": "en",
"task": "translation",
"split": "heldout",
"tokens": 13,
"mean_kl": 0.008009429404846087,
"p95_kl": 0.026895666336722038,
"top1_agreement": 1.0,
"delta_nll": 0.002165151350177041,
"reference_nll": 0.19652510627505373,
"candidate_nll": 0.19869025762523077,
"perplexity_ratio": 1.0021674969829395,
"token_logprob_rmse": 0.04188819246972073,
"p95_abs_token_logprob_delta": 0.08659276918809068
},
{
"id": "held-de-translate",
"language": "de",
"task": "translation",
"split": "heldout",
"tokens": 10,
"mean_kl": 0.0017100095020701194,
"p95_kl": 0.007330455703149122,
"top1_agreement": 1.0,
"delta_nll": 0.011557044823657821,
"reference_nll": 0.08180677501291742,
"candidate_nll": 0.09336381983657524,
"perplexity_ratio": 1.0116240854812253,
"token_logprob_rmse": 0.027525713405869533,
"p95_abs_token_logprob_delta": 0.05976277541536444
},
{
"id": "held-en-window",
"language": "en",
"task": "window-crossing",
"split": "heldout",
"tokens": 7,
"mean_kl": 0.0001233754825823752,
"p95_kl": 0.0004092696933154428,
"top1_agreement": 1.0,
"delta_nll": 3.406186505092825e-05,
"reference_nll": 0.000790756349371949,
"candidate_nll": 0.0008248182144228774,
"perplexity_ratio": 1.000034062445163,
"token_logprob_rmse": 0.0002507851637560797,
"p95_abs_token_logprob_delta": 0.0004831773381365006
},
{
"id": "held-de-window",
"language": "de",
"task": "window-crossing",
"split": "heldout",
"tokens": 7,
"mean_kl": 0.00015027531108951895,
"p95_kl": 0.0005136700549551299,
"top1_agreement": 1.0,
"delta_nll": 0.00012662974242994476,
"reference_nll": 0.0006427372332391178,
"candidate_nll": 0.0007693669756690624,
"perplexity_ratio": 1.0001266377603142,
"token_logprob_rmse": 0.00020826296083695104,
"p95_abs_token_logprob_delta": 0.00040972300826712924
},
{
"id": "held-en-tool",
"language": "en",
"task": "tools",
"split": "heldout",
"tokens": 23,
"mean_kl": 0.0001475047521852483,
"p95_kl": 0.00020519512656532433,
"top1_agreement": 1.0,
"delta_nll": -0.0002006917719244418,
"reference_nll": 0.0004227024073699562,
"candidate_nll": 0.00022201063544551435,
"perplexity_ratio": 0.999799328365322,
"token_logprob_rmse": 0.00107958158592073,
"p95_abs_token_logprob_delta": 0.0004752612138676704
}
],
"calibration": {
"cases": 3,
"tokens": 53,
"mean_kl": 0.012605705573764298,
"top1_agreement": 0.9811320754716981,
"delta_nll": -0.011168876690491877,
"reference_nll": 0.5908046539082561,
"candidate_nll": 0.5796357772177642,
"perplexity_ratio": 0.9888932636516697,
"worst_case_mean_kl": 0.022287621710548947,
"worst_case_perplexity_ratio": 1.0198554341747101,
"passes_diagnostic_gates": true
},
"heldout": {
"cases": 13,
"tokens": 198,
"mean_kl": 0.01912652526808163,
"top1_agreement": 0.9646464646464646,
"delta_nll": 0.014540496989313171,
"reference_nll": 0.5104427418306431,
"candidate_nll": 0.5249832388199562,
"perplexity_ratio": 1.0146467242569452,
"worst_case_mean_kl": 0.06567859401424665,
"worst_case_perplexity_ratio": 1.090229882470684,
"passes_diagnostic_gates": true
},
"by_language": {
"de": {
"cases": 6,
"tokens": 86,
"mean_kl": 0.02168537347165985,
"top1_agreement": 0.9418604651162791,
"delta_nll": 0.006179212399546233,
"reference_nll": 0.5687501790006285,
"candidate_nll": 0.5749293914001747,
"perplexity_ratio": 1.006198343116441,
"worst_case_mean_kl": 0.06567859401424665,
"worst_case_perplexity_ratio": 1.0441077613631318,
"passes_diagnostic_gates": true
},
"en": {
"cases": 7,
"tokens": 112,
"mean_kl": 0.017161695397476927,
"top1_agreement": 0.9821428571428571,
"delta_nll": 0.020960769085027076,
"reference_nll": 0.4656709597179757,
"candidate_nll": 0.4866317288030026,
"perplexity_ratio": 1.0211819889479055,
"worst_case_mean_kl": 0.059990803453216,
"worst_case_perplexity_ratio": 1.090229882470684,
"passes_diagnostic_gates": true
}
},
"passes_all_heldout_gates": true
}
},
"independent_bf16_crosscheck": {
"case_id": "cal-en",
"suite_sha256": "f88d4a83bc2bba693dd3645740725890837eb5d149b9e5a6b9d102f97957d23b",
"independent_reference": "independent PyTorch operators; original BF16 tensors; no TF32",
"tokens": 19,
"full_vocabulary_mean_kl": 0.0019748527884482607,
"top1_agreement": 1.0,
"target_logprob_rmse": 0.09079987054087682,
"target_logprob_max_abs_delta": 0.3578435375267972,
"torch_nll": 0.6771678634604793,
"native_nll": 0.6549558352843168,
"logits_rmse": 0.36495610824735525,
"logits_max_abs_delta": 7.870337247848511,
"torch_logits_sha256": "41e122f3c88551dfab59641fc48785748d3ad70eac2cb3305144bd9484699e1b",
"native_logits_sha256": "58421f32fa07c62e338926ce3a98646b28a221ba8825401732e8fd6ce9d0755e"
},
"validation": {
"repository_gate": "scripts/check.sh",
"status": "passed",
"signed_affine_refit_tests": 3,
"logprob_comparator_tests": 4,
"apple_chat_integration": "passed; pinned tokenizer and 41 MB synthetic mixed-quantized checkpoint, tool history and token logprobs",
"apple_full_checkpoint": "not run; ordinary CLI blocked by less than 16 GB memory admission headroom",
"xctest_count": 5304,
"xctest_skips": 439,
"xctest_failures": 0,
"swift_testing_count": 191,
"gate_log_sha256": "02ba046b6e98edfe17a95476ecfb9d32fcbb3177583e1aa2b5cb32a80da50e32"
}
}