Hugging Face's logo

LordNeel
/
Ornith-1.0-35B-GGUF-llamacpp-tp1

Text Generation
GGUF
llama-cpp
quantized
coding
mtp
qwen3_5_moe
single-gpu
tp1
conversational
Model card Files
xet
 Community
2
LordNeel commited on
Commit
355bc08
·
verified ·
1 Parent(s): 85d31dc

Add SFT strict-output repair plan and artifacts

Browse files
Files changed (24) hide show
  1. ARTIFACT_MANIFEST.json +1992 -300
  2. MODEL_INVESTIGATION_HARNESS.md +24 -0
  3. PROJECT_STATUS_AUDIT.md +3 -1
  4. README.md +152 -416
  5. RELEASE_AUDIT.md +77 -69
  6. SFT_REMEDIATION_PLAN.md +83 -12
  7. configs/sft_repair_blend.yaml +57 -0
  8. configs/training_plan.yaml +4 -0
  9. data/eval/README.md +19 -0
  10. data/eval/coding_sft_repair_eval_messages.jsonl +0 -0
  11. data/train/README.md +41 -0
  12. data/train/coding_sft_repair_messages.jsonl +0 -0
  13. probes/sft-data-validation-prod-repair.json +160 -0
  14. probes/sft-format-audit-prod-repair.json +673 -0
  15. probes/sft-repair-blend-eval.json +21 -0
  16. probes/sft-repair-blend-train.json +21 -0
  17. probes/sft-repair-set.json +39 -0
  18. scripts/build_sft_blend.py +136 -0
  19. scripts/build_sft_repair_set.py +260 -0
  20. scripts/eval_lora_behavior.py +9 -0
  21. scripts/launch_sft_prod_20k_gpu1.sh +78 -0
  22. scripts/render_sft_jsonl.py +98 -0
  23. scripts/train_sft_lora.py +159 -0
  24. scripts/validate_sft_jsonl.py +489 -0
ARTIFACT_MANIFEST.json CHANGED
@@ -1,243 +1,11 @@
1
  {
2
- "audited_remote_head_before_manifest_upload": "78e2847dec3ad84226708b29f0afeb8f751d9d33",
3
- "generated_at": "2026-06-27T23:43:34.525894+00:00",
4
- "harness_bundle": [
5
- {
6
- "blob_id": "6c9743c1480bd2c9bf5308626ffa1bd20eb56830",
7
- "bytes": 7346,
8
- "path": "MODEL_INVESTIGATION_HARNESS.md",
9
- "planned_in_this_upload": true,
10
- "published": true
11
- },
12
- {
13
- "blob_id": "b526e5c40a476b8edb38591dd2d72ad818a087a2",
14
- "bytes": 134,
15
- "path": "requirements-lab.txt",
16
- "planned_in_this_upload": false,
17
- "published": true
18
- },
19
- {
20
- "blob_id": "21ddcf93dfaae55111281e925f0c9974be2efd7b",
21
- "bytes": 4597,
22
- "path": "configs/model_behavior_suite.yaml",
23
- "planned_in_this_upload": false,
24
- "published": true
25
- },
26
- {
27
- "blob_id": "63d4c96c5aaace66eff3a86ce72c379c948422bc",
28
- "bytes": 2287,
29
- "path": "configs/serving_matrix.yaml",
30
- "planned_in_this_upload": false,
31
- "published": true
32
- },
33
- {
34
- "blob_id": "6ed937004209b063a95372ece3422f013996f3fe",
35
- "bytes": 3621,
36
- "path": "configs/quantization_matrix.yaml",
37
- "planned_in_this_upload": false,
38
- "published": true
39
- },
40
- {
41
- "blob_id": "93aa261c54ddf6804d2f62bb2f52e4d3a1d4a668",
42
- "bytes": 842,
43
- "path": "configs/experiment_lanes.yaml",
44
- "planned_in_this_upload": false,
45
- "published": true
46
- },
47
- {
48
- "blob_id": "8fd4bbc2a67c029bf0b360d0901ec200292887e6",
49
- "bytes": 3192,
50
- "path": "configs/training_plan.yaml",
51
- "planned_in_this_upload": false,
52
- "published": true
53
- },
54
- {
55
- "blob_id": "7b1a137d8ffe3d79652141a04cd9477bef67d831",
56
- "bytes": 4573,
57
- "path": "scripts/run_model_investigation.py",
58
- "planned_in_this_upload": false,
59
- "published": true
60
- },
61
- {
62
- "blob_id": "9a1232550356afcbde16a6eaf54a6bdbf466947d",
63
- "bytes": 16017,
64
- "path": "scripts/audit_model.py",
65
- "planned_in_this_upload": false,
66
- "published": true
67
- },
68
- {
69
- "blob_id": "846f1500d0f2aceeb2c97388d3e4b68439742e18",
70
- "bytes": 2200,
71
- "path": "scripts/check_audit_gates.py",
72
- "planned_in_this_upload": false,
73
- "published": true
74
- },
75
- {
76
- "blob_id": "d723430849c22f53b2a4d91a5fea9a427819edf0",
77
- "bytes": 5117,
78
- "path": "scripts/summarize_investigation.py",
79
- "planned_in_this_upload": false,
80
- "published": true
81
- },
82
- {
83
- "blob_id": "700ec81ab96b0573afdac1ffbaa84a9f0f94b903",
84
- "bytes": 16665,
85
- "path": "scripts/probe_model_behavior.py",
86
- "planned_in_this_upload": false,
87
- "published": true
88
- },
89
- {
90
- "blob_id": "3b9acd1e52f8203a59f1d72369f148e2bc672304",
91
- "bytes": 7877,
92
- "path": "scripts/probe_openai_contract.py",
93
- "planned_in_this_upload": false,
94
- "published": true
95
- },
96
- {
97
- "blob_id": "a5d63caae222f5df761668c58d36a158fa73653f",
98
- "bytes": 8169,
99
- "path": "scripts/bench_openai.py",
100
- "planned_in_this_upload": false,
101
- "published": true
102
- },
103
- {
104
- "blob_id": "1fefc910e4b1ebf41ca54c332d404918583f97a6",
105
- "bytes": 4379,
106
- "path": "scripts/collect_next_token_logprobs.py",
107
- "planned_in_this_upload": false,
108
- "published": true
109
- },
110
- {
111
- "blob_id": "aaf11e0827f98fe413959c07296cf1021d4def71",
112
- "bytes": 6926,
113
- "path": "scripts/compare_kld_logprobs.py",
114
- "planned_in_this_upload": false,
115
- "published": true
116
- },
117
- {
118
- "blob_id": "6e674b480252381cd367b480095ffa0d4e65df3a",
119
- "bytes": 5718,
120
- "path": "scripts/check_mtp_support.py",
121
- "planned_in_this_upload": false,
122
- "published": true
123
- },
124
- {
125
- "blob_id": null,
126
- "bytes": null,
127
- "path": "scripts/audit_sft_format.py",
128
- "planned_in_this_upload": true,
129
- "published": false
130
- },
131
- {
132
- "blob_id": "a1df912ce4ac635b80d1927dbe595d42a4c7d685",
133
- "bytes": 22567,
134
- "path": "scripts/audit_gguf_vllm_materialization.py",
135
- "planned_in_this_upload": false,
136
- "published": true
137
- }
138
- ],
139
- "key_benchmark_docs": [
140
- {
141
- "blob_id": "5d2afc62b5b47faeab64cb323a741b1dcbd03bbb",
142
- "bytes": 15106,
143
- "path": "benchmarks/llamacpp-quant-benchmarks.md",
144
- "planned_in_this_upload": false,
145
- "published": true
146
- },
147
- {
148
- "blob_id": "376892b1847c3b98e8bd00b690466970de9fc910",
149
- "bytes": 3294,
150
- "path": "benchmarks/kld-quant-vs-bf16-top64.md",
151
- "planned_in_this_upload": false,
152
- "published": true
153
- },
154
- {
155
- "blob_id": "df0ab1ae4916ec9426ce7b58452ef0a979944af4",
156
- "bytes": 5566,
157
- "path": "benchmarks/llamacpp-iq4-xs-mtp-graft-adaptive-profile.md",
158
- "planned_in_this_upload": false,
159
- "published": true
160
- },
161
- {
162
- "blob_id": "18e0e00637602514a0c8b9d26805fe84e80bc471",
163
- "bytes": 5962,
164
- "path": "benchmarks/sft-prod-20k-preflight.md",
165
- "planned_in_this_upload": false,
166
- "published": true
167
- }
168
- ],
169
- "key_probe_docs": [
170
- {
171
- "blob_id": "470ec1394c86b38eedb2a97ae32530b5a3329f48",
172
- "bytes": 28352,
173
- "path": "probes/investigation-ornith-q4-live.json",
174
- "planned_in_this_upload": false,
175
- "published": true
176
- },
177
- {
178
- "blob_id": "a6fe7fccd66e2d29e1d02571c54fbc1f415cf1fd",
179
- "bytes": 1304,
180
- "path": "probes/investigation-ornith-q4-live.md",
181
- "planned_in_this_upload": false,
182
- "published": true
183
- },
184
- {
185
- "blob_id": "ac4882041cf88837beecde1ead7418fd5262b28c",
186
- "bytes": 1252,
187
- "path": "probes/sft-prod-20k-stopped.json",
188
- "planned_in_this_upload": false,
189
- "published": true
190
- },
191
- {
192
- "blob_id": null,
193
- "bytes": null,
194
- "path": "probes/sft-format-audit-prod.json",
195
- "planned_in_this_upload": true,
196
- "published": false
197
- },
198
- {
199
- "blob_id": "c0376b5f05405e52dc747a1017c1860b45229c34",
200
- "bytes": 21512,
201
- "path": "probes/eval-lora-behavior-checkpoint-8000-fixed.json",
202
- "planned_in_this_upload": false,
203
- "published": true
204
- },
205
- {
206
- "blob_id": "041fb5bb56dc25cf23b634437948011ae77ac1da",
207
- "bytes": 1049,
208
- "path": "probes/eval-lora-behavior-checkpoint-8000-fixed.gate.log",
209
- "planned_in_this_upload": false,
210
- "published": true
211
- }
212
- ],
213
  "known_remaining_gaps": [
214
- "No release-candidate SFT/post-training adapter exists because the 20k run was stopped and checkpoint-8000 failed behavior gates.",
 
215
  "The integrated MTP profile is recommended adaptively; always-on MTP is still not a saturated c16 throughput win.",
216
  "vLLM GGUF remains marked unusable for this model family in the documented probe; llama.cpp is the supported GGUF backend."
217
  ],
218
- "mtp_status": {
219
- "high_concurrency_policy": "adaptive throttle; disable MTP when saturated",
220
- "integrated_graft_artifact": "ornith-1.0-35b-IQ4_XS-MTP-graft-headQ6.gguf",
221
- "low_concurrency_speedup": "c1/128 AR 221.8 tok/s vs MTP 279.4 tok/s; adaptive c1/128 319.5 tok/s",
222
- "reference_report": "benchmarks/llamacpp-iq4-xs-mtp-graft-adaptive-profile.md"
223
- },
224
- "planned_upload_paths": [
225
- "ARTIFACT_MANIFEST.json",
226
- "MODEL_INVESTIGATION_HARNESS.md",
227
- "PROJECT_STATUS_AUDIT.md",
228
- "README.md",
229
- "RELEASE_AUDIT.md",
230
- "SFT_REMEDIATION_PLAN.md",
231
- "probes/sft-format-audit-prod.json",
232
- "scripts/audit_sft_format.py"
233
- ],
234
- "project_status_audit": {
235
- "blob_id": "9d80bde20606281e4014dca83774603f30d71ba3",
236
- "bytes": 5314,
237
- "path": "PROJECT_STATUS_AUDIT.md",
238
- "planned_in_this_upload": true,
239
- "published": true
240
- },
241
  "published_gguf": [
242
  {
243
  "blob_id": "22f67bcceb94b8b5678374fbc318c64f4f09efd0",
@@ -338,74 +106,1998 @@
338
  "published": true
339
  }
340
  ],
341
- "quant_status": {
342
- "kld_report": "benchmarks/kld-quant-vs-bf16-top64.md",
343
- "lowest_memory_validated": "ornith-1.0-35b-Q3_K_M.gguf",
344
- "middle_footprint_validated": "ornith-1.0-35b-IQ4_XS.gguf",
345
- "speed_focused_validated": "ornith-1.0-35b-Q4_K_M.gguf"
346
- },
347
- "readme_reference_check": {
348
- "context_gguf_refs_not_expected_in_repo": [
349
- "ornith-1.0-35b-bf16.gguf"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
350
  ],
351
- "missing_advertised_gguf_refs": [],
352
- "missing_advertised_non_gguf_refs": [],
353
- "missing_non_gguf_refs": []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
354
  },
355
  "repo_id": "LordNeel/Ornith-1.0-35B-GGUF-llamacpp-tp1",
356
- "schema_version": 1,
357
- "serving_status": {
358
- "multi_gpu_serving": "out_of_scope",
359
- "recommended_integrated_mtp_profile": "IQ4_XS-MTP-graft-headQ6 with LLAMA_SPEC_MAX_DRAFTING_SLOTS=1 and SPEC_DRAFT_N_MAX=2",
360
- "recommended_speed_profile": "llama.cpp Q4_K_M with REASONING=off",
361
- "scope": "single_gpu_tp1_only"
362
- },
363
- "sft_remediation": {
364
- "format_audit_report": {
365
- "blob_id": null,
366
- "bytes": null,
367
- "path": "probes/sft-format-audit-prod.json",
368
- "planned_in_this_upload": true,
369
- "published": false
370
  },
371
- "format_audit_script": {
372
- "blob_id": null,
373
- "bytes": null,
374
- "path": "scripts/audit_sft_format.py",
375
- "planned_in_this_upload": true,
376
- "published": false
377
- },
378
- "key_findings": {
379
- "assistant_code_fence_pct": 100.0,
380
- "assistant_contains_heres_pct_approx": 56.0,
381
- "assistant_short_under_80_chars_pct": 0.0,
382
- "rendered_empty_think_pct": 100.0
383
- },
384
- "plan": {
385
- "blob_id": null,
386
- "bytes": null,
 
 
 
 
 
 
387
  "path": "SFT_REMEDIATION_PLAN.md",
388
- "planned_in_this_upload": true,
389
- "published": false
390
- }
391
- },
392
- "sft_status": {
393
- "adapter_released": false,
394
- "checkpoint_9000_exists": false,
395
- "hf_stop_record": "probes/sft-prod-20k-stopped.json",
396
- "latest_durable_checkpoint": "artifacts/train/ornith-35b-coding-lora-prod-20k/checkpoint-8000",
397
- "non_durable_last_step_seen_in_log": 8673,
398
- "release_blocker": "checkpoint-8000 behavior gate failed 4 checks; no adapter should be published from this run",
399
- "training_state": "stopped_by_user"
400
- },
401
- "summary": {
402
- "readme_advertised_file_refs": 66,
403
- "readme_advertised_gguf_refs": 14,
404
- "readme_missing_advertised_gguf_refs": 0,
405
- "readme_missing_advertised_non_gguf_refs": 0,
406
- "readme_missing_non_gguf_refs": 0,
407
- "readme_non_gguf_refs": 52,
408
- "remote_file_count_before_manifest_upload": 200,
409
- "status": "pass"
410
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
411
  }
 
1
  {
2
+ "generated_at": "2026-06-27T23:59:03.593140+00:00",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  "known_remaining_gaps": [
4
+ "No release-candidate SFT/post-training adapter exists because SFT was stopped and checkpoint-8000 failed behavior gates.",
5
+ "The repair blend is prepared and validated CPU-side, but no resumed GPU training has been run from it.",
6
  "The integrated MTP profile is recommended adaptively; always-on MTP is still not a saturated c16 throughput win.",
7
  "vLLM GGUF remains marked unusable for this model family in the documented probe; llama.cpp is the supported GGUF backend."
8
  ],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  "published_gguf": [
10
  {
11
  "blob_id": "22f67bcceb94b8b5678374fbc318c64f4f09efd0",
 
106
  "published": true
107
  }
108
  ],
109
+ "remote_file_count_before_upload": 203,
110
+ "remote_head_before_upload": "85d31dc1d38b8aed268c267550508b0e5a455cce",
111
+ "repair_artifacts": {
112
+ "blend_eval_rows": 2312,
113
+ "blend_rows": 44096,
114
+ "blend_train_format": {
115
+ "assistant_code_fence": 90.7112,
116
+ "assistant_contains_heres": 50.9343,
117
+ "assistant_mentions_json": 3.3382,
118
+ "assistant_short_under_80_chars": 7.5517,
119
+ "assistant_starts_heres": 25.1655,
120
+ "assistant_starts_the_function": 0.6894,
121
+ "assistant_starts_to_solve": 3.057,
122
+ "rendered_contains_qwen_markers": 0.0,
123
+ "rendered_contains_think": 0.0,
124
+ "rendered_empty_think": 0.0
125
+ },
126
+ "config": "configs/sft_repair_blend.yaml",
127
+ "published_synthetic_data": [
128
+ "data/train/coding_sft_repair_messages.jsonl",
129
+ "data/eval/coding_sft_repair_eval_messages.jsonl"
130
+ ],
131
+ "repair_set_format": {
132
+ "assistant_code_fence": 0.0,
133
+ "assistant_contains_heres": 0.0,
134
+ "assistant_mentions_json": 0.0,
135
+ "assistant_short_under_80_chars": 81.2988,
136
+ "assistant_starts_heres": 0.0,
137
+ "assistant_starts_the_function": 0.0,
138
+ "assistant_starts_to_solve": 0.0,
139
+ "rendered_contains_qwen_markers": 0.0,
140
+ "rendered_contains_think": 0.0,
141
+ "rendered_empty_think": 0.0
142
+ },
143
+ "repair_set_rows": {
144
+ "eval": 512,
145
+ "train": 4096
146
+ },
147
+ "reports": [
148
+ "probes/sft-repair-set.json",
149
+ "probes/sft-repair-blend-train.json",
150
+ "probes/sft-repair-blend-eval.json",
151
+ "probes/sft-data-validation-prod-repair.json",
152
+ "probes/sft-format-audit-prod-repair.json"
153
  ],
154
+ "scripts": [
155
+ "scripts/build_sft_repair_set.py",
156
+ "scripts/build_sft_blend.py",
157
+ "scripts/render_sft_jsonl.py",
158
+ "scripts/validate_sft_jsonl.py",
159
+ "scripts/eval_lora_behavior.py",
160
+ "scripts/launch_sft_prod_20k_gpu1.sh"
161
+ ],
162
+ "sft_status": "stopped; no GPU training is running; no LoRA adapter released",
163
+ "validation_issue_count": 0,
164
+ "validation_warnings": [
165
+ {
166
+ "detail": "107 rows exceed 1024 tokens",
167
+ "name": "train:max_length",
168
+ "recommendation": "Chunk, shorten, or raise max_length only if memory allows it.",
169
+ "status": "warn"
170
+ },
171
+ {
172
+ "detail": "7 rows exceed 1024 tokens",
173
+ "name": "eval:max_length",
174
+ "recommendation": "Chunk, shorten, or raise max_length only if memory allows it.",
175
+ "status": "warn"
176
+ }
177
+ ]
178
  },
179
  "repo_id": "LordNeel/Ornith-1.0-35B-GGUF-llamacpp-tp1",
180
+ "staged_file_count_excluding_generated_audit_files": 192,
181
+ "staged_files": [
182
+ {
183
+ "bytes": 8578,
184
+ "path": "MODEL_INVESTIGATION_HARNESS.md",
185
+ "remote_before_upload": {
186
+ "blob_id": "71f275a2bd5cbf215f917156b034efd8d06a489c",
187
+ "bytes": 7695,
188
+ "exists": true
189
+ },
190
+ "sha256": "4b5dd3498ba28a557c1222d945b1dc150f40788523d7258c2854157dd6dafcd4"
 
 
 
191
  },
192
+ {
193
+ "bytes": 6172,
194
+ "path": "PROJECT_STATUS_AUDIT.md",
195
+ "remote_before_upload": {
196
+ "blob_id": "a99b8a6a04d6ac04e277adbbef0742cf5759fb7d",
197
+ "bytes": 5744,
198
+ "exists": true
199
+ },
200
+ "sha256": "46b69c0730ee6d292233f090dd7e970dea49b2917d03f3afc5936fe03def8206"
201
+ },
202
+ {
203
+ "bytes": 7723,
204
+ "path": "README.md",
205
+ "remote_before_upload": {
206
+ "blob_id": "f9608f319b2043d2643ade4f0993092e692f2ed7",
207
+ "bytes": 29986,
208
+ "exists": true
209
+ },
210
+ "sha256": "ed7a12be4abf32d164d300a260c30803c82cdada9af78c4e74bf10a82d084e6e"
211
+ },
212
+ {
213
+ "bytes": 6592,
214
  "path": "SFT_REMEDIATION_PLAN.md",
215
+ "remote_before_upload": {
216
+ "blob_id": "06fdcae256f8588439365b61ac22cdaf9deeb595",
217
+ "bytes": 3496,
218
+ "exists": true
219
+ },
220
+ "sha256": "549d1edbee844234574116ad127903d1bb4a69418a8039f4b6fae7c01b38109b"
221
+ },
222
+ {
223
+ "bytes": 3294,
224
+ "path": "benchmarks/kld-quant-vs-bf16-top64.md",
225
+ "remote_before_upload": {
226
+ "blob_id": "376892b1847c3b98e8bd00b690466970de9fc910",
227
+ "bytes": 3294,
228
+ "exists": true
229
+ },
230
+ "sha256": "82d0722c04bfd08d911342e0599e0fa1b71b104cc832dc8812b3e981a62ceb47"
231
+ },
232
+ {
233
+ "bytes": 5414,
234
+ "path": "benchmarks/llamacpp-iq4-xs-mtp-bootstrap-profile.md",
235
+ "remote_before_upload": {
236
+ "blob_id": "d9ef82a3e750d4e14c1b78b0948cc0b672b54537",
237
+ "bytes": 5414,
238
+ "exists": true
239
+ },
240
+ "sha256": "4cc8f3f7893fc4f9f2386fbbf4ed27cb73ecdf968a429cdeefcad81ba5613006"
241
+ },
242
+ {
243
+ "bytes": 29733,
244
+ "path": "benchmarks/llamacpp-iq4-xs-mtp-chaincorr-profile.md",
245
+ "remote_before_upload": {
246
+ "blob_id": "18d36d59fe066f6065ffc9c417419df9ab2bce9c",
247
+ "bytes": 29733,
248
+ "exists": true
249
+ },
250
+ "sha256": "e26bd22133c7f91755ab3dd25a984e004fd5a77057a64f77a2db3d356d8199ac"
251
+ },
252
+ {
253
+ "bytes": 5566,
254
+ "path": "benchmarks/llamacpp-iq4-xs-mtp-graft-adaptive-profile.md",
255
+ "remote_before_upload": {
256
+ "blob_id": "df0ab1ae4916ec9426ce7b58452ef0a979944af4",
257
+ "bytes": 5566,
258
+ "exists": true
259
+ },
260
+ "sha256": "a3103f4373fd120dcddb3982b7afa66e3e82415ee8f3be97aac4437ce949471e"
261
+ },
262
+ {
263
+ "bytes": 3829,
264
+ "path": "benchmarks/llamacpp-iq4-xs-profile.md",
265
+ "remote_before_upload": {
266
+ "blob_id": "d86d542d50447e01feebc2508a0b21c8cdfcbf7c",
267
+ "bytes": 3829,
268
+ "exists": true
269
+ },
270
+ "sha256": "1b96697d747d54ef1eefb6c6260d4bb33a7924a6b874b382d12efbfa2461c6c6"
271
+ },
272
+ {
273
+ "bytes": 2638,
274
+ "path": "benchmarks/llamacpp-q3-k-m-profile.md",
275
+ "remote_before_upload": {
276
+ "blob_id": "b11c69c02ffbd3f14f98fd1a47010a4a43134b9f",
277
+ "bytes": 2638,
278
+ "exists": true
279
+ },
280
+ "sha256": "f5a3fbb3328dd4df6fa4c676754f867a41bbeeb5ba3f628846725d3abada067d"
281
+ },
282
+ {
283
+ "bytes": 1932,
284
+ "path": "benchmarks/llamacpp-q4-reasoning-off-fix.md",
285
+ "remote_before_upload": {
286
+ "blob_id": "fc3ffff7ad7cd261e62b89fcda7b3d06e663076f",
287
+ "bytes": 1932,
288
+ "exists": true
289
+ },
290
+ "sha256": "4a66511468c6bc150faccd4fcb0dfabbf500f005a0fd8c682acb4877ebf8917c"
291
+ },
292
+ {
293
+ "bytes": 15106,
294
+ "path": "benchmarks/llamacpp-quant-benchmarks.md",
295
+ "remote_before_upload": {
296
+ "blob_id": "5d2afc62b5b47faeab64cb323a741b1dcbd03bbb",
297
+ "bytes": 15106,
298
+ "exists": true
299
+ },
300
+ "sha256": "a3771dd7f81872665dc1e736c2b7bd1dc3b689415720984f550dcbdcfdb80aba"
301
+ },
302
+ {
303
+ "bytes": 540525,
304
+ "path": "benchmarks/raw/kld-native-bf16-top64-32prompts.json",
305
+ "remote_before_upload": {
306
+ "blob_id": "0f06d4902366d440c6d7a78193bcd82397445e7d",
307
+ "bytes": 540525,
308
+ "exists": true
309
+ },
310
+ "sha256": "6561f70553ec5c39127bc5a20faee02e4bc950fe447e5c463d631ff27609d877"
311
+ },
312
+ {
313
+ "bytes": 528560,
314
+ "path": "benchmarks/raw/kld-native-iq4-xs-top64-32prompts.json",
315
+ "remote_before_upload": {
316
+ "blob_id": "3bc36166d34b52507abc2d1d3f454003823a8c65",
317
+ "bytes": 528560,
318
+ "exists": true
319
+ },
320
+ "sha256": "e3b531b145b90f4a2d9c2dbdc8b754b630723cc375e1e49a581efac374cffccf"
321
+ },
322
+ {
323
+ "bytes": 17531,
324
+ "path": "benchmarks/raw/kld-native-iq4-xs-vs-bf16.json",
325
+ "remote_before_upload": {
326
+ "blob_id": "e44ae703feb379132303e961de63478460d3a66f",
327
+ "bytes": 17531,
328
+ "exists": true
329
+ },
330
+ "sha256": "e9f43b3f13496a63e2a99c176cd7ae525eba6e8e18e164f5bd3f9ed6209d900b"
331
+ },
332
+ {
333
+ "bytes": 530982,
334
+ "path": "benchmarks/raw/kld-native-q3-k-m-top64-32prompts.json",
335
+ "remote_before_upload": {
336
+ "blob_id": "da7ef6e6ea29ae5cf5be403b670404986aaadf35",
337
+ "bytes": 530982,
338
+ "exists": true
339
+ },
340
+ "sha256": "340b60468d5d6623f476b5d9e0478189cd1c9e95208207b85d326526ef8e1cdd"
341
+ },
342
+ {
343
+ "bytes": 17496,
344
+ "path": "benchmarks/raw/kld-native-q3-k-m-vs-bf16.json",
345
+ "remote_before_upload": {
346
+ "blob_id": "339a4a765739b3330c0cd9a8cecf3b47794bf27e",
347
+ "bytes": 17496,
348
+ "exists": true
349
+ },
350
+ "sha256": "973a211dda675c26c6e588c0a93d417eed454f0fae3874f2edfd43acaac54ee6"
351
+ },
352
+ {
353
+ "bytes": 528071,
354
+ "path": "benchmarks/raw/kld-native-q4-k-m-top64-32prompts.json",
355
+ "remote_before_upload": {
356
+ "blob_id": "a26028f6d8ef2676d21b1c8a1669ccf6a31f8460",
357
+ "bytes": 528071,
358
+ "exists": true
359
+ },
360
+ "sha256": "7fc99271628c0eef156bee37d8dc15f3826417a5e22d39954fa4c5860917d4a5"
361
+ },
362
+ {
363
+ "bytes": 17562,
364
+ "path": "benchmarks/raw/kld-native-q4-k-m-vs-bf16.json",
365
+ "remote_before_upload": {
366
+ "blob_id": "7a4904241836d5abd7d25332c96d0a9d224b98df",
367
+ "bytes": 17562,
368
+ "exists": true
369
+ },
370
+ "sha256": "a1e5e7e49ecddf51ab642ed83a3616d1b4fafe2a94ce5bb93161dc756a794a48"
371
+ },
372
+ {
373
+ "bytes": 538320,
374
+ "path": "benchmarks/raw/kld-native-q5-k-m-top64-32prompts.json",
375
+ "remote_before_upload": {
376
+ "blob_id": "84ab482383a423c11454aea6201482f5714aef30",
377
+ "bytes": 538320,
378
+ "exists": true
379
+ },
380
+ "sha256": "2aebef6964e648a2c36c3467665b7a95912fa0b0a534cbbc3e6a6b85f1e735cf"
381
+ },
382
+ {
383
+ "bytes": 17572,
384
+ "path": "benchmarks/raw/kld-native-q5-k-m-vs-bf16.json",
385
+ "remote_before_upload": {
386
+ "blob_id": "83801d1b1f08a70cfa7847cace27229581328c73",
387
+ "bytes": 17572,
388
+ "exists": true
389
+ },
390
+ "sha256": "c3dd781285b82d42264836b5d3f2770b99b64edbb5384708d19269dabd84ca58"
391
+ },
392
+ {
393
+ "bytes": 539580,
394
+ "path": "benchmarks/raw/kld-native-q6-k-top64-32prompts.json",
395
+ "remote_before_upload": {
396
+ "blob_id": "765658a1d3f829e6f241afc32565ffebdb3dbf6e",
397
+ "bytes": 539580,
398
+ "exists": true
399
+ },
400
+ "sha256": "5288a90e7900287a0b217906315adbdd80b75421ef6550226d3351d4e3465a26"
401
+ },
402
+ {
403
+ "bytes": 17588,
404
+ "path": "benchmarks/raw/kld-native-q6-k-vs-bf16.json",
405
+ "remote_before_upload": {
406
+ "blob_id": "debba39c4082f6ace6132870addb4d23ca89891f",
407
+ "bytes": 17588,
408
+ "exists": true
409
+ },
410
+ "sha256": "1c1014dec96d0bd6a063e8e05a7ceef16c50118666652fadcef02452854efa59"
411
+ },
412
+ {
413
+ "bytes": 538777,
414
+ "path": "benchmarks/raw/kld-native-q8-0-top64-32prompts.json",
415
+ "remote_before_upload": {
416
+ "blob_id": "bfba8f7fd0f038da40d4c721d132508e72a251d0",
417
+ "bytes": 538777,
418
+ "exists": true
419
+ },
420
+ "sha256": "137e8cd9eba416ea9bd876075aced2062c0fce2571b4c0c4ad28a9a91f2c48df"
421
+ },
422
+ {
423
+ "bytes": 17611,
424
+ "path": "benchmarks/raw/kld-native-q8-0-vs-bf16.json",
425
+ "remote_before_upload": {
426
+ "blob_id": "ba23e8935251dac2198c91192e16453386c24fbc",
427
+ "bytes": 17611,
428
+ "exists": true
429
+ },
430
+ "sha256": "6b7ea9c396c4a74f9ec2da02a8392bf83b6a84db41ca7a7ff7fa140e867b9c33"
431
+ },
432
+ {
433
+ "bytes": 121,
434
+ "path": "benchmarks/raw/kld-native-vram.csv",
435
+ "remote_before_upload": {
436
+ "blob_id": "1c8ef970ff7d1797933a7ade2ba4b4b519012c82",
437
+ "bytes": 121,
438
+ "exists": true
439
+ },
440
+ "sha256": "19c1aab8bf0ff499aba9a5cf01d66a7c0b0952d5ccb273823e28404ea0057934"
441
+ },
442
+ {
443
+ "bytes": 3803,
444
+ "path": "benchmarks/raw/llamacpp-iq4-xs-c1-256.jsonl",
445
+ "remote_before_upload": {
446
+ "blob_id": "12f7cb79599340df380c205783e59326d40db685",
447
+ "bytes": 3803,
448
+ "exists": true
449
+ },
450
+ "sha256": "984754eb43404ba827832e988153e2f62e8bcf65b6bac7e93ddc548d353c454b"
451
+ },
452
+ {
453
+ "bytes": 7176,
454
+ "path": "benchmarks/raw/llamacpp-iq4-xs-c16-256.jsonl",
455
+ "remote_before_upload": {
456
+ "blob_id": "63c33c4cd2dbb9b830e0600698ab945454b5c9d9",
457
+ "bytes": 7176,
458
+ "exists": true
459
+ },
460
+ "sha256": "63915b4a0f7749a5822adeb21c020e54f57a59f6b1d6a69188f8b0df324b53fd"
461
+ },
462
+ {
463
+ "bytes": 3777,
464
+ "path": "benchmarks/raw/llamacpp-iq4-xs-c4-256.jsonl",
465
+ "remote_before_upload": {
466
+ "blob_id": "b4a6396c20f6fd2caa0fbd1e0b12053e47600bd0",
467
+ "bytes": 3777,
468
+ "exists": true
469
+ },
470
+ "sha256": "0540bd2dd3820b45af6f8d78b350f9c91c4face422fb1712538f90b3e46b2b69"
471
+ },
472
+ {
473
+ "bytes": 3780,
474
+ "path": "benchmarks/raw/llamacpp-iq4-xs-c8-256.jsonl",
475
+ "remote_before_upload": {
476
+ "blob_id": "e1cde2a2e9a9711b5a994efcd89cda81e6657c22",
477
+ "bytes": 3780,
478
+ "exists": true
479
+ },
480
+ "sha256": "be391aee1da72d8c9e0e4c09bbf763fd1818ac809e8e77e95530eeeb354a73b1"
481
+ },
482
+ {
483
+ "bytes": 223925,
484
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-h2r10prefix1000-q6k-c16-128-server.log",
485
+ "remote_before_upload": {
486
+ "blob_id": "d607dcc9153e90417adaf4c71717526adba5cc8b",
487
+ "bytes": 223925,
488
+ "exists": true
489
+ },
490
+ "sha256": "4dc7cbbb82837b42170b2df17e053c2eba8ddfe6ce020f246e0731d5cc43ca0f"
491
+ },
492
+ {
493
+ "bytes": 27569,
494
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-h2r10prefix1000-q6k-c16-128-temp0.jsonl",
495
+ "remote_before_upload": {
496
+ "blob_id": "9aa8089daac2f63f09dc41bf0f12c523330cccce",
497
+ "bytes": 27569,
498
+ "exists": true
499
+ },
500
+ "sha256": "da582f2ba8b88bac45b0ac2d8342cc6423ef7aae79eead58cf041307e894c283"
501
+ },
502
+ {
503
+ "bytes": 223923,
504
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-h2r10prefix250-q6k-c16-128-server.log",
505
+ "remote_before_upload": {
506
+ "blob_id": "7e97a255805f67359db1f7f98468a0860f7ec508",
507
+ "bytes": 223923,
508
+ "exists": true
509
+ },
510
+ "sha256": "c7da2d17edd72eb1a162500b2d38f090ed1c999657527a025a3c9a818ef0c5a3"
511
+ },
512
+ {
513
+ "bytes": 27567,
514
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-h2r10prefix250-q6k-c16-128-temp0.jsonl",
515
+ "remote_before_upload": {
516
+ "blob_id": "fc0accd4a6ed8c7ff031969465eed52ed99992c2",
517
+ "bytes": 27567,
518
+ "exists": true
519
+ },
520
+ "sha256": "0ce397e2584b48855378b4afdeb67e127df0d9d213502871f9335932e688c4c8"
521
+ },
522
+ {
523
+ "bytes": 223924,
524
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-h2r10prefix500-q6k-c16-128-server.log",
525
+ "remote_before_upload": {
526
+ "blob_id": "948defd60ea7288e0519a2d5fbd6f1a0f0ce8ef7",
527
+ "bytes": 223924,
528
+ "exists": true
529
+ },
530
+ "sha256": "6226a2505f7b0d45fa48911ddb486dd00530e51c274b1b12637a898a1c64d874"
531
+ },
532
+ {
533
+ "bytes": 27572,
534
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-h2r10prefix500-q6k-c16-128-temp0.jsonl",
535
+ "remote_before_upload": {
536
+ "blob_id": "d8d91c36f89b151611a5195def62ac012e500d54",
537
+ "bytes": 27572,
538
+ "exists": true
539
+ },
540
+ "sha256": "e1476c397955d08ddd9630bfe05f9d08aa361e563d0e1f8d4856949ec16b50ea"
541
+ },
542
+ {
543
+ "bytes": 224324,
544
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-h2r9-q6-nmax2-perhead-defer-c16-128-server.log",
545
+ "remote_before_upload": {
546
+ "blob_id": "e0f88692a01a1369681e34229951d91944394eef",
547
+ "bytes": 224324,
548
+ "exists": true
549
+ },
550
+ "sha256": "c11da52ee1228aa2ad1be4238b1a6fb96f6918d1ce8da07eb45eaaab193298c9"
551
+ },
552
+ {
553
+ "bytes": 27592,
554
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-h2r9-q6-nmax2-perhead-defer-c16-128-temp0.jsonl",
555
+ "remote_before_upload": {
556
+ "blob_id": "46657c5b33ed79cf4e4ee6d51a61473c36aee5ce",
557
+ "bytes": 27592,
558
+ "exists": true
559
+ },
560
+ "sha256": "3c309575058b8cd300b3276cdb6c250cd66d3b15e0c12b8c1809cae1d950c296"
561
+ },
562
+ {
563
+ "bytes": 27562,
564
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-nmax2-perhead-backend-topk1-c16-128-temp0.jsonl",
565
+ "remote_before_upload": {
566
+ "blob_id": "988d49b7da6833cbda47b37641004da1ae0de01f",
567
+ "bytes": 27562,
568
+ "exists": true
569
+ },
570
+ "sha256": "5c1d6bf0f6f4f0180e26bae2262998b6ec90b348d508ae060fba4ba861744af6"
571
+ },
572
+ {
573
+ "bytes": 237737,
574
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-nmax2-perhead-backend-topk1-server.log",
575
+ "remote_before_upload": {
576
+ "blob_id": "516372439d4101908e77a0cc21450baab1c13bb1",
577
+ "bytes": 237737,
578
+ "exists": true
579
+ },
580
+ "sha256": "aa9b0451774e844b074891444a6ac44d891ba58e796c2687383a66a3386dcf80"
581
+ },
582
+ {
583
+ "bytes": 27574,
584
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-nmax2-q6-batched-defer-c16-128-temp0.jsonl",
585
+ "remote_before_upload": {
586
+ "blob_id": "61c29db10412c856adfa97a2645f01337745580b",
587
+ "bytes": 27574,
588
+ "exists": true
589
+ },
590
+ "sha256": "4833bdec51b261415b1e8b242eaba37877c4b1377af01ade17752baf4fd35ab8"
591
+ },
592
+ {
593
+ "bytes": 373194,
594
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-nmax2-q6-batched-defer-server.log",
595
+ "remote_before_upload": {
596
+ "blob_id": "6c8bae1b7725e1d770bbdccb5bee7ff828880ba4",
597
+ "bytes": 373194,
598
+ "exists": true
599
+ },
600
+ "sha256": "d60655422240c3f716a272826da31a8ce59897b0b5da6e699531b45fcf63d439"
601
+ },
602
+ {
603
+ "bytes": 27564,
604
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-nmax3-q6-fast-topk1-c16-128-temp0.jsonl",
605
+ "remote_before_upload": {
606
+ "blob_id": "319ca14c6ab7c6b5abd47179ce52b430b0ec6ad5",
607
+ "bytes": 27564,
608
+ "exists": true
609
+ },
610
+ "sha256": "9e13ef59bca754f9a1ce4566e5d0e8c6125e5045f8d2c9037c20337a3da07307"
611
+ },
612
+ {
613
+ "bytes": 376607,
614
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-nmax3-q6-fast-topk1-server.log",
615
+ "remote_before_upload": {
616
+ "blob_id": "ca56a16b358e34491201605cf4deec2e5e894e93",
617
+ "bytes": 376607,
618
+ "exists": true
619
+ },
620
+ "sha256": "85d80e58eb08a7e49f7bc1844b23d36909c1edf6ae59a830fdb9a07c876aba32"
621
+ },
622
+ {
623
+ "bytes": 3823,
624
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-perhead-q6-nmax2-c1-256.jsonl",
625
+ "remote_before_upload": {
626
+ "blob_id": "a32c868cf7912fa732744025c8494cb07be59e66",
627
+ "bytes": 3823,
628
+ "exists": true
629
+ },
630
+ "sha256": "09110288052799a98cbdeb2ae71d9d0ca134f47601e3d280aecb864a71064787"
631
+ },
632
+ {
633
+ "bytes": 3785,
634
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-perhead-q6-nmax2-c4-256.jsonl",
635
+ "remote_before_upload": {
636
+ "blob_id": "8bc8884b33b7c6ec3fd1369fcfd3da305c2717cc",
637
+ "bytes": 3785,
638
+ "exists": true
639
+ },
640
+ "sha256": "27270dd7abb78ffe8ac9938a3fd2d10891ccef64aacf042580bd8d29e01b111b"
641
+ },
642
+ {
643
+ "bytes": 3774,
644
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-perhead-q6-nmax2-c8-256.jsonl",
645
+ "remote_before_upload": {
646
+ "blob_id": "99accc422b8e2b90f15817ec1de2d718f9a9ffc6",
647
+ "bytes": 3774,
648
+ "exists": true
649
+ },
650
+ "sha256": "31e4a2181dedcc6f1c771359ce81bdcbad236b094113c5aa863b0c621c3c9120"
651
+ },
652
+ {
653
+ "bytes": 84533,
654
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-perhead-q6-nmax2-lowc-server.log",
655
+ "remote_before_upload": {
656
+ "blob_id": "a7f7c798d4045202edfe873305b94b3d2f7ee48c",
657
+ "bytes": 84533,
658
+ "exists": true
659
+ },
660
+ "sha256": "309661ff79be22b92aec281ea1873d94e9dde25d93352bd0c600a7a06576dd21"
661
+ },
662
+ {
663
+ "bytes": 3781,
664
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-seqverify-c1-256.jsonl",
665
+ "remote_before_upload": {
666
+ "blob_id": "2e40066a71538f6e2aaec119ea0afab0aa2dabc6",
667
+ "bytes": 3781,
668
+ "exists": true
669
+ },
670
+ "sha256": "c80b0872423b9be36903306394592b1b87ba208a262d8b2d86a7325114b22508"
671
+ },
672
+ {
673
+ "bytes": 3810,
674
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-seqverify-fast-c1-256.jsonl",
675
+ "remote_before_upload": {
676
+ "blob_id": "abda3eefeec3dba77e0225ac999f23cf6f50cbbe",
677
+ "bytes": 3810,
678
+ "exists": true
679
+ },
680
+ "sha256": "b3e6875e306f6ea623a65846fdb144ec38e3740876e798c661132aee5bd71312"
681
+ },
682
+ {
683
+ "bytes": 27585,
684
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-seqverify-fast-c16-128-r128.jsonl",
685
+ "remote_before_upload": {
686
+ "blob_id": "b624ccb122e4d07e9e4dcebbbfc443fa011fe2c5",
687
+ "bytes": 27585,
688
+ "exists": true
689
+ },
690
+ "sha256": "d446c1b31ea9f04b461ccc41e5cfcca762ab17a88a259f3bcec9d2caf6762dc2"
691
+ },
692
+ {
693
+ "bytes": 3782,
694
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-seqverify-fast-c4-256.jsonl",
695
+ "remote_before_upload": {
696
+ "blob_id": "5cbe5b82c43495b59d2d8e93cdfb84d6c742efb8",
697
+ "bytes": 3782,
698
+ "exists": true
699
+ },
700
+ "sha256": "8c748120730960f9bb4df97d3512ef0de5ab612a7fc4efd4855ce809f1167d12"
701
+ },
702
+ {
703
+ "bytes": 3791,
704
+ "path": "benchmarks/raw/llamacpp-iq4-xs-mtp-seqverify-fast-c8-256.jsonl",
705
+ "remote_before_upload": {
706
+ "blob_id": "a91a90070929f90c618f11050bd9ad2c4ac2821c",
707
+ "bytes": 3791,
708
+ "exists": true
709
+ },
710
+ "sha256": "c3d068af0d2cd699dd27a3a7285952b651438cc314d4c1e8af96698aa96e2e52"
711
+ },
712
+ {
713
+ "bytes": 27601,
714
+ "path": "benchmarks/raw/llamacpp-iq4-xs-target-c16-128-r128.jsonl",
715
+ "remote_before_upload": {
716
+ "blob_id": "00f37c7ff41ad858bc2698206b5a0aad9c139618",
717
+ "bytes": 27601,
718
+ "exists": true
719
+ },
720
+ "sha256": "ffed328c557a9e68ef366e7a1d55b1d6c40e6bc5978b3fe20977535d898c3861"
721
+ },
722
+ {
723
+ "bytes": 535606,
724
+ "path": "benchmarks/raw/llamacpp-iq4-xs-target-c16-128-server.log",
725
+ "remote_before_upload": {
726
+ "blob_id": "f87cd94f830f75eb5c3c8b3d98aba1c301baa67b",
727
+ "bytes": 535606,
728
+ "exists": true
729
+ },
730
+ "sha256": "260775f17dc28040c91fe4d5ec2eaaf1d89bb78f323762bc54d88dfcb22f2adc"
731
+ },
732
+ {
733
+ "bytes": 3814,
734
+ "path": "benchmarks/raw/llamacpp-iq4-xs-target-nocache-c1-256.jsonl",
735
+ "remote_before_upload": {
736
+ "blob_id": "9038a7368dd41de79d069b5061add7778576bb11",
737
+ "bytes": 3814,
738
+ "exists": true
739
+ },
740
+ "sha256": "8c8abd7f0c3056751124c1da90622e75f917d1f5cd14d4613c5c925e400cb259"
741
+ },
742
+ {
743
+ "bytes": 27620,
744
+ "path": "benchmarks/raw/llamacpp-iq4-xs-target-nocache-c16-128-r128.jsonl",
745
+ "remote_before_upload": {
746
+ "blob_id": "407498ae0c818b22a029c2373b1b90d24e70d2f9",
747
+ "bytes": 27620,
748
+ "exists": true
749
+ },
750
+ "sha256": "a04778a5aa8d10e4bc51d2eec9590e44c8ba4e36d83641ef87000a772f733ca9"
751
+ },
752
+ {
753
+ "bytes": 184168,
754
+ "path": "benchmarks/raw/llamacpp-iq4-xs-target-nocache-c16-128-server.log",
755
+ "remote_before_upload": {
756
+ "blob_id": "01498c978ed75dfc6d076aa547f4b69c48840d99",
757
+ "bytes": 184168,
758
+ "exists": true
759
+ },
760
+ "sha256": "353254a8dc21faaa43ceb119d1c880662957e6f1497fcb0bc8bd4bb1d5ed5103"
761
+ },
762
+ {
763
+ "bytes": 3804,
764
+ "path": "benchmarks/raw/llamacpp-iq4-xs-target-nocache-c4-256.jsonl",
765
+ "remote_before_upload": {
766
+ "blob_id": "63dde552bcd9d460e9bba2e7a9e71bdb461b7f54",
767
+ "bytes": 3804,
768
+ "exists": true
769
+ },
770
+ "sha256": "e37fc4b3e76b934e02afc583bee5389a9de09147471dabc3a3f7b86c80e424ac"
771
+ },
772
+ {
773
+ "bytes": 3787,
774
+ "path": "benchmarks/raw/llamacpp-iq4-xs-target-nocache-c8-256.jsonl",
775
+ "remote_before_upload": {
776
+ "blob_id": "7ddb518f391f7c9948a44086413ae095b27924a3",
777
+ "bytes": 3787,
778
+ "exists": true
779
+ },
780
+ "sha256": "4e8fe4a2a1229f4dfb38de28412df5562331d193cb73e068b943c97e72e5d540"
781
+ },
782
+ {
783
+ "bytes": 74853,
784
+ "path": "benchmarks/raw/llamacpp-iq4-xs-target-nocache-lowc-server.log",
785
+ "remote_before_upload": {
786
+ "blob_id": "e7b33283a88c224f8f7a84d6ae295c865494cc8b",
787
+ "bytes": 74853,
788
+ "exists": true
789
+ },
790
+ "sha256": "27d4942bd5e55994bc44fa2dbb735baf6f29747a88a22fcf55c47c8147931e4a"
791
+ },
792
+ {
793
+ "bytes": 584265,
794
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-ar-c16-128-server.log",
795
+ "remote_before_upload": {
796
+ "blob_id": "9e0b06da84c7f1d1b8d12a9c43ed884ea1b3549c",
797
+ "bytes": 584265,
798
+ "exists": true
799
+ },
800
+ "sha256": "e7fcae66fb85541b693204415863d9cda078a123ac3bcec2421d1022c41aaf35"
801
+ },
802
+ {
803
+ "bytes": 27611,
804
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-ar-c16-128-temp0.jsonl",
805
+ "remote_before_upload": {
806
+ "blob_id": "c850fd061d5ee63501cb5e261be85806fbd58196",
807
+ "bytes": 27611,
808
+ "exists": true
809
+ },
810
+ "sha256": "2a3bd26effee4e5764846aea3313764a0325f81bcbd33d4968a3900c5d8a3117"
811
+ },
812
+ {
813
+ "bytes": 41931,
814
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-ar-proctime-trace-c16-32-server.log",
815
+ "remote_before_upload": {
816
+ "blob_id": "8277ed3949fae2c86eefdc5a14ab2c3c3ec69f06",
817
+ "bytes": 41931,
818
+ "exists": true
819
+ },
820
+ "sha256": "35b45995f4502b3dcc1485a9a41d77ab7f16c95f7ab41003325dd53cbf6ac432"
821
+ },
822
+ {
823
+ "bytes": 3763,
824
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-ar-proctime-trace-c16-32.jsonl",
825
+ "remote_before_upload": {
826
+ "blob_id": "d107bfdd1da54d2e04543f95466c68044d13319f",
827
+ "bytes": 3763,
828
+ "exists": true
829
+ },
830
+ "sha256": "f15fc547b5a28629dc14751254eccfda65aca8c5f258b9dab1e844f9b59e71c7"
831
+ },
832
+ {
833
+ "bytes": 28372,
834
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-ar-rollbackdst2-c16-128-server.log",
835
+ "remote_before_upload": {
836
+ "blob_id": "47a08fcc5412884ef9ee252df7cb404bc30196cb",
837
+ "bytes": 28372,
838
+ "exists": true
839
+ },
840
+ "sha256": "dcf7df77e0d28401b84fd59374d004db6426cb044414e59540f7990e98d6e1f2"
841
+ },
842
+ {
843
+ "bytes": 27609,
844
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-ar-rollbackdst2-c16-128.jsonl",
845
+ "remote_before_upload": {
846
+ "blob_id": "7707413671d056a8ef0721c42795e5263b169e65",
847
+ "bytes": 27609,
848
+ "exists": true
849
+ },
850
+ "sha256": "209ab95b9e74f5a8e07166990c4c9277dcaf82357203511e04f1b5c70d73e71f"
851
+ },
852
+ {
853
+ "bytes": 382980,
854
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax1-c16-128-server.log",
855
+ "remote_before_upload": {
856
+ "blob_id": "e10496fca41a7037cf8aa9b09c9c5b08204b577b",
857
+ "bytes": 382980,
858
+ "exists": true
859
+ },
860
+ "sha256": "1c2d49ec9487a565636827a017ecb65480cb1bf9c9f2c43103e047aca46bffc0"
861
+ },
862
+ {
863
+ "bytes": 27555,
864
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax1-c16-128-temp0.jsonl",
865
+ "remote_before_upload": {
866
+ "blob_id": "efb1550d3fd8b0db3cde849f513fa966db757f7e",
867
+ "bytes": 27555,
868
+ "exists": true
869
+ },
870
+ "sha256": "7fe9ac72ab6ddc1ee6f1f605133cecae94ad47a0a302aecb6ca7a64e0e3a8992"
871
+ },
872
+ {
873
+ "bytes": 31944,
874
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax1-fast-topk1-rollbackdst-c16-128-server.log",
875
+ "remote_before_upload": {
876
+ "blob_id": "44d03b52d5b4028f84f12b67b021762e446ff2f4",
877
+ "bytes": 31944,
878
+ "exists": true
879
+ },
880
+ "sha256": "c69cffb486378db47c3aab7a21714c1ddde54309788bd5a9fc3c211f1405ceda"
881
+ },
882
+ {
883
+ "bytes": 27575,
884
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax1-fast-topk1-rollbackdst-c16-128.jsonl",
885
+ "remote_before_upload": {
886
+ "blob_id": "7e04015573299bcdeb8d3547a9216290fa864ca4",
887
+ "bytes": 27575,
888
+ "exists": true
889
+ },
890
+ "sha256": "619f90dfe44a9ef48833cfcbf5683cb45fccdcb59ca2bfb0af46fcccec3ba65c"
891
+ },
892
+ {
893
+ "bytes": 47640,
894
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-batchdepth-trace-c16-32-server.log",
895
+ "remote_before_upload": {
896
+ "blob_id": "a2b9ee68e026751f33dcf92c7808d2f2e2aede8d",
897
+ "bytes": 47640,
898
+ "exists": true
899
+ },
900
+ "sha256": "48cad3e1ac21892f5bd3a123654036ed90d065452d1d8c51b42f29daa1676617"
901
+ },
902
+ {
903
+ "bytes": 3762,
904
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-batchdepth-trace-c16-32.jsonl",
905
+ "remote_before_upload": {
906
+ "blob_id": "dfe744b93232a7cdeb6c955485d1b4e4702a4020",
907
+ "bytes": 3762,
908
+ "exists": true
909
+ },
910
+ "sha256": "9da6413796c7416998030e10d1f3d11e3c85adfb4c886085916e429e5fecc789"
911
+ },
912
+ {
913
+ "bytes": 948002,
914
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-fast-topk1-proctime-c16-128-server.log",
915
+ "remote_before_upload": {
916
+ "blob_id": "3ef9efd16f86320b59c2a84e4a93141d1833e78c",
917
+ "bytes": 948002,
918
+ "exists": true
919
+ },
920
+ "sha256": "d63eeac751919de7aa69b9ff53a049e96154c71c0265dd5499d0014111a8aefd"
921
+ },
922
+ {
923
+ "bytes": 27579,
924
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-fast-topk1-proctime-c16-128.jsonl",
925
+ "remote_before_upload": {
926
+ "blob_id": "33525c555becd9365a59be70e143722002d4aa2f",
927
+ "bytes": 27579,
928
+ "exists": true
929
+ },
930
+ "sha256": "2fbe9d3ca6001459aef942868b24c74ca2f054116b536e718244528dbbb194a2"
931
+ },
932
+ {
933
+ "bytes": 47544,
934
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-fast-topk1-proctime-trace-c16-32-server.log",
935
+ "remote_before_upload": {
936
+ "blob_id": "019bd31cecb2974d67191b86684c77b20d68907d",
937
+ "bytes": 47544,
938
+ "exists": true
939
+ },
940
+ "sha256": "0deb6859d011a9f0de9489f539176971f7d045b1a4c5b6ce1c0f03b07e0c2bb2"
941
+ },
942
+ {
943
+ "bytes": 3767,
944
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-fast-topk1-proctime-trace-c16-32.jsonl",
945
+ "remote_before_upload": {
946
+ "blob_id": "4695a8805e30acbf173fe32f051660bf40287f23",
947
+ "bytes": 3767,
948
+ "exists": true
949
+ },
950
+ "sha256": "5bbfac5026b1d9764544a417a646a8d80682b3de9ead0a4ed79f746dd2e1830a"
951
+ },
952
+ {
953
+ "bytes": 31671,
954
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-fast-topk1-rollbackdst-c16-128-server.log",
955
+ "remote_before_upload": {
956
+ "blob_id": "c88ba84cda5f38c602b21d433109a0e2a27ec0d9",
957
+ "bytes": 31671,
958
+ "exists": true
959
+ },
960
+ "sha256": "a0c722fa65331f33bcffec065150ef7cc066c34c0f12bdbc39eb8bdab1a009a0"
961
+ },
962
+ {
963
+ "bytes": 27576,
964
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-fast-topk1-rollbackdst-c16-128.jsonl",
965
+ "remote_before_upload": {
966
+ "blob_id": "8b770828ba46c3aa550c14514c4b54e788593996",
967
+ "bytes": 27576,
968
+ "exists": true
969
+ },
970
+ "sha256": "a447dbbb838d75861a76430558d95f4938e0b05fc53298994304c86ee5206c98"
971
+ },
972
+ {
973
+ "bytes": 19080,
974
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-fast-topk1-tracev-rollbackdst-lv3-c2-32-server.log",
975
+ "remote_before_upload": {
976
+ "blob_id": "895a8d3f94cea00bb0395ec3e18f393871c753fb",
977
+ "bytes": 19080,
978
+ "exists": true
979
+ },
980
+ "sha256": "b92da2ca30253429ad3d03d45070a31a9e70a6f3fa8853f87182eaf15e67ea9e"
981
+ },
982
+ {
983
+ "bytes": 2083,
984
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-fast-topk1-tracev-rollbackdst-lv3-c2-32.jsonl",
985
+ "remote_before_upload": {
986
+ "blob_id": "a35f701e20150859906241e21610a34229eae950",
987
+ "bytes": 2083,
988
+ "exists": true
989
+ },
990
+ "sha256": "23d28bc0a5dd6a97b98f38134812c3fc6523d25e8d9a5f2c345079bbf1bd7b41"
991
+ },
992
+ {
993
+ "bytes": 392903,
994
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-targetbackend-c16-128-server.log",
995
+ "remote_before_upload": {
996
+ "blob_id": "9f0f27d2323bfd0e37c89b0770d5adcd22886b0f",
997
+ "bytes": 392903,
998
+ "exists": true
999
+ },
1000
+ "sha256": "86a0431c923332d0f86bc9c8d6ba707a16b0ce8dca11e9a33c21454edad8df3e"
1001
+ },
1002
+ {
1003
+ "bytes": 27570,
1004
+ "path": "benchmarks/raw/llamacpp-iq4xs-mtp-graft-headq6-nmax2-targetbackend-c16-128.jsonl",
1005
+ "remote_before_upload": {
1006
+ "blob_id": "d71d88b2777262c9f0a4a3afebe754ca792bba16",
1007
+ "bytes": 27570,
1008
+ "exists": true
1009
+ },
1010
+ "sha256": "f5a0776c7cb98a18a33879d726dcd099431d06b9959d12868ed0ecfb26006abd"
1011
+ },
1012
+ {
1013
+ "bytes": 3807,
1014
+ "path": "benchmarks/raw/llamacpp-q3-k-m-c1-256.jsonl",
1015
+ "remote_before_upload": {
1016
+ "blob_id": "586b312b45dbf8e6b370cac720bda31116844a6e",
1017
+ "bytes": 3807,
1018
+ "exists": true
1019
+ },
1020
+ "sha256": "2c0da0a8565b523b95fe050e6700534f1a49565c30b2a485a4c858b8351dc8c8"
1021
+ },
1022
+ {
1023
+ "bytes": 7175,
1024
+ "path": "benchmarks/raw/llamacpp-q3-k-m-c16-256.jsonl",
1025
+ "remote_before_upload": {
1026
+ "blob_id": "8ac7faff4be54556c079bdff0f48b538d3cb3f29",
1027
+ "bytes": 7175,
1028
+ "exists": true
1029
+ },
1030
+ "sha256": "9ceaa3a73222c5e236dfc83f834e577baaf1e5078b83e794d5f09722334c4dbf"
1031
+ },
1032
+ {
1033
+ "bytes": 3797,
1034
+ "path": "benchmarks/raw/llamacpp-q3-k-m-c4-256.jsonl",
1035
+ "remote_before_upload": {
1036
+ "blob_id": "38c83fee15c472eb9b4c94a5f91bf320481db074",
1037
+ "bytes": 3797,
1038
+ "exists": true
1039
+ },
1040
+ "sha256": "01e14866b431b8cb237cea7fad91ae2bbd9a6188cc8ca2b096097944f2ce3d31"
1041
+ },
1042
+ {
1043
+ "bytes": 3761,
1044
+ "path": "benchmarks/raw/llamacpp-q3-k-m-c8-256.jsonl",
1045
+ "remote_before_upload": {
1046
+ "blob_id": "2cfab637e033f976f1d04c0e745cc91951bb0d69",
1047
+ "bytes": 3761,
1048
+ "exists": true
1049
+ },
1050
+ "sha256": "e00d6d111eccccb2cd6aeb44f560d8fc5091ac7d3a8a25265fe4ca0335bb2a69"
1051
+ },
1052
+ {
1053
+ "bytes": 3807,
1054
+ "path": "benchmarks/raw/llamacpp-q4-c1-256.jsonl",
1055
+ "remote_before_upload": {
1056
+ "blob_id": "a0c966d29753afbcaca269c3f3220373dbe0f2f1",
1057
+ "bytes": 3807,
1058
+ "exists": true
1059
+ },
1060
+ "sha256": "2385b6f04740f2063f7ff03f84754392b5e54668ecf885372430415aa4a6e52a"
1061
+ },
1062
+ {
1063
+ "bytes": 7167,
1064
+ "path": "benchmarks/raw/llamacpp-q4-c16-256.jsonl",
1065
+ "remote_before_upload": {
1066
+ "blob_id": "44567f048292e7eee02aee419a9efe80cd8ecd72",
1067
+ "bytes": 7167,
1068
+ "exists": true
1069
+ },
1070
+ "sha256": "eebc326884540abe38f7fdb6f763e3ed74bccde07f21d959f10a4bd20565cacc"
1071
+ },
1072
+ {
1073
+ "bytes": 7219,
1074
+ "path": "benchmarks/raw/llamacpp-q4-c4-256.jsonl",
1075
+ "remote_before_upload": {
1076
+ "blob_id": "8d1d2938577094056975d996e15b42a76e36d493",
1077
+ "bytes": 7219,
1078
+ "exists": true
1079
+ },
1080
+ "sha256": "07e133e6617c00f9677f27389388a7852075d5ff78f3400d77dc4b0ddecd65e4"
1081
+ },
1082
+ {
1083
+ "bytes": 7174,
1084
+ "path": "benchmarks/raw/llamacpp-q4-c8-256.jsonl",
1085
+ "remote_before_upload": {
1086
+ "blob_id": "5ed014e3212c4127b499a00307b0a2c5e0a97f1d",
1087
+ "bytes": 7174,
1088
+ "exists": true
1089
+ },
1090
+ "sha256": "1bed6575afb5755f09843edbeedfb40ff7acb5a3a542968271a48b87068c11d4"
1091
+ },
1092
+ {
1093
+ "bytes": 3805,
1094
+ "path": "benchmarks/raw/llamacpp-q5-c1-256.jsonl",
1095
+ "remote_before_upload": {
1096
+ "blob_id": "8a35242d5b8a99a1701e89a72c451823a6f5264e",
1097
+ "bytes": 3805,
1098
+ "exists": true
1099
+ },
1100
+ "sha256": "88a2cb26289671cb75e16f17cae02b9e05e3b5c1a7f9f4fc0cc1a3d33cd8667c"
1101
+ },
1102
+ {
1103
+ "bytes": 7161,
1104
+ "path": "benchmarks/raw/llamacpp-q5-c16-256.jsonl",
1105
+ "remote_before_upload": {
1106
+ "blob_id": "d6d6f254fb0cecd8ec27d65c984dca6ef0dffa6f",
1107
+ "bytes": 7161,
1108
+ "exists": true
1109
+ },
1110
+ "sha256": "86aed78128a9a87691f1da7b06e726d10c7ef668f66f1f16db31f33ee3620c79"
1111
+ },
1112
+ {
1113
+ "bytes": 7182,
1114
+ "path": "benchmarks/raw/llamacpp-q5-c4-256.jsonl",
1115
+ "remote_before_upload": {
1116
+ "blob_id": "ba99e3671095443cdc623c8f882ea770937798bf",
1117
+ "bytes": 7182,
1118
+ "exists": true
1119
+ },
1120
+ "sha256": "041349df861777e18d7a03fe9e978034b4b38abbb5b1b204d1358eeb6bd4db39"
1121
+ },
1122
+ {
1123
+ "bytes": 7167,
1124
+ "path": "benchmarks/raw/llamacpp-q5-c8-256.jsonl",
1125
+ "remote_before_upload": {
1126
+ "blob_id": "2189099677423ebf9a736628e9d6917539d19b5a",
1127
+ "bytes": 7167,
1128
+ "exists": true
1129
+ },
1130
+ "sha256": "d21245ab1f6dcf64614dcd542c5c1d7e72d51d8cc7915a14610bdadad17bb65f"
1131
+ },
1132
+ {
1133
+ "bytes": 3807,
1134
+ "path": "benchmarks/raw/llamacpp-q6-c1-256.jsonl",
1135
+ "remote_before_upload": {
1136
+ "blob_id": "f2372a9859300a71623a25c3fe91df626c89c28f",
1137
+ "bytes": 3807,
1138
+ "exists": true
1139
+ },
1140
+ "sha256": "8c4ed15b42aea31588b0b03ef98db4609d46ad93ddff27005e7b6f29c860e637"
1141
+ },
1142
+ {
1143
+ "bytes": 7169,
1144
+ "path": "benchmarks/raw/llamacpp-q6-c16-256.jsonl",
1145
+ "remote_before_upload": {
1146
+ "blob_id": "771c5ad0833226faa610f66b26a23a4b379fab97",
1147
+ "bytes": 7169,
1148
+ "exists": true
1149
+ },
1150
+ "sha256": "d8dbd3f2e84aa83b051567175f23eb5afe78f5439bc6e8f07fd7b355720be073"
1151
+ },
1152
+ {
1153
+ "bytes": 7194,
1154
+ "path": "benchmarks/raw/llamacpp-q6-c4-256.jsonl",
1155
+ "remote_before_upload": {
1156
+ "blob_id": "2204ab4194d33056d44942cf44f2109936cca4cc",
1157
+ "bytes": 7194,
1158
+ "exists": true
1159
+ },
1160
+ "sha256": "c869dbd07c0efe66fdfe4e67ce394c1cae7d0f02e43dd054ac9aabdafec57e8a"
1161
+ },
1162
+ {
1163
+ "bytes": 7165,
1164
+ "path": "benchmarks/raw/llamacpp-q6-c8-256.jsonl",
1165
+ "remote_before_upload": {
1166
+ "blob_id": "704701e3a5302ec34a48b78f08f69e9af9bc7121",
1167
+ "bytes": 7165,
1168
+ "exists": true
1169
+ },
1170
+ "sha256": "7d03d04a1e4fa74440685880d00c0d98bce1580382a14daeeb4869d43cf936a3"
1171
+ },
1172
+ {
1173
+ "bytes": 3807,
1174
+ "path": "benchmarks/raw/llamacpp-q8-c1-256.jsonl",
1175
+ "remote_before_upload": {
1176
+ "blob_id": "9944f6b3d9471822c4b7ce331bb6c5f7b19a89bc",
1177
+ "bytes": 3807,
1178
+ "exists": true
1179
+ },
1180
+ "sha256": "1222b0e21bbefb5d1189b8d9f0343cf85faa7684db39cada951e3cac9965e379"
1181
+ },
1182
+ {
1183
+ "bytes": 7176,
1184
+ "path": "benchmarks/raw/llamacpp-q8-c16-256.jsonl",
1185
+ "remote_before_upload": {
1186
+ "blob_id": "06659cd27308f07772e564a607a8ba5b5fe1a58b",
1187
+ "bytes": 7176,
1188
+ "exists": true
1189
+ },
1190
+ "sha256": "ae0d7743022fdb9e5497fc9e18a35e88f1623f7e2cbd23ac07544f99d46d0c35"
1191
+ },
1192
+ {
1193
+ "bytes": 7185,
1194
+ "path": "benchmarks/raw/llamacpp-q8-c4-256.jsonl",
1195
+ "remote_before_upload": {
1196
+ "blob_id": "d0eb115941c9399b912b944fdc01e1d217387b36",
1197
+ "bytes": 7185,
1198
+ "exists": true
1199
+ },
1200
+ "sha256": "55523a2f6c1af5000012436b37ee999d3b4b34f1e014c4c4a47137c71cc182c0"
1201
+ },
1202
+ {
1203
+ "bytes": 7179,
1204
+ "path": "benchmarks/raw/llamacpp-q8-c8-256.jsonl",
1205
+ "remote_before_upload": {
1206
+ "blob_id": "1009f069552d04c031e8dcc2feb20bdc26176796",
1207
+ "bytes": 7179,
1208
+ "exists": true
1209
+ },
1210
+ "sha256": "c651063aba93e03f12bcc5d527e921b4b93d64ef88c72607d07720bb83255549"
1211
+ },
1212
+ {
1213
+ "bytes": 459524,
1214
+ "path": "benchmarks/raw/llamacpp-wang-ornith-q6-mtp-ar-c16-128-server.log",
1215
+ "remote_before_upload": {
1216
+ "blob_id": "0c6440a9775e2d886bf82088d5a7bde90a7c4562",
1217
+ "bytes": 459524,
1218
+ "exists": true
1219
+ },
1220
+ "sha256": "81526142ae4ffc21e38afb1ec1ec4137a294c0ca44153f613eb442c65b7efa66"
1221
+ },
1222
+ {
1223
+ "bytes": 27615,
1224
+ "path": "benchmarks/raw/llamacpp-wang-ornith-q6-mtp-ar-c16-128-temp0.jsonl",
1225
+ "remote_before_upload": {
1226
+ "blob_id": "e78ca93ef96d68be81267602d3ad4960f8f17f23",
1227
+ "bytes": 27615,
1228
+ "exists": true
1229
+ },
1230
+ "sha256": "e025f7b64762e82b3d801f24a4c0946e88ca68beadf421431ff980758784d031"
1231
+ },
1232
+ {
1233
+ "bytes": 383573,
1234
+ "path": "benchmarks/raw/llamacpp-wang-ornith-q6-mtp-nmax1-c16-128-server.log",
1235
+ "remote_before_upload": {
1236
+ "blob_id": "c35c6c0eca95ed0f02e1ae4033b20078ea6f9f07",
1237
+ "bytes": 383573,
1238
+ "exists": true
1239
+ },
1240
+ "sha256": "a9858d19bd80e25a741d71ce6e12d669d1ec96ec83aa67c32e799dafa8fc02be"
1241
+ },
1242
+ {
1243
+ "bytes": 27564,
1244
+ "path": "benchmarks/raw/llamacpp-wang-ornith-q6-mtp-nmax1-c16-128-temp0.jsonl",
1245
+ "remote_before_upload": {
1246
+ "blob_id": "a18a44c05bbfb33e5d4e1b00532b603c136c6325",
1247
+ "bytes": 27564,
1248
+ "exists": true
1249
+ },
1250
+ "sha256": "bf3954712a92c189a9bd71dd2ac1c0b2d9995e0e74346ed183ada0aca7afb484"
1251
+ },
1252
+ {
1253
+ "bytes": 3785,
1254
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-nmax1-det-capture.json",
1255
+ "remote_before_upload": {
1256
+ "blob_id": "d9cef937bbfcd410d3c49475401060dfe056026f",
1257
+ "bytes": 3785,
1258
+ "exists": true
1259
+ },
1260
+ "sha256": "1eacd053ebdc56dbd39e8f536cb980b4490913df56e6b529f7ade40de22083c6"
1261
+ },
1262
+ {
1263
+ "bytes": 1464,
1264
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-nmax1-det-compare.json",
1265
+ "remote_before_upload": {
1266
+ "blob_id": "d659e3110b45b10b98c61f3e0b60fe94b2507d42",
1267
+ "bytes": 1464,
1268
+ "exists": true
1269
+ },
1270
+ "sha256": "bd630b2c8600e582dbd8cc717702eb96f53a789c809d7ffe5bb1bd21276d35d6"
1271
+ },
1272
+ {
1273
+ "bytes": 3500,
1274
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-no-fa-det-capture.json",
1275
+ "remote_before_upload": {
1276
+ "blob_id": "562303659ac50efafe229baf2ed951083dbd40d4",
1277
+ "bytes": 3500,
1278
+ "exists": true
1279
+ },
1280
+ "sha256": "a3a67b5e8f6ebfb22b738fecdfc430f42bea5c568f052459a7d708c48915251b"
1281
+ },
1282
+ {
1283
+ "bytes": 1470,
1284
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-no-fa-det-compare.json",
1285
+ "remote_before_upload": {
1286
+ "blob_id": "c1244fcaa0603596bdd137b35d2ed6faedfefa3b",
1287
+ "bytes": 1470,
1288
+ "exists": true
1289
+ },
1290
+ "sha256": "7c92acb6e0db6e8b1b9714bead957d19dee1234174da359beb85114e5a2339d1"
1291
+ },
1292
+ {
1293
+ "bytes": 3475,
1294
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-nofast-det-capture.json",
1295
+ "remote_before_upload": {
1296
+ "blob_id": "8846dd0d3bf6764d275976ffba9537bdffbcf6bb",
1297
+ "bytes": 3475,
1298
+ "exists": true
1299
+ },
1300
+ "sha256": "e756476a9aa1c50ff0f1c665c4fb9756b4f2f021184beed95931b932e75cb14d"
1301
+ },
1302
+ {
1303
+ "bytes": 1465,
1304
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-nofast-det-compare.json",
1305
+ "remote_before_upload": {
1306
+ "blob_id": "a885466e2055812c11dffd6d9857928aaeb484c6",
1307
+ "bytes": 1465,
1308
+ "exists": true
1309
+ },
1310
+ "sha256": "d5722b2b7a8bea47f8956aaf1046bee347f33f77a11ee24e3b7f1860614ad2e1"
1311
+ },
1312
+ {
1313
+ "bytes": 3482,
1314
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-nospecbackend-det-capture.json",
1315
+ "remote_before_upload": {
1316
+ "blob_id": "f17b0ad7dc59f179b7d05ffd9e5947f99d3fb5fb",
1317
+ "bytes": 3482,
1318
+ "exists": true
1319
+ },
1320
+ "sha256": "4c1c911da02437210adb5f9fb99c493885ed6685a85e91aa21edc7aab1f673f9"
1321
+ },
1322
+ {
1323
+ "bytes": 1472,
1324
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-nospecbackend-det-compare.json",
1325
+ "remote_before_upload": {
1326
+ "blob_id": "009291e0da2bca3c19880bf24c3b6ad0b7d4bfa9",
1327
+ "bytes": 1472,
1328
+ "exists": true
1329
+ },
1330
+ "sha256": "b37d6df09662ee651f2e94a1dce85acd93fb4e31ae5216689baf41465109bdd0"
1331
+ },
1332
+ {
1333
+ "bytes": 3485,
1334
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-perhead-det-capture.json",
1335
+ "remote_before_upload": {
1336
+ "blob_id": "e2205a8da2e551ffeddf8208b372f671c639066a",
1337
+ "bytes": 3485,
1338
+ "exists": true
1339
+ },
1340
+ "sha256": "5bd3cd20b64d2d965b4f48cc84564c8e04820ef03840fc35281dfb273c09bc3c"
1341
+ },
1342
+ {
1343
+ "bytes": 1466,
1344
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-perhead-det-compare.json",
1345
+ "remote_before_upload": {
1346
+ "blob_id": "14d78960ca5b9cbba8465036fd1d240154a1e962",
1347
+ "bytes": 1466,
1348
+ "exists": true
1349
+ },
1350
+ "sha256": "892303ec126338457cbe986f0c20e1dc76a6bb41e94d27ec7c50c026704cfa60"
1351
+ },
1352
+ {
1353
+ "bytes": 3740,
1354
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-seqverify-det-capture.json",
1355
+ "remote_before_upload": {
1356
+ "blob_id": "bd93a055af0a4d27100cfac6762f2bb85bd2cdd6",
1357
+ "bytes": 3740,
1358
+ "exists": true
1359
+ },
1360
+ "sha256": "dfd66afbfce85aa4347261ba53cb1f5ed2af9c1832b592e3b873a7665c491ec9"
1361
+ },
1362
+ {
1363
+ "bytes": 1465,
1364
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-seqverify-det-compare.json",
1365
+ "remote_before_upload": {
1366
+ "blob_id": "3baea2700f8cc84935a25e961c041c12b304b263",
1367
+ "bytes": 1465,
1368
+ "exists": true
1369
+ },
1370
+ "sha256": "91524dc6d9f799c385416fbe0678999a03ddbab89d8d8fad7ba26d82b5a2c074"
1371
+ },
1372
+ {
1373
+ "bytes": 3745,
1374
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-seqverify-fast-det-capture.json",
1375
+ "remote_before_upload": {
1376
+ "blob_id": "4352a0aa2042ab1180abf1bffaf01a431c1bbafc",
1377
+ "bytes": 3745,
1378
+ "exists": true
1379
+ },
1380
+ "sha256": "92d7cef48661d3585ae8f39d3d937b5cd9732f49db757279de3f3e94e6d3d61a"
1381
+ },
1382
+ {
1383
+ "bytes": 1470,
1384
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-seqverify-fast-det-compare.json",
1385
+ "remote_before_upload": {
1386
+ "blob_id": "05ff091065130f23dff5d04adaa049e8961ec6ad",
1387
+ "bytes": 1470,
1388
+ "exists": true
1389
+ },
1390
+ "sha256": "ac292815bb017d940021d9aabe0086020ebc26651fefe66b445cd432b02c4584"
1391
+ },
1392
+ {
1393
+ "bytes": 3477,
1394
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-standard-det-capture.json",
1395
+ "remote_before_upload": {
1396
+ "blob_id": "430108a805e4a9a361130279dc1b79bfd403d2ef",
1397
+ "bytes": 3477,
1398
+ "exists": true
1399
+ },
1400
+ "sha256": "c286c45f9c78082ed95c36f53bb620c344267b6855c7d0f55b6e23df5c087c40"
1401
+ },
1402
+ {
1403
+ "bytes": 1467,
1404
+ "path": "benchmarks/raw/ornith-iq4-xs-mtp-standard-det-compare.json",
1405
+ "remote_before_upload": {
1406
+ "blob_id": "3f4ec53b705e681e627127b14035ef11118c0281",
1407
+ "bytes": 1467,
1408
+ "exists": true
1409
+ },
1410
+ "sha256": "b5633ed6bb9533de2efe366b499c0c4894867de000f736119083ea3bcb21e58c"
1411
+ },
1412
+ {
1413
+ "bytes": 3733,
1414
+ "path": "benchmarks/raw/ornith-iq4-xs-target-det-capture.json",
1415
+ "remote_before_upload": {
1416
+ "blob_id": "1e35b53919922cf6cd7310cc9ef7554329c7a97b",
1417
+ "bytes": 3733,
1418
+ "exists": true
1419
+ },
1420
+ "sha256": "51fc90847cfde768ac5f577bafb6c8e6b325abbddd41e340809a6ef1f71f8ee5"
1421
+ },
1422
+ {
1423
+ "bytes": 3734,
1424
+ "path": "benchmarks/raw/ornith-iq4-xs-target-det-repeat-capture.json",
1425
+ "remote_before_upload": {
1426
+ "blob_id": "b3603f5d6631215c7e54f3005cdf11763de3da0d",
1427
+ "bytes": 3734,
1428
+ "exists": true
1429
+ },
1430
+ "sha256": "3130916de1334c5d8d3ef908d405163a07e0e5db28f1af88fb5a4916ffcd447e"
1431
+ },
1432
+ {
1433
+ "bytes": 1465,
1434
+ "path": "benchmarks/raw/ornith-iq4-xs-target-det-repeat-compare.json",
1435
+ "remote_before_upload": {
1436
+ "blob_id": "254c2fd19fd39670eaee5828e7986bb98afbba15",
1437
+ "bytes": 1465,
1438
+ "exists": true
1439
+ },
1440
+ "sha256": "72030aed3e2c433ab547fbaa821d08d522404ed602aa7d1a94f3566c9a4fba1a"
1441
+ },
1442
+ {
1443
+ "bytes": 3470,
1444
+ "path": "benchmarks/raw/ornith-iq4-xs-target-no-fa-det-capture.json",
1445
+ "remote_before_upload": {
1446
+ "blob_id": "0c2bed5f0fd20705d2e7efa9535315df9a92c832",
1447
+ "bytes": 3470,
1448
+ "exists": true
1449
+ },
1450
+ "sha256": "0587d9cd065689941d702949d8ed5c4ee4c1367e44cf37693cdc3ffa8e52f78a"
1451
+ },
1452
+ {
1453
+ "bytes": 114484,
1454
+ "path": "benchmarks/raw/quantize-iq4xs-mtp-graft-headq6.log",
1455
+ "remote_before_upload": {
1456
+ "blob_id": "a9bf8546e9d9c5dba797d43bc85b580c2c92702e",
1457
+ "bytes": 114484,
1458
+ "exists": true
1459
+ },
1460
+ "sha256": "c07f6d7d1d2763f016d6d596d9ca4d72515ce94a0929ea06788a7824c04aab83"
1461
+ },
1462
+ {
1463
+ "bytes": 3270,
1464
+ "path": "benchmarks/sft-lora-smoke-bigcode.md",
1465
+ "remote_before_upload": {
1466
+ "blob_id": "a01c654e3754773b502149fac55efb1b4f77d6ce",
1467
+ "bytes": 3270,
1468
+ "exists": true
1469
+ },
1470
+ "sha256": "78a27788b71fee4805a38bcb5c4d4bb86af7f45d933a3ed361109bb920ac519b"
1471
+ },
1472
+ {
1473
+ "bytes": 5962,
1474
+ "path": "benchmarks/sft-prod-20k-preflight.md",
1475
+ "remote_before_upload": {
1476
+ "blob_id": "18e0e00637602514a0c8b9d26805fe84e80bc471",
1477
+ "bytes": 5962,
1478
+ "exists": true
1479
+ },
1480
+ "sha256": "0c67386c20a0af96740677f9ffc9aa48c5cc2f8a04e61e1770114c2cb492c519"
1481
+ },
1482
+ {
1483
+ "bytes": 842,
1484
+ "path": "configs/experiment_lanes.yaml",
1485
+ "remote_before_upload": {
1486
+ "blob_id": "93aa261c54ddf6804d2f62bb2f52e4d3a1d4a668",
1487
+ "bytes": 842,
1488
+ "exists": true
1489
+ },
1490
+ "sha256": "981c9a1efb1bc904092533c3e8fc1fbd0d6698d669e316c2961c3e3c16b7c6f6"
1491
+ },
1492
+ {
1493
+ "bytes": 4597,
1494
+ "path": "configs/model_behavior_suite.yaml",
1495
+ "remote_before_upload": {
1496
+ "blob_id": "21ddcf93dfaae55111281e925f0c9974be2efd7b",
1497
+ "bytes": 4597,
1498
+ "exists": true
1499
+ },
1500
+ "sha256": "cf64f993e6802f7a53869aba46fb8909588f69ca9d8eedfa52c2ff203bb90322"
1501
+ },
1502
+ {
1503
+ "bytes": 3621,
1504
+ "path": "configs/quantization_matrix.yaml",
1505
+ "remote_before_upload": {
1506
+ "blob_id": "6ed937004209b063a95372ece3422f013996f3fe",
1507
+ "bytes": 3621,
1508
+ "exists": true
1509
+ },
1510
+ "sha256": "84efb88c5d571d9942e118cd9cde458fda9ce327b5e71b671c55c9c2836a6a43"
1511
+ },
1512
+ {
1513
+ "bytes": 2287,
1514
+ "path": "configs/serving_matrix.yaml",
1515
+ "remote_before_upload": {
1516
+ "blob_id": "63d4c96c5aaace66eff3a86ce72c379c948422bc",
1517
+ "bytes": 2287,
1518
+ "exists": true
1519
+ },
1520
+ "sha256": "0c8f6d4f79ca9867c417121320020e8d5f3fb08fc6644d3ffc694ad2b671f812"
1521
+ },
1522
+ {
1523
+ "bytes": 3955,
1524
+ "path": "configs/sft_repair_blend.yaml",
1525
+ "remote_before_upload": {
1526
+ "blob_id": null,
1527
+ "bytes": null,
1528
+ "exists": false
1529
+ },
1530
+ "sha256": "a75bea8705aa4d3014fb4abf4828f4ac66b8cd346903f5609847314733511477"
1531
+ },
1532
+ {
1533
+ "bytes": 3432,
1534
+ "path": "configs/training_plan.yaml",
1535
+ "remote_before_upload": {
1536
+ "blob_id": "8fd4bbc2a67c029bf0b360d0901ec200292887e6",
1537
+ "bytes": 3192,
1538
+ "exists": true
1539
+ },
1540
+ "sha256": "c5d54324464029bf291885a3f7785182e6928608d49d97d02dae151cb24ffbd8"
1541
+ },
1542
+ {
1543
+ "bytes": 745,
1544
+ "path": "data/eval/README.md",
1545
+ "remote_before_upload": {
1546
+ "blob_id": null,
1547
+ "bytes": null,
1548
+ "exists": false
1549
+ },
1550
+ "sha256": "b113e75c89902c91b2f1f2008eb1ed194379f26f030d7bcbbd335183267c54f5"
1551
+ },
1552
+ {
1553
+ "bytes": 376090,
1554
+ "path": "data/eval/coding_sft_repair_eval_messages.jsonl",
1555
+ "remote_before_upload": {
1556
+ "blob_id": null,
1557
+ "bytes": null,
1558
+ "exists": false
1559
+ },
1560
+ "sha256": "abf8b701b4b9f8f691f4212aedea20dd2c0f5aeac01e601d713ff127c81bd9be"
1561
+ },
1562
+ {
1563
+ "bytes": 1283,
1564
+ "path": "data/train/README.md",
1565
+ "remote_before_upload": {
1566
+ "blob_id": null,
1567
+ "bytes": null,
1568
+ "exists": false
1569
+ },
1570
+ "sha256": "6f97d0c2f4f0910f6963a542af312c4d65f18f4e71560eb275814aee074a825d"
1571
+ },
1572
+ {
1573
+ "bytes": 3015307,
1574
+ "path": "data/train/coding_sft_repair_messages.jsonl",
1575
+ "remote_before_upload": {
1576
+ "blob_id": null,
1577
+ "bytes": null,
1578
+ "exists": false
1579
+ },
1580
+ "sha256": "bbc6f035b663d5da98e6556b5e40db8bde95ee9665fac009f3072652b1d224c6"
1581
+ },
1582
+ {
1583
+ "bytes": 148531,
1584
+ "path": "patches/llamacpp-mtp-adaptive-serving-and-recurrent-rollback.patch",
1585
+ "remote_before_upload": {
1586
+ "blob_id": "097d6db43a3ed071abb74e3d1d5bb6a89ecad2b1",
1587
+ "bytes": 148531,
1588
+ "exists": true
1589
+ },
1590
+ "sha256": "85ac5b87198808cb0bf484f52ba52b7c696b537af5924c9984cdaf194fa127b4"
1591
+ },
1592
+ {
1593
+ "bytes": 7408,
1594
+ "path": "patches/llamacpp-qwen35moe-mtp-recurrent-row-index.patch",
1595
+ "remote_before_upload": {
1596
+ "blob_id": "420c507b6a95e0a4babfcf951b836fb89063a933",
1597
+ "bytes": 7408,
1598
+ "exists": true
1599
+ },
1600
+ "sha256": "9920acd215e1c76fa99e2744d1b9fea1a1a7f9a83206006e125cbf911b31ff04"
1601
+ },
1602
+ {
1603
+ "bytes": 102858,
1604
+ "path": "patches/llamacpp-server-mtp-seqverify-and-perhead.patch",
1605
+ "remote_before_upload": {
1606
+ "blob_id": "153ce013c233ea95c17d0b58665ef279abf3c4bc",
1607
+ "bytes": 102858,
1608
+ "exists": true
1609
+ },
1610
+ "sha256": "a6d71cc81f15771527c386d7f689b40e2ed4a66fc1a50fb2c2f7d7b29ccb6b11"
1611
+ },
1612
+ {
1613
+ "bytes": 9002,
1614
+ "path": "probes/behavior-llamacpp-iq4-xs-parallel16.json",
1615
+ "remote_before_upload": {
1616
+ "blob_id": "d3935aae6472ed6f6f9e6959c93ed791b0fe1620",
1617
+ "bytes": 9002,
1618
+ "exists": true
1619
+ },
1620
+ "sha256": "bd979c80a913f411c8e4b1182d7dffff1fce32e85e4285ad91202a7eb04531ef"
1621
+ },
1622
+ {
1623
+ "bytes": 9010,
1624
+ "path": "probes/behavior-llamacpp-q3-k-m-parallel16.json",
1625
+ "remote_before_upload": {
1626
+ "blob_id": "942f9ce5b6af97487ab78feb844d8b6009a778f1",
1627
+ "bytes": 9010,
1628
+ "exists": true
1629
+ },
1630
+ "sha256": "05b8fdc912af46101d488ae23f6b7460ce770792cf69b1061edb2ed5a7b90418"
1631
+ },
1632
+ {
1633
+ "bytes": 9029,
1634
+ "path": "probes/behavior-llamacpp-q4-script-default.json",
1635
+ "remote_before_upload": {
1636
+ "blob_id": "7f9609ca10253abc3593d6763faa2d0649974c88",
1637
+ "bytes": 9029,
1638
+ "exists": true
1639
+ },
1640
+ "sha256": "f30e429840b15f2c5b39c83d39a1d2fb0321a586eda1951eaa2df9f49c376b5d"
1641
+ },
1642
+ {
1643
+ "bytes": 15288,
1644
+ "path": "probes/behavior-llamacpp-q4.json",
1645
+ "remote_before_upload": {
1646
+ "blob_id": "9e52b20873ad62f34f12dbcc41c19ec349a72546",
1647
+ "bytes": 15288,
1648
+ "exists": true
1649
+ },
1650
+ "sha256": "359d285644f68caa7deaa929add6f1e49770a1a825383466989c0213885581c3"
1651
+ },
1652
+ {
1653
+ "bytes": 1049,
1654
+ "path": "probes/eval-lora-behavior-checkpoint-8000-fixed.gate.log",
1655
+ "remote_before_upload": {
1656
+ "blob_id": "041fb5bb56dc25cf23b634437948011ae77ac1da",
1657
+ "bytes": 1049,
1658
+ "exists": true
1659
+ },
1660
+ "sha256": "1c848099e35001e1e29ef6dd2e78b986b20fcec1affa5b060c314bbc7b2b7088"
1661
+ },
1662
+ {
1663
+ "bytes": 21512,
1664
+ "path": "probes/eval-lora-behavior-checkpoint-8000-fixed.json",
1665
+ "remote_before_upload": {
1666
+ "blob_id": "c0376b5f05405e52dc747a1017c1860b45229c34",
1667
+ "bytes": 21512,
1668
+ "exists": true
1669
+ },
1670
+ "sha256": "76517d6ffbb33a2db3100b46ef7407e0c6d2ce57332784e1af4fbdfb4188bda0"
1671
+ },
1672
+ {
1673
+ "bytes": 44591,
1674
+ "path": "probes/gguf-materialization-q3-k-m.json",
1675
+ "remote_before_upload": {
1676
+ "blob_id": "673df299dea1d65393687805b61adbec09f4917a",
1677
+ "bytes": 44591,
1678
+ "exists": true
1679
+ },
1680
+ "sha256": "83ff40e6dbc398f1484469ddeb113a4369c7eaff486457e76de85c1effd87895"
1681
+ },
1682
+ {
1683
+ "bytes": 28352,
1684
+ "path": "probes/investigation-ornith-q4-live.json",
1685
+ "remote_before_upload": {
1686
+ "blob_id": "470ec1394c86b38eedb2a97ae32530b5a3329f48",
1687
+ "bytes": 28352,
1688
+ "exists": true
1689
+ },
1690
+ "sha256": "91f3d6d3a51a0b2a6a841e710a55f31a4bb3f8cf452f1eceea039cc26d7f97f8"
1691
+ },
1692
+ {
1693
+ "bytes": 1304,
1694
+ "path": "probes/investigation-ornith-q4-live.md",
1695
+ "remote_before_upload": {
1696
+ "blob_id": "a6fe7fccd66e2d29e1d02571c54fbc1f415cf1fd",
1697
+ "bytes": 1304,
1698
+ "exists": true
1699
+ },
1700
+ "sha256": "9fabb8bdbb0cda54da958e868abbc75c063a510b0f414c1c5c1bd9e03344421f"
1701
+ },
1702
+ {
1703
+ "bytes": 826,
1704
+ "path": "probes/mtp-support-ornith-iq4-xs.json",
1705
+ "remote_before_upload": {
1706
+ "blob_id": "86e138820d98eabdf11686139674b4007af82e55",
1707
+ "bytes": 826,
1708
+ "exists": true
1709
+ },
1710
+ "sha256": "166cdb3141fd55196e86364c441d2b55a31640ae2ff717a0931664e9d0c81d41"
1711
+ },
1712
+ {
1713
+ "bytes": 4390,
1714
+ "path": "probes/probe-llamacpp-iq4-xs-correctness.json",
1715
+ "remote_before_upload": {
1716
+ "blob_id": "19709db97aa20c6554281940880b5ca3ee9b4474",
1717
+ "bytes": 4390,
1718
+ "exists": true
1719
+ },
1720
+ "sha256": "f9e1db3561bfa568a92d9fa590d44eb8bae895c9d7cdf8add34e46a77b53e779"
1721
+ },
1722
+ {
1723
+ "bytes": 4341,
1724
+ "path": "probes/probe-llamacpp-iq4-xs-mtp-bootstrap-correctness.json",
1725
+ "remote_before_upload": {
1726
+ "blob_id": "7b48408c6871d2914b2a0396cd81ccbbc3f3155c",
1727
+ "bytes": 4341,
1728
+ "exists": true
1729
+ },
1730
+ "sha256": "7240aa6f9b8c0b38257e3efd747a99d3ebdbd021147d4ec0a9a3cce2f1d0aaaf"
1731
+ },
1732
+ {
1733
+ "bytes": 4457,
1734
+ "path": "probes/probe-llamacpp-iq4-xs-mtp-bootstrap-iq4draft-nmax1-correctness.json",
1735
+ "remote_before_upload": {
1736
+ "blob_id": "b7d97b77ce6fa6fd2a2a71bdaa521a88a19daf92",
1737
+ "bytes": 4457,
1738
+ "exists": true
1739
+ },
1740
+ "sha256": "1ea6c398d16973bdbb154d79e0f233fd76a79c3c6e2a33704288585cfe75ecf3"
1741
+ },
1742
+ {
1743
+ "bytes": 4403,
1744
+ "path": "probes/probe-llamacpp-iq4-xs-mtp-bootstrap-nmax1-correctness.json",
1745
+ "remote_before_upload": {
1746
+ "blob_id": "dc3e6e93c043f303e73c1a5aa9d88d5b2e4ff981",
1747
+ "bytes": 4403,
1748
+ "exists": true
1749
+ },
1750
+ "sha256": "9bf864843be7674bb02798f2f1ce65eddcc2cda7c0ee484cb0313aec775a2728"
1751
+ },
1752
+ {
1753
+ "bytes": 5833,
1754
+ "path": "probes/probe-llamacpp-q4-correctness.json",
1755
+ "remote_before_upload": {
1756
+ "blob_id": "453a6b70c3fb2af7fd427cefd54c36c3931a0882",
1757
+ "bytes": 5833,
1758
+ "exists": true
1759
+ },
1760
+ "sha256": "d23e14d7c8fcb2fa2e249aaf89bb494e1e9bea9cf10608b12a25fe3b3e8859e3"
1761
+ },
1762
+ {
1763
+ "bytes": 5876,
1764
+ "path": "probes/probe-llamacpp-q5-correctness.json",
1765
+ "remote_before_upload": {
1766
+ "blob_id": "58566533b8dd2dc3de4c1cbd0e5535a18799ca1a",
1767
+ "bytes": 5876,
1768
+ "exists": true
1769
+ },
1770
+ "sha256": "e7cb01b1d2cc93935f7e679c66b357a099ca9ccac48efc9afd4cea01bcdc01dd"
1771
+ },
1772
+ {
1773
+ "bytes": 5820,
1774
+ "path": "probes/probe-llamacpp-q6-correctness.json",
1775
+ "remote_before_upload": {
1776
+ "blob_id": "78c6af38d171ae07f3ee33b62b3d2d8535ad0e27",
1777
+ "bytes": 5820,
1778
+ "exists": true
1779
+ },
1780
+ "sha256": "f93e794a6e38d6a4eee15ac419b42aef59b634feb83b80c293823a9e9b4481f7"
1781
+ },
1782
+ {
1783
+ "bytes": 5840,
1784
+ "path": "probes/probe-llamacpp-q8-correctness.json",
1785
+ "remote_before_upload": {
1786
+ "blob_id": "03221c4f507cd75e4ef02c838d2b9be65e2f8c64",
1787
+ "bytes": 5840,
1788
+ "exists": true
1789
+ },
1790
+ "sha256": "fa01e4d58c37d581786ad78b0b773547acefb1dbce0a08373ca36eb6c7229eb7"
1791
+ },
1792
+ {
1793
+ "bytes": 3925,
1794
+ "path": "probes/sft-data-validation-prod-repair.json",
1795
+ "remote_before_upload": {
1796
+ "blob_id": null,
1797
+ "bytes": null,
1798
+ "exists": false
1799
+ },
1800
+ "sha256": "a19cd71f1338fa6bdfb1547c13b51db726fbf66243e28d431575f377b8d504b0"
1801
+ },
1802
+ {
1803
+ "bytes": 17718,
1804
+ "path": "probes/sft-format-audit-prod-repair.json",
1805
+ "remote_before_upload": {
1806
+ "blob_id": null,
1807
+ "bytes": null,
1808
+ "exists": false
1809
+ },
1810
+ "sha256": "494c16a57b554a7c662e731bca6e6502a3423ab63e35a9ca371d2d37e0c60f09"
1811
+ },
1812
+ {
1813
+ "bytes": 10312,
1814
+ "path": "probes/sft-format-audit-prod.json",
1815
+ "remote_before_upload": {
1816
+ "blob_id": "998fbfd0102c3ade16465d368dd1ba82efdc7b66",
1817
+ "bytes": 10312,
1818
+ "exists": true
1819
+ },
1820
+ "sha256": "8a7b54a9652c7961aca20ba172a5ec4826d299e46af1ed6017373fe35f9ae0c0"
1821
+ },
1822
+ {
1823
+ "bytes": 1252,
1824
+ "path": "probes/sft-prod-20k-stopped.json",
1825
+ "remote_before_upload": {
1826
+ "blob_id": "ac4882041cf88837beecde1ead7418fd5262b28c",
1827
+ "bytes": 1252,
1828
+ "exists": true
1829
+ },
1830
+ "sha256": "bc0563fbdf62434b98eecf6094b34921f1b70cdb13d2614a8eb129d38602ad0b"
1831
+ },
1832
+ {
1833
+ "bytes": 495,
1834
+ "path": "probes/sft-repair-blend-eval.json",
1835
+ "remote_before_upload": {
1836
+ "blob_id": null,
1837
+ "bytes": null,
1838
+ "exists": false
1839
+ },
1840
+ "sha256": "2e585730cfdd855bda70e25ae64ee3833ec61f3e34ba32abc37f6b879996e99a"
1841
+ },
1842
+ {
1843
+ "bytes": 489,
1844
+ "path": "probes/sft-repair-blend-train.json",
1845
+ "remote_before_upload": {
1846
+ "blob_id": null,
1847
+ "bytes": null,
1848
+ "exists": false
1849
+ },
1850
+ "sha256": "5745855fc38abcb13bdb8ada381d1a4f1d3d61279d1c1f6c82f45cdaa5b1ae78"
1851
+ },
1852
+ {
1853
+ "bytes": 1239,
1854
+ "path": "probes/sft-repair-set.json",
1855
+ "remote_before_upload": {
1856
+ "blob_id": null,
1857
+ "bytes": null,
1858
+ "exists": false
1859
+ },
1860
+ "sha256": "a7e8124a4b34c32895a0da66056a2c6d39f751e249ca66b10bf93e27d61eeccf"
1861
+ },
1862
+ {
1863
+ "bytes": 134,
1864
+ "path": "requirements-lab.txt",
1865
+ "remote_before_upload": {
1866
+ "blob_id": "b526e5c40a476b8edb38591dd2d72ad818a087a2",
1867
+ "bytes": 134,
1868
+ "exists": true
1869
+ },
1870
+ "sha256": "3292913fb123daa3431bfc0a9b84301eee2e957abe249d35722ac7dca8d26e11"
1871
+ },
1872
+ {
1873
+ "bytes": 22567,
1874
+ "path": "scripts/audit_gguf_vllm_materialization.py",
1875
+ "remote_before_upload": {
1876
+ "blob_id": "a1df912ce4ac635b80d1927dbe595d42a4c7d685",
1877
+ "bytes": 22567,
1878
+ "exists": true
1879
+ },
1880
+ "sha256": "217a31e4df3817d9b765f9055b8592d94d023b6f6b5ff0a92acdbf2c3eed454a"
1881
+ },
1882
+ {
1883
+ "bytes": 16017,
1884
+ "path": "scripts/audit_model.py",
1885
+ "remote_before_upload": {
1886
+ "blob_id": "9a1232550356afcbde16a6eaf54a6bdbf466947d",
1887
+ "bytes": 16017,
1888
+ "exists": true
1889
+ },
1890
+ "sha256": "dc34af1da98312aa2cbd5bf7886d03c0ced32bb838d6f41d03277a7328d59c7f"
1891
+ },
1892
+ {
1893
+ "bytes": 5139,
1894
+ "path": "scripts/audit_sft_format.py",
1895
+ "remote_before_upload": {
1896
+ "blob_id": "66a3c95ecb74d43984c303b0b6e19820a911e147",
1897
+ "bytes": 5139,
1898
+ "exists": true
1899
+ },
1900
+ "sha256": "471f1929156fd4df794a61f5cf82236e0890eb92d36cdb50cbb4a91372fb3959"
1901
+ },
1902
+ {
1903
+ "bytes": 8169,
1904
+ "path": "scripts/bench_openai.py",
1905
+ "remote_before_upload": {
1906
+ "blob_id": "a5d63caae222f5df761668c58d36a158fa73653f",
1907
+ "bytes": 8169,
1908
+ "exists": true
1909
+ },
1910
+ "sha256": "e699059bedb1e0ae399c95570ae1a1f8cecf986f60ee576b6447d599f04f72a2"
1911
+ },
1912
+ {
1913
+ "bytes": 7615,
1914
+ "path": "scripts/build_bootstrap_mtp_checkpoint.py",
1915
+ "remote_before_upload": {
1916
+ "blob_id": "832542c41db2ba872aba8ee463b3abca415c1338",
1917
+ "bytes": 7615,
1918
+ "exists": true
1919
+ },
1920
+ "sha256": "cf2d52ac14ac23b9c84aa35a8a4c965afed5573f711221411eb93acd634ef526"
1921
+ },
1922
+ {
1923
+ "bytes": 4467,
1924
+ "path": "scripts/build_sft_blend.py",
1925
+ "remote_before_upload": {
1926
+ "blob_id": null,
1927
+ "bytes": null,
1928
+ "exists": false
1929
+ },
1930
+ "sha256": "0a7a463952856e166170ab46a625e22fe73cc1905df33714399af80acdd51ba4"
1931
+ },
1932
+ {
1933
+ "bytes": 10235,
1934
+ "path": "scripts/build_sft_repair_set.py",
1935
+ "remote_before_upload": {
1936
+ "blob_id": null,
1937
+ "bytes": null,
1938
+ "exists": false
1939
+ },
1940
+ "sha256": "835c534808f3fd72c5995dd28e32543fbc6318b5458dd8fd113f1362171547ed"
1941
+ },
1942
+ {
1943
+ "bytes": 2200,
1944
+ "path": "scripts/check_audit_gates.py",
1945
+ "remote_before_upload": {
1946
+ "blob_id": "846f1500d0f2aceeb2c97388d3e4b68439742e18",
1947
+ "bytes": 2200,
1948
+ "exists": true
1949
+ },
1950
+ "sha256": "0bdbf74406a80ce7dd80342095eb06fbe9ec80f8877638d672a26d1e1f2b3285"
1951
+ },
1952
+ {
1953
+ "bytes": 5718,
1954
+ "path": "scripts/check_mtp_support.py",
1955
+ "remote_before_upload": {
1956
+ "blob_id": "6e674b480252381cd367b480095ffa0d4e65df3a",
1957
+ "bytes": 5718,
1958
+ "exists": true
1959
+ },
1960
+ "sha256": "c5f7994e90122971dd6972e7c40855d6afad3e7a8d9de378eab7f378a259e296"
1961
+ },
1962
+ {
1963
+ "bytes": 4379,
1964
+ "path": "scripts/collect_next_token_logprobs.py",
1965
+ "remote_before_upload": {
1966
+ "blob_id": "1fefc910e4b1ebf41ca54c332d404918583f97a6",
1967
+ "bytes": 4379,
1968
+ "exists": true
1969
+ },
1970
+ "sha256": "dcad13feab27fbb4a9d63e95b77649fbf82cf8149fff69a393ed57e56dc96b1f"
1971
+ },
1972
+ {
1973
+ "bytes": 6926,
1974
+ "path": "scripts/compare_kld_logprobs.py",
1975
+ "remote_before_upload": {
1976
+ "blob_id": "aaf11e0827f98fe413959c07296cf1021d4def71",
1977
+ "bytes": 6926,
1978
+ "exists": true
1979
+ },
1980
+ "sha256": "371da2dd45b2a240c5cfb329a6a2c97d90dae7bf56422e655e62151e842f25c9"
1981
+ },
1982
+ {
1983
+ "bytes": 11075,
1984
+ "path": "scripts/eval_lora_behavior.py",
1985
+ "remote_before_upload": {
1986
+ "blob_id": "964d10591be26f0b144d5934db14834aad110d30",
1987
+ "bytes": 10715,
1988
+ "exists": true
1989
+ },
1990
+ "sha256": "47726dab593caa4863efcf2d1f33404cc23143811799e3b1fccfd9000ad24995"
1991
+ },
1992
+ {
1993
+ "bytes": 2313,
1994
+ "path": "scripts/launch_sft_prod_20k_gpu1.sh",
1995
+ "remote_before_upload": {
1996
+ "blob_id": null,
1997
+ "bytes": null,
1998
+ "exists": false
1999
+ },
2000
+ "sha256": "2c879a9b608eee2d9dbfb96c20fdca5f2ccfb495efcef62affbaa84aa4c7a6ca"
2001
+ },
2002
+ {
2003
+ "bytes": 16665,
2004
+ "path": "scripts/probe_model_behavior.py",
2005
+ "remote_before_upload": {
2006
+ "blob_id": "700ec81ab96b0573afdac1ffbaa84a9f0f94b903",
2007
+ "bytes": 16665,
2008
+ "exists": true
2009
+ },
2010
+ "sha256": "80bf223764395794ce3359860e58742a6798c6b4c5815ecbf649d39cbf0b0060"
2011
+ },
2012
+ {
2013
+ "bytes": 7877,
2014
+ "path": "scripts/probe_openai_contract.py",
2015
+ "remote_before_upload": {
2016
+ "blob_id": "3b9acd1e52f8203a59f1d72369f148e2bc672304",
2017
+ "bytes": 7877,
2018
+ "exists": true
2019
+ },
2020
+ "sha256": "dd4187dcfa8973dc120dd492b09d7c6fb172461b22911e2675b253ba09e9de94"
2021
+ },
2022
+ {
2023
+ "bytes": 6082,
2024
+ "path": "scripts/probe_openai_determinism.py",
2025
+ "remote_before_upload": {
2026
+ "blob_id": "50a6c2c88371612ad72c9bf3a79f05785ad66fc2",
2027
+ "bytes": 6082,
2028
+ "exists": true
2029
+ },
2030
+ "sha256": "89f2dbe49d563b5537c935d54be3682b7d23e3dbdab2c793f63e7125454646f4"
2031
+ },
2032
+ {
2033
+ "bytes": 3115,
2034
+ "path": "scripts/render_sft_jsonl.py",
2035
+ "remote_before_upload": {
2036
+ "blob_id": null,
2037
+ "bytes": null,
2038
+ "exists": false
2039
+ },
2040
+ "sha256": "7495bdc31867c2c6cc52fb761bf4321b62376f3a8a98f699999b6a7b41b731f7"
2041
+ },
2042
+ {
2043
+ "bytes": 4573,
2044
+ "path": "scripts/run_model_investigation.py",
2045
+ "remote_before_upload": {
2046
+ "blob_id": "7b1a137d8ffe3d79652141a04cd9477bef67d831",
2047
+ "bytes": 4573,
2048
+ "exists": true
2049
+ },
2050
+ "sha256": "b1dab876292279a13380d0e6045c94b3077a8501ae8f836682e447dc84361b3c"
2051
+ },
2052
+ {
2053
+ "bytes": 4382,
2054
+ "path": "scripts/serve_llamacpp_gpu0.sh",
2055
+ "remote_before_upload": {
2056
+ "blob_id": "a12b1a8154ba4603d8c61d316a8efbf8eb2378f9",
2057
+ "bytes": 4382,
2058
+ "exists": true
2059
+ },
2060
+ "sha256": "02992430ab841bc77586a63f9e4756b070abfc1eac8ba0042cb1db55e3261709"
2061
+ },
2062
+ {
2063
+ "bytes": 171,
2064
+ "path": "scripts/serve_llamacpp_gpu1.sh",
2065
+ "remote_before_upload": {
2066
+ "blob_id": "548586de5c0b343bafdfaa28fab7138e28e9f0f5",
2067
+ "bytes": 171,
2068
+ "exists": true
2069
+ },
2070
+ "sha256": "d1d1c7d102bc40255352cd176a3e44a97e84b3ff85557c3e4a9a3475c625ae33"
2071
+ },
2072
+ {
2073
+ "bytes": 5117,
2074
+ "path": "scripts/summarize_investigation.py",
2075
+ "remote_before_upload": {
2076
+ "blob_id": "d723430849c22f53b2a4d91a5fea9a427819edf0",
2077
+ "bytes": 5117,
2078
+ "exists": true
2079
+ },
2080
+ "sha256": "2125556f7b4f9eea5fe4b925ac5022632a5a2b8ff42c54fca20d80185cfc1e14"
2081
+ },
2082
+ {
2083
+ "bytes": 6488,
2084
+ "path": "scripts/train_sft_lora.py",
2085
+ "remote_before_upload": {
2086
+ "blob_id": null,
2087
+ "bytes": null,
2088
+ "exists": false
2089
+ },
2090
+ "sha256": "340edc3ee4e104e9bf8145195b05ce9f106c63f3b128c32d181ed0512140868a"
2091
+ },
2092
+ {
2093
+ "bytes": 15231,
2094
+ "path": "scripts/validate_sft_jsonl.py",
2095
+ "remote_before_upload": {
2096
+ "blob_id": null,
2097
+ "bytes": null,
2098
+ "exists": false
2099
+ },
2100
+ "sha256": "b1db75a43b9b3415cba337b868638a496f960cf492bbdfa4c31ad978c277908c"
2101
+ }
2102
+ ]
2103
  }
MODEL_INVESTIGATION_HARNESS.md CHANGED
@@ -177,6 +177,28 @@ SFT data must pass validation before loading the 35B model:
177
  --output runs/sft-format-audit-prod.json
178
  ```
179
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
180
  ```bash
181
  .venv-train/bin/python scripts/validate_sft_jsonl.py \
182
  --train-jsonl data/train/coding_sft_prod.jsonl \
@@ -200,6 +222,8 @@ Current stopped-run evidence:
200
  - `runs/ornith-35b-coding-lora-prod-20k.stopped.json`
201
  - `artifacts/train/ornith-35b-coding-lora-prod-20k/checkpoint-8000`
202
  - `runs/sft-format-audit-prod.json`
 
 
203
  - `runs/eval-lora-behavior-checkpoint-8000-fixed.json`
204
  - `runs/eval-lora-behavior-checkpoint-8000-fixed.gate.log`
205
 
 
177
  --output runs/sft-format-audit-prod.json
178
  ```
179
 
180
+ For the stopped Ornith SFT run, the CPU-side repair path is:
181
+
182
+ ```bash
183
+ .venv-lab/bin/python scripts/build_sft_repair_set.py \
184
+ --train-size 4096 \
185
+ --eval-size 512 \
186
+ --seed 29 \
187
+ --train-output data/train/coding_sft_repair_messages.jsonl \
188
+ --eval-output data/eval/coding_sft_repair_eval_messages.jsonl \
189
+ --report-output runs/sft-repair-set.json
190
+
191
+ .venv-lab/bin/python scripts/build_sft_blend.py \
192
+ --source data/train/coding_sft_prod_messages.jsonl=40000:base_coding \
193
+ --source data/train/coding_sft_repair_messages.jsonl=4096:strict_repair \
194
+ --output data/train/coding_sft_prod_repair_messages.jsonl \
195
+ --report-output runs/sft-repair-blend-train.json \
196
+ --seed 31
197
+ ```
198
+
199
+ See `configs/sft_repair_blend.yaml` for the full train/eval render, validation,
200
+ and audit command sequence.
201
+
202
  ```bash
203
  .venv-train/bin/python scripts/validate_sft_jsonl.py \
204
  --train-jsonl data/train/coding_sft_prod.jsonl \
 
222
  - `runs/ornith-35b-coding-lora-prod-20k.stopped.json`
223
  - `artifacts/train/ornith-35b-coding-lora-prod-20k/checkpoint-8000`
224
  - `runs/sft-format-audit-prod.json`
225
+ - `runs/sft-data-validation-prod-repair.json`
226
+ - `runs/sft-format-audit-prod-repair.json`
227
  - `runs/eval-lora-behavior-checkpoint-8000-fixed.json`
228
  - `runs/eval-lora-behavior-checkpoint-8000-fixed.gate.log`
229
 
PROJECT_STATUS_AUDIT.md CHANGED
@@ -26,6 +26,7 @@ release-candidate adapter.
26
  | Make quants usable on HF | Complete | HF repo `LordNeel/Ornith-1.0-35B-GGUF-llamacpp-tp1`, `RELEASE_AUDIT.md` | Remote README advertised 62 files with 0 missing after readback verification. |
27
  | Fine-tune on coding data and post-train | Incomplete by user stop | `runs/ornith-35b-coding-lora-prod-20k.stopped.json`, `hf_upload/probes/sft-prod-20k-stopped.json`, `benchmarks/sft-prod-20k-preflight.md` | Production LoRA run stopped at user request. Latest durable checkpoint is `checkpoint-8000`; training log reached step 8673 but no `checkpoint-9000` exists. |
28
  | Diagnose SFT gate failure | Complete | `runs/sft-format-audit-prod.json`, `SFT_REMEDIATION_PLAN.md`, `scripts/audit_sft_format.py` | Format audit found 100% code-fenced answers, about 56% `Here's` answers, 0 short exact-answer rows, and empty rendered think blocks in every rendered row. |
 
29
  | Release fine-tuned adapter | Not achieved | `runs/eval-lora-behavior-checkpoint-8000-fixed.json`, `runs/eval-lora-behavior-checkpoint-8000-fixed.gate.log` | Checkpoint-8000 loaded cleanly but failed 4 behavior checks, including high-severity strict-output failures. No adapter should be published from this run. |
30
  | HF model cards/docs updated | Complete | Remote commit `84f7d57948815525e5ed2c0a8423a80f8cf38f71` before this status audit | README, release audit, artifact manifest, benchmark docs, probes, scripts, configs, and harness were uploaded and read back. This status audit creates a later commit when published. |
31
  | Keep GPUs idle after SFT stop | Complete at audit time | `nvidia-smi`: GPU0 10 MiB / 0%, GPU1 2 MiB / 0%; process scan empty | No `train_sft_lora`, watcher, eval, server, quantize, or convert processes were running. |
@@ -47,7 +48,8 @@ release-candidate adapter.
47
  1. Fine-tuning/post-training needs a new or resumed training plan if a release
48
  adapter is still desired. The stopped run should resume only from
49
  `checkpoint-8000`, and later checkpoints must pass `configs/model_behavior_suite.yaml`.
50
- Before spending GPU time, rebuild or repair the blend per `SFT_REMEDIATION_PLAN.md`.
 
51
  2. Always-on MTP should not be promoted for saturated batches unless a future
52
  runtime change beats matched AR throughput at c16.
53
  3. vLLM GGUF should remain unsupported for this model family until its corrupted
 
26
  | Make quants usable on HF | Complete | HF repo `LordNeel/Ornith-1.0-35B-GGUF-llamacpp-tp1`, `RELEASE_AUDIT.md` | Remote README advertised 62 files with 0 missing after readback verification. |
27
  | Fine-tune on coding data and post-train | Incomplete by user stop | `runs/ornith-35b-coding-lora-prod-20k.stopped.json`, `hf_upload/probes/sft-prod-20k-stopped.json`, `benchmarks/sft-prod-20k-preflight.md` | Production LoRA run stopped at user request. Latest durable checkpoint is `checkpoint-8000`; training log reached step 8673 but no `checkpoint-9000` exists. |
28
  | Diagnose SFT gate failure | Complete | `runs/sft-format-audit-prod.json`, `SFT_REMEDIATION_PLAN.md`, `scripts/audit_sft_format.py` | Format audit found 100% code-fenced answers, about 56% `Here's` answers, 0 short exact-answer rows, and empty rendered think blocks in every rendered row. |
29
+ | Prepare CPU-side SFT repair path | Complete | `configs/sft_repair_blend.yaml`, `scripts/build_sft_repair_set.py`, `scripts/build_sft_blend.py`, `runs/sft-data-validation-prod-repair.json`, `runs/sft-format-audit-prod-repair.json` | Generated a strict-output repair component and blended 44,096 train / 2,312 eval rendered rows. Validation has no blocking issues; SFT remains stopped. |
30
  | Release fine-tuned adapter | Not achieved | `runs/eval-lora-behavior-checkpoint-8000-fixed.json`, `runs/eval-lora-behavior-checkpoint-8000-fixed.gate.log` | Checkpoint-8000 loaded cleanly but failed 4 behavior checks, including high-severity strict-output failures. No adapter should be published from this run. |
31
  | HF model cards/docs updated | Complete | Remote commit `84f7d57948815525e5ed2c0a8423a80f8cf38f71` before this status audit | README, release audit, artifact manifest, benchmark docs, probes, scripts, configs, and harness were uploaded and read back. This status audit creates a later commit when published. |
32
  | Keep GPUs idle after SFT stop | Complete at audit time | `nvidia-smi`: GPU0 10 MiB / 0%, GPU1 2 MiB / 0%; process scan empty | No `train_sft_lora`, watcher, eval, server, quantize, or convert processes were running. |
 
48
  1. Fine-tuning/post-training needs a new or resumed training plan if a release
49
  adapter is still desired. The stopped run should resume only from
50
  `checkpoint-8000`, and later checkpoints must pass `configs/model_behavior_suite.yaml`.
51
+ Before spending GPU time, use the prepared repair blend in
52
+ `configs/sft_repair_blend.yaml` and `SFT_REMEDIATION_PLAN.md`.
53
  2. Always-on MTP should not be promoted for saturated batches unless a future
54
  runtime change beats matched AR throughput at c16.
55
  3. vLLM GGUF should remain unsupported for this model family until its corrupted
README.md CHANGED
@@ -1,466 +1,202 @@
1
- ---
2
- base_model:
3
- - deepreinforce-ai/Ornith-1.0-35B
4
- tags:
5
- - gguf
6
- - llama.cpp
7
- - coding
8
- - quantized
9
- - single-gpu
10
- - openai-compatible
11
- - mtp
12
- - speculative-decoding
13
- library_name: gguf
14
- ---
15
-
16
- # Ornith-1.0-35B GGUF Single-GPU Serving Package
17
-
18
- This repo packages validated GGUF quants of `deepreinforce-ai/Ornith-1.0-35B` with a single-GPU llama.cpp serving profile and benchmark artifacts.
19
-
20
- Validated on 2026-06-26 with one RTX PRO 6000 Blackwell Max-Q 96GB GPU, `tp=1`, using `llama.cpp` CUDA server. Updated on 2026-06-27 with a local IQ4_XS quant, KLD probe, trained experimental MTP drafts, an integrated grafted-MTP runtime comparison, and an adaptive single-GPU MTP serving profile.
21
-
22
- ## Files
23
-
24
- - `ornith-1.0-35b-Q4_K_M.gguf`
25
- - `ornith-1.0-35b-Q3_K_M.gguf`
26
- - `ornith-1.0-35b-IQ4_XS.gguf`
27
- - `ornith-1.0-35b-IQ4_XS-MTP-graft-headQ6.gguf`
28
- - `ornith-1.0-35b-mtp-bootstrap-layer39-IQ4_XS.gguf`
29
- - `ornith-1.0-35b-mtp-chaincorr-h2r9fw125-h1r2-h3r6ce375-lr5e7-Q6_K.gguf`
30
- - `ornith-1.0-35b-mtp-chaincorr-h2r9fw125-h1r2-h3r6ce375-lr5e7-Q5_K_M.gguf`
31
- - `ornith-1.0-35b-mtp-chaincorr-h2r9fw125-h1r2-h3r6ce375-lr5e7-IQ4_XS.gguf`
32
- - `ornith-1.0-35b-mtp-chaincorr-h2r10prefix250-h1r2-h3r8fw-lr5e6-Q6_K.gguf`
33
- - `ornith-1.0-35b-mtp-chaincorr-h2r10prefix500-h1r2-h3r8fw-lr5e6-Q6_K.gguf`
34
- - `ornith-1.0-35b-mtp-chaincorr-h2r10prefix1000-h1r2-h3r8fw-lr5e6-Q6_K.gguf`
35
- - `ornith-1.0-35b-Q5_K_M.gguf`
36
- - `ornith-1.0-35b-Q6_K.gguf`
37
- - `ornith-1.0-35b-Q8_0.gguf`
38
- - `ARTIFACT_MANIFEST.json`
39
- - `RELEASE_AUDIT.md`
40
- - `PROJECT_STATUS_AUDIT.md`
41
- - `SFT_REMEDIATION_PLAN.md`
42
- - `MODEL_INVESTIGATION_HARNESS.md`
43
- - `requirements-lab.txt`
44
- - `configs/model_behavior_suite.yaml`
45
- - `configs/serving_matrix.yaml`
46
- - `configs/quantization_matrix.yaml`
47
- - `configs/experiment_lanes.yaml`
48
- - `configs/training_plan.yaml`
49
- - `scripts/serve_llamacpp_gpu0.sh`
50
- - `scripts/serve_llamacpp_gpu1.sh`
51
- - `scripts/run_model_investigation.py`
52
- - `scripts/audit_model.py`
53
- - `scripts/check_audit_gates.py`
54
- - `scripts/summarize_investigation.py`
55
- - `scripts/probe_model_behavior.py`
56
- - `scripts/probe_openai_contract.py`
57
- - `scripts/bench_openai.py`
58
- - `scripts/collect_next_token_logprobs.py`
59
- - `scripts/compare_kld_logprobs.py`
60
- - `scripts/check_mtp_support.py`
61
- - `scripts/audit_sft_format.py`
62
- - `scripts/audit_gguf_vllm_materialization.py`
63
- - `scripts/build_bootstrap_mtp_checkpoint.py`
64
- - `patches/llamacpp-qwen35moe-mtp-recurrent-row-index.patch`
65
- - `benchmarks/llamacpp-quant-benchmarks.md`
66
- - `patches/llamacpp-mtp-adaptive-serving-and-recurrent-rollback.patch`
67
- - `benchmarks/kld-quant-vs-bf16-top64.md`
68
- - `benchmarks/sft-lora-smoke-bigcode.md`
69
- - `benchmarks/sft-prod-20k-preflight.md`
70
- - `probes/sft-prod-20k-stopped.json`
71
- - `probes/sft-format-audit-prod.json`
72
- - `benchmarks/llamacpp-q3-k-m-profile.md`
73
- - `benchmarks/llamacpp-iq4-xs-profile.md`
74
- - `benchmarks/llamacpp-iq4-xs-mtp-bootstrap-profile.md`
75
- - `benchmarks/llamacpp-iq4-xs-mtp-chaincorr-profile.md`
76
- - `benchmarks/llamacpp-iq4-xs-mtp-graft-adaptive-profile.md`
77
- - `benchmarks/llamacpp-q4-reasoning-off-fix.md`
78
- - `probes/behavior-llamacpp-q3-k-m-parallel16.json`
79
- - `probes/probe-llamacpp-iq4-xs-correctness.json`
80
- - `probes/behavior-llamacpp-iq4-xs-parallel16.json`
81
- - `probes/probe-llamacpp-iq4-xs-mtp-bootstrap-correctness.json`
82
- - `probes/probe-llamacpp-iq4-xs-mtp-bootstrap-nmax1-correctness.json`
83
- - `probes/probe-llamacpp-iq4-xs-mtp-bootstrap-iq4draft-nmax1-correctness.json`
84
- - `probes/mtp-support-ornith-iq4-xs.json`
85
- - `probes/gguf-materialization-q3-k-m.json`
86
- - `probes/behavior-llamacpp-q4.json`
87
- - `probes/behavior-llamacpp-q4-script-default.json`
88
- - `probes/investigation-ornith-q4-live.md`
89
- - `probes/investigation-ornith-q4-live.json`
90
-
91
- ## Recommended Quant
92
-
93
- Use `Q4_K_M` for speed-focused serving. It remains the fastest validated GGUF in the benchmark profile.
94
-
95
- Use `Q3_K_M` when footprint matters more than peak throughput. It is a custom quant generated from the upstream BF16 GGUF, validated through the materialization audit and behavior suite, and is about 20.8% smaller on disk than Q4_K_M.
96
-
97
- Use `IQ4_XS` when you want a middle-footprint quant: smaller than Q4_K_M with better next-token KLD than Q3_K_M, but slower than Q4_K_M in the short-context throughput profile.
98
-
99
- Use the trained Q6_K MTP draft only for MTP experimentation and regression work. It makes llama.cpp `draft-mtp` operational with 67.19-67.22% draft-token acceptance at `n_max=2`, but it is not yet a production throughput speedup: the fastest measured IQ4_XS target plus Q6_K draft profile reached 288.67 tok/s at c16 with per-head MTP contexts, versus 595.89 tok/s for the matched IQ4_XS target-only c16/128 no-prompt-cache profile. Low-concurrency checks show the same pattern: the MTP profile is near target-only at c1, then falls behind at c4/c8/c16. The opt-in sequential target-verification patch fixes the deterministic output mismatch and matches target-only exactly on the 3-prompt probe, but it is slower than target-only at c1/c4/c8/c16. The smaller Q5_K_M and IQ4_XS MTP drafts are included as experimental footprint variants but are not recommended. The later `h2r10prefix` head-1 refinement drafts are also experimental: they trained cleanly offline, but benchmarked at only 283.74-284.62 tok/s and 65.79-65.91% draft-token acceptance, so they were not promoted over the h2r9 Q6_K profile.
100
-
101
- `ornith-1.0-35b-IQ4_XS-MTP-graft-headQ6.gguf` is a separate integrated MTP artifact. It was requantized from the MIT-licensed `wang-yang/Ornith-1.0-35B-MTP-GGUF` grafted Q6_K GGUF to IQ4_XS while preserving the grafted `blk.40.*` MTP tensors at Q6_K/F16/F32. Metadata/tensor verification matches the wang-yang native graft structure: both are `qwen35moe`, `qwen35moe.block_count=41`, `qwen35moe.nextn_predict_layers=1`, and one appended `blk.40.*` MTP block. The difference is quantization and runtime policy, not the graft layout: wang-yang is Q6_K (`general.file_type=18`, 29.2 GB decimal), while this file is IQ4_XS body plus preserved Q6_K/F16/F32 MTP block (`general.file_type=30`, 19.6 GB decimal; 18.9 GB for the plain IQ4_XS body).
102
-
103
- The wang-yang card reports its best MTP result on an M3 Max single-prompt real code continuation: AR tg128 66.6 tok/s, `draft-mtp n_max=1` 83.8 tok/s, 1.26x, 92.2% acceptance. Local RTX PRO 6000 Blackwell Max-Q results show the same low-concurrency behavior for this IQ4_XS graft: c1/128 AR measured 221.8 tok/s, `draft-mtp n_max=2` measured 279.4 tok/s (1.26x), and the adaptive profile below measured 319.5 tok/s with cumulative acceptance by position `(0.953, 0.865)` over four c1 prompts.
104
-
105
- High-concurrency serving behaves differently. At c16/64, target-only AR measured 568.6 tok/s. Always-on active MTP still loses on this saturated shape: the best active-MTP c16/64 profile measured 323.1 tok/s despite high acceptance. The practical single-GPU serving profile is therefore adaptive: keep grafted MTP active for low-concurrency slots, and automatically disable it as concurrency rises. With `LLAMA_SPEC_MAX_DRAFTING_SLOTS=1`, the c16/64 workload measured 564.5 tok/s in the first validation and 592.4 tok/s in the later threshold sweep, effectively matching target-only, while low-concurrency runs kept MTP active. A threshold sweep found `1` and `2` are both safe, but `4` hurts c4 throughput, so this repo recommends `LLAMA_SPEC_MAX_DRAFTING_SLOTS=1` as the conservative default. See `benchmarks/llamacpp-iq4-xs-mtp-graft-adaptive-profile.md`.
106
-
107
- ## Quant Provenance
108
-
109
- All files are GGUF artifacts for `deepreinforce-ai/Ornith-1.0-35B`.
110
-
111
- | File | Quant | Provenance | Quantization detail |
112
- |---|---|---|---|
113
- | `ornith-1.0-35b-Q3_K_M.gguf` | Q3_K_M | Locally produced by this lab from the upstream BF16 GGUF | `llama-quantize ornith-1.0-35b-bf16.gguf ornith-1.0-35b-Q3_K_M.gguf Q3_K_M`; quantizer reported source 66,152.24 MiB / 16.01 BPW and quantized size 15,977.65 MiB / 3.87 BPW |
114
- | `ornith-1.0-35b-IQ4_XS.gguf` | IQ4_XS | Locally produced by this lab from the upstream BF16 GGUF | `llama-quantize ornith-1.0-35b-bf16.gguf ornith-1.0-35b-IQ4_XS.gguf IQ4_XS 32`; no importance matrix; quantizer reported source 66,152.24 MiB / 16.01 BPW and quantized size 18,051.46 MiB / 4.37 BPW |
115
- | `ornith-1.0-35b-IQ4_XS-MTP-graft-headQ6.gguf` | IQ4_XS body + Q6_K grafted MTP head | Experimental local requant from `wang-yang/Ornith-1.0-35B-MTP-GGUF` Q6_K-MTP, which grafts same-architecture MTP tensors into Ornith | `llama-quantize --allow-requantize --tensor-type-file runs/tensor-types-mtp-blk40-preserve.txt Ornith-1.0-35B-Q6_K-MTP.gguf ornith-1.0-35b-IQ4_XS-MTP-graft-headQ6.gguf IQ4_XS 24`; `blk.40.*` preserved as Q6_K/F16/F32; quantizer reported source 27,844.13 MiB / 6.58 BPW and quantized size 18,712.85 MiB / 4.42 BPW |
116
- | `ornith-1.0-35b-mtp-bootstrap-layer39-IQ4_XS.gguf` | IQ4_XS draft MTP | Locally produced by this lab from the released HF safetensors | `build_bootstrap_mtp_checkpoint.py --source-layer 39 --eh-proj-init hidden`, then `convert_hf_to_gguf.py --mtp --outtype bf16`, then `llama-quantize ... IQ4_XS 32`; draft quantizer reported source 3,552.05 MiB / 16.00 BPW and quantized size 1,085.37 MiB / 4.89 BPW |
117
- | `ornith-1.0-35b-mtp-chaincorr-h2r9fw125-h1r2-h3r6ce375-lr5e7-Q6_K.gguf` | Q6_K draft MTP | Locally trained by this lab from target hidden-state captures, then converted to GGUF | Chain-corrected Qwen3.5 MoE MTP draft; `llama-quantize ... Q6_K`; source draft 8,388.17 MiB / 16.01 BPW; estimated draft load 3,176.45 MiB |
118
- | `ornith-1.0-35b-mtp-chaincorr-h2r9fw125-h1r2-h3r6ce375-lr5e7-Q5_K_M.gguf` | Q5_K_M draft MTP | Locally trained by this lab from target hidden-state captures, then converted to GGUF | Same trained draft as above, quantized with `llama-quantize ... Q5_K_M`; quantized size 3,090.01 MiB / 5.90 BPW; estimated draft load 2,885.59 MiB |
119
- | `ornith-1.0-35b-mtp-chaincorr-h2r9fw125-h1r2-h3r6ce375-lr5e7-IQ4_XS.gguf` | IQ4_XS draft MTP | Locally trained by this lab from target hidden-state captures, then converted to GGUF | Same trained draft as above, quantized with `llama-quantize ... IQ4_XS`; quantized size 2,374.93 MiB / 4.53 BPW; estimated draft load 2,246.29 MiB |
120
- | `ornith-1.0-35b-mtp-chaincorr-h2r10prefix250-h1r2-h3r8fw-lr5e6-Q6_K.gguf` | Q6_K draft MTP | Experimental head-1 chain-prefix refinement from the h2r9/h3r8 draft lineage | `train_mtp_chain_prefix.py --head-index 1 --learning-rate 5e-6 --max-steps 1000`, snapshot step 250; then `convert_hf_to_gguf.py --mtp --outtype bf16` and `llama-quantize ... Q6_K`; source 8,388.17 MiB / 16.01 BPW; quantized 3,445.28 MiB / 6.57 BPW |
121
- | `ornith-1.0-35b-mtp-chaincorr-h2r10prefix500-h1r2-h3r8fw-lr5e6-Q6_K.gguf` | Q6_K draft MTP | Experimental head-1 chain-prefix refinement from the h2r9/h3r8 draft lineage | Same training run, snapshot step 500; converted with `convert_hf_to_gguf.py --mtp --outtype bf16` and quantized with `llama-quantize ... Q6_K`; source 8,388.17 MiB / 16.01 BPW; quantized 3,445.28 MiB / 6.57 BPW |
122
- | `ornith-1.0-35b-mtp-chaincorr-h2r10prefix1000-h1r2-h3r8fw-lr5e6-Q6_K.gguf` | Q6_K draft MTP | Experimental head-1 chain-prefix refinement from the h2r9/h3r8 draft lineage | Same training run, final step 1000; converted with `convert_hf_to_gguf.py --mtp --outtype bf16` and quantized with `llama-quantize ... Q6_K`; source 8,388.17 MiB / 16.01 BPW; quantized 3,445.28 MiB / 6.57 BPW |
123
- | `ornith-1.0-35b-Q4_K_M.gguf` | Q4_K_M | Mirrored from `deepreinforce-ai/Ornith-1.0-35B-GGUF`, then validated here | Upstream llama.cpp GGUF K-quant artifact; exact upstream quantization command was not republished in this repo |
124
- | `ornith-1.0-35b-Q5_K_M.gguf` | Q5_K_M | Mirrored from `deepreinforce-ai/Ornith-1.0-35B-GGUF`, then validated here | Upstream llama.cpp GGUF K-quant artifact; exact upstream quantization command was not republished in this repo |
125
- | `ornith-1.0-35b-Q6_K.gguf` | Q6_K | Mirrored from `deepreinforce-ai/Ornith-1.0-35B-GGUF`, then validated here | Upstream llama.cpp GGUF K-quant artifact; exact upstream quantization command was not republished in this repo |
126
- | `ornith-1.0-35b-Q8_0.gguf` | Q8_0 | Mirrored from `deepreinforce-ai/Ornith-1.0-35B-GGUF`, then validated here | Upstream llama.cpp GGUF Q8_0 artifact; exact upstream quantization command was not republished in this repo |
127
-
128
- The locally produced Q3_K_M and IQ4_XS files were generated with the CPU `llama-quantize`
129
- binary built from the checked-out `llama.cpp` source in this lab. The input BF16
130
- GGUF was downloaded from `deepreinforce-ai/Ornith-1.0-35B-GGUF`:
131
- `ornith-1.0-35b-bf16.gguf`.
132
-
133
- ## KLD Probe
134
-
135
- Each quant was compared to the upstream BF16 GGUF baseline with a corrected
136
- native llama.cpp next-token top-64 `KL(P_bf16 || P_quant)` probe over 32 coding
137
- prompts. The probe used `/completion` with `n_predict=1`, `temperature=-1`,
138
- `n_probs=64`, `cache_prompt=false`, and token-ID based matching. The BF16
139
- baseline loaded 64.57 GiB VRAM in this KLD profile.
140
-
141
- | Quant | KLD profile VRAM GiB | Mean KLD nats | P50 nats | P95 nats | Max nats | Top-1 match |
142
- |---|---:|---:|---:|---:|---:|---:|
143
- | Q3_K_M | 16.32 | 0.3620 | 0.2548 | 1.1077 | 1.3730 | 84.4% |
144
- | IQ4_XS | 19.34 | 0.1426 | 0.0868 | 0.3195 | 0.6586 | 84.4% |
145
- | Q4_K_M | 20.35 | 0.0864 | 0.0379 | 0.2877 | 0.4503 | 90.6% |
146
- | Q5_K_M | 23.61 | 0.0354 | 0.0235 | 0.0943 | 0.2497 | 93.8% |
147
- | Q6_K | 27.07 | 0.0165 | 0.0092 | 0.0513 | 0.0586 | 100.0% |
148
- | Q8_0 | 34.77 | 0.0108 | 0.0052 | 0.0440 | 0.0590 | 96.9% |
149
-
150
- Q8_0 had the lowest mean KLD in this corrected top-64 probe. Q6_K was nearly as
151
- close and had perfect top-1 agreement across all prompts. IQ4_XS landed between
152
- Q3_K_M and Q4_K_M: materially closer to BF16 than Q3_K_M, but not as close as
153
- Q4_K_M. See
154
- `benchmarks/kld-quant-vs-bf16-top64.md` for method caveats and raw artifact
155
- paths.
156
-
157
- ## Serving
158
-
159
- Fast short-context throughput profile:
160
 
161
  ```bash
162
- QUANT=Q4_K_M PORT=8002 CTX_SIZE=8192 PARALLEL=16 \
163
- scripts/serve_llamacpp_gpu0.sh
164
  ```
165
 
166
- Lowest-memory profile:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
 
168
  ```bash
169
- QUANT=Q3_K_M PORT=8002 CTX_SIZE=8192 PARALLEL=16 \
170
- scripts/serve_llamacpp_gpu0.sh
 
 
171
  ```
172
 
173
- Middle-footprint profile:
174
 
175
  ```bash
176
- QUANT=IQ4_XS PORT=8002 CTX_SIZE=8192 PARALLEL=16 \
177
- scripts/serve_llamacpp_gpu0.sh
178
  ```
179
 
180
- Experimental trained-MTP profile:
181
 
182
  ```bash
183
- SPEC_DRAFT_BACKEND_SAMPLING=1 \
184
- MTP_PER_HEAD_CONTEXT=1 \
185
- LLAMA_MTP_FAST_BACKEND_SAMPLE=1 \
186
- LLAMA_MTP_DRAFT_TOP_K=1 LLAMA_MTP_DRAFT_TOP_P=1 LLAMA_MTP_DRAFT_TEMP=1 \
187
- QUANT=IQ4_XS PORT=8002 CTX_SIZE=8192 PARALLEL=16 \
188
- SPEC_TYPE=draft-mtp \
189
- SPEC_DRAFT_MODEL=ornith-1.0-35b-mtp-chaincorr-h2r9fw125-h1r2-h3r6ce375-lr5e7-Q6_K.gguf \
190
- SPEC_DRAFT_N_MAX=2 SPEC_DRAFT_N_MIN=0 \
191
- scripts/serve_llamacpp_gpu0.sh
192
  ```
193
 
194
- Experimental integrated graft MTP profile:
 
 
 
195
 
196
  ```bash
197
- SPEC_DRAFT_BACKEND_SAMPLING=1 \
198
- LLAMA_SPEC_MAX_DRAFTING_SLOTS=1 \
199
- LLAMA_MTP_FAST_BACKEND_SAMPLE=1 \
200
- LLAMA_MTP_DRAFT_TOP_K=1 LLAMA_MTP_DRAFT_TOP_P=1 LLAMA_MTP_DRAFT_TEMP=1 \
201
- MODEL_PATH=ornith-1.0-35b-IQ4_XS-MTP-graft-headQ6.gguf \
202
- PORT=8002 CTX_SIZE=8192 PARALLEL=16 \
203
- SPEC_TYPE=draft-mtp SPEC_DRAFT_N_MAX=2 SPEC_DRAFT_N_MIN=0 \
204
- scripts/serve_llamacpp_gpu0.sh
205
  ```
206
 
207
- This integrated graft profile avoids an external draft model and reaches high
208
- acceptance. With `LLAMA_SPEC_MAX_DRAFTING_SLOTS=1`, it uses MTP for low
209
- concurrency and falls back to AR when the server is saturated. In the first
210
- validated c16/64 profile this measured 564.5 tok/s versus 568.6 tok/s
211
- target-only, while the c1/128 profile kept MTP active and measured 319.5 tok/s.
212
- A later c1/c2/c4/c8/c16 threshold sweep measured 271.1/316.5/424.1/564.2/592.4
213
- tok/s at threshold 1, 267.0/313.3/422.9/558.2/592.4 tok/s at threshold 2, and
214
- 269.2/313.1/328.5/547.6/591.5 tok/s at threshold 4. Keep threshold 1 as the
215
- default; threshold 2 is acceptable, but threshold 4 is too permissive. If the
216
- throttle is disabled, active MTP is still slower than target-only on c16.
217
 
218
- Coding-serving profile with 16 slots and about 8k tokens per slot:
219
 
220
  ```bash
221
- QUANT=Q4_K_M PORT=8002 CTX_SIZE=131072 PARALLEL=16 \
222
- scripts/serve_llamacpp_gpu0.sh
 
 
223
  ```
224
 
225
- Use `scripts/serve_llamacpp_gpu1.sh` for the second physical GPU. The wrappers accept `QUANT`, `MODEL_PATH`, `PORT`, `ALIAS`, `CTX_SIZE`, `PARALLEL`, `BATCH_SIZE`, `UBATCH_SIZE`, `CONT_BATCHING`, `METRICS`, `BACKEND_SAMPLING`, `REASONING`, `REASONING_FORMAT`, `REASONING_BUDGET`, `REASONING_BUDGET_MESSAGE`, `SPEC_TYPE`, `SPEC_DRAFT_MODEL`, `SPEC_DRAFT_HF`, `SPEC_DRAFT_NGL`, `SPEC_DRAFT_N_MAX`, `SPEC_DRAFT_N_MIN`, `SPEC_DRAFT_P_MIN`, `SPEC_DRAFT_DEVICE`, and `SPEC_DRAFT_BACKEND_SAMPLING`.
226
 
227
- The default is `REASONING=off`. This is the usable coding-serving profile: with llama.cpp reasoning left on/auto, simple requests can spend the whole short response budget in parsed `reasoning_content` and return empty final `content`. Set `REASONING=on` only for explicit thinking-mode experiments.
228
-
229
- ## Correctness
 
 
230
 
231
- All uploaded quants passed an OpenAI-compatible correctness gate through llama.cpp:
232
 
233
- - `/v1/models` listed the served alias.
234
- - `/v1/chat/completions` returned coherent text.
235
- - `/v1/completions` returned coherent text.
236
- - Output matched `def add(` and `return a + b`.
237
 
238
- The same model's GGUF path was not usable in the tested vLLM build: Q4_K_M loaded but produced corrupted output. Use llama.cpp for these GGUF files unless vLLM GGUF support for this architecture is fixed.
239
 
240
- The Q4 llama.cpp profile also found and fixed a reasoning-mode serving issue.
241
- With reasoning left on/auto, short coding requests could finish by length with
242
- empty final `content`; the patched default `REASONING=off` profile passes the
243
- behavior suite with 14/14 checks and 0 issues. See
244
- `benchmarks/llamacpp-q4-reasoning-off-fix.md`.
 
 
 
245
 
246
- The custom Q3_K_M quant passed the same behavior suite on the exact 16-slot
247
- serving profile with 14/14 checks and 0 issues. See
248
- `benchmarks/llamacpp-q3-k-m-profile.md`.
249
 
250
- The local IQ4_XS quant also passed the OpenAI-compatible correctness gate and
251
- the same 16-slot behavior suite with 14/14 checks and 0 issues. See
252
- `benchmarks/llamacpp-iq4-xs-profile.md`.
 
 
 
253
 
254
- The MTP profiles also passed the OpenAI-compatible correctness gate. The
255
- bootstrap profile is preserved as a historical loadability baseline, while the
256
- trained chain-corrected Q6_K draft is the current MTP development profile. See
257
- `benchmarks/llamacpp-iq4-xs-mtp-bootstrap-profile.md` and
258
- `benchmarks/llamacpp-iq4-xs-mtp-chaincorr-profile.md`.
259
 
260
- ## Investigation Harness
 
 
 
 
 
 
 
 
 
261
 
262
- The reusable investigation harness combines metadata/tokenizer/config checks
263
- with live OpenAI-compatible behavior probes. A live Q4_K_M llama.cpp run passed
264
- 27/27 combined checks with 0 behavior issues. The only remaining issues were
265
- low-severity metadata notes:
266
 
267
- - `generation-eos-includes-pad`: generation config EOS list includes the PAD token.
268
- - `missing-config-torch-dtype`: model config does not declare `torch_dtype`, so serving/quantization commands should pass dtype explicitly.
 
269
 
270
- See `probes/investigation-ornith-q4-live.md` and
271
- `probes/investigation-ornith-q4-live.json`.
272
 
273
- The reusable harness bundle is included in this repo:
 
 
 
 
 
274
 
275
- - Workflow entry point: `MODEL_INVESTIGATION_HARNESS.md`
276
- - Lightweight dependencies: `requirements-lab.txt`
277
- - Behavior suite: `configs/model_behavior_suite.yaml`
278
- - Core runner: `scripts/run_model_investigation.py`
279
- - Metadata/config audit: `scripts/audit_model.py`
280
- - OpenAI-compatible behavior probe: `scripts/probe_model_behavior.py`
281
- - Server contract probe: `scripts/probe_openai_contract.py`
282
- - Throughput benchmark: `scripts/bench_openai.py`
283
- - Native llama.cpp KLD collectors: `scripts/collect_next_token_logprobs.py` and `scripts/compare_kld_logprobs.py`
284
- - MTP loadability check: `scripts/check_mtp_support.py`
285
- - Gate and summary helpers: `scripts/check_audit_gates.py` and `scripts/summarize_investigation.py`
286
 
287
- Minimal metadata-only run:
288
 
289
  ```bash
290
- python -m venv .venv-lab
291
- . .venv-lab/bin/activate
292
- pip install -r requirements-lab.txt
293
 
294
- python scripts/run_model_investigation.py \
295
- --model deepreinforce-ai/Ornith-1.0-35B \
296
- --gguf deepreinforce-ai/Ornith-1.0-35B-GGUF \
297
- --output runs/investigation-ornith-metadata.json
298
 
299
- python scripts/check_audit_gates.py runs/investigation-ornith-metadata.json
 
300
  ```
301
 
302
- Add `--base-url` and `--served-model` to the runner when a llama.cpp, vLLM, or
303
- SGLang OpenAI-compatible server is live. That adds the behavior suite to the
304
- same report.
305
-
306
- ## MTP Status
307
-
308
- The released HF config advertises `mtp_num_hidden_layers=1`, but the released
309
- HF safetensors and GGUF artifacts do not contain a loadable MTP/NextN head. The
310
- source snapshot has no `mtp`, `nextn`, `draft`, or `eagle` tensor names, and the
311
- GGUF metadata has no `nextn` keys. A llama.cpp `convert_hf_to_gguf.py --mtp
312
- --dry-run` only planned embeddings, output head, and output norm, not the
313
- Qwen3.5 MoE MTP block tensors required by llama.cpp.
314
-
315
- Status of the released upstream checkpoint: `config_only_not_loadable`.
316
-
317
- To make MTP loadable, this lab first created
318
- `ornith-1.0-35b-mtp-bootstrap-layer39-IQ4_XS.gguf`, a separate llama.cpp MTP
319
- draft GGUF. It copies full-attention trunk layer 39 into `model.mtp.layers.0`,
320
- adds the missing `mtp.fc` and MTP input norms, converts with
321
- `convert_hf_to_gguf.py --mtp`, and quantizes the draft to IQ4_XS. That artifact
322
- loads with `--spec-type draft-mtp --spec-draft-model ...` and passes the
323
- OpenAI-compatible correctness gate, but only accepted 1.21% of draft tokens.
324
-
325
- This lab then trained a chain-corrected Qwen3.5 MoE MTP draft from target
326
- hidden-state captures and exported Q6_K, Q5_K_M, and IQ4_XS draft GGUFs. The
327
- current best MTP profile is IQ4_XS target plus Q6_K trained draft with
328
- `n_max=2`, per-head MTP contexts, draft backend sampling, and fast top-k=1
329
- draft sampling:
330
-
331
- - c16 throughput: 288.67 tok/s
332
- - draft-token acceptance: 67.19%
333
- - acceptance by position: `(0.718, 0.621)`
334
- - approximate loaded VRAM: about 25.5 GiB
335
-
336
- This is operational but still not a production MTP speedup. The refreshed
337
- IQ4_XS target-only profile reaches 595.89 tok/s on the matched c16/128
338
- no-prompt-cache benchmark shape, 586.57 tok/s on the same shape with default
339
- prompt cache enabled, and 482.68 tok/s on the earlier c16/256 refreshed
340
- benchmark. The remaining blocker is runtime efficiency: llama.cpp currently
341
- executes draft and target verification mostly sequentially with host-side
342
- sampling/control overhead, and target-only c16 already saturates the GPU better.
343
- The measured per-head Q6_K MTP profile does not beat target-only at c1, c4, c8,
344
- or c16, so do not use MTP for production throughput yet.
345
-
346
- A later head-1 chain-prefix refinement sweep produced three Q6_K draft GGUFs at
347
- 250, 500, and 1000 training steps. These are included for reproducibility, but
348
- they are not promoted: on the same IQ4_XS target, `n_max=2`, per-head-context
349
- c16/128 benchmark they measured 284.62, 283.74, and 284.00 tok/s respectively.
350
- Their final server-side draft-token acceptance was 65.91%, 65.82%, and 65.79%,
351
- with acceptance by position ending around `(0.693, 0.621)`, `(0.692, 0.620)`,
352
- and `(0.692, 0.620)`. The h2r9 Q6_K draft remains the best measured MTP
353
- development profile at 288.67 tok/s and 67.19% acceptance.
354
-
355
- A deterministic-output probe found an additional MTP correctness issue in the
356
- batched target-verification path: target-only IQ4_XS repeated exactly across
357
- server restarts at temperature 0, but normal MTP, per-head MTP, `n_max=1`, and
358
- flash-attn-off MTP diverged from target-only on two of three coding prompts. The
359
- patched runtime now includes `LLAMA_SPEC_VERIFY_SEQUENTIAL=1`, an efficient
360
- sequential target-verification mode. It matched target-only exactly on the
361
- three-prompt probe and avoids the older full checkpoint replay path, but it is a
362
- correctness/debug mode rather than a speed profile: c1/c4/c8 measured
363
- 192.48/165.63/167.47 tok/s on the 256-token low-concurrency prompt mix, and
364
- c16/128 measured 169.24 tok/s. The matched target-only no-prompt-cache profile
365
- measured 239.91/431.95/470.81/595.89 tok/s.
366
- See `probes/mtp-support-ornith-iq4-xs.json`, `benchmarks/llamacpp-iq4-xs-profile.md`,
367
- `benchmarks/llamacpp-iq4-xs-mtp-bootstrap-profile.md`, and
368
- `benchmarks/llamacpp-iq4-xs-mtp-chaincorr-profile.md`.
369
-
370
- The integrated graft MTP artifact is structurally the same MTP layout used by
371
- `wang-yang/Ornith-1.0-35B-MTP-GGUF`: `qwen35moe.block_count=41`,
372
- `qwen35moe.nextn_predict_layers=1`, and one appended `blk.40.*` MTP block.
373
- The production-serving lesson is workload-specific: MTP is a clear c1 latency
374
- and throughput win, but always-on MTP hurts saturated c16 batches. The current
375
- recommended MTP serving profile is adaptive throttling with
376
- `LLAMA_SPEC_MAX_DRAFTING_SLOTS=1`. See
377
- `benchmarks/llamacpp-iq4-xs-mtp-graft-adaptive-profile.md`.
378
-
379
- ## Fine-Tuning Smoke
380
-
381
- A one-step LoRA SFT smoke run succeeded on GPU1 using
382
- `bigcode/self-oss-instruct-sc2-exec-filter-50k` with license metadata required.
383
- This validates the local training stack and data pipeline only. No
384
- capability-bearing fine-tuned adapter is released in this repo.
385
-
386
- Smoke metrics: train loss 1.1419, eval loss 1.1048, train mean token accuracy
387
- 0.7574, eval mean token accuracy 0.7442. See
388
- `benchmarks/sft-lora-smoke-bigcode.md`.
389
-
390
- The production coding SFT run was launched locally as a 20,000-step LoRA run
391
- over 44,000 train rows and 2,000 eval rows from
392
- `bigcode/self-oss-instruct-sc2-exec-filter-50k`, then stopped at user request on
393
- 2026-06-27. No adapter from that run is released in this repo because the
394
- latest durable checkpoint did not pass behavior gates. See
395
- `benchmarks/sft-prod-20k-preflight.md`.
396
-
397
- Stopped-run update, 2026-06-27: `checkpoint-8000` is the latest durable
398
- checkpoint. It loaded cleanly after the PEFT adapter-key mapping fix and
399
- reached train/eval losses 0.48498/0.47880 with train/eval mean token accuracy
400
- 0.84811/0.85095. It is not a release candidate: the local behavior gate still
401
- reported 4 failed checks, including two high-severity issues for unbounded
402
- reasoning/template leakage on strict prompts. The training log reached step
403
- 8673 before termination, but no `checkpoint-9000` was written, so steps after
404
- 8000 are not recoverable as model state. The stop record is
405
- `probes/sft-prod-20k-stopped.json`.
406
-
407
- ## Benchmark Summary
408
-
409
- Benchmark profile:
410
 
411
  ```bash
412
- CUDA_VISIBLE_DEVICES=1 \
413
- HOST=127.0.0.1 PORT=<port> QUANT=<quant> \
414
- CTX_SIZE=8192 PARALLEL=16 BATCH_SIZE=4096 UBATCH_SIZE=512 \
415
- CONT_BATCHING=1 METRICS=1 \
416
- scripts/serve_llamacpp_gpu1.sh
 
417
  ```
418
 
419
- `CTX_SIZE=8192 PARALLEL=16` gives 512 context tokens per slot in llama.cpp server. Use `CTX_SIZE=131072 PARALLEL=16` for about 8k tokens per slot.
420
-
421
- | Quant | GGUF GiB | Loaded VRAM GiB | c1 tok/s | c1 p95 TTFT ms | c4 tok/s | c4 p95 TTFT ms | c8 tok/s | c8 p95 TTFT ms | c16 tok/s | c16 p95 TTFT ms |
422
- |---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
423
- | Q3_K_M | 15.61 | 17.27 | 240.5 | 77.9 | 422.0 | 170.7 | 464.4 | 344.9 | 493.0 | 493.7 |
424
- | IQ4_XS | 17.64 | 19.34 | 234.1 | 75.1 | 297.7 | 159.7 | 411.5 | 330.2 | 476.0 | 541.5 |
425
- | Q4_K_M | 19.71 | 21.31 | 243.3 | 76.3 | 458.3 | 192.3 | 615.0 | 361.8 | 655.6 | 650.0 |
426
- | Q5_K_M | 23.03 | 24.65 | 236.7 | 75.1 | 311.0 | 198.8 | 439.0 | 383.6 | 638.6 | 620.4 |
427
- | Q6_K | 26.56 | 28.03 | 225.9 | 76.8 | 295.8 | 194.2 | 409.6 | 394.8 | 603.3 | 657.4 |
428
- | Q8_0 | 34.37 | 35.72 | 208.5 | 76.9 | 281.5 | 190.8 | 405.8 | 389.1 | 601.4 | 725.8 |
429
-
430
- Full raw benchmark details are in `benchmarks/llamacpp-quant-benchmarks.md`.
431
-
432
- Experimental MTP profiles on the same IQ4_XS target:
433
-
434
- | Profile | Loaded VRAM GiB | c1 tok/s | c4 tok/s | c8 tok/s | c16 tok/s | Draft acceptance |
435
- |---|---:|---:|---:|---:|---:|---:|
436
- | IQ4_XS target only, original sweep | 19.34 | 234.1 | 297.7 | 411.5 | 476.0 | n/a |
437
- | IQ4_XS target only, matched no-prompt-cache profile | 19.34 | 239.91 | 431.95 | 470.81 | 595.89 | n/a |
438
- | IQ4_XS + BF16 MTP draft, n_max=3 | 25.12 | 95.5 | 97.0 | 102.1 | 108.2 | 0.47% |
439
- | IQ4_XS + BF16 MTP draft, n_max=1 | 23.03 | 153.6 | 147.2 | 154.3 | 159.5 | 1.07% |
440
- | IQ4_XS + IQ4_XS MTP draft, n_max=1 | 21.32 | 155.1 | 151.6 | 155.8 | 162.7 | 1.21% |
441
- | IQ4_XS + Q6_K trained MTP draft, n_max=2, per-head contexts | ~25.5 | 235.99 | 245.28 | 247.82 | 288.67 | 67.19% c16; 54.45% mixed c1/c4/c8 aggregate |
442
- | IQ4_XS + Q6_K trained MTP draft, n_max=2, efficient sequential target verification | ~24.4 | 192.48 | 165.63 | 167.47 | 169.24 | exact target-only output on 3-prompt determinism probe; c16 aggregate token acceptance 61.51%, position `(0.683, 0.540)` |
443
- | IQ4_XS integrated graft MTP, adaptive throttle, n_max=2 | ~20-21 | 319.53 | n/t | n/t | 564.48 | c1 cumulative position `(0.953, 0.865)`; c16 throttle disables MTP and matches target-only |
444
- | IQ4_XS integrated graft MTP, always-on active, n_max=2, target+draft backend sampling | ~20-21 | 279.36 | n/t | n/t | 323.12 | c1 speedup 1.26x vs AR; c16 still slower than target-only despite high acceptance |
445
- | IQ4_XS + Q6_K trained MTP draft, n_max=2 | 22.44 | n/t | n/t | n/t | 286.77 | 67.22% |
446
- | IQ4_XS + Q5_K_M trained MTP draft, n_max=2 | 22.16 | n/t | n/t | n/t | 285.32 | 66.61% |
447
- | IQ4_XS + IQ4_XS trained MTP draft, n_max=2 | 21.53 | n/t | n/t | n/t | 265.06 | 59.20% |
448
- | IQ4_XS + Q6_K trained MTP draft, n_max=1 | 22.44 | n/t | n/t | n/t | 262.76 | 68.12% |
449
- | IQ4_XS + Q6_K trained MTP draft, n_max=2, batched deferred replay | 22.44 | n/t | n/t | n/t | 282.51 | 67.10% |
450
- | IQ4_XS + Q6_K trained MTP draft, n_max=3 | 22.44 | n/t | n/t | n/t | 270.19 | 58.54% |
451
-
452
- `n/t` means not tested in the latest trained-MTP loop; those rows were measured
453
- only on the c16 profile used for the MTP fix loop. The matched no-prompt-cache
454
- target-only row is the fair comparison for the per-head MTP row because both
455
- disable prompt-cache reuse.
456
-
457
- Determinism/correctness raw artifacts are in `benchmarks/raw/ornith-iq4-xs-*-det-*.json`.
458
- The probe script is `scripts/probe_openai_determinism.py`. The llama.cpp server
459
- patch used for per-head MTP and sequential verification is provided at
460
- `patches/llamacpp-server-mtp-seqverify-and-perhead.patch`.
461
-
462
- Note: Q3_K_M was benchmarked after the serving script default changed to
463
- `REASONING=off`; the original Q4_K_M through Q8_0 throughput rows are preserved
464
- from the earlier sweep. The usable serving profile in this repo defaults to
465
- `REASONING=off` because reasoning-on/auto can spend short responses in parsed
466
- `reasoning_content` and return empty final `content`.
 
1
+ # Ornith 35B Lab
2
+
3
+ This workspace tracks optimization work for `deepreinforce-ai/Ornith-1.0-35B`.
4
+
5
+ ## Scope
6
+
7
+ - Serving is single GPU only: `tp=1`, one model copy per GPU.
8
+ - GPU0 is the serving speed lane.
9
+ - GPU1 is an independent experiment lane for quantization, fine-tuning, or audit jobs.
10
+ - Multi-GPU serving profiles are intentionally out of scope for this model version.
11
+
12
+ ## Initial Tracks
13
+
14
+ 1. Baseline and tune single-GPU serving with SGLang, vLLM, and llama.cpp GGUF.
15
+ 2. Build a reusable model audit harness for tokenizer, chat template, special-token, reasoning, and tool-call issues.
16
+ 3. Evaluate existing GGUF quantizations, then test smaller or higher-throughput formats.
17
+ 4. Prepare a coding-data fine-tuning and post-training pipeline.
18
+
19
+ Current reusable workflow: [MODEL_INVESTIGATION_HARNESS.md](MODEL_INVESTIGATION_HARNESS.md).
20
+ Current requirement audit: [PROJECT_STATUS_AUDIT.md](PROJECT_STATUS_AUDIT.md).
21
+ SFT remediation plan: [SFT_REMEDIATION_PLAN.md](SFT_REMEDIATION_PLAN.md).
22
+
23
+ ## Current Serving Baseline
24
+
25
+ - Best current GGUF serving profile: llama.cpp `Q4_K_M` `tp=1` with `REASONING=off`.
26
+ - Integrated MTP serving profile: `IQ4_XS-MTP-graft-headQ6` with adaptive throttling, `LLAMA_SPEC_MAX_DRAFTING_SLOTS=1`; this keeps low-concurrency MTP speedups and avoids the saturated c16 loss.
27
+ - Earlier vLLM BF16 `tp=1` and SGLang BF16 `tp=1` profiles are preserved as baseline notes, but the publishable model package is the llama.cpp GGUF stack.
28
+ - The default llama.cpp script uses `REASONING=off` because the model otherwise spends simple coding prompts in `reasoning_content` before producing final `content`.
29
+ - Custom Q3_K_M `tp=1` is now validated as the lowest-memory GGUF option: 15.61 GiB on disk, 17.27 GiB loaded VRAM in the short-context benchmark profile, and 0 behavior-suite issues. Q4_K_M remains faster.
30
+ - Custom IQ4_XS `tp=1` is validated as the middle-footprint GGUF option and is the base body for the integrated MTP graft artifact.
31
+ - vLLM GGUF Q4_K_M currently loads but has broken output and is not benchmark-valid.
32
+ - Profile notes: `runs/vllm-initial-profile.md` and `runs/sglang-initial-profile.md`.
33
+ - Quantized control note: `runs/llamacpp-q4-control.md`.
34
+ - GGUF/vLLM materialization audit: `runs/gguf-vllm-materialization-q4.md`.
35
+ - Reusable benchmark summary:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
36
 
37
  ```bash
38
+ .venv-lab/bin/python scripts/summarize_benchmarks.py runs/*-seq-c*-256.jsonl
 
39
  ```
40
 
41
+ ## GPU1 Experiment Lane
42
+
43
+ - Lane config: `configs/experiment_lanes.yaml`.
44
+ - Quantization matrix: `configs/quantization_matrix.yaml`.
45
+ - Custom Q3_K_M profile: `runs/llamacpp-q3-k-m-profile.md`.
46
+ - Training plan: `configs/training_plan.yaml`.
47
+ - Source-backed run note: `runs/gpu1-quant-train-plan.md`.
48
+
49
+ ## Hardware Baseline
50
+
51
+ - GPU0: NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 97,887 MiB VRAM.
52
+ - GPU1: NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 97,887 MiB VRAM.
53
+ - Driver: 580.159.03, CUDA runtime reported by driver: 13.0.
54
+ - Host had no `nvcc` and no `cmake` at project start. A local `.venv-build` now provides CMake, Ninja, and CUDA 13.2 wheels for the llama.cpp CUDA build.
55
+
56
+ ## Quick Start
57
+
58
+ Create the lightweight lab environment:
59
 
60
  ```bash
61
+ cd /home/ripper/ornith-35b-lab
62
+ uv venv .venv-lab --python 3.12
63
+ source .venv-lab/bin/activate
64
+ uv pip install -r requirements-lab.txt
65
  ```
66
 
67
+ Run the metadata audit without downloading full model weights:
68
 
69
  ```bash
70
+ python scripts/audit_model.py --output runs/audit-ornith-35b.json
71
+ .venv-lab/bin/python scripts/check_audit_gates.py runs/audit-ornith-35b.json
72
  ```
73
 
74
+ Run the combined reusable investigation report:
75
 
76
  ```bash
77
+ .venv-lab/bin/python scripts/run_model_investigation.py \
78
+ --output runs/investigation-ornith-metadata.json
79
+ .venv-lab/bin/python scripts/summarize_investigation.py \
80
+ runs/investigation-ornith-metadata.json \
81
+ --output runs/investigation-ornith-metadata.md
 
 
 
 
82
  ```
83
 
84
+ Add `--base-url` and `--served-model` when a live OpenAI-compatible backend is
85
+ running to include the behavior suite in the same report.
86
+
87
+ Run reusable behavior probes against any OpenAI-compatible backend:
88
 
89
  ```bash
90
+ .venv-lab/bin/python scripts/probe_model_behavior.py \
91
+ --base-url http://localhost:8000/v1 \
92
+ --model Ornith-1.0-35B \
93
+ --output runs/behavior-vllm-bf16.json
 
 
 
 
94
  ```
95
 
96
+ The suite lives in `configs/model_behavior_suite.yaml` and is designed to be
97
+ copied to other models. It turns model bugs into named, repeatable checks for
98
+ coding correctness, JSON following, chat-template leakage, decode corruption,
99
+ and optional tool-call handling.
 
 
 
 
 
 
100
 
101
+ Run the GGUF/vLLM materialization audit for a quantized artifact:
102
 
103
  ```bash
104
+ .venv-lab/bin/python scripts/audit_gguf_vllm_materialization.py \
105
+ --model deepreinforce-ai/Ornith-1.0-35B \
106
+ --gguf deepreinforce-ai/Ornith-1.0-35B-GGUF:Q4_K_M \
107
+ --output runs/gguf-vllm-materialization-q4.json
108
  ```
109
 
110
+ Start one serving backend on GPU0 after installing that backend in its own environment:
111
 
112
+ ```bash
113
+ CUDA_VISIBLE_DEVICES=0 scripts/serve_vllm_gpu0.sh
114
+ CUDA_VISIBLE_DEVICES=0 scripts/serve_sglang_gpu0.sh
115
+ CUDA_VISIBLE_DEVICES=0 scripts/serve_llamacpp_gpu0.sh
116
+ ```
117
 
118
+ Serve the custom local Q3_K_M GGUF:
119
 
120
+ ```bash
121
+ QUANT=Q3_K_M PORT=8002 CTX_SIZE=8192 PARALLEL=16 \
122
+ scripts/serve_llamacpp_gpu0.sh
123
+ ```
124
 
125
+ Run a smoke benchmark against an OpenAI-compatible server:
126
 
127
+ ```bash
128
+ python scripts/bench_openai.py \
129
+ --base-url http://localhost:8000/v1 \
130
+ --model Ornith-1.0-35B \
131
+ --concurrency 1 \
132
+ --max-tokens 256 \
133
+ --output runs/smoke-bench.jsonl
134
+ ```
135
 
136
+ Probe the OpenAI-compatible server contract after each backend/profile change:
 
 
137
 
138
+ ```bash
139
+ .venv-lab/bin/python scripts/probe_openai_contract.py \
140
+ --base-url http://localhost:8000/v1 \
141
+ --model Ornith-1.0-35B \
142
+ --output runs/server-contract.json
143
+ ```
144
 
145
+ Prepare and gate SFT data before launching GPU training:
 
 
 
 
146
 
147
+ ```bash
148
+ .venv-train/bin/python scripts/render_sft_jsonl.py \
149
+ --input data/train/coding_sft_messages.jsonl \
150
+ --output data/train/coding_sft.jsonl
151
+
152
+ .venv-train/bin/python scripts/validate_sft_jsonl.py \
153
+ --train-jsonl data/train/coding_sft.jsonl \
154
+ --eval-jsonl data/eval/coding_sft_eval.jsonl \
155
+ --output runs/sft-data-validation.json
156
+ ```
157
 
158
+ `scripts/train_sft_lora.py` runs the same validation before importing Torch and
159
+ loading the 35B model unless `--skip-data-validation` is passed.
 
 
160
 
161
+ The production SFT run was stopped at user request. Latest durable checkpoint:
162
+ `artifacts/train/ornith-35b-coding-lora-prod-20k/checkpoint-8000`; stop record:
163
+ `runs/ornith-35b-coding-lora-prod-20k.stopped.json`.
164
 
165
+ CPU-side SFT remediation is prepared but not running. The repair path is defined
166
+ in `configs/sft_repair_blend.yaml` and uses:
167
 
168
+ - `scripts/build_sft_repair_set.py`
169
+ - `scripts/build_sft_blend.py`
170
+ - `data/train/coding_sft_prod_repair.jsonl`
171
+ - `data/eval/coding_sft_prod_repair_eval.jsonl`
172
+ - `runs/sft-data-validation-prod-repair.json`
173
+ - `runs/sft-format-audit-prod-repair.json`
174
 
175
+ The blended repair split has 44,096 train rows, 2,312 eval rows, and no blocking
176
+ validation issues. Do not resume SFT unless explicitly requested.
 
 
 
 
 
 
 
 
 
177
 
178
+ Run the production SFT monitor only for a resumed run:
179
 
180
  ```bash
181
+ scripts/monitor_sft_prod.sh
182
+ ```
 
183
 
184
+ Start the automatic checkpoint behavior gate watcher only for a resumed run:
 
 
 
185
 
186
+ ```bash
187
+ scripts/watch_sft_checkpoint_gate.sh
188
  ```
189
 
190
+ Evaluate a LoRA checkpoint against the reusable behavior suite:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
191
 
192
  ```bash
193
+ CUDA_VISIBLE_DEVICES=0 \
194
+ .venv-train/bin/python scripts/eval_lora_behavior.py \
195
+ --base-model deepreinforce-ai/Ornith-1.0-35B \
196
+ --adapter artifacts/train/ornith-35b-coding-lora-prod-20k/checkpoint-<step> \
197
+ --suite configs/model_behavior_suite.yaml \
198
+ --output runs/eval-lora-behavior-checkpoint-<step>.json
199
  ```
200
 
201
+ The LoRA evaluator defaults to final-answer mode, matching llama.cpp
202
+ `REASONING=off`. Add `--enable-thinking` only for explicit reasoning-mode tests.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
RELEASE_AUDIT.md CHANGED
@@ -1,37 +1,13 @@
1
  # Release Audit
2
 
3
- Generated: 2026-06-27T23:43:34.525894+00:00
4
 
5
  - HF repo: `LordNeel/Ornith-1.0-35B-GGUF-llamacpp-tp1`
6
- - Audited remote head before this audit upload: `78e2847dec3ad84226708b29f0afeb8f751d9d33`
7
- - Remote files before this audit upload: 200
8
- - README advertised files: 66 checked
9
- - README advertised GGUF files: 14 checked, 0 missing
10
- - README advertised non-GGUF files: 52 checked, 0 missing after planned upload
11
- - README broader docs/config/script references: 52 checked, 0 missing after planned upload
12
- - Overall reference status: `pass`
13
-
14
- ## Published GGUF Artifacts
15
-
16
- | File | Published | GiB | Bytes |
17
- |---|---:|---:|---:|
18
- | `ornith-1.0-35b-IQ4_XS-MTP-graft-headQ6.gguf` | yes | 18.285 | 19632837760 |
19
- | `ornith-1.0-35b-IQ4_XS.gguf` | yes | 17.639 | 18939311680 |
20
- | `ornith-1.0-35b-Q3_K_M.gguf` | yes | 15.613 | 16764763712 |
21
- | `ornith-1.0-35b-Q4_K_M.gguf` | yes | 19.713 | 21166757760 |
22
- | `ornith-1.0-35b-Q5_K_M.gguf` | yes | 23.031 | 24729130848 |
23
- | `ornith-1.0-35b-Q6_K.gguf` | yes | 26.556 | 28514152288 |
24
- | `ornith-1.0-35b-Q8_0.gguf` | yes | 34.369 | 36903138880 |
25
- | `ornith-1.0-35b-mtp-bootstrap-layer39-IQ4_XS.gguf` | yes | 1.07 | 1149036128 |
26
- | `ornith-1.0-35b-mtp-chaincorr-h2r10prefix1000-h1r2-h3r8fw-lr5e6-Q6_K.gguf` | yes | 3.375 | 3623587040 |
27
- | `ornith-1.0-35b-mtp-chaincorr-h2r10prefix250-h1r2-h3r8fw-lr5e6-Q6_K.gguf` | yes | 3.375 | 3623586912 |
28
- | `ornith-1.0-35b-mtp-chaincorr-h2r10prefix500-h1r2-h3r8fw-lr5e6-Q6_K.gguf` | yes | 3.375 | 3623586912 |
29
- | `ornith-1.0-35b-mtp-chaincorr-h2r9fw125-h1r2-h3r6ce375-lr5e7-IQ4_XS.gguf` | yes | 2.329 | 2501241952 |
30
- | `ornith-1.0-35b-mtp-chaincorr-h2r9fw125-h1r2-h3r6ce375-lr5e7-Q5_K_M.gguf` | yes | 3.028 | 3251055712 |
31
- | `ornith-1.0-35b-mtp-chaincorr-h2r9fw125-h1r2-h3r6ce375-lr5e7-Q6_K.gguf` | yes | 3.375 | 3623586912 |
32
-
33
- Context-only GGUF mentions not expected to exist in this repo, such as upstream/source inputs:
34
- - `ornith-1.0-35b-bf16.gguf`
35
 
36
  ## Current Recommendations
37
 
@@ -42,41 +18,24 @@ Context-only GGUF mentions not expected to exist in this repo, such as upstream/
42
  - MTP serving policy: adaptive throttle with `LLAMA_SPEC_MAX_DRAFTING_SLOTS=1`; always-on MTP is not recommended for saturated c16 batches.
43
  - Multi-GPU serving remains out of scope; all serving profiles are single-GPU `tp=1`.
44
 
45
- ## Harness Status
46
-
47
- | Path | Published | Planned in audit upload | Bytes |
48
- |---|---:|---:|---:|
49
- | `MODEL_INVESTIGATION_HARNESS.md` | yes | yes | 7346 |
50
- | `requirements-lab.txt` | yes | no | 134 |
51
- | `configs/model_behavior_suite.yaml` | yes | no | 4597 |
52
- | `configs/serving_matrix.yaml` | yes | no | 2287 |
53
- | `configs/quantization_matrix.yaml` | yes | no | 3621 |
54
- | `configs/experiment_lanes.yaml` | yes | no | 842 |
55
- | `configs/training_plan.yaml` | yes | no | 3192 |
56
- | `scripts/run_model_investigation.py` | yes | no | 4573 |
57
- | `scripts/audit_model.py` | yes | no | 16017 |
58
- | `scripts/check_audit_gates.py` | yes | no | 2200 |
59
- | `scripts/summarize_investigation.py` | yes | no | 5117 |
60
- | `scripts/probe_model_behavior.py` | yes | no | 16665 |
61
- | `scripts/probe_openai_contract.py` | yes | no | 7877 |
62
- | `scripts/bench_openai.py` | yes | no | 8169 |
63
- | `scripts/collect_next_token_logprobs.py` | yes | no | 4379 |
64
- | `scripts/compare_kld_logprobs.py` | yes | no | 6926 |
65
- | `scripts/check_mtp_support.py` | yes | no | 5718 |
66
- | `scripts/audit_sft_format.py` | no | yes | n/a |
67
- | `scripts/audit_gguf_vllm_materialization.py` | yes | no | 22567 |
68
-
69
- ## Project Status Audit
70
-
71
- - Requirement-by-requirement status audit: `PROJECT_STATUS_AUDIT.md`.
72
- - Full original goal is not marked complete because the fine-tuning/post-training deliverable has no release-candidate adapter.
73
-
74
- ## SFT Remediation
75
-
76
- - Remediation plan: `SFT_REMEDIATION_PLAN.md`.
77
- - Format audit report: `probes/sft-format-audit-prod.json`.
78
- - Format audit tool: `scripts/audit_sft_format.py`.
79
- - Key finding: production SFT data is structurally valid but format-skewed toward fenced explanatory answers and lacks strict short-output examples.
80
 
81
  ## SFT Status
82
 
@@ -84,13 +43,62 @@ Context-only GGUF mentions not expected to exist in this repo, such as upstream/
84
  - Latest durable checkpoint: `checkpoint-8000`.
85
  - Training log reached step `8673`, but no `checkpoint-9000` was written.
86
  - No SFT adapter is released from this run.
87
- - Release blocker: `checkpoint-8000` failed the local behavior gate with 4 failed checks.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
 
89
  ## Remaining Gaps
90
 
91
- - No release-candidate SFT/post-training adapter exists because the 20k run was stopped and checkpoint-8000 failed behavior gates.
92
- - The integrated MTP profile is recommended adaptively; always-on MTP is still not a saturated c16 throughput win.
93
- - vLLM GGUF remains marked unusable for this model family in the documented probe; llama.cpp is the supported GGUF backend.
94
 
95
  ## Machine-Readable Manifest
96
 
 
1
  # Release Audit
2
 
3
+ Generated: 2026-06-27T23:59:03.593140+00:00
4
 
5
  - HF repo: `LordNeel/Ornith-1.0-35B-GGUF-llamacpp-tp1`
6
+ - Remote head before this upload: `85d31dc1d38b8aed268c267550508b0e5a455cce`
7
+ - Remote files before this upload: 203
8
+ - Staged support files excluding generated audit files: 192
9
+ - Published GGUF files observed remotely: 14
10
+ - Overall status: `pass for serving/quant/MTP support files; SFT adapter remains unreleased`
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
 
12
  ## Current Recommendations
13
 
 
18
  - MTP serving policy: adaptive throttle with `LLAMA_SPEC_MAX_DRAFTING_SLOTS=1`; always-on MTP is not recommended for saturated c16 batches.
19
  - Multi-GPU serving remains out of scope; all serving profiles are single-GPU `tp=1`.
20
 
21
+ ## SFT Remediation Update
22
+
23
+ | Item | Result | Evidence |
24
+ |---|---:|---|
25
+ | Synthetic repair train rows | 4096 | `probes/sft-repair-set.json` |
26
+ | Synthetic repair eval rows | 512 | `probes/sft-repair-set.json` |
27
+ | Repair assistant code fences | 0.0% | `probes/sft-format-audit-prod-repair.json` |
28
+ | Repair assistant `Here's` rate | 0.0% | `probes/sft-format-audit-prod-repair.json` |
29
+ | Repair assistant reasoning-tag rate | 0.0% | `probes/sft-format-audit-prod-repair.json` |
30
+ | Blended rendered train rows | 44096 | `probes/sft-data-validation-prod-repair.json` |
31
+ | Blended rendered eval rows | 2312 | `probes/sft-data-validation-prod-repair.json` |
32
+ | Validation blocking issues | 0 | `probes/sft-data-validation-prod-repair.json` |
33
+ | Validation warnings | 2 | train:max_length: 107 rows exceed 1024 tokens; eval:max_length: 7 rows exceed 1024 tokens |
34
+ | Blended assistant code fences | 90.7112% | `probes/sft-format-audit-prod-repair.json` |
35
+ | Blended assistant `Here's` rate | 50.9343% | `probes/sft-format-audit-prod-repair.json` |
36
+ | Blended short answers under 80 chars | 7.5517% | `probes/sft-format-audit-prod-repair.json` |
37
+
38
+ The synthetic repair message files are uploaded under `data/train/` and `data/eval/`. The full blended production-derived files are not uploaded; regenerate them with `configs/sft_repair_blend.yaml`.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
 
40
  ## SFT Status
41
 
 
43
  - Latest durable checkpoint: `checkpoint-8000`.
44
  - Training log reached step `8673`, but no `checkpoint-9000` was written.
45
  - No SFT adapter is released from this run.
46
+ - The repair blend is CPU-prepared only; no resumed GPU training has been launched.
47
+
48
+ ## Harness And Support Files
49
+
50
+ | Path | Bytes | Remote existed before upload |
51
+ |---|---:|---:|
52
+ | `MODEL_INVESTIGATION_HARNESS.md` | 8578 | yes |
53
+ | `PROJECT_STATUS_AUDIT.md` | 6172 | yes |
54
+ | `README.md` | 7723 | yes |
55
+ | `SFT_REMEDIATION_PLAN.md` | 6592 | yes |
56
+ | `configs/experiment_lanes.yaml` | 842 | yes |
57
+ | `configs/model_behavior_suite.yaml` | 4597 | yes |
58
+ | `configs/quantization_matrix.yaml` | 3621 | yes |
59
+ | `configs/serving_matrix.yaml` | 2287 | yes |
60
+ | `configs/sft_repair_blend.yaml` | 3955 | no |
61
+ | `configs/training_plan.yaml` | 3432 | yes |
62
+ | `data/eval/README.md` | 745 | no |
63
+ | `data/eval/coding_sft_repair_eval_messages.jsonl` | 376090 | no |
64
+ | `data/train/README.md` | 1283 | no |
65
+ | `data/train/coding_sft_repair_messages.jsonl` | 3015307 | no |
66
+ | `probes/sft-data-validation-prod-repair.json` | 3925 | no |
67
+ | `probes/sft-format-audit-prod-repair.json` | 17718 | no |
68
+ | `probes/sft-format-audit-prod.json` | 10312 | yes |
69
+ | `probes/sft-prod-20k-stopped.json` | 1252 | yes |
70
+ | `probes/sft-repair-blend-eval.json` | 495 | no |
71
+ | `probes/sft-repair-blend-train.json` | 489 | no |
72
+ | `probes/sft-repair-set.json` | 1239 | no |
73
+ | `scripts/audit_gguf_vllm_materialization.py` | 22567 | yes |
74
+ | `scripts/audit_model.py` | 16017 | yes |
75
+ | `scripts/audit_sft_format.py` | 5139 | yes |
76
+ | `scripts/bench_openai.py` | 8169 | yes |
77
+ | `scripts/build_bootstrap_mtp_checkpoint.py` | 7615 | yes |
78
+ | `scripts/build_sft_blend.py` | 4467 | no |
79
+ | `scripts/build_sft_repair_set.py` | 10235 | no |
80
+ | `scripts/check_audit_gates.py` | 2200 | yes |
81
+ | `scripts/check_mtp_support.py` | 5718 | yes |
82
+ | `scripts/collect_next_token_logprobs.py` | 4379 | yes |
83
+ | `scripts/compare_kld_logprobs.py` | 6926 | yes |
84
+ | `scripts/eval_lora_behavior.py` | 11075 | yes |
85
+ | `scripts/launch_sft_prod_20k_gpu1.sh` | 2313 | no |
86
+ | `scripts/probe_model_behavior.py` | 16665 | yes |
87
+ | `scripts/probe_openai_contract.py` | 7877 | yes |
88
+ | `scripts/probe_openai_determinism.py` | 6082 | yes |
89
+ | `scripts/render_sft_jsonl.py` | 3115 | no |
90
+ | `scripts/run_model_investigation.py` | 4573 | yes |
91
+ | `scripts/serve_llamacpp_gpu0.sh` | 4382 | yes |
92
+ | `scripts/serve_llamacpp_gpu1.sh` | 171 | yes |
93
+ | `scripts/summarize_investigation.py` | 5117 | yes |
94
+ | `scripts/train_sft_lora.py` | 6488 | no |
95
+ | `scripts/validate_sft_jsonl.py` | 15231 | no |
96
 
97
  ## Remaining Gaps
98
 
99
+ - No release-candidate SFT/post-training adapter exists.
100
+ - Future SFT must be explicitly resumed and must pass `configs/model_behavior_suite.yaml` before any adapter upload.
101
+ - vLLM GGUF remains unsupported for this model family until corrupted output is fixed and revalidated.
102
 
103
  ## Machine-Readable Manifest
104
 
SFT_REMEDIATION_PLAN.md CHANGED
@@ -48,22 +48,85 @@ Recommended blend:
48
  - Optional small negative/contrastive set for no markdown fences, no reasoning
49
  preambles, and no template markers.
50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
  Recommended gates before using GPU:
52
 
53
  ```bash
54
- .venv-lab/bin/python scripts/audit_sft_format.py \
55
- --input data/train/coding_sft_prod_messages.jsonl \
56
- --input data/eval/coding_sft_prod_eval_messages.jsonl \
57
- --input data/train/coding_sft_prod.jsonl \
58
- --input data/eval/coding_sft_prod_eval.jsonl \
59
- --output runs/sft-format-audit-prod.json
60
-
61
- .venv-train/bin/python scripts/validate_sft_jsonl.py \
62
- --train-jsonl data/train/coding_sft_prod.jsonl \
63
- --eval-jsonl data/eval/coding_sft_prod_eval.jsonl \
64
- --output runs/sft-data-validation-bigcode-prod-44k.json \
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
  --require-license \
66
- --min-train-rows 44000
 
 
 
 
 
 
 
 
 
67
  ```
68
 
69
  Required release gate for every future checkpoint:
@@ -77,6 +140,11 @@ CUDA_VISIBLE_DEVICES=0 \
77
  --output runs/eval-lora-behavior-checkpoint-<step>.json
78
  ```
79
 
 
 
 
 
 
80
  ## Release Rule
81
 
82
  No LoRA adapter should be uploaded unless high/critical behavior checks pass.
@@ -91,3 +159,6 @@ For this project, that means at minimum:
91
 
92
  No fine-tuned adapter is released. The only durable SFT checkpoint is
93
  `checkpoint-8000`, and it is not a release candidate.
 
 
 
 
48
  - Optional small negative/contrastive set for no markdown fences, no reasoning
49
  preambles, and no template markers.
50
 
51
+ CPU-side repair artifacts are now prepared for this path:
52
+
53
+ - `scripts/build_sft_repair_set.py`
54
+ - `scripts/build_sft_blend.py`
55
+ - `configs/sft_repair_blend.yaml`
56
+ - `data/train/coding_sft_repair_messages.jsonl`
57
+ - `data/eval/coding_sft_repair_eval_messages.jsonl`
58
+ - `data/train/coding_sft_prod_repair_messages.jsonl`
59
+ - `data/eval/coding_sft_prod_repair_eval_messages.jsonl`
60
+ - `data/train/coding_sft_prod_repair.jsonl`
61
+ - `data/eval/coding_sft_prod_repair_eval.jsonl`
62
+ - `runs/sft-repair-set.json`
63
+ - `runs/sft-repair-blend-train.json`
64
+ - `runs/sft-repair-blend-eval.json`
65
+ - `runs/sft-data-validation-prod-repair.json`
66
+ - `runs/sft-format-audit-prod-repair.json`
67
+
68
+ The generated repair component has 4,096 train rows and 512 eval rows. It has
69
+ 0 code fences, 0 `Here's`/`Here is` preambles, 0 reasoning tags, and about 81%
70
+ short assistant answers under 80 characters. The blended train split has 44,096
71
+ rows and validates with no blocking issues; 107 train rows and 7 eval rows are
72
+ longer than 1024 tokens, so they are warnings only. The blended train format
73
+ signals improve from the stopped production mix:
74
+
75
+ - code fences: 100% -> 90.71%
76
+ - `Here's`/`Here is`: about 56% -> 50.93%
77
+ - short exact answers under 80 chars: 0% -> 7.55%
78
+
79
  Recommended gates before using GPU:
80
 
81
  ```bash
82
+ .venv-lab/bin/python scripts/build_sft_repair_set.py \
83
+ --train-size 4096 \
84
+ --eval-size 512 \
85
+ --seed 29 \
86
+ --train-output data/train/coding_sft_repair_messages.jsonl \
87
+ --eval-output data/eval/coding_sft_repair_eval_messages.jsonl \
88
+ --report-output runs/sft-repair-set.json
89
+
90
+ .venv-lab/bin/python scripts/build_sft_blend.py \
91
+ --source data/train/coding_sft_prod_messages.jsonl=40000:base_coding \
92
+ --source data/train/coding_sft_repair_messages.jsonl=4096:strict_repair \
93
+ --output data/train/coding_sft_prod_repair_messages.jsonl \
94
+ --report-output runs/sft-repair-blend-train.json \
95
+ --seed 31
96
+
97
+ .venv-lab/bin/python scripts/build_sft_blend.py \
98
+ --source data/eval/coding_sft_prod_eval_messages.jsonl=1800:base_coding \
99
+ --source data/eval/coding_sft_repair_eval_messages.jsonl=512:strict_repair \
100
+ --output data/eval/coding_sft_prod_repair_eval_messages.jsonl \
101
+ --report-output runs/sft-repair-blend-eval.json \
102
+ --seed 37
103
+
104
+ .venv-lab/bin/python scripts/render_sft_jsonl.py \
105
+ --input data/train/coding_sft_prod_repair_messages.jsonl \
106
+ --output data/train/coding_sft_prod_repair.jsonl \
107
+ --force
108
+
109
+ .venv-lab/bin/python scripts/render_sft_jsonl.py \
110
+ --input data/eval/coding_sft_prod_repair_eval_messages.jsonl \
111
+ --output data/eval/coding_sft_prod_repair_eval.jsonl \
112
+ --force
113
+
114
+ .venv-lab/bin/python scripts/validate_sft_jsonl.py \
115
+ --train-jsonl data/train/coding_sft_prod_repair.jsonl \
116
+ --eval-jsonl data/eval/coding_sft_prod_repair_eval.jsonl \
117
+ --max-length 1024 \
118
+ --min-train-rows 44096 \
119
  --require-license \
120
+ --output runs/sft-data-validation-prod-repair.json
121
+
122
+ .venv-lab/bin/python scripts/audit_sft_format.py \
123
+ --input data/train/coding_sft_repair_messages.jsonl \
124
+ --input data/eval/coding_sft_repair_eval_messages.jsonl \
125
+ --input data/train/coding_sft_prod_repair_messages.jsonl \
126
+ --input data/eval/coding_sft_prod_repair_eval_messages.jsonl \
127
+ --input data/train/coding_sft_prod_repair.jsonl \
128
+ --input data/eval/coding_sft_prod_repair_eval.jsonl \
129
+ --output runs/sft-format-audit-prod-repair.json
130
  ```
131
 
132
  Required release gate for every future checkpoint:
 
140
  --output runs/eval-lora-behavior-checkpoint-<step>.json
141
  ```
142
 
143
+ `scripts/eval_lora_behavior.py` now defaults to final-answer mode by passing
144
+ `enable_thinking=False` into the chat template. Use `--enable-thinking` only for
145
+ explicit reasoning-mode experiments. This matches the published llama.cpp
146
+ `REASONING=off` serving profile.
147
+
148
  ## Release Rule
149
 
150
  No LoRA adapter should be uploaded unless high/critical behavior checks pass.
 
159
 
160
  No fine-tuned adapter is released. The only durable SFT checkpoint is
161
  `checkpoint-8000`, and it is not a release candidate.
162
+
163
+ SFT remains stopped until explicitly resumed. The prepared repair blend is CPU
164
+ work only and does not consume GPU memory.
configs/sft_repair_blend.yaml ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ base: deepreinforce-ai/Ornith-1.0-35B
3
+ policy: single_gpu_lora_repair_after_checkpoint_8000
4
+
5
+ purpose:
6
+ summary: Repair strict-output regressions observed in checkpoint-8000 before spending more GPU time.
7
+ failed_gates:
8
+ - short_python_add finished by length
9
+ - json_instruction_following did not parse and used markdown
10
+ - template_marker_leak returned an explanatory preamble instead of exactly ready
11
+
12
+ data:
13
+ base_messages_train: data/train/coding_sft_prod_messages.jsonl
14
+ base_messages_eval: data/eval/coding_sft_prod_eval_messages.jsonl
15
+ repair_messages_train: data/train/coding_sft_repair_messages.jsonl
16
+ repair_messages_eval: data/eval/coding_sft_repair_eval_messages.jsonl
17
+ blended_messages_train: data/train/coding_sft_prod_repair_messages.jsonl
18
+ blended_messages_eval: data/eval/coding_sft_prod_repair_eval_messages.jsonl
19
+ rendered_train: data/train/coding_sft_prod_repair.jsonl
20
+ rendered_eval: data/eval/coding_sft_prod_repair_eval.jsonl
21
+ blend_counts:
22
+ train:
23
+ base_coding: 40000
24
+ strict_repair: 4096
25
+ eval:
26
+ base_coding: 1800
27
+ strict_repair: 512
28
+
29
+ preflight_commands:
30
+ - .venv-lab/bin/python scripts/build_sft_repair_set.py --train-size 4096 --eval-size 512 --seed 29 --train-output data/train/coding_sft_repair_messages.jsonl --eval-output data/eval/coding_sft_repair_eval_messages.jsonl --report-output runs/sft-repair-set.json
31
+ - .venv-lab/bin/python scripts/build_sft_blend.py --source data/train/coding_sft_prod_messages.jsonl=40000:base_coding --source data/train/coding_sft_repair_messages.jsonl=4096:strict_repair --output data/train/coding_sft_prod_repair_messages.jsonl --report-output runs/sft-repair-blend-train.json --seed 31
32
+ - .venv-lab/bin/python scripts/build_sft_blend.py --source data/eval/coding_sft_prod_eval_messages.jsonl=1800:base_coding --source data/eval/coding_sft_repair_eval_messages.jsonl=512:strict_repair --output data/eval/coding_sft_prod_repair_eval_messages.jsonl --report-output runs/sft-repair-blend-eval.json --seed 37
33
+ - .venv-lab/bin/python scripts/render_sft_jsonl.py --input data/train/coding_sft_prod_repair_messages.jsonl --output data/train/coding_sft_prod_repair.jsonl --force
34
+ - .venv-lab/bin/python scripts/render_sft_jsonl.py --input data/eval/coding_sft_prod_repair_eval_messages.jsonl --output data/eval/coding_sft_prod_repair_eval.jsonl --force
35
+ - .venv-lab/bin/python scripts/validate_sft_jsonl.py --train-jsonl data/train/coding_sft_prod_repair.jsonl --eval-jsonl data/eval/coding_sft_prod_repair_eval.jsonl --max-length 1024 --min-train-rows 44096 --require-license --output runs/sft-data-validation-prod-repair.json
36
+ - .venv-lab/bin/python scripts/audit_sft_format.py --input data/train/coding_sft_repair_messages.jsonl --input data/eval/coding_sft_repair_eval_messages.jsonl --input data/train/coding_sft_prod_repair_messages.jsonl --input data/eval/coding_sft_prod_repair_eval_messages.jsonl --output runs/sft-format-audit-prod-repair.json
37
+
38
+ future_training_command:
39
+ note: Do not run until the user explicitly resumes SFT work.
40
+ command: >
41
+ CUDA_VISIBLE_DEVICES=1 RUN_NAME=ornith-35b-coding-lora-prod-repair-20k
42
+ OUTPUT_DIR=artifacts/train/ornith-35b-coding-lora-prod-repair-20k
43
+ TRAIN_JSONL=data/train/coding_sft_prod_repair.jsonl
44
+ EVAL_JSONL=data/eval/coding_sft_prod_repair_eval.jsonl
45
+ MIN_TRAIN_ROWS=44096
46
+ VALIDATION_REPORT=runs/sft-train-validation-prod-repair.json
47
+ RESUME_FROM_CHECKPOINT=artifacts/train/ornith-35b-coding-lora-prod-20k/checkpoint-8000
48
+ scripts/launch_sft_prod_20k_gpu1.sh
49
+
50
+ release_gate:
51
+ evaluator_mode: final_answer_default
52
+ command: >
53
+ CUDA_VISIBLE_DEVICES=0 .venv-train/bin/python scripts/eval_lora_behavior.py
54
+ --base-model deepreinforce-ai/Ornith-1.0-35B
55
+ --adapter artifacts/train/ornith-35b-coding-lora-prod-repair-20k/checkpoint-<step>
56
+ --suite configs/model_behavior_suite.yaml
57
+ --output runs/eval-lora-behavior-prod-repair-checkpoint-<step>.json
configs/training_plan.yaml CHANGED
@@ -9,12 +9,15 @@ sft:
9
  script: scripts/train_sft_lora.py
10
  data_validation_script: scripts/validate_sft_jsonl.py
11
  render_script: scripts/render_sft_jsonl.py
 
12
  output_dir: artifacts/train/ornith-35b-coding-lora-sft
13
  data:
14
  train_jsonl: data/train/coding_sft_prod.jsonl
15
  eval_jsonl: data/eval/coding_sft_prod_eval.jsonl
16
  text_field: text
17
  format_note: pre-render each row with the model chat template before training, or provide a TRL conversational dataset
 
 
18
  preflight_commands:
19
  - .venv-train/bin/python scripts/prepare_coding_sft_data.py --dataset bigcode/self-oss-instruct-sc2-exec-filter-50k --train-size 44000 --eval-size 2000 --seed 23 --train-output data/train/coding_sft_prod_messages.jsonl --eval-output data/eval/coding_sft_prod_eval_messages.jsonl --report-output runs/coding-sft-prod-source-prep.json
20
  - .venv-train/bin/python scripts/render_sft_jsonl.py --input data/train/coding_sft_prod_messages.jsonl --output data/train/coding_sft_prod.jsonl --force
@@ -74,6 +77,7 @@ quality_gates:
74
  after_sft:
75
  - no_chat_template_regression
76
  - model_behavior_suite_passes_high_severity_gates
 
77
  - short_coding_bench_improves_or_ties
78
  - compile_rate_improves_or_ties
79
  - no_major_reasoning_format_regression
 
9
  script: scripts/train_sft_lora.py
10
  data_validation_script: scripts/validate_sft_jsonl.py
11
  render_script: scripts/render_sft_jsonl.py
12
+ repair_blend_config: configs/sft_repair_blend.yaml
13
  output_dir: artifacts/train/ornith-35b-coding-lora-sft
14
  data:
15
  train_jsonl: data/train/coding_sft_prod.jsonl
16
  eval_jsonl: data/eval/coding_sft_prod_eval.jsonl
17
  text_field: text
18
  format_note: pre-render each row with the model chat template before training, or provide a TRL conversational dataset
19
+ stopped_run_repair_jsonl: data/train/coding_sft_prod_repair.jsonl
20
+ stopped_run_repair_eval_jsonl: data/eval/coding_sft_prod_repair_eval.jsonl
21
  preflight_commands:
22
  - .venv-train/bin/python scripts/prepare_coding_sft_data.py --dataset bigcode/self-oss-instruct-sc2-exec-filter-50k --train-size 44000 --eval-size 2000 --seed 23 --train-output data/train/coding_sft_prod_messages.jsonl --eval-output data/eval/coding_sft_prod_eval_messages.jsonl --report-output runs/coding-sft-prod-source-prep.json
23
  - .venv-train/bin/python scripts/render_sft_jsonl.py --input data/train/coding_sft_prod_messages.jsonl --output data/train/coding_sft_prod.jsonl --force
 
77
  after_sft:
78
  - no_chat_template_regression
79
  - model_behavior_suite_passes_high_severity_gates
80
+ - strict_output_repair_cases_pass
81
  - short_coding_bench_improves_or_ties
82
  - compile_rate_improves_or_ties
83
  - no_major_reasoning_format_regression
data/eval/README.md ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Evaluation Data
2
+
3
+ Place held-out SFT validation JSONL at `coding_sft_eval.jsonl`.
4
+
5
+ Keep this split repository-family-disjoint from training data when possible.
6
+ Do not include benchmark answers in training data.
7
+
8
+ Use the same rendered `{"text": ...}` format as the training split. If the raw
9
+ eval split is stored as structured messages, render it with
10
+ `scripts/render_sft_jsonl.py` before running validation or training.
11
+
12
+ Stopped-run repair artifacts:
13
+
14
+ - `coding_sft_repair_eval_messages.jsonl`: synthetic strict-output repair eval
15
+ messages.
16
+ - `coding_sft_prod_repair_eval_messages.jsonl`: 1,800 production eval message
17
+ rows plus 512 strict repair rows.
18
+ - `coding_sft_prod_repair_eval.jsonl`: rendered eval split for a future resumed
19
+ run.
data/eval/coding_sft_repair_eval_messages.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
data/train/README.md ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Training Data
2
+
3
+ Place rendered SFT JSONL at `coding_sft.jsonl`.
4
+
5
+ Expected minimal format for `scripts/train_sft_lora.py`:
6
+
7
+ ```jsonl
8
+ {"text": "<full rendered chat transcript ending with the assistant answer>"}
9
+ ```
10
+
11
+ For tool-use or multi-turn training, keep a raw copy with structured messages
12
+ and generate this rendered file as a build artifact so template changes are
13
+ auditable.
14
+
15
+ Raw message format:
16
+
17
+ ```jsonl
18
+ {"id":"example-1","license":"mit","messages":[{"role":"user","content":"Write add(a, b)."},{"role":"assistant","content":"def add(a, b):\n return a + b"}]}
19
+ ```
20
+
21
+ Render and validate:
22
+
23
+ ```bash
24
+ .venv-train/bin/python scripts/render_sft_jsonl.py \
25
+ --input data/train/coding_sft_messages.jsonl \
26
+ --output data/train/coding_sft.jsonl
27
+
28
+ .venv-train/bin/python scripts/validate_sft_jsonl.py \
29
+ --train-jsonl data/train/coding_sft.jsonl \
30
+ --eval-jsonl data/eval/coding_sft_eval.jsonl
31
+ ```
32
+
33
+ Stopped-run repair artifacts:
34
+
35
+ - `coding_sft_repair_messages.jsonl`: synthetic strict-output repair messages.
36
+ - `coding_sft_prod_repair_messages.jsonl`: 40,000 production message rows plus
37
+ 4,096 strict repair rows.
38
+ - `coding_sft_prod_repair.jsonl`: rendered training split for a future resumed
39
+ run.
40
+
41
+ Do not launch SFT from these files until the user explicitly resumes training.
data/train/coding_sft_repair_messages.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
probes/sft-data-validation-prod-repair.json ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checks": [
3
+ {
4
+ "detail": "all lines parsed",
5
+ "name": "train:jsonl_parse",
6
+ "recommendation": null,
7
+ "status": "pass"
8
+ },
9
+ {
10
+ "detail": "rows=44096",
11
+ "name": "train:rows_present",
12
+ "recommendation": null,
13
+ "status": "pass"
14
+ },
15
+ {
16
+ "detail": "all rows have text",
17
+ "name": "train:text_field_present",
18
+ "recommendation": null,
19
+ "status": "pass"
20
+ },
21
+ {
22
+ "detail": "all text rows are nonempty",
23
+ "name": "train:nonempty_text",
24
+ "recommendation": null,
25
+ "status": "pass"
26
+ },
27
+ {
28
+ "detail": "all rows look rendered",
29
+ "name": "train:rendered_template_present",
30
+ "recommendation": null,
31
+ "status": "pass"
32
+ },
33
+ {
34
+ "detail": "no obvious secret patterns found",
35
+ "name": "train:no_obvious_secrets",
36
+ "recommendation": null,
37
+ "status": "pass"
38
+ },
39
+ {
40
+ "detail": "no exact normalized duplicates",
41
+ "name": "train:duplicates",
42
+ "recommendation": null,
43
+ "status": "pass"
44
+ },
45
+ {
46
+ "detail": "all rows have license metadata",
47
+ "name": "train:license_present",
48
+ "recommendation": null,
49
+ "status": "pass"
50
+ },
51
+ {
52
+ "detail": "107 rows exceed 1024 tokens",
53
+ "name": "train:max_length",
54
+ "recommendation": "Chunk, shorten, or raise max_length only if memory allows it.",
55
+ "status": "warn"
56
+ },
57
+ {
58
+ "detail": "usable_text_rows=44096, required=44096",
59
+ "name": "train:min_rows",
60
+ "recommendation": null,
61
+ "status": "pass"
62
+ },
63
+ {
64
+ "detail": "all lines parsed",
65
+ "name": "eval:jsonl_parse",
66
+ "recommendation": null,
67
+ "status": "pass"
68
+ },
69
+ {
70
+ "detail": "rows=2312",
71
+ "name": "eval:rows_present",
72
+ "recommendation": null,
73
+ "status": "pass"
74
+ },
75
+ {
76
+ "detail": "all rows have text",
77
+ "name": "eval:text_field_present",
78
+ "recommendation": null,
79
+ "status": "pass"
80
+ },
81
+ {
82
+ "detail": "all text rows are nonempty",
83
+ "name": "eval:nonempty_text",
84
+ "recommendation": null,
85
+ "status": "pass"
86
+ },
87
+ {
88
+ "detail": "all rows look rendered",
89
+ "name": "eval:rendered_template_present",
90
+ "recommendation": null,
91
+ "status": "pass"
92
+ },
93
+ {
94
+ "detail": "no obvious secret patterns found",
95
+ "name": "eval:no_obvious_secrets",
96
+ "recommendation": null,
97
+ "status": "pass"
98
+ },
99
+ {
100
+ "detail": "no exact normalized duplicates",
101
+ "name": "eval:duplicates",
102
+ "recommendation": null,
103
+ "status": "pass"
104
+ },
105
+ {
106
+ "detail": "all rows have license metadata",
107
+ "name": "eval:license_present",
108
+ "recommendation": null,
109
+ "status": "pass"
110
+ },
111
+ {
112
+ "detail": "7 rows exceed 1024 tokens",
113
+ "name": "eval:max_length",
114
+ "recommendation": "Chunk, shorten, or raise max_length only if memory allows it.",
115
+ "status": "warn"
116
+ },
117
+ {
118
+ "detail": "no exact normalized overlap",
119
+ "name": "train_eval:no_exact_overlap",
120
+ "recommendation": null,
121
+ "status": "pass"
122
+ }
123
+ ],
124
+ "inputs": {
125
+ "eval_jsonl": "data/eval/coding_sft_prod_repair_eval.jsonl",
126
+ "max_length": 1024,
127
+ "model": "deepreinforce-ai/Ornith-1.0-35B",
128
+ "require_license": true,
129
+ "require_rendered": true,
130
+ "text_field": "text",
131
+ "train_jsonl": "data/train/coding_sft_prod_repair.jsonl"
132
+ },
133
+ "issues": [],
134
+ "summaries": {
135
+ "eval": {
136
+ "duplicate_groups": 0,
137
+ "rows": 2312,
138
+ "token_lengths": {
139
+ "max": 1493,
140
+ "mean": 293.49,
141
+ "min": 61,
142
+ "p50": 279,
143
+ "p95": 584
144
+ },
145
+ "usable_text_rows": 2312
146
+ },
147
+ "train": {
148
+ "duplicate_groups": 0,
149
+ "rows": 44096,
150
+ "token_lengths": {
151
+ "max": 1863,
152
+ "mean": 326.26,
153
+ "min": 60,
154
+ "p50": 309,
155
+ "p95": 599
156
+ },
157
+ "usable_text_rows": 44096
158
+ }
159
+ }
160
+ }
probes/sft-format-audit-prod-repair.json ADDED
@@ -0,0 +1,673 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "files": [
3
+ {
4
+ "assistant_length_chars": {
5
+ "max": 199,
6
+ "min": 2,
7
+ "p50": 56,
8
+ "p95": 198
9
+ },
10
+ "counts": {
11
+ "assistant_code_fence": 0,
12
+ "assistant_contains_heres": 0,
13
+ "assistant_mentions_json": 0,
14
+ "assistant_short_under_80_chars": 3330,
15
+ "assistant_starts_heres": 0,
16
+ "assistant_starts_the_function": 0,
17
+ "assistant_starts_to_solve": 0,
18
+ "rendered_contains_qwen_markers": 0,
19
+ "rendered_contains_think": 0,
20
+ "rendered_empty_think": 0
21
+ },
22
+ "exact_phrase_counts": {
23
+ "Here's a thinking process": 0,
24
+ "Return only": 0,
25
+ "exactly one word": 0
26
+ },
27
+ "path": "data/train/coding_sft_repair_messages.jsonl",
28
+ "percentages": {
29
+ "assistant_code_fence": 0.0,
30
+ "assistant_contains_heres": 0.0,
31
+ "assistant_mentions_json": 0.0,
32
+ "assistant_short_under_80_chars": 81.2988,
33
+ "assistant_starts_heres": 0.0,
34
+ "assistant_starts_the_function": 0.0,
35
+ "assistant_starts_to_solve": 0.0,
36
+ "rendered_contains_qwen_markers": 0.0,
37
+ "rendered_contains_think": 0.0,
38
+ "rendered_empty_think": 0.0
39
+ },
40
+ "rows": 4096,
41
+ "top_openers": [
42
+ {
43
+ "count": 411,
44
+ "pct": 10.0342,
45
+ "text": "The model output is"
46
+ },
47
+ {
48
+ "count": 68,
49
+ "pct": 1.6602,
50
+ "text": "no"
51
+ },
52
+ {
53
+ "count": 63,
54
+ "pct": 1.5381,
55
+ "text": "python"
56
+ },
57
+ {
58
+ "count": 62,
59
+ "pct": 1.5137,
60
+ "text": "pass"
61
+ },
62
+ {
63
+ "count": 61,
64
+ "pct": 1.4893,
65
+ "text": "ready"
66
+ },
67
+ {
68
+ "count": 61,
69
+ "pct": 1.4893,
70
+ "text": "invalid"
71
+ },
72
+ {
73
+ "count": 60,
74
+ "pct": 1.4648,
75
+ "text": "fail"
76
+ },
77
+ {
78
+ "count": 59,
79
+ "pct": 1.4404,
80
+ "text": "yes"
81
+ },
82
+ {
83
+ "count": 58,
84
+ "pct": 1.416,
85
+ "text": "done"
86
+ },
87
+ {
88
+ "count": 58,
89
+ "pct": 1.416,
90
+ "text": "valid"
91
+ },
92
+ {
93
+ "count": 58,
94
+ "pct": 1.416,
95
+ "text": "ok"
96
+ },
97
+ {
98
+ "count": 1,
99
+ "pct": 0.0244,
100
+ "text": "def first_or_none_15(items): return items[0]"
101
+ },
102
+ {
103
+ "count": 1,
104
+ "pct": 0.0244,
105
+ "text": "{\"case\":817,\"language\":\"typescript\",\"mode\":\"strict\",\"ok\":true}"
106
+ },
107
+ {
108
+ "count": 1,
109
+ "pct": 0.0244,
110
+ "text": "def dedupe_preserve_order_10(items): seen ="
111
+ },
112
+ {
113
+ "count": 1,
114
+ "pct": 0.0244,
115
+ "text": "def first_or_none_128(items): return items[0]"
116
+ },
117
+ {
118
+ "count": 1,
119
+ "pct": 0.0244,
120
+ "text": "def dedupe_preserve_order_273(items): seen ="
121
+ },
122
+ {
123
+ "count": 1,
124
+ "pct": 0.0244,
125
+ "text": "def first_or_none_30(items): return items[0]"
126
+ },
127
+ {
128
+ "count": 1,
129
+ "pct": 0.0244,
130
+ "text": "def is_even_158(value): return value"
131
+ },
132
+ {
133
+ "count": 1,
134
+ "pct": 0.0244,
135
+ "text": "def dedupe_preserve_order_222(items): seen ="
136
+ },
137
+ {
138
+ "count": 1,
139
+ "pct": 0.0244,
140
+ "text": "def add_40(a, b): return"
141
+ }
142
+ ]
143
+ },
144
+ {
145
+ "assistant_length_chars": {
146
+ "max": 199,
147
+ "min": 2,
148
+ "p50": 56,
149
+ "p95": 198
150
+ },
151
+ "counts": {
152
+ "assistant_code_fence": 0,
153
+ "assistant_contains_heres": 0,
154
+ "assistant_mentions_json": 0,
155
+ "assistant_short_under_80_chars": 414,
156
+ "assistant_starts_heres": 0,
157
+ "assistant_starts_the_function": 0,
158
+ "assistant_starts_to_solve": 0,
159
+ "rendered_contains_qwen_markers": 0,
160
+ "rendered_contains_think": 0,
161
+ "rendered_empty_think": 0
162
+ },
163
+ "exact_phrase_counts": {
164
+ "Here's a thinking process": 0,
165
+ "Return only": 0,
166
+ "exactly one word": 0
167
+ },
168
+ "path": "data/eval/coding_sft_repair_eval_messages.jsonl",
169
+ "percentages": {
170
+ "assistant_code_fence": 0.0,
171
+ "assistant_contains_heres": 0.0,
172
+ "assistant_mentions_json": 0.0,
173
+ "assistant_short_under_80_chars": 80.8594,
174
+ "assistant_starts_heres": 0.0,
175
+ "assistant_starts_the_function": 0.0,
176
+ "assistant_starts_to_solve": 0.0,
177
+ "rendered_contains_qwen_markers": 0.0,
178
+ "rendered_contains_think": 0.0,
179
+ "rendered_empty_think": 0.0
180
+ },
181
+ "rows": 512,
182
+ "top_openers": [
183
+ {
184
+ "count": 49,
185
+ "pct": 9.5703,
186
+ "text": "The model output is"
187
+ },
188
+ {
189
+ "count": 11,
190
+ "pct": 2.1484,
191
+ "text": "ok"
192
+ },
193
+ {
194
+ "count": 11,
195
+ "pct": 2.1484,
196
+ "text": "valid"
197
+ },
198
+ {
199
+ "count": 11,
200
+ "pct": 2.1484,
201
+ "text": "done"
202
+ },
203
+ {
204
+ "count": 10,
205
+ "pct": 1.9531,
206
+ "text": "yes"
207
+ },
208
+ {
209
+ "count": 9,
210
+ "pct": 1.7578,
211
+ "text": "fail"
212
+ },
213
+ {
214
+ "count": 8,
215
+ "pct": 1.5625,
216
+ "text": "invalid"
217
+ },
218
+ {
219
+ "count": 8,
220
+ "pct": 1.5625,
221
+ "text": "ready"
222
+ },
223
+ {
224
+ "count": 7,
225
+ "pct": 1.3672,
226
+ "text": "pass"
227
+ },
228
+ {
229
+ "count": 6,
230
+ "pct": 1.1719,
231
+ "text": "python"
232
+ },
233
+ {
234
+ "count": 1,
235
+ "pct": 0.1953,
236
+ "text": "def dedupe_preserve_order_194(items): seen ="
237
+ },
238
+ {
239
+ "count": 1,
240
+ "pct": 0.1953,
241
+ "text": "{\"case\":551,\"language\":\"bash\",\"mode\":\"strict\",\"ok\":true}"
242
+ },
243
+ {
244
+ "count": 1,
245
+ "pct": 0.1953,
246
+ "text": "{\"case\":846,\"language\":\"python\",\"mode\":\"strict\",\"ok\":false}"
247
+ },
248
+ {
249
+ "count": 1,
250
+ "pct": 0.1953,
251
+ "text": "def add_181(a, b): return"
252
+ },
253
+ {
254
+ "count": 1,
255
+ "pct": 0.1953,
256
+ "text": "def safe_get_16(mapping, key, default=None):"
257
+ },
258
+ {
259
+ "count": 1,
260
+ "pct": 0.1953,
261
+ "text": "def safe_get_171(mapping, key, default=None):"
262
+ },
263
+ {
264
+ "count": 1,
265
+ "pct": 0.1953,
266
+ "text": "def first_or_none_168(items): return items[0]"
267
+ },
268
+ {
269
+ "count": 1,
270
+ "pct": 0.1953,
271
+ "text": "{\"case\":881,\"language\":\"bash\",\"mode\":\"strict\",\"ok\":true}"
272
+ },
273
+ {
274
+ "count": 1,
275
+ "pct": 0.1953,
276
+ "text": "{\"case\":865,\"language\":\"typescript\",\"mode\":\"strict\",\"ok\":true}"
277
+ },
278
+ {
279
+ "count": 1,
280
+ "pct": 0.1953,
281
+ "text": "{\"case\":68,\"language\":\"rust\",\"mode\":\"strict\",\"ok\":true}"
282
+ }
283
+ ]
284
+ },
285
+ {
286
+ "assistant_length_chars": {
287
+ "max": 5168,
288
+ "min": 2,
289
+ "p50": 670,
290
+ "p95": 1655
291
+ },
292
+ "counts": {
293
+ "assistant_code_fence": 40000,
294
+ "assistant_contains_heres": 22460,
295
+ "assistant_mentions_json": 1472,
296
+ "assistant_short_under_80_chars": 3330,
297
+ "assistant_starts_heres": 11097,
298
+ "assistant_starts_the_function": 304,
299
+ "assistant_starts_to_solve": 1348,
300
+ "rendered_contains_qwen_markers": 0,
301
+ "rendered_contains_think": 0,
302
+ "rendered_empty_think": 0
303
+ },
304
+ "exact_phrase_counts": {
305
+ "Here's a thinking process": 0,
306
+ "Return only": 1,
307
+ "exactly one word": 0
308
+ },
309
+ "path": "data/train/coding_sft_prod_repair_messages.jsonl",
310
+ "percentages": {
311
+ "assistant_code_fence": 90.7112,
312
+ "assistant_contains_heres": 50.9343,
313
+ "assistant_mentions_json": 3.3382,
314
+ "assistant_short_under_80_chars": 7.5517,
315
+ "assistant_starts_heres": 25.1655,
316
+ "assistant_starts_the_function": 0.6894,
317
+ "assistant_starts_to_solve": 3.057,
318
+ "rendered_contains_qwen_markers": 0.0,
319
+ "rendered_contains_think": 0.0,
320
+ "rendered_empty_think": 0.0
321
+ },
322
+ "rows": 44096,
323
+ "top_openers": [
324
+ {
325
+ "count": 3771,
326
+ "pct": 8.5518,
327
+ "text": "Here's how you can"
328
+ },
329
+ {
330
+ "count": 1884,
331
+ "pct": 4.2725,
332
+ "text": "Here's how you could"
333
+ },
334
+ {
335
+ "count": 1869,
336
+ "pct": 4.2385,
337
+ "text": "Here is a Python"
338
+ },
339
+ {
340
+ "count": 1756,
341
+ "pct": 3.9822,
342
+ "text": "Here is the implementation"
343
+ },
344
+ {
345
+ "count": 1402,
346
+ "pct": 3.1794,
347
+ "text": "Here's how we can"
348
+ },
349
+ {
350
+ "count": 1222,
351
+ "pct": 2.7712,
352
+ "text": "To solve this problem,"
353
+ },
354
+ {
355
+ "count": 963,
356
+ "pct": 2.1839,
357
+ "text": "To write a Python"
358
+ },
359
+ {
360
+ "count": 847,
361
+ "pct": 1.9208,
362
+ "text": "You can create a"
363
+ },
364
+ {
365
+ "count": 797,
366
+ "pct": 1.8074,
367
+ "text": "Here's an example implementation"
368
+ },
369
+ {
370
+ "count": 552,
371
+ "pct": 1.2518,
372
+ "text": "Here is the implementation:"
373
+ },
374
+ {
375
+ "count": 454,
376
+ "pct": 1.0296,
377
+ "text": "Here is how you"
378
+ },
379
+ {
380
+ "count": 452,
381
+ "pct": 1.025,
382
+ "text": "To create a Python"
383
+ },
384
+ {
385
+ "count": 443,
386
+ "pct": 1.0046,
387
+ "text": "To implement this function,"
388
+ },
389
+ {
390
+ "count": 411,
391
+ "pct": 0.9321,
392
+ "text": "The model output is"
393
+ },
394
+ {
395
+ "count": 382,
396
+ "pct": 0.8663,
397
+ "text": "Here is an example"
398
+ },
399
+ {
400
+ "count": 376,
401
+ "pct": 0.8527,
402
+ "text": "Here's an implementation of"
403
+ },
404
+ {
405
+ "count": 363,
406
+ "pct": 0.8232,
407
+ "text": "Here's an example of"
408
+ },
409
+ {
410
+ "count": 320,
411
+ "pct": 0.7257,
412
+ "text": "Here's the implementation of"
413
+ },
414
+ {
415
+ "count": 297,
416
+ "pct": 0.6735,
417
+ "text": "Here's a possible implementation"
418
+ },
419
+ {
420
+ "count": 285,
421
+ "pct": 0.6463,
422
+ "text": "To write a function"
423
+ }
424
+ ]
425
+ },
426
+ {
427
+ "assistant_length_chars": {
428
+ "max": 3836,
429
+ "min": 2,
430
+ "p50": 571,
431
+ "p95": 1589
432
+ },
433
+ "counts": {
434
+ "assistant_code_fence": 1800,
435
+ "assistant_contains_heres": 1005,
436
+ "assistant_mentions_json": 63,
437
+ "assistant_short_under_80_chars": 414,
438
+ "assistant_starts_heres": 479,
439
+ "assistant_starts_the_function": 13,
440
+ "assistant_starts_to_solve": 62,
441
+ "rendered_contains_qwen_markers": 0,
442
+ "rendered_contains_think": 0,
443
+ "rendered_empty_think": 0
444
+ },
445
+ "exact_phrase_counts": {
446
+ "Here's a thinking process": 0,
447
+ "Return only": 0,
448
+ "exactly one word": 0
449
+ },
450
+ "path": "data/eval/coding_sft_prod_repair_eval_messages.jsonl",
451
+ "percentages": {
452
+ "assistant_code_fence": 77.8547,
453
+ "assistant_contains_heres": 43.4689,
454
+ "assistant_mentions_json": 2.7249,
455
+ "assistant_short_under_80_chars": 17.9066,
456
+ "assistant_starts_heres": 20.718,
457
+ "assistant_starts_the_function": 0.5623,
458
+ "assistant_starts_to_solve": 2.6817,
459
+ "rendered_contains_qwen_markers": 0.0,
460
+ "rendered_contains_think": 0.0,
461
+ "rendered_empty_think": 0.0
462
+ },
463
+ "rows": 2312,
464
+ "top_openers": [
465
+ {
466
+ "count": 168,
467
+ "pct": 7.2664,
468
+ "text": "Here's how you can"
469
+ },
470
+ {
471
+ "count": 78,
472
+ "pct": 3.3737,
473
+ "text": "Here is a Python"
474
+ },
475
+ {
476
+ "count": 76,
477
+ "pct": 3.2872,
478
+ "text": "Here's how you could"
479
+ },
480
+ {
481
+ "count": 76,
482
+ "pct": 3.2872,
483
+ "text": "Here is the implementation"
484
+ },
485
+ {
486
+ "count": 60,
487
+ "pct": 2.5952,
488
+ "text": "Here's how we can"
489
+ },
490
+ {
491
+ "count": 55,
492
+ "pct": 2.3789,
493
+ "text": "To solve this problem,"
494
+ },
495
+ {
496
+ "count": 50,
497
+ "pct": 2.1626,
498
+ "text": "To write a Python"
499
+ },
500
+ {
501
+ "count": 49,
502
+ "pct": 2.1194,
503
+ "text": "The model output is"
504
+ },
505
+ {
506
+ "count": 40,
507
+ "pct": 1.7301,
508
+ "text": "You can create a"
509
+ },
510
+ {
511
+ "count": 33,
512
+ "pct": 1.4273,
513
+ "text": "Here's an example implementation"
514
+ },
515
+ {
516
+ "count": 29,
517
+ "pct": 1.2543,
518
+ "text": "Here is the implementation:"
519
+ },
520
+ {
521
+ "count": 22,
522
+ "pct": 0.9516,
523
+ "text": "To create a Python"
524
+ },
525
+ {
526
+ "count": 21,
527
+ "pct": 0.9083,
528
+ "text": "Here's the implementation of"
529
+ },
530
+ {
531
+ "count": 20,
532
+ "pct": 0.8651,
533
+ "text": "To implement this function,"
534
+ },
535
+ {
536
+ "count": 20,
537
+ "pct": 0.8651,
538
+ "text": "Here is an example"
539
+ },
540
+ {
541
+ "count": 19,
542
+ "pct": 0.8218,
543
+ "text": "Here is how you"
544
+ },
545
+ {
546
+ "count": 18,
547
+ "pct": 0.7785,
548
+ "text": "To check if a"
549
+ },
550
+ {
551
+ "count": 17,
552
+ "pct": 0.7353,
553
+ "text": "Here's an example of"
554
+ },
555
+ {
556
+ "count": 16,
557
+ "pct": 0.692,
558
+ "text": "Here's an implementation of"
559
+ },
560
+ {
561
+ "count": 15,
562
+ "pct": 0.6488,
563
+ "text": "To write a function"
564
+ }
565
+ ]
566
+ },
567
+ {
568
+ "assistant_length_chars": {
569
+ "max": 7218,
570
+ "min": 317,
571
+ "p50": 1324,
572
+ "p95": 2495
573
+ },
574
+ "counts": {
575
+ "assistant_code_fence": 40000,
576
+ "assistant_contains_heres": 22615,
577
+ "assistant_mentions_json": 2626,
578
+ "assistant_short_under_80_chars": 0,
579
+ "assistant_starts_heres": 0,
580
+ "assistant_starts_the_function": 0,
581
+ "assistant_starts_to_solve": 0,
582
+ "rendered_contains_qwen_markers": 44096,
583
+ "rendered_contains_think": 44096,
584
+ "rendered_empty_think": 44096
585
+ },
586
+ "exact_phrase_counts": {
587
+ "Here's a thinking process": 0,
588
+ "Return only": 2264,
589
+ "exactly one word": 150
590
+ },
591
+ "path": "data/train/coding_sft_prod_repair.jsonl",
592
+ "percentages": {
593
+ "assistant_code_fence": 90.7112,
594
+ "assistant_contains_heres": 51.2858,
595
+ "assistant_mentions_json": 5.9552,
596
+ "assistant_short_under_80_chars": 0.0,
597
+ "assistant_starts_heres": 0.0,
598
+ "assistant_starts_the_function": 0.0,
599
+ "assistant_starts_to_solve": 0.0,
600
+ "rendered_contains_qwen_markers": 100.0,
601
+ "rendered_contains_think": 100.0,
602
+ "rendered_empty_think": 100.0
603
+ },
604
+ "rows": 44096,
605
+ "top_openers": [
606
+ {
607
+ "count": 44096,
608
+ "pct": 100.0,
609
+ "text": "<|im_start|>system You are a"
610
+ }
611
+ ]
612
+ },
613
+ {
614
+ "assistant_length_chars": {
615
+ "max": 6162,
616
+ "min": 318,
617
+ "p50": 1212,
618
+ "p95": 2469
619
+ },
620
+ "counts": {
621
+ "assistant_code_fence": 1800,
622
+ "assistant_contains_heres": 1009,
623
+ "assistant_mentions_json": 194,
624
+ "assistant_short_under_80_chars": 0,
625
+ "assistant_starts_heres": 0,
626
+ "assistant_starts_the_function": 0,
627
+ "assistant_starts_to_solve": 0,
628
+ "rendered_contains_qwen_markers": 2312,
629
+ "rendered_contains_think": 2312,
630
+ "rendered_empty_think": 2312
631
+ },
632
+ "exact_phrase_counts": {
633
+ "Here's a thinking process": 0,
634
+ "Return only": 279,
635
+ "exactly one word": 23
636
+ },
637
+ "path": "data/eval/coding_sft_prod_repair_eval.jsonl",
638
+ "percentages": {
639
+ "assistant_code_fence": 77.8547,
640
+ "assistant_contains_heres": 43.6419,
641
+ "assistant_mentions_json": 8.391,
642
+ "assistant_short_under_80_chars": 0.0,
643
+ "assistant_starts_heres": 0.0,
644
+ "assistant_starts_the_function": 0.0,
645
+ "assistant_starts_to_solve": 0.0,
646
+ "rendered_contains_qwen_markers": 100.0,
647
+ "rendered_contains_think": 100.0,
648
+ "rendered_empty_think": 100.0
649
+ },
650
+ "rows": 2312,
651
+ "top_openers": [
652
+ {
653
+ "count": 2312,
654
+ "pct": 100.0,
655
+ "text": "<|im_start|>system You are a"
656
+ }
657
+ ]
658
+ }
659
+ ],
660
+ "inputs": [
661
+ "data/train/coding_sft_repair_messages.jsonl",
662
+ "data/eval/coding_sft_repair_eval_messages.jsonl",
663
+ "data/train/coding_sft_prod_repair_messages.jsonl",
664
+ "data/eval/coding_sft_prod_repair_eval_messages.jsonl",
665
+ "data/train/coding_sft_prod_repair.jsonl",
666
+ "data/eval/coding_sft_prod_repair_eval.jsonl"
667
+ ],
668
+ "interpretation": [
669
+ "High code-fence and explanatory-opener rates can conflict with strict final-answer gates.",
670
+ "A lack of short exact-answer rows can weaken exact-output behavior.",
671
+ "Rendered empty thinking blocks should be reviewed if the target serving profile disables reasoning."
672
+ ]
673
+ }
probes/sft-repair-blend-eval.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "duplicates_dropped": 0,
3
+ "output": "data/eval/coding_sft_prod_repair_eval_messages.jsonl",
4
+ "rows_before_dedupe": 2312,
5
+ "rows_written": 2312,
6
+ "seed": 37,
7
+ "sources": [
8
+ {
9
+ "available": 2000,
10
+ "label": "base_coding",
11
+ "path": "data/eval/coding_sft_prod_eval_messages.jsonl",
12
+ "selected": 1800
13
+ },
14
+ {
15
+ "available": 512,
16
+ "label": "strict_repair",
17
+ "path": "data/eval/coding_sft_repair_eval_messages.jsonl",
18
+ "selected": 512
19
+ }
20
+ ]
21
+ }
probes/sft-repair-blend-train.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "duplicates_dropped": 0,
3
+ "output": "data/train/coding_sft_prod_repair_messages.jsonl",
4
+ "rows_before_dedupe": 44096,
5
+ "rows_written": 44096,
6
+ "seed": 31,
7
+ "sources": [
8
+ {
9
+ "available": 44000,
10
+ "label": "base_coding",
11
+ "path": "data/train/coding_sft_prod_messages.jsonl",
12
+ "selected": 40000
13
+ },
14
+ {
15
+ "available": 4096,
16
+ "label": "strict_repair",
17
+ "path": "data/train/coding_sft_repair_messages.jsonl",
18
+ "selected": 4096
19
+ }
20
+ ]
21
+ }
probes/sft-repair-set.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eval": {
3
+ "assistant_contains_code_fence": 0,
4
+ "assistant_contains_heres": 0,
5
+ "assistant_contains_think": 0,
6
+ "assistant_short_under_80_chars": 414,
7
+ "categories": {
8
+ "code_only_no_fence": 251,
9
+ "exact_short_answer": 82,
10
+ "json_only": 130,
11
+ "plain_final_answer": 49
12
+ },
13
+ "rows": 512
14
+ },
15
+ "interpretation": [
16
+ "Synthetic repair rows target strict final-answer behavior after checkpoint-8000 failed behavior gates.",
17
+ "Assistant messages intentionally contain no markdown fences, no reasoning tags, and no explanatory preambles.",
18
+ "Use this as a small repair component in a larger coding-data blend, not as a standalone release dataset."
19
+ ],
20
+ "outputs": {
21
+ "eval": "data/eval/coding_sft_repair_eval_messages.jsonl",
22
+ "report": "runs/sft-repair-set.json",
23
+ "train": "data/train/coding_sft_repair_messages.jsonl"
24
+ },
25
+ "seed": 29,
26
+ "train": {
27
+ "assistant_contains_code_fence": 0,
28
+ "assistant_contains_heres": 0,
29
+ "assistant_contains_think": 0,
30
+ "assistant_short_under_80_chars": 3330,
31
+ "categories": {
32
+ "code_only_no_fence": 2057,
33
+ "exact_short_answer": 608,
34
+ "json_only": 1020,
35
+ "plain_final_answer": 411
36
+ },
37
+ "rows": 4096
38
+ }
39
+ }
scripts/build_sft_blend.py ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Build a deterministic JSONL blend from multiple SFT message or text files."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import hashlib
8
+ import json
9
+ import random
10
+ import re
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+
15
+ def normalize_text(text: str) -> str:
16
+ return re.sub(r"\s+", " ", text).strip().lower()
17
+
18
+
19
+ def row_text(row: dict[str, Any]) -> str:
20
+ messages = row.get("messages")
21
+ if isinstance(messages, list):
22
+ return json.dumps(messages, ensure_ascii=False, sort_keys=True)
23
+ return str(row.get("text") or "")
24
+
25
+
26
+ def stable_hash(row: dict[str, Any]) -> str:
27
+ return hashlib.sha256(normalize_text(row_text(row)).encode("utf-8")).hexdigest()
28
+
29
+
30
+ def read_jsonl(path: Path) -> list[dict[str, Any]]:
31
+ rows: list[dict[str, Any]] = []
32
+ for line_no, line in enumerate(path.read_text().splitlines(), start=1):
33
+ if not line.strip():
34
+ continue
35
+ row = json.loads(line)
36
+ if not isinstance(row, dict):
37
+ raise ValueError(f"{path}:{line_no}: expected object row")
38
+ rows.append(row)
39
+ return rows
40
+
41
+
42
+ def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
43
+ path.parent.mkdir(parents=True, exist_ok=True)
44
+ with path.open("w", encoding="utf-8") as handle:
45
+ for row in rows:
46
+ handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
47
+
48
+
49
+ def parse_source(spec: str) -> tuple[Path, int, str]:
50
+ label = ""
51
+ if ":" in spec:
52
+ before, label = spec.rsplit(":", 1)
53
+ else:
54
+ before = spec
55
+ if "=" not in before:
56
+ raise ValueError(f"source must be PATH=COUNT[:LABEL], got {spec!r}")
57
+ path_text, count_text = before.rsplit("=", 1)
58
+ count = int(count_text)
59
+ if count < 0:
60
+ raise ValueError(f"source count must be non-negative, got {spec!r}")
61
+ path = Path(path_text)
62
+ return path, count, label or path.stem
63
+
64
+
65
+ def selected_rows(path: Path, count: int, label: str, rng: random.Random) -> tuple[list[dict[str, Any]], dict[str, Any]]:
66
+ rows = read_jsonl(path)
67
+ if count > len(rows):
68
+ raise RuntimeError(f"{path} has {len(rows)} rows, cannot select {count}")
69
+ indices = list(range(len(rows)))
70
+ rng.shuffle(indices)
71
+ chosen = []
72
+ for source_index in indices[:count]:
73
+ row = dict(rows[source_index])
74
+ row["blend_source"] = label
75
+ row["blend_source_index"] = source_index
76
+ row["blend_source_path"] = str(path)
77
+ chosen.append(row)
78
+ return chosen, {"available": len(rows), "label": label, "path": str(path), "selected": count}
79
+
80
+
81
+ def main() -> int:
82
+ parser = argparse.ArgumentParser()
83
+ parser.add_argument(
84
+ "--source",
85
+ action="append",
86
+ required=True,
87
+ help="Input source as PATH=COUNT[:LABEL]. Repeat for each source.",
88
+ )
89
+ parser.add_argument("--output", type=Path, required=True)
90
+ parser.add_argument("--report-output", type=Path, required=True)
91
+ parser.add_argument("--seed", type=int, default=31)
92
+ parser.add_argument("--dedupe", action="store_true", default=True)
93
+ parser.add_argument("--allow-duplicates", action="store_false", dest="dedupe")
94
+ args = parser.parse_args()
95
+
96
+ rng = random.Random(args.seed)
97
+ all_rows: list[dict[str, Any]] = []
98
+ source_reports = []
99
+ for spec in args.source:
100
+ path, count, label = parse_source(spec)
101
+ rows, report = selected_rows(path, count, label, rng)
102
+ all_rows.extend(rows)
103
+ source_reports.append(report)
104
+
105
+ rows_before_dedupe = len(all_rows)
106
+ duplicate_count = 0
107
+ if args.dedupe:
108
+ seen: set[str] = set()
109
+ deduped = []
110
+ for row in all_rows:
111
+ digest = stable_hash(row)
112
+ if digest in seen:
113
+ duplicate_count += 1
114
+ continue
115
+ seen.add(digest)
116
+ deduped.append(row)
117
+ all_rows = deduped
118
+
119
+ rng.shuffle(all_rows)
120
+ write_jsonl(args.output, all_rows)
121
+ report = {
122
+ "duplicates_dropped": duplicate_count,
123
+ "output": str(args.output),
124
+ "rows_before_dedupe": rows_before_dedupe,
125
+ "rows_written": len(all_rows),
126
+ "seed": args.seed,
127
+ "sources": source_reports,
128
+ }
129
+ args.report_output.parent.mkdir(parents=True, exist_ok=True)
130
+ args.report_output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
131
+ print(json.dumps(report, indent=2, sort_keys=True))
132
+ return 0
133
+
134
+
135
+ if __name__ == "__main__":
136
+ raise SystemExit(main())
scripts/build_sft_repair_set.py ADDED
@@ -0,0 +1,260 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Build a deterministic strict-output repair set for SFT remediation."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import hashlib
8
+ import json
9
+ import random
10
+ from collections import Counter
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+
15
+ SYSTEM_MESSAGE = (
16
+ "You are a precise coding assistant. Follow output format constraints exactly. "
17
+ "Do not include reasoning, markdown fences, or preambles unless explicitly requested."
18
+ )
19
+
20
+
21
+ def stable_hash(*parts: str) -> str:
22
+ digest = hashlib.sha256()
23
+ for part in parts:
24
+ digest.update(part.encode("utf-8"))
25
+ digest.update(b"\0")
26
+ return digest.hexdigest()
27
+
28
+
29
+ def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
30
+ path.parent.mkdir(parents=True, exist_ok=True)
31
+ with path.open("w", encoding="utf-8") as handle:
32
+ for row in rows:
33
+ handle.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n")
34
+
35
+
36
+ def make_row(category: str, index: int, user: str, assistant: str) -> dict[str, Any]:
37
+ digest = stable_hash(category, str(index), user, assistant)
38
+ return {
39
+ "license": "synthetic-project",
40
+ "messages": [
41
+ {"role": "system", "content": SYSTEM_MESSAGE},
42
+ {"role": "user", "content": user},
43
+ {"role": "assistant", "content": assistant},
44
+ ],
45
+ "repair_category": category,
46
+ "sha1": hashlib.sha1(f"{category}\0{user}\0{assistant}".encode("utf-8")).hexdigest(),
47
+ "source_dataset": "ornith_sft_strict_repair_synthetic",
48
+ "source_id": digest[:16],
49
+ "source_preset": "strict_output_repair",
50
+ "source_split": "synthetic",
51
+ }
52
+
53
+
54
+ def code_only_row(index: int) -> dict[str, Any]:
55
+ variants = [
56
+ (
57
+ "add_{n}",
58
+ "Return only Python code. Define a function `{name}(a, b)` that returns `a + b`. "
59
+ "No markdown, no explanation.",
60
+ "def {name}(a, b):\n return a + b",
61
+ ),
62
+ (
63
+ "is_even_{n}",
64
+ "Return only code. Write Python function `{name}(value)` that returns True when "
65
+ "`value` is even.",
66
+ "def {name}(value):\n return value % 2 == 0",
67
+ ),
68
+ (
69
+ "clamp_{n}",
70
+ "Return only code with no code fence. Write `{name}(value, low, high)` to clamp "
71
+ "`value` into the inclusive range.",
72
+ "def {name}(value, low, high):\n return max(low, min(value, high))",
73
+ ),
74
+ (
75
+ "first_or_none_{n}",
76
+ "Return only raw Python code. Define `{name}(items)` that returns the first item or "
77
+ "None for an empty sequence.",
78
+ "def {name}(items):\n return items[0] if items else None",
79
+ ),
80
+ (
81
+ "count_matches_{n}",
82
+ "Return only Python code. Define `{name}(items, target)` that counts values equal "
83
+ "to `target`.",
84
+ "def {name}(items, target):\n return sum(1 for item in items if item == target)",
85
+ ),
86
+ (
87
+ "safe_get_{n}",
88
+ "Return only code, no prose. Write `{name}(mapping, key, default=None)` that returns "
89
+ "the mapped value or default.",
90
+ "def {name}(mapping, key, default=None):\n return mapping.get(key, default)",
91
+ ),
92
+ (
93
+ "reverse_words_{n}",
94
+ "Return only Python code. Define `{name}(text)` that reverses word order while "
95
+ "preserving each word.",
96
+ "def {name}(text):\n return \" \".join(reversed(text.split()))",
97
+ ),
98
+ (
99
+ "dedupe_preserve_order_{n}",
100
+ "Return only raw Python code. Write `{name}(items)` that removes duplicates while "
101
+ "preserving order.",
102
+ "def {name}(items):\n"
103
+ " seen = set()\n"
104
+ " result = []\n"
105
+ " for item in items:\n"
106
+ " if item not in seen:\n"
107
+ " seen.add(item)\n"
108
+ " result.append(item)\n"
109
+ " return result",
110
+ ),
111
+ ]
112
+ stem, prompt_template, answer_template = variants[index % len(variants)]
113
+ name = stem.format(n=index // len(variants))
114
+ return make_row("code_only_no_fence", index, prompt_template.format(name=name), answer_template.format(name=name))
115
+
116
+
117
+ def json_only_row(index: int) -> dict[str, Any]:
118
+ language = ["python", "typescript", "rust", "go", "sql", "bash"][index % 6]
119
+ ok = index % 3 != 0
120
+ payload = {"case": index, "language": language, "mode": "strict", "ok": ok}
121
+ compact = json.dumps(payload, separators=(",", ":"), sort_keys=True)
122
+ if index % 5 == 0:
123
+ user = f"Return only this compact JSON object with no markdown and no extra whitespace: {compact}"
124
+ assistant = compact
125
+ else:
126
+ user = (
127
+ "Return a compact JSON object only. Fields: "
128
+ f"language={language}, ok={str(ok).lower()}, case={index}, mode=strict. "
129
+ "No markdown fence."
130
+ )
131
+ assistant = compact
132
+ return make_row("json_only", index, user, assistant)
133
+
134
+
135
+ def exact_short_row(index: int) -> dict[str, Any]:
136
+ values = ["ready", "done", "yes", "no", "pass", "fail", "python", "valid", "invalid", "ok"]
137
+ value = values[index % len(values)]
138
+ variants = [
139
+ f"Validation item {index}: reply with exactly one word, {value}. Do not include any special tokens.",
140
+ f"Validation item {index}: return exactly `{value}` and nothing else.",
141
+ f"Validation item {index}: answer with the single token {value}. No punctuation.",
142
+ f"Validation item {index}: output only this word: {value}",
143
+ ]
144
+ return make_row("exact_short_answer", index, variants[index % len(variants)], value)
145
+
146
+
147
+ def plain_sentence_row(index: int) -> dict[str, Any]:
148
+ adjectives = ["readable", "valid", "deterministic", "compact", "complete", "clear"]
149
+ adjective = adjectives[index % len(adjectives)]
150
+ user = (
151
+ f"For validation item {index}, in one short plain English sentence, say that the model output is {adjective}. "
152
+ "No markdown and no reasoning."
153
+ )
154
+ assistant = f"The model output is {adjective}."
155
+ return make_row("plain_final_answer", index, user, assistant)
156
+
157
+
158
+ GENERATORS = {
159
+ "code_only_no_fence": code_only_row,
160
+ "json_only": json_only_row,
161
+ "exact_short_answer": exact_short_row,
162
+ "plain_final_answer": plain_sentence_row,
163
+ }
164
+
165
+
166
+ def category_for_position(position: int) -> str:
167
+ mod = position % 20
168
+ if mod < 10:
169
+ return "code_only_no_fence"
170
+ if mod < 15:
171
+ return "json_only"
172
+ if mod < 18:
173
+ return "exact_short_answer"
174
+ return "plain_final_answer"
175
+
176
+
177
+ def build_rows(total: int, seed: int) -> list[dict[str, Any]]:
178
+ rng = random.Random(seed)
179
+ counters: Counter[str] = Counter()
180
+ rows: list[dict[str, Any]] = []
181
+ seen: set[str] = set()
182
+
183
+ for position in range(total):
184
+ category = category_for_position(position)
185
+ index = counters[category]
186
+ counters[category] += 1
187
+ row = GENERATORS[category](index)
188
+ digest = stable_hash(json.dumps(row["messages"], sort_keys=True))
189
+ if digest in seen:
190
+ raise RuntimeError(f"duplicate generated row: category={category} index={index}")
191
+ seen.add(digest)
192
+ rows.append(row)
193
+
194
+ rng.shuffle(rows)
195
+ return rows
196
+
197
+
198
+ def assistant_text(row: dict[str, Any]) -> str:
199
+ return "\n".join(
200
+ str(message.get("content") or "")
201
+ for message in row["messages"]
202
+ if isinstance(message, dict) and message.get("role") == "assistant"
203
+ )
204
+
205
+
206
+ def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]:
207
+ counts = Counter(str(row["repair_category"]) for row in rows)
208
+ assistant_values = [assistant_text(row) for row in rows]
209
+ return {
210
+ "rows": len(rows),
211
+ "categories": dict(sorted(counts.items())),
212
+ "assistant_contains_code_fence": sum("```" in text for text in assistant_values),
213
+ "assistant_contains_heres": sum("Here's" in text or "Here is" in text for text in assistant_values),
214
+ "assistant_contains_think": sum("<think>" in text or "</think>" in text for text in assistant_values),
215
+ "assistant_short_under_80_chars": sum(len(text.strip()) < 80 for text in assistant_values),
216
+ }
217
+
218
+
219
+ def main() -> int:
220
+ parser = argparse.ArgumentParser()
221
+ parser.add_argument("--train-output", type=Path, default=Path("data/train/coding_sft_repair_messages.jsonl"))
222
+ parser.add_argument("--eval-output", type=Path, default=Path("data/eval/coding_sft_repair_eval_messages.jsonl"))
223
+ parser.add_argument("--report-output", type=Path, default=Path("runs/sft-repair-set.json"))
224
+ parser.add_argument("--train-size", type=int, default=4096)
225
+ parser.add_argument("--eval-size", type=int, default=512)
226
+ parser.add_argument("--seed", type=int, default=29)
227
+ args = parser.parse_args()
228
+
229
+ if args.train_size <= 0 or args.eval_size < 0:
230
+ raise ValueError("train-size must be positive and eval-size must be non-negative")
231
+
232
+ rows = build_rows(args.train_size + args.eval_size, args.seed)
233
+ train_rows = rows[: args.train_size]
234
+ eval_rows = rows[args.train_size :]
235
+ write_jsonl(args.train_output, train_rows)
236
+ write_jsonl(args.eval_output, eval_rows)
237
+
238
+ report = {
239
+ "seed": args.seed,
240
+ "outputs": {
241
+ "train": str(args.train_output),
242
+ "eval": str(args.eval_output),
243
+ "report": str(args.report_output),
244
+ },
245
+ "train": summarize(train_rows),
246
+ "eval": summarize(eval_rows),
247
+ "interpretation": [
248
+ "Synthetic repair rows target strict final-answer behavior after checkpoint-8000 failed behavior gates.",
249
+ "Assistant messages intentionally contain no markdown fences, no reasoning tags, and no explanatory preambles.",
250
+ "Use this as a small repair component in a larger coding-data blend, not as a standalone release dataset.",
251
+ ],
252
+ }
253
+ args.report_output.parent.mkdir(parents=True, exist_ok=True)
254
+ args.report_output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
255
+ print(json.dumps(report, indent=2, sort_keys=True))
256
+ return 0
257
+
258
+
259
+ if __name__ == "__main__":
260
+ raise SystemExit(main())
scripts/eval_lora_behavior.py CHANGED
@@ -131,6 +131,7 @@ def generate_case(
131
  case: dict[str, Any],
132
  defaults: dict[str, Any],
133
  max_input_length: int,
 
134
  ) -> dict[str, Any]:
135
  max_new_tokens = int(case.get("max_tokens", defaults.get("max_tokens", 192)))
136
  temperature = float(case.get("temperature", defaults.get("temperature", 0.0)))
@@ -140,6 +141,7 @@ def generate_case(
140
  messages,
141
  tokenize=False,
142
  add_generation_prompt=True,
 
143
  )
144
  encoded = tokenizer(
145
  prompt,
@@ -220,6 +222,11 @@ def main() -> int:
220
  parser.add_argument("--max-input-length", type=int, default=4096)
221
  parser.add_argument("--include-disabled", action="store_true")
222
  parser.add_argument("--include-tag", action="append")
 
 
 
 
 
223
  parser.add_argument(
224
  "--blocking-severity",
225
  action="append",
@@ -253,6 +260,7 @@ def main() -> int:
253
  case=case,
254
  defaults=defaults,
255
  max_input_length=args.max_input_length,
 
256
  )
257
  check_reports = []
258
  for check_def in case.get("checks") or []:
@@ -301,6 +309,7 @@ def main() -> int:
301
  "adapter": str(args.adapter) if args.adapter else None,
302
  "dtype": args.dtype,
303
  "device_index": args.device_index,
 
304
  },
305
  "summary": {
306
  "cases": len(case_reports),
 
131
  case: dict[str, Any],
132
  defaults: dict[str, Any],
133
  max_input_length: int,
134
+ enable_thinking: bool,
135
  ) -> dict[str, Any]:
136
  max_new_tokens = int(case.get("max_tokens", defaults.get("max_tokens", 192)))
137
  temperature = float(case.get("temperature", defaults.get("temperature", 0.0)))
 
141
  messages,
142
  tokenize=False,
143
  add_generation_prompt=True,
144
+ enable_thinking=enable_thinking,
145
  )
146
  encoded = tokenizer(
147
  prompt,
 
222
  parser.add_argument("--max-input-length", type=int, default=4096)
223
  parser.add_argument("--include-disabled", action="store_true")
224
  parser.add_argument("--include-tag", action="append")
225
+ parser.add_argument(
226
+ "--enable-thinking",
227
+ action="store_true",
228
+ help="Open generation in thinking mode. Default matches the final-answer serving profile.",
229
+ )
230
  parser.add_argument(
231
  "--blocking-severity",
232
  action="append",
 
260
  case=case,
261
  defaults=defaults,
262
  max_input_length=args.max_input_length,
263
+ enable_thinking=args.enable_thinking,
264
  )
265
  check_reports = []
266
  for check_def in case.get("checks") or []:
 
309
  "adapter": str(args.adapter) if args.adapter else None,
310
  "dtype": args.dtype,
311
  "device_index": args.device_index,
312
+ "enable_thinking": args.enable_thinking,
313
  },
314
  "summary": {
315
  "cases": len(case_reports),
scripts/launch_sft_prod_20k_gpu1.sh ADDED
@@ -0,0 +1,78 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+
4
+ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
5
+ cd "${REPO_ROOT}"
6
+
7
+ RUN_NAME="${RUN_NAME:-ornith-35b-coding-lora-prod-20k}"
8
+ OUTPUT_DIR="${OUTPUT_DIR:-artifacts/train/${RUN_NAME}}"
9
+ LOG_PATH="${LOG_PATH:-runs/${RUN_NAME}.log}"
10
+ PID_PATH="${PID_PATH:-runs/${RUN_NAME}.pid}"
11
+ COMMAND_PATH="${COMMAND_PATH:-runs/${RUN_NAME}.command.txt}"
12
+ RESUME_FROM_CHECKPOINT="${RESUME_FROM_CHECKPOINT:-}"
13
+ TRAIN_JSONL="${TRAIN_JSONL:-data/train/coding_sft_prod.jsonl}"
14
+ EVAL_JSONL="${EVAL_JSONL:-data/eval/coding_sft_prod_eval.jsonl}"
15
+ MIN_TRAIN_ROWS="${MIN_TRAIN_ROWS:-44000}"
16
+ VALIDATION_REPORT="${VALIDATION_REPORT:-runs/sft-train-validation-bigcode-prod-44k.json}"
17
+
18
+ if [[ -f "${PID_PATH}" ]]; then
19
+ old_pid="$(cat "${PID_PATH}")"
20
+ if [[ -n "${old_pid}" ]] && kill -0 "${old_pid}" 2>/dev/null; then
21
+ echo "training is already running: pid=${old_pid}"
22
+ exit 0
23
+ fi
24
+ fi
25
+
26
+ mkdir -p "$(dirname "${LOG_PATH}")" "$(dirname "${PID_PATH}")" "${OUTPUT_DIR}"
27
+
28
+ cmd=(
29
+ env
30
+ "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-1}"
31
+ "TOKENIZERS_PARALLELISM=false"
32
+ "PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
33
+ .venv-train/bin/python scripts/train_sft_lora.py
34
+ --model deepreinforce-ai/Ornith-1.0-35B
35
+ --train-jsonl "${TRAIN_JSONL}"
36
+ --eval-jsonl "${EVAL_JSONL}"
37
+ --output "${OUTPUT_DIR}"
38
+ --max-length 1024
39
+ --per-device-train-batch-size 1
40
+ --per-device-eval-batch-size 8
41
+ --gradient-accumulation-steps 8
42
+ --learning-rate 1e-4
43
+ --warmup-ratio 0.03
44
+ --lr-scheduler-type cosine
45
+ --weight-decay 0.0
46
+ --max-grad-norm 1.0
47
+ --num-train-epochs 1
48
+ --max-steps 20000
49
+ --save-steps 1000
50
+ --eval-steps 1000
51
+ --logging-steps 25
52
+ --save-total-limit 5
53
+ --lora-r 16
54
+ --lora-alpha 32
55
+ --lora-dropout 0.05
56
+ --loss-type nll
57
+ --router-aux-loss-coef 0.0
58
+ --seed 23
59
+ --require-license
60
+ --min-train-rows "${MIN_TRAIN_ROWS}"
61
+ --validation-report "${VALIDATION_REPORT}"
62
+ )
63
+
64
+ if [[ -n "${RESUME_FROM_CHECKPOINT}" ]]; then
65
+ cmd+=(--resume-from-checkpoint "${RESUME_FROM_CHECKPOINT}")
66
+ fi
67
+
68
+ printf '%q ' "${cmd[@]}" > "${COMMAND_PATH}"
69
+ printf '\n' >> "${COMMAND_PATH}"
70
+
71
+ setsid "${cmd[@]}" > "${LOG_PATH}" 2>&1 < /dev/null &
72
+ pid="$!"
73
+ echo "${pid}" > "${PID_PATH}"
74
+
75
+ echo "started ${RUN_NAME}"
76
+ echo "pid=${pid}"
77
+ echo "log=${LOG_PATH}"
78
+ echo "output=${OUTPUT_DIR}"
scripts/render_sft_jsonl.py ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Render structured chat-message JSONL into the model chat template."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import json
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from transformers import AutoTokenizer
12
+
13
+
14
+ def read_jsonl(path: Path) -> list[dict[str, Any]]:
15
+ rows = []
16
+ for line_no, line in enumerate(path.read_text().splitlines(), start=1):
17
+ if not line.strip():
18
+ continue
19
+ row = json.loads(line)
20
+ if not isinstance(row, dict):
21
+ raise ValueError(f"{path}:{line_no} expected object row, got {type(row).__name__}")
22
+ row["_line_no"] = line_no
23
+ rows.append(row)
24
+ return rows
25
+
26
+
27
+ def render_row(
28
+ row: dict[str, Any],
29
+ tokenizer: Any,
30
+ text_field: str,
31
+ keep_messages: bool,
32
+ force: bool,
33
+ enable_thinking: bool,
34
+ ) -> dict[str, Any]:
35
+ messages = row.get("messages")
36
+ if isinstance(row.get(text_field), str) and not force:
37
+ rendered = str(row[text_field])
38
+ else:
39
+ if not isinstance(messages, list):
40
+ raise ValueError(
41
+ f"line {row.get('_line_no')} needs a messages list or existing `{text_field}`"
42
+ )
43
+ rendered = tokenizer.apply_chat_template(
44
+ messages,
45
+ tokenize=False,
46
+ add_generation_prompt=False,
47
+ enable_thinking=enable_thinking,
48
+ )
49
+
50
+ output = {
51
+ key: value
52
+ for key, value in row.items()
53
+ if key != "_line_no" and (keep_messages or key != "messages")
54
+ }
55
+ output[text_field] = rendered
56
+ return output
57
+
58
+
59
+ def main() -> int:
60
+ parser = argparse.ArgumentParser()
61
+ parser.add_argument("--input", type=Path, required=True)
62
+ parser.add_argument("--output", type=Path, required=True)
63
+ parser.add_argument("--model", default="deepreinforce-ai/Ornith-1.0-35B")
64
+ parser.add_argument("--text-field", default="text")
65
+ parser.add_argument("--keep-messages", action="store_true")
66
+ parser.add_argument(
67
+ "--enable-thinking",
68
+ action="store_true",
69
+ help="Pass enable_thinking=True to chat templates that support it. Defaults to final-answer mode.",
70
+ )
71
+ parser.add_argument(
72
+ "--force",
73
+ action="store_true",
74
+ help="Render messages even when the row already has the text field.",
75
+ )
76
+ args = parser.parse_args()
77
+
78
+ tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
79
+ rows = read_jsonl(args.input)
80
+ args.output.parent.mkdir(parents=True, exist_ok=True)
81
+ with args.output.open("w", encoding="utf-8") as handle:
82
+ for row in rows:
83
+ rendered = render_row(
84
+ row=row,
85
+ tokenizer=tokenizer,
86
+ text_field=args.text_field,
87
+ keep_messages=args.keep_messages,
88
+ force=args.force,
89
+ enable_thinking=args.enable_thinking,
90
+ )
91
+ handle.write(json.dumps(rendered, ensure_ascii=False, sort_keys=True) + "\n")
92
+
93
+ print(f"wrote {len(rows)} rows to {args.output}")
94
+ return 0
95
+
96
+
97
+ if __name__ == "__main__":
98
+ raise SystemExit(main())
scripts/train_sft_lora.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Single-GPU LoRA SFT starter for coding data."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import json
8
+ from pathlib import Path
9
+
10
+
11
+ def main() -> int:
12
+ parser = argparse.ArgumentParser()
13
+ parser.add_argument("--model", default="deepreinforce-ai/Ornith-1.0-35B")
14
+ parser.add_argument("--train-jsonl", type=Path, required=True)
15
+ parser.add_argument("--eval-jsonl", type=Path)
16
+ parser.add_argument("--output", required=True)
17
+ parser.add_argument("--text-field", default="text")
18
+ parser.add_argument("--max-length", type=int, default=4096)
19
+ parser.add_argument("--packing", action="store_true")
20
+ parser.add_argument("--padding-free", action="store_true")
21
+ parser.add_argument("--per-device-train-batch-size", type=int, default=1)
22
+ parser.add_argument("--per-device-eval-batch-size", type=int, default=8)
23
+ parser.add_argument("--gradient-accumulation-steps", type=int, default=16)
24
+ parser.add_argument("--learning-rate", type=float, default=2e-4)
25
+ parser.add_argument("--warmup-ratio", type=float, default=0.0)
26
+ parser.add_argument("--weight-decay", type=float, default=0.0)
27
+ parser.add_argument("--max-grad-norm", type=float, default=1.0)
28
+ parser.add_argument("--lr-scheduler-type", default="linear")
29
+ parser.add_argument("--num-train-epochs", type=float, default=1.0)
30
+ parser.add_argument("--max-steps", type=int, default=-1)
31
+ parser.add_argument("--save-steps", type=int, default=200)
32
+ parser.add_argument("--eval-steps", type=int, default=200)
33
+ parser.add_argument("--logging-steps", type=int, default=10)
34
+ parser.add_argument("--save-total-limit", type=int, default=3)
35
+ parser.add_argument("--optim", default="adamw_torch")
36
+ parser.add_argument("--loss-type", choices=["nll", "dft", "chunked_nll"], default="nll")
37
+ parser.add_argument("--router-aux-loss-coef", type=float, default=0.0)
38
+ parser.add_argument("--lora-r", type=int, default=16)
39
+ parser.add_argument("--lora-alpha", type=int, default=32)
40
+ parser.add_argument("--lora-dropout", type=float, default=0.05)
41
+ parser.add_argument("--seed", type=int, default=17)
42
+ parser.add_argument("--resume-from-checkpoint")
43
+ parser.add_argument("--skip-data-validation", action="store_true")
44
+ parser.add_argument("--validation-report", type=Path)
45
+ parser.add_argument("--min-train-rows", type=int, default=1)
46
+ parser.add_argument("--allow-unrendered", action="store_true")
47
+ parser.add_argument("--require-license", action="store_true")
48
+ args = parser.parse_args()
49
+
50
+ if not args.skip_data_validation:
51
+ from validate_sft_jsonl import validate_files
52
+
53
+ report = validate_files(
54
+ train_jsonl=args.train_jsonl,
55
+ eval_jsonl=args.eval_jsonl,
56
+ text_field=args.text_field,
57
+ model=args.model,
58
+ max_length=args.max_length,
59
+ min_train_rows=args.min_train_rows,
60
+ require_rendered=not args.allow_unrendered,
61
+ require_license=args.require_license,
62
+ )
63
+ if args.validation_report:
64
+ args.validation_report.parent.mkdir(parents=True, exist_ok=True)
65
+ args.validation_report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n")
66
+ blocking = [
67
+ issue for issue in report["issues"] if issue["severity"] in {"critical", "high"}
68
+ ]
69
+ if blocking:
70
+ for issue in report["issues"]:
71
+ if issue["severity"] in {"critical", "high"}:
72
+ print(f"{issue['severity']} {issue['issue_id']}: {issue['title']}")
73
+ print(f" evidence: {issue['evidence']}")
74
+ print(f" recommendation: {issue['recommendation']}")
75
+ return 2
76
+
77
+ import torch
78
+ from datasets import load_dataset
79
+ from peft import LoraConfig
80
+ from transformers import AutoTokenizer
81
+ from trl import SFTConfig, SFTTrainer
82
+
83
+ data_files = {"train": str(args.train_jsonl)}
84
+ if args.eval_jsonl is not None:
85
+ data_files["eval"] = str(args.eval_jsonl)
86
+
87
+ dataset = load_dataset("json", data_files=data_files)
88
+ eval_dataset = dataset.get("eval")
89
+ tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
90
+
91
+ peft_config = LoraConfig(
92
+ r=args.lora_r,
93
+ lora_alpha=args.lora_alpha,
94
+ lora_dropout=args.lora_dropout,
95
+ bias="none",
96
+ task_type="CAUSAL_LM",
97
+ target_modules=[
98
+ "q_proj",
99
+ "k_proj",
100
+ "v_proj",
101
+ "o_proj",
102
+ "gate_proj",
103
+ "up_proj",
104
+ "down_proj",
105
+ ],
106
+ )
107
+
108
+ training_args = SFTConfig(
109
+ output_dir=args.output,
110
+ dataset_text_field=args.text_field,
111
+ max_length=args.max_length,
112
+ packing=args.packing,
113
+ eval_packing=args.packing,
114
+ padding_free=args.padding_free,
115
+ per_device_train_batch_size=args.per_device_train_batch_size,
116
+ per_device_eval_batch_size=args.per_device_eval_batch_size,
117
+ gradient_accumulation_steps=args.gradient_accumulation_steps,
118
+ learning_rate=args.learning_rate,
119
+ warmup_ratio=args.warmup_ratio,
120
+ weight_decay=args.weight_decay,
121
+ max_grad_norm=args.max_grad_norm,
122
+ lr_scheduler_type=args.lr_scheduler_type,
123
+ num_train_epochs=args.num_train_epochs,
124
+ max_steps=args.max_steps,
125
+ seed=args.seed,
126
+ bf16=True,
127
+ gradient_checkpointing=True,
128
+ gradient_checkpointing_kwargs={"use_reentrant": False},
129
+ eval_strategy="steps" if eval_dataset is not None else "no",
130
+ eval_steps=args.eval_steps if eval_dataset is not None else None,
131
+ save_steps=args.save_steps,
132
+ save_total_limit=args.save_total_limit,
133
+ logging_steps=args.logging_steps,
134
+ optim=args.optim,
135
+ loss_type=args.loss_type,
136
+ router_aux_loss_coef=args.router_aux_loss_coef,
137
+ report_to="none",
138
+ model_init_kwargs={
139
+ "dtype": torch.bfloat16,
140
+ "trust_remote_code": True,
141
+ "device_map": {"": 0},
142
+ },
143
+ )
144
+
145
+ trainer = SFTTrainer(
146
+ model=args.model,
147
+ args=training_args,
148
+ train_dataset=dataset["train"],
149
+ eval_dataset=eval_dataset,
150
+ processing_class=tokenizer,
151
+ peft_config=peft_config,
152
+ )
153
+ trainer.train(resume_from_checkpoint=args.resume_from_checkpoint)
154
+ trainer.save_model(args.output)
155
+ return 0
156
+
157
+
158
+ if __name__ == "__main__":
159
+ raise SystemExit(main())
scripts/validate_sft_jsonl.py ADDED
@@ -0,0 +1,489 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """Validate rendered SFT JSONL before spending GPU time."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import argparse
7
+ import hashlib
8
+ import json
9
+ import re
10
+ import statistics
11
+ from dataclasses import asdict, dataclass
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+
16
+ DEFAULT_BLOCKING_SEVERITIES = {"critical", "high"}
17
+ SECRET_PATTERNS = {
18
+ "hf_token": re.compile(r"\bhf_[A-Za-z0-9]{20,}\b"),
19
+ "openai_key": re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"),
20
+ "aws_access_key": re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
21
+ "private_key": re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
22
+ }
23
+
24
+
25
+ @dataclass
26
+ class Check:
27
+ name: str
28
+ status: str
29
+ detail: str
30
+ recommendation: str | None = None
31
+
32
+
33
+ @dataclass
34
+ class Issue:
35
+ issue_id: str
36
+ severity: str
37
+ component: str
38
+ title: str
39
+ evidence: str
40
+ recommendation: str
41
+
42
+
43
+ def add_check(
44
+ checks: list[Check],
45
+ name: str,
46
+ ok: bool,
47
+ detail: str,
48
+ recommendation: str | None = None,
49
+ warn: bool = False,
50
+ ) -> None:
51
+ checks.append(
52
+ Check(
53
+ name=name,
54
+ status="pass" if ok else ("warn" if warn else "fail"),
55
+ detail=detail,
56
+ recommendation=None if ok else recommendation,
57
+ )
58
+ )
59
+
60
+
61
+ def add_issue(
62
+ issues: list[Issue],
63
+ condition: bool,
64
+ issue_id: str,
65
+ severity: str,
66
+ component: str,
67
+ title: str,
68
+ evidence: str,
69
+ recommendation: str,
70
+ ) -> None:
71
+ if condition:
72
+ issues.append(
73
+ Issue(
74
+ issue_id=issue_id,
75
+ severity=severity,
76
+ component=component,
77
+ title=title,
78
+ evidence=evidence,
79
+ recommendation=recommendation,
80
+ )
81
+ )
82
+
83
+
84
+ def normalize_text(text: str) -> str:
85
+ return re.sub(r"\s+", " ", text).strip().lower()
86
+
87
+
88
+ def text_hash(text: str) -> str:
89
+ return hashlib.sha256(normalize_text(text).encode()).hexdigest()
90
+
91
+
92
+ def percentile(values: list[int], pct: float) -> int | None:
93
+ if not values:
94
+ return None
95
+ values = sorted(values)
96
+ idx = min(len(values) - 1, max(0, round((pct / 100) * (len(values) - 1))))
97
+ return values[idx]
98
+
99
+
100
+ def load_tokenizer(model: str | None):
101
+ if not model:
102
+ return None
103
+ from transformers import AutoTokenizer
104
+
105
+ return AutoTokenizer.from_pretrained(model, trust_remote_code=True)
106
+
107
+
108
+ def read_jsonl(path: Path, split: str, checks: list[Check], issues: list[Issue]) -> list[dict[str, Any]]:
109
+ if not path.exists():
110
+ add_check(
111
+ checks,
112
+ f"{split}:file_exists",
113
+ False,
114
+ f"{path} does not exist",
115
+ "Create the JSONL file or point the command to the correct split.",
116
+ )
117
+ add_issue(
118
+ issues,
119
+ True,
120
+ f"{split}:missing-file",
121
+ "critical",
122
+ "data",
123
+ f"{split} JSONL is missing",
124
+ str(path),
125
+ "Create the split before launching training.",
126
+ )
127
+ return []
128
+
129
+ rows: list[dict[str, Any]] = []
130
+ parse_errors = []
131
+ for line_no, line in enumerate(path.read_text().splitlines(), start=1):
132
+ if not line.strip():
133
+ continue
134
+ try:
135
+ row = json.loads(line)
136
+ except json.JSONDecodeError as exc:
137
+ parse_errors.append(f"line {line_no}: {exc}")
138
+ continue
139
+ if not isinstance(row, dict):
140
+ parse_errors.append(f"line {line_no}: row is {type(row).__name__}, expected object")
141
+ continue
142
+ row["_line_no"] = line_no
143
+ rows.append(row)
144
+
145
+ add_check(
146
+ checks,
147
+ f"{split}:jsonl_parse",
148
+ not parse_errors,
149
+ "all lines parsed" if not parse_errors else "; ".join(parse_errors[:5]),
150
+ "Fix malformed JSONL rows before training.",
151
+ )
152
+ add_issue(
153
+ issues,
154
+ bool(parse_errors),
155
+ f"{split}:json-parse-errors",
156
+ "critical",
157
+ "data",
158
+ f"{split} JSONL has parse errors",
159
+ "; ".join(parse_errors[:10]),
160
+ "Fix malformed rows and re-run validation.",
161
+ )
162
+ add_check(
163
+ checks,
164
+ f"{split}:rows_present",
165
+ bool(rows),
166
+ f"rows={len(rows)}",
167
+ "Add at least one valid row.",
168
+ )
169
+ return rows
170
+
171
+
172
+ def validate_split(
173
+ rows: list[dict[str, Any]],
174
+ split: str,
175
+ text_field: str,
176
+ tokenizer: Any,
177
+ max_length: int,
178
+ require_rendered: bool,
179
+ require_license: bool,
180
+ checks: list[Check],
181
+ issues: list[Issue],
182
+ ) -> dict[str, Any]:
183
+ texts: list[str] = []
184
+ token_lengths: list[int] = []
185
+ empty_lines: list[int] = []
186
+ missing_text_lines: list[int] = []
187
+ missing_template_lines: list[int] = []
188
+ missing_license_lines: list[int] = []
189
+ too_long_lines: list[int] = []
190
+ secret_hits: list[str] = []
191
+ hashes: dict[str, list[int]] = {}
192
+
193
+ for row in rows:
194
+ line_no = int(row.get("_line_no", 0))
195
+ text = row.get(text_field)
196
+ if not isinstance(text, str):
197
+ missing_text_lines.append(line_no)
198
+ continue
199
+ if not text.strip():
200
+ empty_lines.append(line_no)
201
+ continue
202
+ if require_rendered and (
203
+ "<|im_start|>" not in text or "<|im_start|>assistant" not in text
204
+ ):
205
+ missing_template_lines.append(line_no)
206
+ if require_license and not row.get("license"):
207
+ missing_license_lines.append(line_no)
208
+ for name, pattern in SECRET_PATTERNS.items():
209
+ if pattern.search(text):
210
+ secret_hits.append(f"line {line_no}: {name}")
211
+ digest = text_hash(text)
212
+ hashes.setdefault(digest, []).append(line_no)
213
+ texts.append(text)
214
+ if tokenizer is not None:
215
+ length = len(tokenizer.encode(text, add_special_tokens=False))
216
+ token_lengths.append(length)
217
+ if length > max_length:
218
+ too_long_lines.append(line_no)
219
+
220
+ duplicate_groups = [lines for lines in hashes.values() if len(lines) > 1]
221
+ add_check(
222
+ checks,
223
+ f"{split}:text_field_present",
224
+ not missing_text_lines,
225
+ "all rows have text" if not missing_text_lines else f"missing on lines={missing_text_lines[:20]}",
226
+ f"Render structured messages to `{text_field}` before training.",
227
+ )
228
+ add_check(
229
+ checks,
230
+ f"{split}:nonempty_text",
231
+ not empty_lines,
232
+ "all text rows are nonempty" if not empty_lines else f"empty on lines={empty_lines[:20]}",
233
+ "Drop or repair empty rows.",
234
+ )
235
+ add_check(
236
+ checks,
237
+ f"{split}:rendered_template_present",
238
+ not missing_template_lines,
239
+ "all rows look rendered" if not missing_template_lines else f"missing template markers on lines={missing_template_lines[:20]}",
240
+ "Use scripts/render_sft_jsonl.py so data follows the model chat template.",
241
+ warn=not require_rendered,
242
+ )
243
+ add_check(
244
+ checks,
245
+ f"{split}:no_obvious_secrets",
246
+ not secret_hits,
247
+ "no obvious secret patterns found" if not secret_hits else "; ".join(secret_hits[:10]),
248
+ "Remove secrets from training data and rotate any exposed credentials.",
249
+ )
250
+ add_check(
251
+ checks,
252
+ f"{split}:duplicates",
253
+ not duplicate_groups,
254
+ "no exact normalized duplicates"
255
+ if not duplicate_groups
256
+ else f"duplicate groups={duplicate_groups[:10]}",
257
+ "Deduplicate the split before training.",
258
+ warn=True,
259
+ )
260
+
261
+ if require_license:
262
+ add_check(
263
+ checks,
264
+ f"{split}:license_present",
265
+ not missing_license_lines,
266
+ "all rows have license metadata"
267
+ if not missing_license_lines
268
+ else f"missing license on lines={missing_license_lines[:20]}",
269
+ "Keep source/license metadata for every row.",
270
+ )
271
+
272
+ if tokenizer is not None:
273
+ too_long_fraction = len(too_long_lines) / max(1, len(texts))
274
+ add_check(
275
+ checks,
276
+ f"{split}:max_length",
277
+ not too_long_lines,
278
+ "all rows fit max_length"
279
+ if not too_long_lines
280
+ else f"{len(too_long_lines)} rows exceed {max_length} tokens",
281
+ "Chunk, shorten, or raise max_length only if memory allows it.",
282
+ warn=too_long_fraction < 0.05,
283
+ )
284
+ add_issue(
285
+ issues,
286
+ too_long_fraction >= 0.05,
287
+ f"{split}:too-many-long-rows",
288
+ "high",
289
+ "data",
290
+ f"{split} has too many rows longer than max_length",
291
+ f"{len(too_long_lines)}/{max(1, len(texts))} rows exceed {max_length}",
292
+ "Chunk long examples or adjust the training sequence length deliberately.",
293
+ )
294
+
295
+ add_issue(
296
+ issues,
297
+ bool(missing_text_lines),
298
+ f"{split}:missing-text-field",
299
+ "high",
300
+ "data",
301
+ f"{split} rows are missing `{text_field}`",
302
+ f"lines={missing_text_lines[:50]}",
303
+ "Render structured messages to the training text field first.",
304
+ )
305
+ add_issue(
306
+ issues,
307
+ bool(empty_lines),
308
+ f"{split}:empty-text",
309
+ "high",
310
+ "data",
311
+ f"{split} rows have empty text",
312
+ f"lines={empty_lines[:50]}",
313
+ "Drop or repair empty rows.",
314
+ )
315
+ add_issue(
316
+ issues,
317
+ require_rendered and bool(missing_template_lines),
318
+ f"{split}:not-rendered",
319
+ "high",
320
+ "data",
321
+ f"{split} rows do not look rendered with the chat template",
322
+ f"lines={missing_template_lines[:50]}",
323
+ "Run scripts/render_sft_jsonl.py before training.",
324
+ )
325
+ add_issue(
326
+ issues,
327
+ bool(secret_hits),
328
+ f"{split}:secret-patterns",
329
+ "critical",
330
+ "data",
331
+ f"{split} appears to contain secrets",
332
+ "; ".join(secret_hits[:20]),
333
+ "Remove the rows and rotate any exposed credentials before training.",
334
+ )
335
+
336
+ return {
337
+ "rows": len(rows),
338
+ "usable_text_rows": len(texts),
339
+ "duplicate_groups": len(duplicate_groups),
340
+ "token_lengths": {
341
+ "min": min(token_lengths) if token_lengths else None,
342
+ "mean": round(statistics.mean(token_lengths), 2) if token_lengths else None,
343
+ "p50": percentile(token_lengths, 50),
344
+ "p95": percentile(token_lengths, 95),
345
+ "max": max(token_lengths) if token_lengths else None,
346
+ },
347
+ "hashes": hashes,
348
+ }
349
+
350
+
351
+ def validate_files(
352
+ train_jsonl: Path,
353
+ eval_jsonl: Path | None,
354
+ text_field: str = "text",
355
+ model: str | None = None,
356
+ max_length: int = 4096,
357
+ min_train_rows: int = 1,
358
+ require_rendered: bool = True,
359
+ require_license: bool = False,
360
+ ) -> dict[str, Any]:
361
+ checks: list[Check] = []
362
+ issues: list[Issue] = []
363
+ tokenizer = load_tokenizer(model)
364
+
365
+ train_rows = read_jsonl(train_jsonl, "train", checks, issues)
366
+ train_summary = validate_split(
367
+ train_rows,
368
+ "train",
369
+ text_field,
370
+ tokenizer,
371
+ max_length,
372
+ require_rendered,
373
+ require_license,
374
+ checks,
375
+ issues,
376
+ )
377
+ add_check(
378
+ checks,
379
+ "train:min_rows",
380
+ train_summary["usable_text_rows"] >= min_train_rows,
381
+ f"usable_text_rows={train_summary['usable_text_rows']}, required={min_train_rows}",
382
+ "Add enough high-quality training rows before launching SFT.",
383
+ )
384
+ add_issue(
385
+ issues,
386
+ train_summary["usable_text_rows"] < min_train_rows,
387
+ "train:too-few-rows",
388
+ "high",
389
+ "data",
390
+ "Training split is too small",
391
+ f"usable_text_rows={train_summary['usable_text_rows']}, required={min_train_rows}",
392
+ "Add more validated coding SFT rows.",
393
+ )
394
+
395
+ eval_summary = None
396
+ if eval_jsonl is not None:
397
+ eval_rows = read_jsonl(eval_jsonl, "eval", checks, issues)
398
+ eval_summary = validate_split(
399
+ eval_rows,
400
+ "eval",
401
+ text_field,
402
+ tokenizer,
403
+ max_length,
404
+ require_rendered,
405
+ require_license,
406
+ checks,
407
+ issues,
408
+ )
409
+ overlap = sorted(set(train_summary["hashes"]) & set(eval_summary["hashes"]))
410
+ add_check(
411
+ checks,
412
+ "train_eval:no_exact_overlap",
413
+ not overlap,
414
+ "no exact normalized overlap" if not overlap else f"overlap_count={len(overlap)}",
415
+ "Keep eval repository-family-disjoint from train; remove exact overlaps immediately.",
416
+ )
417
+ add_issue(
418
+ issues,
419
+ bool(overlap),
420
+ "train-eval:exact-overlap",
421
+ "high",
422
+ "data",
423
+ "Train and eval contain exact normalized overlaps",
424
+ f"overlap_count={len(overlap)}",
425
+ "Remove overlapping examples and rebuild the held-out split.",
426
+ )
427
+
428
+ return {
429
+ "inputs": {
430
+ "train_jsonl": str(train_jsonl),
431
+ "eval_jsonl": str(eval_jsonl) if eval_jsonl else None,
432
+ "text_field": text_field,
433
+ "model": model,
434
+ "max_length": max_length,
435
+ "require_rendered": require_rendered,
436
+ "require_license": require_license,
437
+ },
438
+ "summaries": {
439
+ "train": {k: v for k, v in train_summary.items() if k != "hashes"},
440
+ "eval": {k: v for k, v in (eval_summary or {}).items() if k != "hashes"}
441
+ if eval_summary
442
+ else None,
443
+ },
444
+ "checks": [asdict(check) for check in checks],
445
+ "issues": [asdict(issue) for issue in issues],
446
+ }
447
+
448
+
449
+ def main() -> int:
450
+ parser = argparse.ArgumentParser()
451
+ parser.add_argument("--train-jsonl", type=Path, required=True)
452
+ parser.add_argument("--eval-jsonl", type=Path)
453
+ parser.add_argument("--text-field", default="text")
454
+ parser.add_argument("--model", default="deepreinforce-ai/Ornith-1.0-35B")
455
+ parser.add_argument("--max-length", type=int, default=4096)
456
+ parser.add_argument("--min-train-rows", type=int, default=1)
457
+ parser.add_argument("--allow-unrendered", action="store_true")
458
+ parser.add_argument("--require-license", action="store_true")
459
+ parser.add_argument("--skip-tokenizer", action="store_true")
460
+ parser.add_argument("--output", type=Path)
461
+ parser.add_argument(
462
+ "--blocking-severity",
463
+ action="append",
464
+ choices=["low", "medium", "high", "critical"],
465
+ )
466
+ args = parser.parse_args()
467
+
468
+ report = validate_files(
469
+ train_jsonl=args.train_jsonl,
470
+ eval_jsonl=args.eval_jsonl,
471
+ text_field=args.text_field,
472
+ model=None if args.skip_tokenizer else args.model,
473
+ max_length=args.max_length,
474
+ min_train_rows=args.min_train_rows,
475
+ require_rendered=not args.allow_unrendered,
476
+ require_license=args.require_license,
477
+ )
478
+ text = json.dumps(report, indent=2, sort_keys=True)
479
+ print(text)
480
+ if args.output:
481
+ args.output.parent.mkdir(parents=True, exist_ok=True)
482
+ args.output.write_text(text + "\n")
483
+
484
+ blocking = set(args.blocking_severity or DEFAULT_BLOCKING_SEVERITIES)
485
+ return 1 if any(issue["severity"] in blocking for issue in report["issues"]) else 0
486
+
487
+
488
+ if __name__ == "__main__":
489
+ raise SystemExit(main())