| { |
| "schema": "tr-hash-agentic-tokenizer-balanced-2b-v1", |
| "target_tokens": 2000000000, |
| "retained_tokens": 1999999889, |
| "general_tokens": 1199999967, |
| "agentic_tokens": 799999922, |
| "reference_tokenizer_sha256": "86784d23456d5013b644b92d80ee161b1303f0570f45d52295580a2c96832bed", |
| "sources": [ |
| { |
| "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/00-dclm_general/00-dclm_general.jsonl", |
| "bucket": "general", |
| "target_tokens": 400000000, |
| "retained_tokens": 399999995, |
| "retained_records": 318300, |
| "scanned_records": 1094143, |
| "source_file_sha256": "2ec435d1e2ae3d08ab380f6c5e8ccc3fcf0110a3b30b9539c941c3e7fca25684" |
| }, |
| { |
| "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/01-fineweb_edu_general/00-fineweb_edu_general.jsonl", |
| "bucket": "general", |
| "target_tokens": 500000000, |
| "retained_tokens": 499999997, |
| "retained_records": 522326, |
| "scanned_records": 1281788, |
| "source_file_sha256": "ff68a72136d4e98dd72f56e839babe1432b33255060facd26d5e462d2f123a37" |
| }, |
| { |
| "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/02-cosmopedia_general/00-cosmopedia_general.jsonl", |
| "bucket": "general", |
| "target_tokens": 300000000, |
| "retained_tokens": 299999975, |
| "retained_records": 401460, |
| "scanned_records": 1810174, |
| "source_file_sha256": "51a0b2a12e52610799ab589cbd0c4650857b29e237b3e473cf25065085fe9460" |
| }, |
| { |
| "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/03-stack_edu_agentic/00-stack_edu_agentic.jsonl", |
| "bucket": "agentic", |
| "target_tokens": 400000000, |
| "retained_tokens": 399999999, |
| "retained_records": 196562, |
| "scanned_records": 310272, |
| "source_file_sha256": "619d123fa01cc59cb51a0675b1abaf6932bba630003dfd40574280d24f2bfd6c" |
| }, |
| { |
| "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/04-fineweb_edu_agentic/00-fineweb_edu_agentic.jsonl", |
| "bucket": "agentic", |
| "target_tokens": 200000000, |
| "retained_tokens": 199999991, |
| "retained_records": 84506, |
| "scanned_records": 229888, |
| "source_file_sha256": "ed67215e2788e2b9cf2e9601abd72c89848766b62fb98d76f7dda0acc7562b96" |
| }, |
| { |
| "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/05-finemath_agentic/00-finemath_agentic.jsonl", |
| "bucket": "agentic", |
| "target_tokens": 100000000, |
| "retained_tokens": 99999943, |
| "retained_records": 36882, |
| "scanned_records": 185852, |
| "source_file_sha256": "b2c74fe27f82dbffdfe4c36ee01ef219feb1aeb699eb9e22408791f29b1af6a4" |
| }, |
| { |
| "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/06-infiwebmath_agentic/00-infiwebmath_agentic.jsonl", |
| "bucket": "agentic", |
| "target_tokens": 100000000, |
| "retained_tokens": 99999989, |
| "retained_records": 34244, |
| "scanned_records": 170939, |
| "source_file_sha256": "131498929c1ea647bedd2de6509337c1fa2a28221814f67e9e46e87f9116ebba" |
| } |
| ] |
| } |
|
|
