{
  "schema": "tr-hash-agentic-tokenizer-balanced-2b-v1",
  "target_tokens": 2000000000,
  "retained_tokens": 1999999889,
  "general_tokens": 1199999967,
  "agentic_tokens": 799999922,
  "reference_tokenizer_sha256": "86784d23456d5013b644b92d80ee161b1303f0570f45d52295580a2c96832bed",
  "sources": [
    {
      "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/00-dclm_general/00-dclm_general.jsonl",
      "bucket": "general",
      "target_tokens": 400000000,
      "retained_tokens": 399999995,
      "retained_records": 318300,
      "scanned_records": 1094143,
      "source_file_sha256": "2ec435d1e2ae3d08ab380f6c5e8ccc3fcf0110a3b30b9539c941c3e7fca25684"
    },
    {
      "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/01-fineweb_edu_general/00-fineweb_edu_general.jsonl",
      "bucket": "general",
      "target_tokens": 500000000,
      "retained_tokens": 499999997,
      "retained_records": 522326,
      "scanned_records": 1281788,
      "source_file_sha256": "ff68a72136d4e98dd72f56e839babe1432b33255060facd26d5e462d2f123a37"
    },
    {
      "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/02-cosmopedia_general/00-cosmopedia_general.jsonl",
      "bucket": "general",
      "target_tokens": 300000000,
      "retained_tokens": 299999975,
      "retained_records": 401460,
      "scanned_records": 1810174,
      "source_file_sha256": "51a0b2a12e52610799ab589cbd0c4650857b29e237b3e473cf25065085fe9460"
    },
    {
      "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/03-stack_edu_agentic/00-stack_edu_agentic.jsonl",
      "bucket": "agentic",
      "target_tokens": 400000000,
      "retained_tokens": 399999999,
      "retained_records": 196562,
      "scanned_records": 310272,
      "source_file_sha256": "619d123fa01cc59cb51a0675b1abaf6932bba630003dfd40574280d24f2bfd6c"
    },
    {
      "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/04-fineweb_edu_agentic/00-fineweb_edu_agentic.jsonl",
      "bucket": "agentic",
      "target_tokens": 200000000,
      "retained_tokens": 199999991,
      "retained_records": 84506,
      "scanned_records": 229888,
      "source_file_sha256": "ed67215e2788e2b9cf2e9601abd72c89848766b62fb98d76f7dda0acc7562b96"
    },
    {
      "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/05-finemath_agentic/00-finemath_agentic.jsonl",
      "bucket": "agentic",
      "target_tokens": 100000000,
      "retained_tokens": 99999943,
      "retained_records": 36882,
      "scanned_records": 185852,
      "source_file_sha256": "b2c74fe27f82dbffdfe4c36ee01ef219feb1aeb699eb9e22408791f29b1af6a4"
    },
    {
      "path": "/workspace/builds/tr-hash-agentic-tokenizer-10b/parts/06-infiwebmath_agentic/00-infiwebmath_agentic.jsonl",
      "bucket": "agentic",
      "target_tokens": 100000000,
      "retained_tokens": 99999989,
      "retained_records": 34244,
      "scanned_records": 170939,
      "source_file_sha256": "131498929c1ea647bedd2de6509337c1fa2a28221814f67e9e46e87f9116ebba"
    }
  ]
}