{
  "schema": "tr-hash-agentic-tokenizer-benchmark-v1",
  "suite": "fixed multilingual-agentic suite",
  "categories": {
    "english": {
      "characters": 311,
      "utf8_bytes": 311,
      "tokens": {
        "tr_hash_agentic_32k": 59,
        "r50k_base": 58,
        "cl100k_base": 55,
        "o200k_base": 57
      }
    },
    "french": {
      "characters": 329,
      "utf8_bytes": 339,
      "tokens": {
        "tr_hash_agentic_32k": 100,
        "r50k_base": 96,
        "cl100k_base": 80,
        "o200k_base": 62
      }
    },
    "code": {
      "characters": 319,
      "utf8_bytes": 319,
      "tokens": {
        "tr_hash_agentic_32k": 90,
        "r50k_base": 115,
        "cl100k_base": 69,
        "o200k_base": 70
      }
    },
    "json_tool": {
      "characters": 195,
      "utf8_bytes": 195,
      "tokens": {
        "tr_hash_agentic_32k": 71,
        "r50k_base": 67,
        "cl100k_base": 62,
        "o200k_base": 62
      }
    },
    "math": {
      "characters": 205,
      "utf8_bytes": 215,
      "tokens": {
        "tr_hash_agentic_32k": 87,
        "r50k_base": 90,
        "cl100k_base": 93,
        "o200k_base": 92
      }
    },
    "emoji": {
      "characters": 105,
      "utf8_bytes": 208,
      "tokens": {
        "tr_hash_agentic_32k": 113,
        "r50k_base": 106,
        "cl100k_base": 100,
        "o200k_base": 75
      }
    },
    "agentic_markers": {
      "characters": 420,
      "utf8_bytes": 420,
      "tokens": {
        "tr_hash_agentic_32k": 66,
        "r50k_base": 149,
        "cl100k_base": 122,
        "o200k_base": 122
      }
    }
  },
  "natural_text_total": {
    "tr_hash_agentic_32k": {
      "characters": 1464,
      "utf8_bytes": 1587,
      "tokens": 520,
      "characters_per_token": 2.8153846153846156,
      "bytes_per_token": 3.0519230769230767,
      "vocab_size": 32000
    },
    "r50k_base": {
      "characters": 1464,
      "utf8_bytes": 1587,
      "tokens": 532,
      "characters_per_token": 2.7518796992481205,
      "bytes_per_token": 2.9830827067669174,
      "vocab_size": 50257
    },
    "cl100k_base": {
      "characters": 1464,
      "utf8_bytes": 1587,
      "tokens": 459,
      "characters_per_token": 3.189542483660131,
      "bytes_per_token": 3.457516339869281,
      "vocab_size": 100277
    },
    "o200k_base": {
      "characters": 1464,
      "utf8_bytes": 1587,
      "tokens": 418,
      "characters_per_token": 3.5023923444976077,
      "bytes_per_token": 3.7966507177033493,
      "vocab_size": 200019
    }
  },
  "including_native_markers_total": {
    "tr_hash_agentic_32k": {
      "characters": 1884,
      "utf8_bytes": 2007,
      "tokens": 586,
      "characters_per_token": 3.2150170648464163,
      "bytes_per_token": 3.4249146757679183,
      "vocab_size": 32000
    },
    "r50k_base": {
      "characters": 1884,
      "utf8_bytes": 2007,
      "tokens": 681,
      "characters_per_token": 2.7665198237885464,
      "bytes_per_token": 2.947136563876652,
      "vocab_size": 50257
    },
    "cl100k_base": {
      "characters": 1884,
      "utf8_bytes": 2007,
      "tokens": 581,
      "characters_per_token": 3.242685025817556,
      "bytes_per_token": 3.4543889845094666,
      "vocab_size": 100277
    },
    "o200k_base": {
      "characters": 1884,
      "utf8_bytes": 2007,
      "tokens": 540,
      "characters_per_token": 3.488888888888889,
      "bytes_per_token": 3.716666666666667,
      "vocab_size": 200019
    }
  },
  "tr_hash_roundtrip": {
    "english": true,
    "french": true,
    "code": true,
    "json_tool": true,
    "math": true,
    "emoji": true,
    "agentic_markers": true
  }
}