from __future__ import annotations from dataclasses import asdict, replace import hashlib import json import random import pytest from glyph_ascii_v1 import ( GLYPH_ASCII_V1_GRAMMAR_ID, GLYPH_ASCII_V1_MAX_IDENTIFIER_BYTES, GLYPH_ASCII_V1_SYMBOLS, GlyphAsciiV1AtomProof, GlyphAsciiV1Error, GlyphAsciiV1Proof, glyph_ascii_v1_asset_manifest_sha256, glyph_ascii_v1_identifier_kind, inverse_glyph_ascii_v1, render_glyph_ascii_v1, ) from production import normalize_spoken_forms, plan_generation_chunks def _asset_manifest() -> dict[str, str]: return { symbol.asset_id: hashlib.sha256( f"test-asset:{symbol.asset_id}".encode("ascii") ).hexdigest() for symbol in GLYPH_ASCII_V1_SYMBOLS } ASSET_MANIFEST = _asset_manifest() def _render( raw: str, *, absolute_raw_start: int = 0, canonical_start: int = 0, ) -> tuple[str, GlyphAsciiV1Proof]: return render_glyph_ascii_v1( raw, asset_entry_sha256_by_asset_id=ASSET_MANIFEST, absolute_raw_start=absolute_raw_start, canonical_start=canonical_start, ) def _inverse( proof: GlyphAsciiV1Proof, *, manifest: dict[str, str] | None = None, ) -> str: return inverse_glyph_ascii_v1( proof, asset_entry_sha256_by_asset_id=( ASSET_MANIFEST if manifest is None else manifest ), ) def test_exact_32_symbol_bijection_and_frozen_tokens(): expected_characters = "abcdefghijklmnopqrstuvwxyz@.-_:/" expected_tokens = ( "第一個英文字母", "第二個英文字母", "第三個英文字母", "第四個英文字母", "第五個英文字母", "第六個英文字母", "第七個英文字母", "第八個英文字母", "第九個英文字母", "第十個英文字母", "第十一個英文字母", "第十二個英文字母", "第十三個英文字母", "第十四個英文字母", "第十五個英文字母", "第十六個英文字母", "第十七個英文字母", "第十八個英文字母", "第十九個英文字母", "第二十個英文字母", "第二十一個英文字母", "第二十二個英文字母", "第二十三個英文字母", "第二十四個英文字母", "第二十五個英文字母", "第二十六個英文字母", "網路符號小老鼠", "網路符號點", "這個符號叫做橫線", "這個符號叫做底線", "這個符號叫做冒號", "網路符號斜線", ) expected_asset_ids = tuple( f"letter-{character}" for character in "abcdefghijklmnopqrstuvwxyz" ) + ( "symbol-at", "symbol-dot", "symbol-hyphen", "symbol-underscore", "symbol-colon", "symbol-slash", ) assert len(GLYPH_ASCII_V1_SYMBOLS) == 32 assert "".join(chr(symbol.raw_byte) for symbol in GLYPH_ASCII_V1_SYMBOLS) == ( expected_characters ) assert tuple(symbol.canonical_token for symbol in GLYPH_ASCII_V1_SYMBOLS) == ( expected_tokens ) assert tuple(symbol.asset_id for symbol in GLYPH_ASCII_V1_SYMBOLS) == ( expected_asset_ids ) assert len({symbol.raw_byte for symbol in GLYPH_ASCII_V1_SYMBOLS}) == 32 assert len({symbol.canonical_token for symbol in GLYPH_ASCII_V1_SYMBOLS}) == 32 assert len({symbol.asset_id for symbol in GLYPH_ASCII_V1_SYMBOLS}) == 32 @pytest.mark.parametrize( ("raw", "kind"), ( ("a@b.c", "email"), ("a.b_c-d@e-f.g-h", "email"), ("http://a.b", "url"), ("https://a-b.c-d/e_f.g-h/i", "url"), ), ) def test_strict_examples_roundtrip_with_range_and_manifest_binding(raw, kind): absolute_raw_start = 19 canonical_start = 31 canonical, proof = _render( raw, absolute_raw_start=absolute_raw_start, canonical_start=canonical_start, ) symbols = {symbol.raw_byte: symbol for symbol in GLYPH_ASCII_V1_SYMBOLS} assert glyph_ascii_v1_identifier_kind(raw) == kind assert proof.grammar_id == GLYPH_ASCII_V1_GRAMMAR_ID assert proof.identifier_kind == kind assert proof.absolute_raw_start == absolute_raw_start assert proof.absolute_raw_end == absolute_raw_start + len(raw.encode("ascii")) assert proof.canonical_start == canonical_start assert proof.canonical_end == canonical_start + len(canonical) assert proof.raw_sha256 == hashlib.sha256(raw.encode("ascii")).hexdigest() assert proof.canonical_sha256 == hashlib.sha256( canonical.encode("utf-8") ).hexdigest() assert proof.asset_manifest_sha256 == glyph_ascii_v1_asset_manifest_sha256( ASSET_MANIFEST ) assert canonical == " ".join( symbols[raw_byte].canonical_token for raw_byte in raw.encode("ascii") ) assert " " not in canonical assert _inverse(proof) == raw for ordinal, (raw_byte, atom) in enumerate(zip(raw.encode("ascii"), proof.atoms)): symbol = symbols[raw_byte] assert atom.ordinal == ordinal assert ( atom.absolute_raw_start, atom.absolute_raw_end, ) == ( absolute_raw_start + ordinal, absolute_raw_start + ordinal + 1, ) assert ( atom.identifier_raw_start, atom.identifier_raw_end, ) == (ordinal, ordinal + 1) assert canonical[ atom.canonical_start - canonical_start : atom.canonical_end - canonical_start ] == atom.canonical_token assert atom.raw_byte == raw_byte assert atom.canonical_token == symbol.canonical_token assert atom.asset_id == symbol.asset_id assert atom.manifest_entry_sha256 == ASSET_MANIFEST[symbol.asset_id] def _alpha(rng: random.Random, *, minimum: int = 1, maximum: int = 8) -> str: return "".join( rng.choice("abcdefghijklmnopqrstuvwxyz") for _ in range(rng.randint(minimum, maximum)) ) def _joined_alpha( rng: random.Random, separators: str, *, maximum_parts: int = 4, ) -> str: output = _alpha(rng) for _ in range(rng.randint(0, maximum_parts - 1)): output += rng.choice(separators) + _alpha(rng) return output def _random_email(rng: random.Random) -> str: local = _joined_alpha(rng, "._-") labels = [_joined_alpha(rng, "-", maximum_parts=3) for _ in range(rng.randint(2, 5))] return f"{local}@{'.'.join(labels)}" def _random_url(rng: random.Random) -> str: scheme = rng.choice(("http", "https")) labels = [_joined_alpha(rng, "-", maximum_parts=3) for _ in range(rng.randint(2, 5))] path = "" if rng.choice((False, True)): segments = [ _joined_alpha(rng, "-_.", maximum_parts=4) for _ in range(rng.randint(1, 4)) ] path = "/" + "/".join(segments) return f"{scheme}://{'.'.join(labels)}{path}" def test_seeded_property_roundtrip_for_random_valid_identifiers(): rng = random.Random(0xB10E_5A11) for _ in range(1_000): raw = _random_email(rng) if rng.randrange(2) == 0 else _random_url(rng) raw_offset = rng.randrange(0, 1_000) canonical_offset = rng.randrange(0, 1_000) canonical, proof = _render( raw, absolute_raw_start=raw_offset, canonical_start=canonical_offset, ) assert _inverse(proof) == raw assert len(proof.atoms) == len(raw.encode("ascii")) assert canonical.split(" ") == [ atom.canonical_token for atom in proof.atoms ] assert proof.absolute_raw_end - proof.absolute_raw_start == len(raw) @pytest.mark.parametrize( "raw", ( "", "Mead@forest.org", "mead@Forest.org", "MEAD@FOREST.ORG", "HTTP://forest.org", "https://forest.org/Path", "méad@forest.org", "梅@forest.org", "mead@forest.org", "xn--caf-dma@example.org", "mead@xn--caf-dma.org", "mead@\u202eforest.org", "mead@forest.org\u2066", "mead@forest.org\x00", "mead@forest.org\n", "mead@forest.org\t", "mead1@forest.org", "mead@forest2.org", "https://forest2.org", "https://forest.org/path2", "https://forest.org/path?query", "https://forest.org/path#fragment", "https://forest.org/pa%74h", "https://forest.org:443/path", "https://user@forest.org/path", "https://user:pass@forest.org/path", "ftp://forest.org/path", "forest.org/path", "mailto:mead@forest.org", ".mead@forest.org", "mead.@forest.org", "mead..voice@forest.org", "mead__voice@forest.org", "mead--voice@forest.org", "mead@.forest.org", "mead@forest.org.", "mead@forest..org", "mead@-forest.org", "mead@forest-.org", "https:///forest.org", "https://forest.org/", "https://forest.org//path", "https://forest.org/.", "https://forest.org/..", "https://forest.org/.path", "https://forest.org/path.", "https://forest.org/path..next", "https://forest.org/-path", "https://forest.org/path-", "https://forest.org/_path", "https://forest.org/path_", "a", "a.b", "@", "://", ), ) def test_unsafe_or_out_of_grammar_inputs_are_rejected(raw): with pytest.raises(GlyphAsciiV1Error): _render(raw) def test_cap_overflow_is_rejected_without_truncation(): # 509 letters + "@a.b" is one byte beyond the frozen 512-byte cap. raw = "a" * (GLYPH_ASCII_V1_MAX_IDENTIFIER_BYTES - 3) + "@a.b" assert len(raw.encode("ascii")) == GLYPH_ASCII_V1_MAX_IDENTIFIER_BYTES + 1 with pytest.raises(GlyphAsciiV1Error, match="cap"): _render(raw) @pytest.mark.parametrize("value", (b"a@b.c", None, 7, True)) def test_non_string_raw_input_is_rejected(value): with pytest.raises(GlyphAsciiV1Error): _render(value) # type: ignore[arg-type] @pytest.mark.parametrize( ("keyword", "value"), ( ("absolute_raw_start", -1), ("absolute_raw_start", True), ("absolute_raw_start", 1.5), ("canonical_start", -1), ("canonical_start", False), ("canonical_start", "1"), ), ) def test_invalid_external_offsets_are_rejected(keyword, value): kwargs = {keyword: value} with pytest.raises(GlyphAsciiV1Error): render_glyph_ascii_v1( "a@b.c", asset_entry_sha256_by_asset_id=ASSET_MANIFEST, **kwargs, ) def test_manifest_must_bind_exactly_32_valid_entries(): missing = dict(ASSET_MANIFEST) missing.pop(next(iter(missing))) extra = dict(ASSET_MANIFEST) extra["glyph_ascii_v1_extra"] = "0" * 64 invalid_digest = dict(ASSET_MANIFEST) invalid_digest[next(iter(invalid_digest))] = "A" * 64 for manifest in (missing, extra, invalid_digest): with pytest.raises(GlyphAsciiV1Error): render_glyph_ascii_v1( "a@b.c", asset_entry_sha256_by_asset_id=manifest, ) with pytest.raises(GlyphAsciiV1Error): render_glyph_ascii_v1( "a@b.c", asset_entry_sha256_by_asset_id=[], # type: ignore[arg-type] ) def _proof_with_atoms( proof: GlyphAsciiV1Proof, atoms: tuple[GlyphAsciiV1AtomProof, ...] | list[GlyphAsciiV1AtomProof], ) -> GlyphAsciiV1Proof: return replace(proof, atoms=atoms) # type: ignore[arg-type] def test_atom_reorder_omit_duplicate_substitute_and_borrow_are_rejected(): _, proof = _render("mead@forest.org", absolute_raw_start=7, canonical_start=11) _, donor = _render("owl@island.net", absolute_raw_start=7, canonical_start=11) atoms = proof.atoms tampered = ( _proof_with_atoms(proof, (atoms[1], atoms[0], *atoms[2:])), _proof_with_atoms(proof, (*atoms[:3], *atoms[4:])), _proof_with_atoms(proof, (*atoms[:3], atoms[3], *atoms[3:])), _proof_with_atoms( proof, ( replace( atoms[0], raw_byte=ord("z"), canonical_token="第二十六個英文字母", asset_id="letter-z", manifest_entry_sha256=ASSET_MANIFEST["letter-z"], ), *atoms[1:], ), ), _proof_with_atoms(proof, (donor.atoms[0], *atoms[1:])), ) for forged in tampered: with pytest.raises(GlyphAsciiV1Error): _inverse(forged) @pytest.mark.parametrize( "mutate", ( lambda atom: replace(atom, ordinal=atom.ordinal + 1), lambda atom: replace(atom, ordinal=True), lambda atom: replace(atom, absolute_raw_start=atom.absolute_raw_start + 1), lambda atom: replace(atom, absolute_raw_end=atom.absolute_raw_end + 1), lambda atom: replace( atom, identifier_raw_start=atom.identifier_raw_start + 1 ), lambda atom: replace(atom, identifier_raw_end=atom.identifier_raw_end + 1), lambda atom: replace(atom, canonical_start=atom.canonical_start + 1), lambda atom: replace(atom, canonical_end=atom.canonical_end + 1), lambda atom: replace(atom, raw_byte=True), lambda atom: replace(atom, raw_byte=ord("z")), lambda atom: replace(atom, canonical_token="第二十六個英文字母"), lambda atom: replace(atom, canonical_token=[]), lambda atom: replace(atom, asset_id="letter-z"), lambda atom: replace(atom, asset_id=[]), lambda atom: replace(atom, manifest_entry_sha256="0" * 64), lambda atom: replace(atom, manifest_entry_sha256="A" * 64), ), ) def test_atom_gap_overlap_type_hash_and_mapping_tampering_is_rejected(mutate): _, proof = _render("mead@forest.org", absolute_raw_start=7, canonical_start=11) index = 2 atoms = list(proof.atoms) atoms[index] = mutate(atoms[index]) with pytest.raises(GlyphAsciiV1Error): _inverse(_proof_with_atoms(proof, tuple(atoms))) @pytest.mark.parametrize( "mutate", ( lambda proof: replace(proof, grammar_id="glyph_ascii_v2"), lambda proof: replace(proof, identifier_kind="url"), lambda proof: replace(proof, absolute_raw_start=True), lambda proof: replace(proof, absolute_raw_start=proof.absolute_raw_start + 1), lambda proof: replace(proof, absolute_raw_end=proof.absolute_raw_end + 1), lambda proof: replace(proof, canonical_start=False), lambda proof: replace(proof, canonical_start=proof.canonical_start + 1), lambda proof: replace(proof, canonical_end=proof.canonical_end + 1), lambda proof: replace(proof, raw_sha256="0" * 64), lambda proof: replace(proof, canonical_sha256="0" * 64), lambda proof: replace(proof, asset_manifest_sha256="0" * 64), lambda proof: replace(proof, raw_sha256="A" * 64), lambda proof: replace(proof, atoms=()), lambda proof: replace(proof, atoms=list(proof.atoms)), ), ) def test_header_range_hash_type_and_expected_proof_tampering_is_rejected(mutate): _, proof = _render("mead@forest.org", absolute_raw_start=7, canonical_start=11) with pytest.raises(GlyphAsciiV1Error): _inverse(mutate(proof)) def test_manifest_substitution_is_rejected_by_entry_and_manifest_hashes(): _, proof = _render("mead@forest.org") changed = dict(ASSET_MANIFEST) changed[proof.atoms[0].asset_id] = hashlib.sha256(b"different").hexdigest() with pytest.raises(GlyphAsciiV1Error): _inverse(proof, manifest=changed) def test_inverse_rejects_non_proof_and_subclass_types(): class ForgedProof(GlyphAsciiV1Proof): pass _, proof = _render("mead@forest.org") forged = ForgedProof(**asdict(proof)) with pytest.raises(GlyphAsciiV1Error): _inverse(forged) with pytest.raises(GlyphAsciiV1Error): inverse_glyph_ascii_v1( # type: ignore[arg-type] object(), asset_entry_sha256_by_asset_id=ASSET_MANIFEST, ) def test_legacy_normalize_and_generation_plan_byte_for_byte_regression(): """The isolated core must not alter the frozen legacy frontend or planner.""" samples = ( "今天測試一般句子,保持原有規劃。", "聯絡 mead@forestmail.org。", "請開啟 https://audiocoast.org/tour。", ) output = [] for raw in samples: normalized = normalize_spoken_forms(raw, locale="zh-TW") output.append( { "raw": raw, "normalized": normalized, "plan": [ asdict(spec) for spec in plan_generation_chunks(raw, normalized) ], } ) serialized = json.dumps( output, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ).encode("utf-8") assert len(serialized) == 3_941 assert ( hashlib.sha256(serialized).hexdigest() == "cea66a03b4ac1381f1acbc240e73d08f0b992f2a5efd4af1bc855b55d7bb50ee" )