import math from dataclasses import replace import numpy as np import pytest import torch from torch import nn from production import ( NetworkFragmentProof, StopHysteresisController, active_pace_correction_speed, candidate_local_score, candidate_transition_score, canonicalize_asr_network_fragments, coalesce_text_chunks, compare_asr_text, contains_naturalized_url_spoken_form, contains_network_identifier, count_network_endpoint_duration_units, count_speech_units, duration_hard_stop_steps, decode_pcm16, email_domain_mail_generation_variant, encode_pcm16, effective_generation_cfg, endpoint_generation_plan, ensure_terminal_punctuation, finish_audio, fade_internal_edges, fade_variable_internal_edges, join_audio_chunks, join_audio_chunks_variable, inverse_network_url_rendering, normalize_asr_spoken_forms, mandarin_acoustic_units, network_protected_spoken_spans, naturalized_url_rendering_proof, normalize_spoken_forms, normalize_tts_eval_text, normalize_tts_text, plan_generation_chunks, pcm16_audio_output, pcm16_verification_waveform, select_candidate_sequence, select_generation_cps, split_leading_clause, split_quality_text_for_tts, strip_unspoken_cjk_quotes_for_generation, split_text_for_tts, target_cps_min_len, target_cps_steps, target_pace_speed, ) _SPOKEN_HTTPS = "H T T P S" _SPOKEN_PATH = "path" def test_eval_only_pronoun_homophones_do_not_change_model_input_text(): text = "她提醒妳,它、牠和祂都在這裡,再把她的東西放得穩穩地。" assert normalize_tts_text(text) == text assert normalize_tts_eval_text("她提醒妳,它、牠和祂都在這裡。") == ( "他提醒你,他、他和他都在這裡。" ) assert normalize_tts_eval_text("的地得在再") == "的地得在再" def test_asr_comparison_accepts_only_verified_pronoun_homophones(): comparison = compare_asr_text("她提醒我", "他提醒我") assert comparison.passed assert comparison.cer == 0.0 assert comparison.prefix_cer == 0.0 assert comparison.suffix_cer == 0.0 assert comparison.extra_tail_units == 0 assert not compare_asr_text("他在這裡", "他再這裡").passed def test_asr_endpoint_gate_uses_tone_preserving_mandarin_acoustics(): assert mandarin_acoustic_units("陶藝") == mandarin_acoustic_units("陶逸") assert mandarin_acoustic_units("陶藝") != mandarin_acoustic_units("陶一") assert mandarin_acoustic_units("改到") != mandarin_acoustic_units("改造") assert mandarin_acoustic_units("里") == mandarin_acoustic_units("裡") assert mandarin_acoustic_units("tour") == ("t", "o", "u", "r") target = "陶藝課今天改到二樓,學員先等候老師。" homophone = compare_asr_text( target, "陶逸課今天改到二樓,學員先等候老師。", max_cer=0.20, ) assert homophone.passed assert homophone.cer > 0.0 assert homophone.prefix_cer == 0.0 wrong_tone = compare_asr_text( target, "陶一課今天改到二樓,學員先等候老師。", max_cer=0.20, ) assert not wrong_tone.passed assert wrong_tone.prefix_cer > 0.0 different_syllable = compare_asr_text( target, "陶藝課今天買到二樓,學員先等候老師。", max_cer=0.20, ) assert not different_syllable.passed assert different_syllable.prefix_cer > 0.0 whole_cer_still_hard = compare_asr_text("里訂", "禮定", max_cer=0.20) assert whole_cer_still_hard.prefix_cer == 0.0 assert whole_cer_still_hard.suffix_cer == 0.0 assert whole_cer_still_hard.cer == 1.0 assert not whole_cer_still_hard.passed def test_asr_acoustic_endpoint_gate_keeps_ascii_identifiers_literal(): target = "請寄信到 T O U R 點 H E L P 小老鼠 E X A M P L E 點 T W。" assert compare_asr_text(target, target, max_cer=0.20).passed replaced_tld = target.replace("T W", "C O M") comparison = compare_asr_text(target, replaced_tld, max_cer=0.20) assert not comparison.passed assert comparison.suffix_cer > 0.0 @pytest.mark.parametrize( "mutated_url", [ "htps://museum.example.tw/path", "https://museum.exampl1.tw/path", "https://museum.example.t", "https://museun.example.tw/path", "https://museuman.example.tw/path", ], ids=["htps", "exampl1", "tld_path_omission", "museun", "museuman"], ) def test_network_protected_span_cannot_be_hidden_by_whole_cer(mutated_url): target_url = "https://museum.example.tw/path" raw_target = ( f"請先閱讀 {target_url},確認展覽時間與集合位置後再回覆。" ) target = normalize_spoken_forms(raw_target) exact = compare_asr_text(target, target, max_cer=0.20) assert exact.passed assert exact.network_protected_spans == 1 assert exact.network_protected_spans_passed is True transcript = normalize_spoken_forms( raw_target.replace(target_url, mutated_url) ) comparison = compare_asr_text( target, transcript, max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.cer <= 0.20 assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed def test_network_protected_span_accepts_exact_raw_url_and_email(): for target in ( "請看 https://museum.example.tw/path,完成後回覆。", "請寄到 museum@example.tw,完成後回覆。", ): comparison = compare_asr_text(target, normalize_spoken_forms(target), max_cer=0.20) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed @pytest.mark.parametrize("insertion", ("加", "X", "§")) @pytest.mark.parametrize("side", ("before", "after")) def test_network_boundary_provenance_keeps_adjacent_outside_insertions_outside( insertion, side, ): target = normalize_spoken_forms( "前方甲乙,https://example.tw/path,後方丙丁。" ) transcript = ( target.replace("甲乙,", f"甲乙{insertion},") if side == "before" else target.replace(",後方", f",{insertion}後方") ) comparison = compare_asr_text( target, transcript, max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed @pytest.mark.parametrize( "mutate", ( lambda text: text.replace( f",{_SPOKEN_HTTPS}", f",X {_SPOKEN_HTTPS}" ), lambda text: text.replace( f",{_SPOKEN_HTTPS}", f",§ {_SPOKEN_HTTPS}" ), lambda text: text.replace(f"{_SPOKEN_PATH},", f"{_SPOKEN_PATH} X,"), lambda text: text.replace(f"{_SPOKEN_PATH},", f"{_SPOKEN_PATH}§,"), ), ) def test_network_boundary_provenance_rejects_insertions_inside_span_edges(mutate): target = normalize_spoken_forms( "前方甲乙,https://example.tw/path,後方丙丁。" ) comparison = compare_asr_text( target, mutate(target), max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed @pytest.mark.parametrize("boundary", (";", "…", "—", "。", "︰")) @pytest.mark.parametrize("side", ("before", "after")) def test_network_boundary_sentinel_accepts_outside_insertions_after_delimiter_substitution( boundary, side, ): target = normalize_spoken_forms( "前方甲乙,https://example.tw/path,後方丙丁。" ) transcript = ( target.replace("甲乙,", f"甲乙X{boundary}") if side == "before" else target.replace(",後方", f"{boundary}X後方") ) comparison = compare_asr_text( target, transcript, max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed @pytest.mark.parametrize("boundary", (";", "…", "—", "。", "︰")) @pytest.mark.parametrize("side", ("before", "after")) def test_network_boundary_sentinel_rejects_inside_insertions_after_delimiter_substitution( boundary, side, ): target = normalize_spoken_forms( "前方甲乙,https://example.tw/path,後方丙丁。" ) transcript = ( target.replace( f",{_SPOKEN_HTTPS}", f"{boundary}X {_SPOKEN_HTTPS}" ) if side == "before" else target.replace(f"{_SPOKEN_PATH},", f"{_SPOKEN_PATH}X{boundary}") ) comparison = compare_asr_text( target, transcript, max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed @pytest.mark.parametrize("side", ("before", "after")) def test_network_boundary_sentinel_allows_pure_delimiter_omission(side): target = normalize_spoken_forms( "前方甲乙,https://example.tw/path,後方丙丁。" ) transcript = ( target.replace(f"甲乙,{_SPOKEN_HTTPS}", f"甲乙{_SPOKEN_HTTPS}") if side == "before" else target.replace(f"{_SPOKEN_PATH},後方", f"{_SPOKEN_PATH}後方") ) comparison = compare_asr_text(target, transcript, max_cer=0.20) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed @pytest.mark.parametrize("insertion", ("X", "XY", "加", "額外", "§")) @pytest.mark.parametrize("side", ("before", "after")) def test_network_boundary_sentinel_fails_closed_when_omission_is_lexically_ambiguous( insertion, side, ): target = normalize_spoken_forms( "前方甲乙,https://example.tw/path,後方丙丁。" ) transcript = ( target.replace( f",{_SPOKEN_HTTPS}", f"{insertion} {_SPOKEN_HTTPS}" ) if side == "before" else target.replace(f"{_SPOKEN_PATH},", f"{_SPOKEN_PATH}{insertion}") ) comparison = compare_asr_text( target, transcript, max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed def test_network_boundary_sentinel_attributes_raw_asr_trailing_edits(): target = normalize_spoken_forms( "請看,https://example.tw/path,回覆。" ) outside = compare_asr_text( target, "請看,https://example.tw/path. X回覆。", max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) inside = compare_asr_text( target, "請看,https://example.tw/pathX. 回覆。", max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert outside.network_protected_spans_passed is True assert outside.passed assert inside.network_protected_spans_passed is False assert not inside.passed def test_network_boundary_sentinel_attributes_raw_asr_leading_edits(): target = normalize_spoken_forms( "X,https://example.tw/path,回覆。" ) outside = compare_asr_text( target, "X. https://example.tw/path,回覆。", max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) inside = compare_asr_text( target, ". Xhttps://example.tw/path,回覆。", max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert outside.network_protected_spans_passed is True assert outside.passed assert inside.network_protected_spans_passed is False assert not inside.passed @pytest.mark.parametrize( "target", ( "https://example.tw/v1", "https://example.tw/a?x=2&n=3", "https://api.example.com/v1/items?q=RTX-5090&n=2", ), ) def test_network_protected_span_preserves_complete_literal_url_before_fragments(target): comparison = compare_asr_text(target, target, max_cer=0.20) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed def test_asr_network_fragment_punctuation_is_target_proven_only(): assert normalize_asr_spoken_forms( "Tour.help", "tour 點 help", ) == "Tour 點 help" assert normalize_asr_spoken_forms( "小老鼠islandmuseum.tw", "小老鼠 islandmuseum 點 T W。", ) == "小老鼠islandmuseum 點 tw" # A target without a spoken network separator cannot authorize the rewrite. assert normalize_asr_spoken_forms("版本v1.2", "版本v一點二") != "版本v1 點 2" def test_asr_xiaolaoshu_is_authorized_by_complete_target_email_coverage(): target = "請寄信到 museum@example.tw,完成後回覆。" transcript = "請寄信到 museum Xiaolaoshu example.tw,完成後回覆。" normalized = normalize_asr_spoken_forms(transcript, target) comparison = compare_asr_text(target, transcript, max_cer=0.20) assert "Xiaolaoshu" not in normalized assert normalized.count("小老鼠") == 1 assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed def test_asr_xiaolaoshu_wrong_domain_cannot_borrow_email_proof(): target = "請寄信到 museum@example.tw,完成後回覆。" transcript = "請寄信到 museum Xiaolaoshu example.com,完成後回覆。" normalized = normalize_asr_spoken_forms(transcript, target) comparison = compare_asr_text( target, transcript, max_cer=1.0, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert "Xiaolaoshu" in normalized assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed def test_asr_xiaolaoshu_network_insertion_is_not_exact_coverage(): target = "請寄信到 museum@example.tw,完成後回覆。" transcript = "請寄信到 museum Xiaolaoshu exam§ple.tw,完成後回覆。" normalized = normalize_asr_spoken_forms(transcript, target) comparison = compare_asr_text( target, transcript, max_cer=1.0, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert "Xiaolaoshu" in normalized assert comparison.network_protected_spans_passed is False assert not comparison.passed def test_asr_xiaolaoshu_literal_network_label_is_not_rewritten(): target = "請看 https://example.tw/xiaolaoshu,完成後回覆。" transcript = ( "請看 H T T P S 冒號 斜線 斜線 example 點 T W " "斜線 Xiaolaoshu,完成後回覆。" ) normalized = normalize_asr_spoken_forms(transcript, target) assert "小老鼠" not in normalized assert compare_asr_text(target, transcript, max_cer=0.20).passed @pytest.mark.parametrize( ("target", "transcript"), ( ( "https://example.tw/a@b", "H T T P S 冒號 斜線 斜線 example 點 T W 斜線 aXiaolaoshub", ), ( "www.example.tw/a@b", "W W W 點 example 點 T W 斜線 aXiaolaoshub", ), ), ) def test_asr_xiaolaoshu_alias_cannot_borrow_url_path_at_sign(target, transcript): normalized = normalize_asr_spoken_forms(transcript, target) comparison = compare_asr_text( target, transcript, max_cer=1.0, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert "小老鼠" not in normalized assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed def test_asr_xiaolaoshu_no_space_email_alias_gets_token_boundaries(): target = "請寄到 tour.help@IslandMuseum.tw。" transcript = "請寄到 tour.helpXiaolaoshuIslandMuseum.tw。" normalized = normalize_asr_spoken_forms(transcript, target) assert "xiaolaoshu" not in normalized.casefold() assert " 小老鼠 " in normalized assert compare_asr_text(target, transcript, max_cer=0.20).passed @pytest.mark.parametrize( "transcript", ( # The ASR removes the prose/local boundary. Because the target domain # begins with ``i``, this also exercises the ambiguous no-space split. "若要更換導覽場次 請寄信到tour.helpXiaolaoshuislandmuseum.tw", # Some pinned decodes add a trailing ``i`` to the pinyin alias. The # doubled ``ii`` is alias-final ``i`` plus domain-initial ``i``. "若要更換導覽場次 請寄信到tour.helpXiaolaoshuiislandmuseum.tw", ), ) def test_asr_email_alias_is_exact_target_bound_without_prose_boundary(transcript): target = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" normalized = normalize_asr_spoken_forms(transcript, target) comparison = compare_asr_text(target, transcript, max_cer=0.20) assert "xiaolaoshu" not in normalized.casefold() assert normalized.count("小老鼠") == 1 assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed @pytest.mark.parametrize( "mutated_email", ( "tour.helpXiaolaoshuiislandmuseum.com", "tour.helpXiaolaoshuiislandmuseurn.tw", "tour.helpXiaolaoshuiislandmuseum.t", "tour.helpXiaolaoshuiislandmuseum.twz", "tour.helpXiaolaoshuiislandmuseum.tw/x", "tour.helpXiaolaoshuiislandmuseum.tw?x", "tour.helpXiaolaoshuiislandmuseum.tw#x", "xtour.helpXiaolaoshuiislandmuseum.tw", "tour.helpxXiaolaoshuiislandmuseum.tw", ), ) def test_asr_email_alias_rejects_identifier_and_suffix_mutations(mutated_email): target = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" transcript = f"若要更換導覽場次 請寄信到{mutated_email}" normalized = normalize_asr_spoken_forms(transcript, target) comparison = compare_asr_text( target, transcript, max_cer=1.0, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert "xiaolaoshu" in normalized.casefold() assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed def test_asr_email_alias_cannot_borrow_exact_target_from_url_path(): target = ( "請寄到 tour.help@islandmuseum.tw," "再看 https://example.tw/tour.help@islandmuseum.tw。" ) transcript = ( "請寄到," "再看https://example.tw/tour.helpXiaolaoshuiislandmuseum.tw" ) normalized = normalize_asr_spoken_forms(transcript, target) comparison = compare_asr_text( target, transcript, max_cer=1.0, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert "xiaolaoshu" in normalized.casefold() assert comparison.network_protected_spans == 2 assert comparison.network_protected_spans_passed is False assert not comparison.passed def test_asr_email_alias_occurrence_cannot_cover_duplicate_target_emails(): target = ( "請寄到 tour.help@islandmuseum.tw," "並副本到 tour.help@islandmuseum.tw。" ) transcript = "請寄到tour.helpXiaolaoshuiislandmuseum.tw,並副本到。" normalized = normalize_asr_spoken_forms(transcript, target) comparison = compare_asr_text( target, transcript, max_cer=1.0, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert normalized.count("小老鼠") == 1 assert comparison.network_protected_spans == 2 assert comparison.network_protected_spans_passed is False assert not comparison.passed def test_asr_xiaolaoshu_keeps_overlapping_email_proofs_independent(): target = "請寄到 a@example.tw 與 xa@example.tw。" transcript = "請寄到 aXiaolaoshuexample.tw 與 xa@example.tw。" normalized = normalize_asr_spoken_forms(transcript, target) assert "xiaolaoshu" not in normalized.casefold() assert compare_asr_text(target, transcript, max_cer=0.20).passed def test_asr_xiaolaoshu_rejects_suffix_of_longer_email_local(): target = "請寄到 a@example.tw。" transcript = "請寄到 xaXiaolaoshuexample.tw。" normalized = normalize_asr_spoken_forms(transcript, target) comparison = compare_asr_text( target, transcript, max_cer=1.0, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert "xiaolaoshu" in normalized.casefold() assert comparison.network_protected_spans_passed is False assert not comparison.passed def test_asr_xiaolaoshu_accepts_www_email_local_part(): target = "www.foo@example.tw" transcript = "W W W 點 fooXiaolaoshuexample 點 T W" normalized = normalize_asr_spoken_forms(transcript, target) assert "xiaolaoshu" not in normalized.casefold() assert compare_asr_text(target, transcript, max_cer=0.20).passed def test_literal_mouse_and_snack_prose_is_not_a_network_identifier(): target = ( "孩子今天看到 小老鼠 吃了 點 心,覺得很可愛。" "這是一般文字而不是信箱。" ) transcript = target.replace("小老鼠", "Xiaolaoshu") assert network_protected_spoken_spans(target) == () assert split_text_for_tts(target, max_chars=80, min_chunk_chars=12) == [target] assert select_generation_cps(target, cjk_cps=5.2, ascii_cps=4.6) == 5.2 assert "xiaolaoshu" in normalize_asr_spoken_forms(transcript, target).casefold() assert not compare_asr_text(target, transcript, max_cer=0.20).passed def test_email_ampersand_cannot_be_dropped_below_whole_cer_limit(): target = "一般文字中放入 a&b@example.tw 之後繼續說明完整內容。" transcript = target.replace("a&b@example.tw", "b@example.tw") comparison = compare_asr_text(target, transcript, max_cer=0.20) assert comparison.cer < 0.20 assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed def test_two_email_aliases_joined_by_he_are_proven_independently(): target = "a@example.tw 和 b@example.tw" transcript = "aXiaolaoshuexample.tw 和 bXiaolaoshuexample.tw" normalized = normalize_asr_spoken_forms(transcript, target) assert normalized.count("小老鼠") == 2 assert "xiaolaoshu" not in normalized.casefold() assert compare_asr_text(target, transcript, max_cer=0.20).passed @pytest.mark.parametrize( ("target", "transcript"), ( ("6g@example.tw", "6g Xiaolaoshu example 点 T W"), ( "https://x.tw/8l", "H T T P S 冒号 斜线 斜线 X 点 T W 斜线 8l", ), ( "https://x.tw/12.5%", "H T T P S 冒号 斜线 斜线 X 点 T W 斜线 12.5%", ), ), ) def test_network_tokens_bypass_prose_unit_normalization(target, transcript): comparison = compare_asr_text(target, transcript, max_cer=0.20) assert comparison.cer == 0.0 assert comparison.network_protected_spans_passed is True assert comparison.passed def test_simplified_network_letter_reading_matches_ascii_i(): target = "https://i.tw" transcript = "H T T P S 冒号 斜线 斜线 爱 点 T W" assert compare_asr_text(target, transcript, max_cer=0.20).passed @pytest.mark.parametrize("transcript", ("https://x.tw/愛", "https://x.tw/i")) def test_non_ascii_iri_is_explicitly_unsupported_and_fails_closed(transcript): comparison = compare_asr_text( "https://x.tw/愛", transcript, max_cer=1.0, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed @pytest.mark.parametrize( ("transcript", "expected_prefix_cer", "expected_suffix_cer"), ( ( "前方甲乙X,H T T P S 冒號 斜線 斜線 orange 點 " "example 點 T W 斜線 road," "後方丙丁。", 1.0 / 6.0, 0.0, ), ( "前方甲乙,H T T P S 冒號 斜線 斜線 orange 點 " "example 點 T W 斜線 road," "X後方丙丁。", 0.0, 1.0 / 6.0, ), ), ) def test_mixed_network_boundary_insertion_does_not_lexicalize_frontend_comma( transcript, expected_prefix_cer, expected_suffix_cer, ): target = "前方甲乙,https://orange.example.tw/road,後方丙丁。" comparison = compare_asr_text( target, transcript, max_cer=0.10, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.cer == pytest.approx(1.0 / 42.0) assert comparison.prefix_cer == pytest.approx(expected_prefix_cer) assert comparison.suffix_cer == pytest.approx(expected_suffix_cer) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed def test_asr_xiaolaoshu_prose_insertion_stays_literal_while_email_is_exact(): target = "請寄信到 museum@example.tw,完成後回覆。" transcript = "Xiaolaoshu,請寄信到 museum@example.tw,完成後回覆。" normalized = normalize_asr_spoken_forms(transcript, target) assert normalized.startswith("Xiaolaoshu,") assert normalized.count("小老鼠") == 1 assert not compare_asr_text(target, transcript, max_cer=0.20).passed def test_asr_xiaolaoshu_without_email_target_is_not_rewritten(): target = "今天說明網路用語。" transcript = "今天說明 Xiaolaoshu 網路用語。" normalized = normalize_asr_spoken_forms(transcript, target) assert "Xiaolaoshu" in normalized assert "小老鼠" not in normalized class _SequenceStopHead(nn.Module): def __init__(self, probabilities): super().__init__() self.probabilities = iter(probabilities) def forward(self, hidden): probability = next(self.probabilities) logit = math.log(probability / (1.0 - probability)) return torch.tensor([[0.0, logit]], dtype=hidden.dtype) def test_stop_controller_requires_two_eligible_hits(): controller = StopHysteresisController( _SequenceStopHead([0.9, 0.9, 0.9, 0.9]), threshold=0.65, consecutive=2, ) controller.begin(min_len=1) decisions = [controller(torch.zeros(1, 2)).argmax(dim=-1).item() for _ in range(4)] assert decisions == [0, 0, 0, 1] def test_stop_controller_can_stop_on_first_confident_hit(): controller = StopHysteresisController( _SequenceStopHead([0.01, 0.01, 0.9]), threshold=0.50, consecutive=1, ) controller.begin(min_len=1) decisions = [controller(torch.zeros(1, 2)).argmax(dim=-1).item() for _ in range(3)] assert decisions == [0, 0, 1] def test_stop_controller_preserves_threshold_reason_at_the_hard_cap(): controller = StopHysteresisController( _SequenceStopHead([0.01, 0.01, 0.01, 0.9]), threshold=0.50, late_threshold=0.05, consecutive=1, ) controller.begin(min_len=2, expected_steps=3, hard_stop_steps=4) decisions = [ controller(torch.zeros(1, 2)).argmax(dim=-1).item() for _ in range(4) ] assert decisions == [0, 0, 0, 1] assert controller.last_stop_reason == "stop_threshold" def test_stop_controller_relaxes_near_endpoint_and_enforces_hard_stop(): relaxed = StopHysteresisController( _SequenceStopHead([0.6] * 10), threshold=0.65, late_threshold=0.50, consecutive=2, ) relaxed.begin(min_len=6, expected_steps=10, hard_stop_steps=12) decisions = [relaxed(torch.zeros(1, 2)).argmax(dim=-1).item() for _ in range(10)] assert decisions[-2:] == [0, 1] forced = StopHysteresisController( _SequenceStopHead([0.01] * 4), threshold=0.65, late_threshold=0.50, consecutive=2, ) forced.begin(min_len=1, expected_steps=3, hard_stop_steps=4) decisions = [forced(torch.zeros(1, 2)).argmax(dim=-1).item() for _ in range(4)] assert decisions == [0, 0, 0, 1] assert forced.last_generated_steps == 4 assert forced.last_stop_reason == "hard_stop" assert len(forced.last_probabilities) == 4 def test_stop_controller_can_relax_before_the_text_pace_cap(): controller = StopHysteresisController( _SequenceStopHead([0.04] * 9 + [0.06]), threshold=0.50, late_threshold=0.05, consecutive=1, late_start_ratio=0.75, late_full_ratio=0.95, ) controller.begin(min_len=2, expected_steps=10, hard_stop_steps=11) decisions = [controller(torch.zeros(1, 2)).argmax(dim=-1).item() for _ in range(10)] assert decisions == [0, 0, 0, 0, 0, 0, 0, 0, 0, 1] def test_stop_controller_does_not_accept_weak_signal_before_late_window(): controller = StopHysteresisController( _SequenceStopHead([0.10] * 8), threshold=0.50, late_threshold=0.05, consecutive=1, late_start_ratio=0.75, late_full_ratio=0.95, ) controller.begin(min_len=2, expected_steps=10, hard_stop_steps=11) decisions = [controller(torch.zeros(1, 2)).argmax(dim=-1).item() for _ in range(8)] assert decisions == [0, 0, 0, 0, 0, 0, 0, 0] def test_split_preserves_punctuation_and_minimum_chunk_size(): text = ( "第一句很短。第二句也不算長。第三句需要再多一些文字,才能測試切段是否正確。" "第四句也要完整保留標點符號。" ) chunks = split_text_for_tts(text, max_chars=30, min_chunk_chars=12) assert "".join(chunks) == text assert all(count_speech_units(chunk) <= 30 for chunk in chunks) assert all(count_speech_units(chunk) >= 12 for chunk in chunks) assert chunks[0].endswith("。") def test_quality_split_uses_smaller_chunks_for_punctuated_medium_text(): text = "兩張票合計新台幣二千六百八十,租借望遠鏡另收日圓四千五百。" chunks = split_quality_text_for_tts(text) assert chunks == [ "兩張票合計新台幣二千六百八十,", "租借望遠鏡另收日圓四千五百。", ] assert "".join(chunks) == text assert all(12 <= count_speech_units(chunk) <= 24 for chunk in chunks) def test_quality_split_latency_profile_keeps_ordinary_27_unit_request_whole(): text = "兩張票合計新台幣二千六百八十,租借望遠鏡另收日圓四千五百。" chunks = split_quality_text_for_tts( text, medium_max_units=64, medium_chunk_units=32, ) assert chunks == [text] def test_quality_split_keeps_target_proven_currency_amounts_atomic(): raw = "兩張船票合計 NT$2,680,租借望遠鏡另收 JPY 4,500。" text = normalize_spoken_forms(raw) chunks = split_quality_text_for_tts(text) assert chunks == [ "兩張船票合計 新台幣二千六百八十,", "租借望遠鏡另收 日圓四千五百。", ] assert "".join(chunks) == text assert all(12 <= count_speech_units(chunk) <= 24 for chunk in chunks) def test_quality_split_keeps_unpunctuated_and_short_clause_text_unchanged(): unpunctuated = "晨間船班將於二零二九年六月八日上午六點二十五分離港" short_clause = "木工師傅先量好桌腳高度,接著才挑選合適的榫接方式。" assert split_quality_text_for_tts(unpunctuated) == [unpunctuated] assert split_quality_text_for_tts(short_clause) == [short_clause] def test_quality_split_does_not_bisect_decimal_when_trailing_clause_is_short(): text = "今年濕地的候鳥數量增加百分之十七點二五,幼鳥比例也同步上升。" chunks = split_quality_text_for_tts(text) assert chunks == [text] assert all("十七點二五" in chunk for chunk in chunks) def test_quality_split_keeps_real_sentence_boundary_after_quote_normalization(): text = "園丁提醒:「剛換盆的蘭花不要立刻施肥。」新進志工把這句話寫進值班簿。" chunks = split_quality_text_for_tts( strip_unspoken_cjk_quotes_for_generation(text) ) assert chunks == [ "園丁提醒:剛換盆的蘭花不要立刻施肥。", "新進志工把這句話寫進值班簿。", ] def test_generation_frontend_removes_unspoken_cjk_quotes_only(): assert strip_unspoken_cjk_quotes_for_generation( "園丁提醒:「剛換盆的蘭花不要立刻施肥。」" ) == "園丁提醒:剛換盆的蘭花不要立刻施肥。" assert strip_unspoken_cjk_quotes_for_generation('The "quoted" word stays.') == ( 'The "quoted" word stays.' ) def test_split_uses_speech_units_instead_of_expanded_codepoints(): text = ( "若系統顯示 H T T P S 冒號 斜線 斜線 weatherstation 點 sample 點 tw " "訊號超過 百分之六十,工作人員就改走安全替代路線,並且重新確認集合位置。" ) assert len(text) > 80 assert count_speech_units(text) <= 80 chunks = split_text_for_tts(text, max_chars=80, min_chunk_chars=12) assert chunks == [ "若系統顯示 H T T P S 冒號 斜線 斜線 weatherstation 點 sample 點 tw " "訊號超過 百分之六十,", "工作人員就改走安全替代路線,並且重新確認集合位置。", ] assert [count_speech_units(chunk) for chunk in chunks] == [34, 23] @pytest.mark.parametrize( ("text_id", "raw_text", "expected_chunks"), ( ( "H01", "燈亮之後才開門,離開前記得關好後窗。", ("燈亮之後才開門,離開前記得關好後窗。",), ), ( "H02", "陶藝課今天改到二樓,學員可以先在走廊等候老師。", ("陶藝課今天改到二樓,學員可以先在走廊等候老師。",), ), ( "H08", "控制器版本 v5.4.2 將搭配模組 NX-730 進行相容性測試。", ("控制器版本五點四點二 將搭配模組 N X 七三零 進行相容性測試。",), ), ( "H09", "阿嬤笑著說:「蘿蔔糕要趁熱吃。」大家聽完都把筷子準備好了。", ("阿嬤笑著說:「蘿蔔糕要趁熱吃。」大家聽完都把筷子準備好了。",), ), ( "H10", ( "清晨開館以前,水族館人員會先量測各池的水溫與鹽度," "再觀察魚群是否正常進食。確認照明、循環馬達和緊急電源" "都沒有異常後,才會打開入口讓第一批遊客進場。" ), ( "清晨開館以前,水族館人員會先量測各池的水溫與鹽度," "再觀察魚群是否正常進食。", "確認照明、循環馬達和緊急電源都沒有異常後," "才會打開入口讓第一批遊客進場。", ), ), ), ) def test_frontend_clause_split_preserves_nonstructured_holdout_chunking( text_id, raw_text, expected_chunks, ): del text_id normalized = normalize_spoken_forms(raw_text) assert split_text_for_tts(normalized, max_chars=80, min_chunk_chars=12) == list( expected_chunks ) def test_frontend_clause_split_matrix_for_structured_and_network_holdouts(): raw_h11 = ( "山區步道的志工預計在 2028/10/21 上午 07:15 集合," "先用定位器 AX-520 核對座標,再分組檢查木棧道、里程牌與飲水站。" "若氣象網站 https://trailweather.example.tw 顯示降雨機率超過 65%," "領隊就取消高海拔路線,改走較短的林間環線。" "途中若發現落石或樹枝阻斷通行,請拍照並寄到 " "patrol@forestmail.tw,不要自行搬動大型障礙物。" "所有隊員回到登山口後,還要清點無線電與急救包," "確認沒有任何人落單,才結束當天的巡查。" ) matrix = { "H05": ( "這批咖啡豆重 2.75 kg,會員價是 NT$1,480,回饋比例為 6.5%。", ( "這批咖啡豆重 二點七五公斤,", "會員價是 新台幣一千四百八十,回饋比例為 百分之六點五。", ), (12, 24), ), "H06": ( "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。", ( "若要更換導覽場次,請寄信到,tour 點 help 小老鼠 " "island museum 點 T W。", ), (25,), ), "H07": ( "潮汐預報可查詢 https://coastwatch.example.tw/tide。", ( "潮汐預報可查詢,安全網址是 coast watch 點 example 點 " "T W,路徑是 T I D E,網址輸入完畢。", ), (35,), ), "H11": ( raw_h11, ( "山區步道的志工預計在 二零二八年十月二十一日 上午 " "七點十五分 集合,", "先用定位器 A X 五二零 核對座標,再分組檢查木棧道、" "里程牌與飲水站。", "若氣象網站,安全網址是 trail weather 點 example 點 " "T W,網址輸入完畢,", "顯示降雨機率超過 百分之六十五,", "領隊就取消高海拔路線,改走較短的林間環線。", "途中若發現落石或樹枝阻斷通行,", "請拍照並寄到,patrol 小老鼠 forest mail 點 T W," "不要自行搬動大型障礙物。", "所有隊員回到登山口後,還要清點無線電與急救包," "確認沒有任何人落單,才結束當天的巡查。", ), (30, 29, 26, 14, 19, 14, 28, 38), ), } for raw_text, expected_chunks, expected_units in matrix.values(): normalized = normalize_spoken_forms(raw_text) chunks = split_text_for_tts(normalized, max_chars=80, min_chunk_chars=12) assert chunks == list(expected_chunks) assert "".join(chunks) == normalized assert tuple(count_speech_units(chunk) for chunk in chunks) == expected_units assert all(12 <= count_speech_units(chunk) <= 80 for chunk in chunks) for protected_span in network_protected_spoken_spans(normalized): protected_chunks = [chunk for chunk in chunks if protected_span in chunk] assert len(protected_chunks) == 1 assert select_generation_cps( protected_chunks[0], cjk_cps=5.2, ascii_cps=4.6, ) == 4.6 def test_generation_only_network_component_planner_holdout_matrix(): raw_h11 = ( "山區步道的志工預計在 2028/10/21 上午 07:15 集合," "先用定位器 AX-520 核對座標,再分組檢查木棧道、里程牌與飲水站。" "若氣象網站 https://trailweather.example.tw 顯示降雨機率超過 65%," "領隊就取消高海拔路線,改走較短的林間環線。" "途中若發現落石或樹枝阻斷通行,請拍照並寄到 " "patrol@forestmail.tw,不要自行搬動大型障礙物。" "所有隊員回到登山口後,還要清點無線電與急救包," "確認沒有任何人落單,才結束當天的巡查。" ) matrix = { "H06": ( "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。", (25,), (), ), "H07": ( "潮汐預報可查詢 https://coastwatch.example.tw/tide。", (35,), (), ), "H11": ( raw_h11, (30, 29, 26, 33, 14, 28, 38), (), ), } for raw, expected_units, mandatory_after in matrix.values(): normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert tuple(count_speech_units(spec.text) for spec in specs) == expected_units assert normalize_tts_text(" ".join(spec.text for spec in specs)) == normalized assert all( spec.source_end == following.source_start for spec, following in zip(specs, specs[1:]) ) assert all( (8 if spec.network_conditioned else 12) <= count_speech_units(spec.text) <= (36 if spec.network_conditioned else 80) for spec in specs ) assert tuple( index for index, spec in enumerate(specs) if spec.boundary_after == "network_internal" ) == mandatory_after for spec in specs: assert len(spec.network_span_indices) == len( spec.network_full_spoken_proofs ) assert len(spec.network_span_indices) == len( spec.network_fragment_proofs ) for proof in spec.network_fragment_proofs: assert ( spec.text[proof.chunk_start : proof.chunk_end] == proof.full_spoken_proof[ proof.parent_start : proof.parent_end ] ) for span_index in { index for spec in specs for index in spec.network_span_indices }: ranges = sorted( (proof.parent_start, proof.parent_end, proof.full_spoken_proof) for spec in specs for proof in spec.network_fragment_proofs if proof.span_index == span_index ) assert ranges[0][0] == 0 assert ranges[-1][1] == len(ranges[0][2]) assert all( left_end == right_start for (_, left_end, _), (right_start, _, _) in zip( ranges, ranges[1:], ) ) h11_specs = plan_generation_chunks( raw_h11, normalize_spoken_forms(raw_h11), ) assert h11_specs[2].text.endswith("網址輸入完畢,") assert not h11_specs[3].text.startswith((",", ",")) assert h11_specs[2].network_fragment_proofs[0].parent_start == 0 assert h11_specs[2].network_fragment_proofs[0].parent_end == len( h11_specs[2].network_fragment_proofs[0].full_spoken_proof ) assert h11_specs[5].network_fragment_proofs[0].parent_start == 0 assert h11_specs[5].network_fragment_proofs[0].parent_end == len( h11_specs[5].network_fragment_proofs[0].full_spoken_proof ) def test_network_request_bounded_ordinary_envelope_splits_long_final_clause(): raw = ( "請寄到 patrol@forestmail.tw。" "所有隊員回到登山口後,還要清點無線電與急救包," "確認沒有任何人落單,才結束當天的巡查。" ) target = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, target, ordinary_max_units=36) assert tuple(count_speech_units(spec.text) for spec in specs) == (14, 21, 17) assert specs[0].network_conditioned assert not specs[1].network_conditioned assert not specs[2].network_conditioned assert specs[1].boundary_after == "semantic" assert specs[2].boundary_after == "none" assert "".join(spec.text for spec in specs) == target @pytest.mark.parametrize( ("terminal_units", "expected_units"), ( (36, (14, 36)), (37, (14, 25, 12)), ), ) def test_network_request_terminal_ordinary_clause_splits_only_above_36_units( terminal_units, expected_units, ): raw = "請寄到 patrol@forestmail.tw。" + "甲" * terminal_units + "。" target = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, target, ordinary_max_units=36) assert tuple(count_speech_units(spec.text) for spec in specs) == expected_units assert specs[0].network_conditioned assert all(not spec.network_conditioned for spec in specs[1:]) assert all( left.source_end == right.source_start for left, right in zip(specs, specs[1:], strict=False) ) assert specs[0].source_start == 0 assert specs[-1].source_end == len(target) assert all(spec.boundary_after == "semantic" for spec in specs[:-1]) assert specs[-1].boundary_after == "none" assert max(count_speech_units(spec.text) for spec in specs) <= 36 assert "".join(spec.text for spec in specs) == target def test_generation_network_planner_prefers_context_before_identifier_start(): raw = ( "途中若發現落石或樹枝阻斷通行,請拍照並寄到 " "patrol@forestmail.tw,不要自行搬動大型障礙物。" ) normalized = normalize_spoken_forms(raw) repeated = tuple(plan_generation_chunks(raw, normalized) for _ in range(5)) assert all(specs == repeated[0] for specs in repeated[1:]) specs = repeated[0] assert tuple(spec.text for spec in specs) == ( "途中若發現落石或樹枝阻斷通行,", "請拍照並寄到,patrol 小老鼠 forest mail 點 T W," "不要自行搬動大型障礙物。", ) assert tuple(count_speech_units(spec.text) for spec in specs) == (14, 28) assert "".join(spec.text for spec in specs) == normalized assert not specs[0].network_conditioned assert specs[1].network_span_indices == (0,) proof = specs[1].network_fragment_proofs[0] assert proof.chunk_start == len("請拍照並寄到,") assert proof.chunk_end == proof.chunk_start + len(proof.full_spoken_proof) assert proof.parent_start == 0 assert proof.parent_end == len(proof.full_spoken_proof) assert ( specs[1].text[proof.chunk_start : proof.chunk_end] == proof.full_spoken_proof ) def test_email_domain_mail_generation_variant_is_range_and_parent_proof_bound(): raw = ( "請把 mail 記在備註,再寄到 patrol@forestmail.tw," "最後瀏覽 https://forestmail.tw。" ) target = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, target) variants = tuple( email_domain_mail_generation_variant( spec.text, spec.network_fragment_proofs, ) for spec in specs ) joined = "".join(variants) assert "把 mail 記在備註" in joined assert "patrol 小老鼠 forest M A I L 點 T W" in joined assert "forest mail" in joined assert count_speech_units(joined) == count_speech_units(target) + 3 email_spec = next( spec for spec in specs if "小老鼠" in spec.text and "mail" in spec.text ) proof = next( proof for proof in email_spec.network_fragment_proofs if proof.identifier_kind == "email" ) with pytest.raises(ValueError, match="does not bind"): forged = ( f"{email_spec.text[:proof.chunk_start]}X" f"{email_spec.text[proof.chunk_start + 1:]}" ) email_domain_mail_generation_variant( forged, (proof,), ) def test_email_domain_mail_generation_variant_keeps_canonical_verifier_target(): raw = "請寄到 patrol@forestmail.tw。" target = normalize_spoken_forms(raw) spec = plan_generation_chunks(raw, target)[0] variant = email_domain_mail_generation_variant( spec.text, spec.network_fragment_proofs, ) assert variant != target assert "forest M A I L 點 T W" in variant assert compare_asr_text(target, "請寄到 patrol@forestmail.tw。").passed assert not compare_asr_text(target, "請寄到 patrol@forestmale.tw。").passed assert not compare_asr_text(target, "請寄到 patrol@forestmail.com。").passed def test_generation_network_planner_request_start_does_not_invent_context(): raw = ( "patrol@forestmail.tw,不要自行搬動大型障礙物," "途中若發現落石或樹枝阻斷通行,請拍照保存。" ) normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert tuple(spec.text for spec in specs) == ( "patrol 小老鼠 forest mail 點 T W,不要自行搬動大型障礙物,", "途中若發現落石或樹枝阻斷通行,請拍照保存。", ) assert tuple(count_speech_units(spec.text) for spec in specs) == (22, 19) assert specs[0].source_start == 0 assert specs[0].text.startswith( "patrol 小老鼠 forest mail 點 T W" ) assert "".join(spec.text for spec in specs) == normalized proof = specs[0].network_fragment_proofs[0] assert proof.chunk_start == 0 assert proof.parent_start == 0 assert proof.parent_end == len(proof.full_spoken_proof) assert specs[0].text[: proof.chunk_end] == proof.full_spoken_proof def _tour_help_fragment_case(prefix: str = "請核對,", suffix: str = ",完成。"): fragment = "tour 點 help" full_proof = normalize_spoken_forms("tour.help@islandmuseum.tw") parent_start = full_proof.index(fragment) target = f"{prefix}{fragment}{suffix}" chunk_start = len(prefix) return target, NetworkFragmentProof( span_index=0, chunk_start=chunk_start, chunk_end=chunk_start + len(fragment), parent_start=parent_start, parent_end=parent_start + len(fragment), full_spoken_proof=full_proof, ) @pytest.mark.parametrize("literal", ("tour.help", "Tour.Help", "t o u r . h e l p")) def test_local_network_fragment_canonicalizes_only_exact_typed_ascii(literal): target, proof = _tour_help_fragment_case() transcript = f"請核對,{literal},完成。" evidence = canonicalize_asr_network_fragments(transcript, target, (proof,)) assert evidence.passed is True assert evidence.protected_range_count == 1 assert "tour 點 help" in evidence.transcript_text assert compare_asr_text(target, evidence.transcript_text, max_cer=0.20).passed def test_local_network_fragment_normalizes_ascii_sentence_boundaries_first(): target, proof = _tour_help_fragment_case() evidence = canonicalize_asr_network_fragments( "請核對,tour.help,完成.", target, (proof,), ) assert evidence.transcript_text == target assert evidence.passed is True @pytest.mark.parametrize( "literal", ( "tour.herp", # wrong content "tourhelp", # missing separator "tour.hel", # missing suffix letter "help.tour", # reordered labels "xtour.help", # extra prefix content "tour.help/path", # extra suffix content "tour.help.help", # repeated content ), ) def test_local_network_fragment_rejects_wrong_missing_reordered_or_extra_content( literal, ): target, proof = _tour_help_fragment_case() transcript = f"請核對,{literal},完成。" evidence = canonicalize_asr_network_fragments(transcript, target, (proof,)) assert evidence.passed is False assert evidence.protected_range_count == 1 @pytest.mark.parametrize("symbol", ("\\", "<", ">", "§", "🙂")) @pytest.mark.parametrize("side", ("prefix", "suffix")) def test_local_network_fragment_rejects_adjacent_unknown_symbols(symbol, side): target, proof = _tour_help_fragment_case() literal = f"{symbol}tour.help" if side == "prefix" else f"tour.help{symbol}" evidence = canonicalize_asr_network_fragments( f"請核對,{literal},完成。", target, (proof,), ) assert evidence.protected_range_count == 1 assert evidence.passed is False def _pure_label_fragment_case(label: str): fragment = f"{label} " full_proof = normalize_spoken_forms(f"{label}@islandmuseum.tw") parent_start = full_proof.index(fragment) prefix = "請核對," target = f"{prefix}{fragment},完成。" chunk_start = len(prefix) return target, NetworkFragmentProof( span_index=0, chunk_start=chunk_start, chunk_end=chunk_start + len(fragment), parent_start=parent_start, parent_end=parent_start + len(fragment), full_spoken_proof=full_proof, ) @pytest.mark.parametrize("label", ("tour", "a")) @pytest.mark.parametrize("side", ("prefix", "suffix")) @pytest.mark.parametrize( "symbol", ( "/", "\\", "@", "-", "_", "%", "$", "~", "|", "=", "&", "#", "+", "*", "<", ">", "§", "🙂", ), ) def test_pure_label_network_fragment_rejects_adjacent_identifier_symbols( label, side, symbol, ): target, proof = _pure_label_fragment_case(label) literal = f"{symbol}{label}" if side == "prefix" else f"{label}{symbol}" evidence = canonicalize_asr_network_fragments( f"請核對,{literal},完成。", target, (proof,), ) assert evidence.protected_range_count == 1 assert evidence.passed is False @pytest.mark.parametrize("label", ("tour", "a")) def test_pure_label_network_fragment_accepts_ascii_sentence_punctuation(label): target, proof = _pure_label_fragment_case(label) evidence = canonicalize_asr_network_fragments( f"請核對,{label},完成.", target, (proof,), ) assert evidence.transcript_text == target assert evidence.passed is True def test_local_network_fragment_exact_gate_is_stricter_than_whole_chunk_cer(): prefix = "甲" * 30 + "," suffix = "," + "乙" * 30 + "。" target, proof = _tour_help_fragment_case(prefix, suffix) wrong_spoken = target.replace("help", "herp", 1) # One protected middle error is below the ordinary whole-chunk CER limit # and outside the prefix/suffix windows, but the range gate still rejects. assert compare_asr_text(target, wrong_spoken, max_cer=0.20).passed evidence = canonicalize_asr_network_fragments( wrong_spoken, target, (proof,), ) assert evidence.passed is False def test_local_network_fragment_provenance_is_range_bound_and_fails_closed(): target, proof = _tour_help_fragment_case() forged = NetworkFragmentProof( span_index=proof.span_index, chunk_start=proof.chunk_start + 1, chunk_end=proof.chunk_end, parent_start=proof.parent_start, parent_end=proof.parent_end - 1, full_spoken_proof=proof.full_spoken_proof, ) with pytest.raises(ValueError, match="does not bind"): canonicalize_asr_network_fragments("tour.help", target, (forged,)) with pytest.raises(ValueError, match="invalid type"): canonicalize_asr_network_fragments("tour.help", target, (object(),)) def test_local_network_fragment_cannot_borrow_repeated_plain_spoken_text(): fragment = "tour 點 help" prefix = f"先照字面念{fragment},再核對," target, proof = _tour_help_fragment_case(prefix, "。") transcript = "先照字面念 tour.help,再核對,tour.herp。" evidence = canonicalize_asr_network_fragments(transcript, target, (proof,)) assert evidence.passed is False def _hybrid_tour_help_fragment_case(prefix: str = "請核對,"): fragment = "tour 點 help" full_proof = normalize_spoken_forms("tour.help@islandmuseum.tw") parent_start = full_proof.index(fragment) target = f"{prefix}{fragment},完成。" chunk_start = len(prefix) return target, NetworkFragmentProof( span_index=0, chunk_start=chunk_start, chunk_end=chunk_start + len(fragment), parent_start=parent_start, parent_end=parent_start + len(fragment), full_spoken_proof=full_proof, ) def test_local_network_fragment_accepts_exact_range_bound_hybrid_lexical_atoms(): target, proof = _hybrid_tour_help_fragment_case() evidence = canonicalize_asr_network_fragments( "請核對,Tour.Help,完成。", target, (proof,), ) assert evidence.passed is True assert evidence.transcript_text == target def test_real_network_fragment_proofs_reject_repeat_wrong_range_and_reorder(): raw = "請查 https://a.tw 並寄信到 a@b.co。" spec = plan_generation_chunks(raw, normalize_spoken_forms(raw))[0] first, second = spec.network_fragment_proofs exact = canonicalize_asr_network_fragments( spec.text, spec.text, (first, second), ) assert exact.passed is True with pytest.raises(ValueError, match="bind|offset|range"): canonicalize_asr_network_fragments( spec.text, spec.text, (first, first), ) with pytest.raises(ValueError, match="bind|offset|range"): canonicalize_asr_network_fragments( spec.text, spec.text, (second, first), ) with pytest.raises(ValueError, match="bind|offset|range"): canonicalize_asr_network_fragments( spec.text, spec.text, ( replace( first, chunk_start=first.chunk_start + 1, ), second, ), ) def _oversized_email_domain_spec(domain: str): raw = f"請寄信到 {'a' * 120}@{domain}。" specs = plan_generation_chunks(raw, normalize_spoken_forms(raw)) assert len(specs) == 2 assert specs[0].boundary_after == "network_internal" assert specs[1].boundary_after == "none" assert specs[1].network_fragment_proofs[0].parent_start > 0 return specs[1] @pytest.mark.parametrize( "transcript", ( "小老鼠islandmuseum.tw", "@ islandmuseum 點 T W", "小老鼠 I S L A N D M U S E U M . T W", "小老鼠 islandmuseum 点 T W", ), ) def test_local_network_fragment_accepts_exact_mixed_email_domain_render(transcript): domain = _oversized_email_domain_spec("islandmuseum.tw") evidence = canonicalize_asr_network_fragments( transcript, domain.text, domain.network_fragment_proofs, ) assert evidence.passed is True assert compare_asr_text(domain.text, evidence.transcript_text).passed @pytest.mark.parametrize("render", ("spoken", "spoken_without_commas", "raw")) def test_local_naturalized_url_accepts_only_proof_bound_exact_renders(render): raw = "潮汐預報可查詢 https://coastwatch.example.tw/tide。" target = normalize_spoken_forms(raw) spec = plan_generation_chunks(raw, target)[0] if render == "spoken": transcript = target elif render == "spoken_without_commas": transcript = target.replace(",路徑是", "路徑是").replace( ",網址輸入完畢", "網址輸入完畢", ) else: transcript = raw evidence = canonicalize_asr_network_fragments( transcript, spec.text, spec.network_fragment_proofs, ) assert evidence.passed is True assert compare_asr_text(spec.text, evidence.transcript_text).passed def test_local_network_fragment_accepts_exact_mixed_long_form_email_domain(): domain = _oversized_email_domain_spec("forestmail.tw") evidence = canonicalize_asr_network_fragments( "小老鼠forestmail.tw", domain.text, domain.network_fragment_proofs, ) assert evidence.passed is True assert compare_asr_text(domain.text, evidence.transcript_text).passed @pytest.mark.parametrize( "transcript", ( "若要更換導覽場次請寄信到tour.help@islandmuseum.tw", "若要更換導覽場次請寄信到tour.help小老鼠islandmuseum.tw", "若要更換導覽場次請寄信到tour.helpXiaolaoshuislandmuseum.tw", "若要更換導覽場次請寄信到tour.helpXiaolaoshuiislandmuseum.tw", ), ) def test_atomic_email_chunk_accepts_only_exact_proof_bound_mixed_render(transcript): raw = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" target = normalize_spoken_forms(raw) spec = plan_generation_chunks(raw, target)[0] evidence = canonicalize_asr_network_fragments( transcript, target, spec.network_fragment_proofs, ) assert evidence.passed is True assert compare_asr_text(target, evidence.transcript_text).passed def test_atomic_email_chunk_rejects_mixed_render_mutation(): raw = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" target = normalize_spoken_forms(raw) spec = plan_generation_chunks(raw, target)[0] evidence = canonicalize_asr_network_fragments( "若要更換導覽場次請寄信到tour.help@islandmuseum.com", target, spec.network_fragment_proofs, ) assert evidence.passed is False @pytest.mark.parametrize( "transcript", ( "若要更換導覽場次請寄信到tour.helpXiaolaoshuiislandmuseum.com", "若要更換導覽場次請寄信到tour.helpXiaolaoshuiislandmuseurn.tw", "若要更換導覽場次請寄信到tour.helpXiaolaoshuiislandmuseum.t", "若要更換導覽場次請寄信到tour.helpXiaolaoshuiislandmuseum.twz", "若要更換導覽場次請寄信到tour.helpXiaolaoshuiislandmuseum.tw/x", "若要更換導覽場次請寄信到tour.helpXiaolaoshuiislandmuseum.tw?x", "若要更換導覽場次請寄信到tour.helpXiaolaoshuiislandmuseum.tw#x", "若要更換導覽場次請寄信到xtour.helpXiaolaoshuiislandmuseum.tw", "若要更換導覽場次請寄信到tour.helpxXiaolaoshuiislandmuseum.tw", "若要更換導覽場次請寄信到tour.helpXiaolaoshuixislandmuseum.tw", ), ) def test_atomic_email_alias_rejects_content_and_boundary_mutations(transcript): raw = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" target = normalize_spoken_forms(raw) spec = plan_generation_chunks(raw, target)[0] evidence = canonicalize_asr_network_fragments( transcript, target, spec.network_fragment_proofs, ) assert evidence.protected_range_count == 1 assert evidence.passed is False def test_atomic_email_alias_cannot_borrow_proof_from_url_path(): raw = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" target = normalize_spoken_forms(raw) spec = plan_generation_chunks(raw, target)[0] evidence = canonicalize_asr_network_fragments( ( "若要更換導覽場次請寄信到" "https://example.tw/tour.helpXiaolaoshuiislandmuseum.tw" ), target, spec.network_fragment_proofs, ) assert evidence.protected_range_count == 1 assert evidence.passed is False @pytest.mark.parametrize( "transcript", ( "小老鼠islandmuseurn.tw", "小老鼠islandmuseumtw", "小老鼠islandmuseum.t", "小老鼠islandmuseum..tw", "小老鼠islandmuseum.tw/x", "x小老鼠islandmuseum.tw", "小老鼠islandmuseum.twz", ), ) def test_mixed_email_domain_render_rejects_mutations(transcript): domain = _oversized_email_domain_spec("islandmuseum.tw") evidence = canonicalize_asr_network_fragments( transcript, domain.text, domain.network_fragment_proofs, ) assert evidence.passed is False @pytest.mark.parametrize( "symbol", ("/", ".", "-", "\\", "<", ">", "§", "🙂"), ) @pytest.mark.parametrize("location", ("label", "separator")) def test_mixed_email_domain_rejects_internal_symbol_insertions(symbol, location): domain = _oversized_email_domain_spec("islandmuseum.tw") if location == "label": transcript = f"小老鼠islandmu{symbol}seum.tw" else: transcript = f"小老鼠islandmuseum點{symbol}TW" evidence = canonicalize_asr_network_fragments( transcript, domain.text, domain.network_fragment_proofs, ) assert evidence.passed is False @pytest.mark.parametrize( "invisible", ("\u200b", "\u200c", "\u200d", "\u202e", "\x00"), ) def test_mixed_email_domain_rejects_hidden_control_insertions(invisible): domain = _oversized_email_domain_spec("islandmuseum.tw") transcript = f"小老鼠islandmu{invisible}seum.tw" evidence = canonicalize_asr_network_fragments( transcript, domain.text, domain.network_fragment_proofs, ) assert evidence.passed is False def test_mixed_email_domain_cannot_borrow_plain_duplicate_around_mutation(): fragment = "小老鼠 island museum 點 T W" full_proof = normalize_spoken_forms("tour.help@islandmuseum.tw") parent_start = full_proof.index(fragment) prefix = f"先照字面念{fragment},再核對," target = f"{prefix}{fragment}。" proof = NetworkFragmentProof( span_index=0, chunk_start=len(prefix), chunk_end=len(prefix) + len(fragment), parent_start=parent_start, parent_end=parent_start + len(fragment), full_spoken_proof=full_proof, ) transcript = ( "先照字面念小老鼠islandmuseum.tw,再核對," "小老鼠islandmu/seum.tw。" ) evidence = canonicalize_asr_network_fragments(transcript, target, (proof,)) assert evidence.passed is False @pytest.mark.parametrize( "mutate", ( lambda text: text.replace("安全網址是 ", ""), lambda text: text.replace("安全網址是", "網址是"), lambda text: text.replace("example 點 ", ""), lambda text: text.replace(" D", ""), lambda text: text.replace(",網址輸入完畢", ""), lambda text: text.replace("example", "example X"), lambda text: text.replace("coast watch", "coast weather"), ), ) def test_naturalized_url_render_rejects_mutations(mutate): raw = "潮汐預報可查詢 https://coastwatch.example.tw/tide。" target = normalize_spoken_forms(raw) spec = plan_generation_chunks(raw, target)[0] evidence = canonicalize_asr_network_fragments( mutate(target), spec.text, spec.network_fragment_proofs, ) assert evidence.passed is False def test_naturalized_url_raw_render_rejects_adjacent_ascii_atom_exhaustively(): raw = "潮汐預報可查詢 https://coastwatch.example.tw/tide。" spec = plan_generation_chunks(raw, normalize_spoken_forms(raw))[0] exact = "https://coastwatch.example.tw/tide" for whitespace in (" ", " ", "\t", "\n"): for atom in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789": for transcript in ( raw.replace(exact, f"{atom}{whitespace}{exact}"), raw.replace(exact, f"{exact}{whitespace}{atom}"), ): evidence = canonicalize_asr_network_fragments( transcript, spec.text, spec.network_fragment_proofs, ) assert evidence.passed is False, (whitespace, atom, transcript) @pytest.mark.parametrize( "literal", ( "tours.help", "tour.helps", "our.help", "tour.hepl", "xtour.help", "tour.help/x", ), ) def test_hybrid_lexical_network_atoms_reject_substrings_and_mutations(literal): target, proof = _hybrid_tour_help_fragment_case() evidence = canonicalize_asr_network_fragments( f"請核對,{literal},完成。", target, (proof,), ) assert evidence.passed is False def _duplicate_network_fragment_case(*, protected_first: bool): fragment = "點 欸" full_proof = "H T T P S 冒號 斜線 斜線 tour 點 欸" prefix = "甲" * 20 suffix = "乙" * 20 + "。" target = f"{prefix}{fragment},{fragment}{suffix}" if protected_first: chunk_start = len(prefix) else: chunk_start = len(prefix) + len(fragment) + 1 parent_start = full_proof.index(fragment) return target, prefix, suffix, NetworkFragmentProof( span_index=0, chunk_start=chunk_start, chunk_end=chunk_start + len(fragment), parent_start=parent_start, parent_end=parent_start + len(fragment), full_spoken_proof=full_proof, ) @pytest.mark.parametrize("protected_first", (False, True)) def test_network_fragment_duplicate_cannot_borrow_any_optimal_alignment( protected_first, ): target, prefix, suffix, proof = _duplicate_network_fragment_case( protected_first=protected_first ) # The lone literal can align equally well to either duplicate. The local # proof must fail because at least one optimal path deletes the protected # occurrence, regardless of deterministic backtrace tie-breaking. evidence = canonicalize_asr_network_fragments( f"{prefix}.a{suffix}", target, (proof,), ) assert compare_asr_text( target, evidence.transcript_text, max_cer=0.20, ).passed assert evidence.passed is False @pytest.mark.parametrize("protected_first", (False, True)) def test_network_fragment_duplicate_exact_two_occurrences_remain_valid( protected_first, ): target, prefix, suffix, proof = _duplicate_network_fragment_case( protected_first=protected_first ) evidence = canonicalize_asr_network_fragments( f"{prefix}.a,.a{suffix}", target, (proof,), ) assert evidence.transcript_text == target assert evidence.passed is True @pytest.mark.parametrize("boundary", ("start", "end")) def test_network_fragment_rejects_insertions_on_closed_range_boundary(boundary): prefix = "甲" * 30 + "," suffix = "," + "乙" * 30 + "。" target, proof = _tour_help_fragment_case(prefix, suffix) offset = proof.chunk_start if boundary == "start" else proof.chunk_end transcript = target[:offset] + "錯" + target[offset:] evidence = canonicalize_asr_network_fragments( transcript, target, (proof,), ) assert compare_asr_text(target, transcript, max_cer=0.20).passed assert evidence.passed is False def test_generation_network_planner_keeps_ordinary_semantic_chunks_unchanged(): raw = ( "清晨開館以前,水族館人員會先量測各池的水溫與鹽度," "再觀察魚群是否正常進食。確認設備都正常後才開門。" ) normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert [spec.text for spec in specs] == split_text_for_tts( normalized, max_chars=80, min_chunk_chars=12, ) assert all(not spec.network_conditioned for spec in specs) def test_generation_network_planner_fails_closed_on_invalid_provenance(): raw = "請寄到 tour.help@islandmuseum.tw。" different = normalize_spoken_forms( "請寄到 tour.help@differentmuseum.tw。" ) with pytest.raises(ValueError, match="does not match"): plan_generation_chunks(raw, different) iri = "請查詢 https://example.tw/愛。" with pytest.raises(ValueError, match="non-ASCII IRI"): plan_generation_chunks(iri, normalize_spoken_forms(iri)) emoji_iri = "請查詢 https://example.tw/🐦。" with pytest.raises(ValueError, match="non-ASCII IRI"): plan_generation_chunks(emoji_iri, normalize_spoken_forms(emoji_iri)) oversized = "請查詢 https://" + "a" * 150 + ".tw。" with pytest.raises(ValueError, match="indivisible network component"): plan_generation_chunks(oversized, normalize_spoken_forms(oversized)) with pytest.raises(ValueError, match="chunk limits are inconsistent"): plan_generation_chunks( "請查 https://example.tw。", normalize_spoken_forms("請查 https://example.tw。"), min_units=12, network_min_units=13, ) @pytest.mark.parametrize("raw", ("a@b.co", "https://a.tw")) def test_short_network_identifier_relaxes_only_infeasible_preferred_cut(raw): normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert len(specs) == 1 spec = specs[0] assert spec.text == normalized assert spec.source_start == 0 assert spec.source_end == len(normalized) assert spec.network_conditioned assert spec.boundary_after == "none" assert spec.network_span_indices == (0,) assert spec.network_full_spoken_proofs == (normalized,) assert len(spec.network_fragment_proofs) == 1 proof = spec.network_fragment_proofs[0] assert (proof.chunk_start, proof.chunk_end) == (0, len(normalized)) assert (proof.parent_start, proof.parent_end) == (0, len(normalized)) assert proof.full_spoken_proof == normalized def test_generation_planner_keeps_bounded_network_identifier_atomic(): raw = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert tuple(spec.text for spec in specs) == (normalized,) assert tuple(count_speech_units(spec.text) for spec in specs) == (25,) assert specs[0].boundary_after == "none" assert specs[0].network_conditioned assert len(specs[0].network_fragment_proofs) == 1 proof = specs[0].network_fragment_proofs[0] assert proof.parent_start == 0 assert proof.parent_end == len(proof.full_spoken_proof) assert specs[0].text[proof.chunk_start : proof.chunk_end] == ( proof.full_spoken_proof ) def test_generation_planner_can_split_proven_contextual_atomic_email_separator(): raw = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks( raw, normalized, split_atomic_email_at_separator=True, ) assert tuple(spec.text for spec in specs) == ( "若要更換導覽場次,請寄信到,tour 點 help ", "小老鼠 island museum 點 T W。", ) assert tuple(count_speech_units(spec.text) for spec in specs) == (15, 10) assert tuple(spec.boundary_after for spec in specs) == ( "network_internal", "none", ) assert "".join(spec.text for spec in specs) == normalized first, second = ( spec.network_fragment_proofs[0] for spec in specs ) assert first.identifier_kind == second.identifier_kind == "email" assert first.span_index == second.span_index == 0 assert first.full_spoken_proof == second.full_spoken_proof assert first.parent_start == 0 assert first.parent_end == second.parent_start assert second.parent_end == len(second.full_spoken_proof) def test_generation_planner_splits_seven_unit_contextual_email_prose(): raw = "團體導覽請聯絡 booking.team@harborarts.tw。" normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks( raw, normalized, split_atomic_email_at_separator=True, ) assert tuple(count_speech_units(spec.text) for spec in specs) == (11, 9) assert tuple(spec.boundary_after for spec in specs) == ( "network_internal", "none", ) assert "".join(spec.text for spec in specs) == normalized assert specs[0].text.endswith("booking 點 team ") assert specs[1].text.startswith("小老鼠 harbor arts 點 T W") def test_email_domain_uses_unique_email_only_lexical_segmentation(): assert normalize_spoken_forms("booking.team@harborarts.tw") == ( "booking 點 team 小老鼠 harbor arts 點 T W" ) assert normalize_spoken_forms("https://harborarts.tw/tour") == ( "H T T P S 冒號 斜線 斜線 harborarts 點 T W 斜線 tour" ) @pytest.mark.parametrize( "transcript", ( "小老鼠harborarts.tw", "小老鼠HarborArts.TW", "Xiaolaoshuharbor arts.tw", "小老鼠 HARBOR ARTS 點 T W", ), ) def test_email_only_segmented_domain_keeps_exact_parent_proof(transcript): raw = "團體導覽請聯絡 booking.team@harborarts.tw。" target = normalize_spoken_forms(raw) right = plan_generation_chunks( raw, target, split_atomic_email_at_separator=True, )[1] evidence = canonicalize_asr_network_fragments( transcript, right.text, right.network_fragment_proofs, ) assert evidence.passed is True assert compare_asr_text(right.text, evidence.transcript_text).passed @pytest.mark.parametrize( "transcript", ( "小老鼠harbourarts.tw", "小老鼠harborart.tw", "小老鼠harborarts.com", "小老鼠harbor-arts.tw", "小老鼠harborarts.tw/x", "x小老鼠harborarts.tw", "小老鼠harborarts..tw", ), ) def test_email_only_segmented_domain_rejects_identifier_mutation(transcript): raw = "團體導覽請聯絡 booking.team@harborarts.tw。" target = normalize_spoken_forms(raw) right = plan_generation_chunks( raw, target, split_atomic_email_at_separator=True, )[1] evidence = canonicalize_asr_network_fragments( transcript, right.text, right.network_fragment_proofs, ) assert evidence.passed is False def test_generation_planner_contextual_email_split_rebalances_h11_sentence(): raw = ( "途中若發現落石或樹枝阻斷通行,請拍照並寄到 " "patrol@forestmail.tw,不要自行搬動大型障礙物。" ) normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks( raw, normalized, split_atomic_email_at_separator=True, ) assert tuple(count_speech_units(spec.text) for spec in specs) == (22, 20) assert tuple(spec.boundary_after for spec in specs) == ( "network_internal", "none", ) assert specs[0].text.endswith("patrol ") assert specs[1].text.startswith("小老鼠 forest mail 點 T W") assert "".join(spec.text for spec in specs) == normalized def test_contextual_email_split_local_proofs_verify_each_exact_fragment(): raw = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" target = normalize_spoken_forms(raw) left, right = plan_generation_chunks( raw, target, split_atomic_email_at_separator=True, ) left_evidence = canonicalize_asr_network_fragments( "若要更換導覽場次請寄信到tour.help", left.text, left.network_fragment_proofs, ) right_evidence = canonicalize_asr_network_fragments( "Xiaolaoshuislandmuseum.tw", right.text, right.network_fragment_proofs, ) assert left_evidence.passed is True assert right_evidence.passed is True assert canonicalize_asr_network_fragments( left.text, left.text, left.network_fragment_proofs, ).passed is True assert canonicalize_asr_network_fragments( right.text, right.text, right.network_fragment_proofs, ).passed is True assert canonicalize_asr_network_fragments( "小老鼠islandmuseum.com", right.text, right.network_fragment_proofs, ).passed is False def test_contextual_email_split_mail_variant_remains_fragment_proof_bound(): raw = ( "途中若發現落石或樹枝阻斷通行,請拍照並寄到 " "patrol@forestmail.tw,不要自行搬動大型障礙物。" ) target = normalize_spoken_forms(raw) left, right = plan_generation_chunks( raw, target, split_atomic_email_at_separator=True, ) left_variant = email_domain_mail_generation_variant( left.text, left.network_fragment_proofs, ) right_variant = email_domain_mail_generation_variant( right.text, right.network_fragment_proofs, ) assert left_variant == left.text assert "forest M A I L 點 T W" in right_variant assert right_variant != right.text assert "".join((left.text, right.text)) == target proof = right.network_fragment_proofs[0] assert proof.identifier_kind == "email" assert proof.parent_start == proof.full_spoken_proof.index("小老鼠") @pytest.mark.parametrize( "raw", ( "請寄到 a@b.co。", "tour.help@islandmuseum.tw。", "請把地址記成 tour.help@islandmuseum.tw。", ( "請寄到 tour.help@islandmuseum.tw 並副本到 " "patrol@forestmail.tw。" ), ), ) def test_contextual_atomic_email_split_rejects_unproven_or_short_context(raw): normalized = normalize_spoken_forms(raw) default = plan_generation_chunks(raw, normalized) requested = plan_generation_chunks( raw, normalized, split_atomic_email_at_separator=True, ) assert requested == default assert all(spec.boundary_after != "network_internal" for spec in requested) def test_contextual_atomic_email_split_does_not_change_atomic_url_plan(): raw = "潮汐預報可查詢 https://coastwatch.example.tw/tide。" normalized = normalize_spoken_forms(raw) assert plan_generation_chunks( raw, normalized, split_atomic_email_at_separator=True, ) == plan_generation_chunks(raw, normalized) def test_contextual_atomic_email_split_policy_must_be_boolean(): raw = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" with pytest.raises(ValueError, match="must be boolean"): plan_generation_chunks( raw, normalize_spoken_forms(raw), split_atomic_email_at_separator=1, ) def test_generation_planner_preserves_component_cut_for_oversized_identifier(): raw = f"請查詢 https://{'a' * 96}.example.tw/path。" normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert tuple(count_speech_units(spec.text) for spec in specs) == (14, 33) assert tuple(spec.boundary_after for spec in specs) == ( "network_internal", "none", ) assert "".join(spec.text for spec in specs) == normalized fragments = tuple( proof for spec in specs for proof in spec.network_fragment_proofs ) assert len(fragments) == 2 assert count_speech_units(fragments[0].full_spoken_proof) > 36 assert fragments[0].full_spoken_proof == fragments[1].full_spoken_proof assert fragments[0].parent_start == 0 assert fragments[0].parent_end == fragments[1].parent_start assert fragments[1].parent_end == len(fragments[1].full_spoken_proof) def test_hybrid_network_frontend_keeps_labels_lexical_and_typed_atoms_explicit(): assert normalize_spoken_forms( "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" ) == ( "若要更換導覽場次,請寄信到,tour 點 help 小老鼠 " "island museum 點 T W。" ) assert normalize_spoken_forms( "潮汐預報可查詢 https://coastwatch.example.tw/tide。" ) == ( "潮汐預報可查詢,安全網址是 coast watch 點 example 點 " "T W,路徑是 T I D E,網址輸入完畢。" ) assert normalize_spoken_forms( "HTTPS://WWW.Example.COM/API/v1?ID=RTX5090" ) == ( "H T T P S 冒號 斜線 斜線 W W W 點 Example 點 C O M 斜線 " "A P I 斜線 v 一 問號 I D 等於 R T X 五零九零" ) def test_naturalized_url_proof_is_reversible_and_covers_every_raw_render_range(): raw = "https://coastwatch.example.tw/tide" expected = ( "安全網址是 coast watch 點 example 點 T W,路徑是 " "T I D E,網址輸入完畢" ) proof = naturalized_url_rendering_proof(raw) assert proof is not None assert proof.raw_identifier == raw assert proof.spoken_text == expected assert inverse_network_url_rendering(proof) == raw assert contains_naturalized_url_spoken_form(expected) assert network_protected_spoken_spans(expected) == (expected,) assert network_protected_spoken_spans(f"不{expected}") == () assert {atom.component for atom in proof.atoms} >= { "scheme", "host_label", "host_dot", "path_slash", "path_atom", "terminator", } assert tuple(atom.spoken_start for atom in proof.atoms) == ( 0, *(atom.spoken_end for atom in proof.atoms[:-1]), ) assert "".join( atom.raw_text for atom in proof.atoms if atom.component != "terminator" ) == raw assert "".join(atom.spoken_text for atom in proof.atoms) == expected for atom in proof.atoms: assert raw[atom.raw_start : atom.raw_end] == atom.raw_text assert expected[atom.spoken_start : atom.spoken_end] == atom.spoken_text def test_naturalized_root_url_is_reversible_atomic_and_mutation_safe(): raw_url = "https://trailweather.example.tw" raw_text = f"若氣象網站 {raw_url} 顯示降雨。" spoken = ( "安全網址是 trail weather 點 example 點 T W,網址輸入完畢" ) proof = naturalized_url_rendering_proof(raw_url) target = normalize_spoken_forms(raw_text) specs = plan_generation_chunks(raw_text, target) assert proof is not None assert proof.spoken_text == spoken assert inverse_network_url_rendering(proof) == raw_url assert target == f"若氣象網站,{spoken},顯示降雨。" assert len(specs) == 1 assert specs[0].network_conditioned fragment = specs[0].network_fragment_proofs[0] assert fragment.parent_start == 0 assert fragment.parent_end == len(spoken) assert fragment.raw_identifier == raw_url exact = canonicalize_asr_network_fragments( "若氣象網站安全網址是trailweather.example.tw網址輸入完畢顯示降雨", target, specs[0].network_fragment_proofs, ) assert exact.passed is True for transcript in ( "若氣象網站安全網址是trailweather.example.com網址輸入完畢顯示降雨", "若氣象網站安全網址是trailweather.example.tw顯示降雨", "若氣象網站網址是trailweather.example.tw網址輸入完畢顯示降雨", ): assert canonicalize_asr_network_fragments( transcript, target, specs[0].network_fragment_proofs, ).passed is False for suffix in (".evil", "/tide", "?q=1", "#fragment", ":443", "-evil", "x"): transcript = f"若氣象網站 {raw_url}{suffix} 顯示降雨。" assert canonicalize_asr_network_fragments( transcript, target, specs[0].network_fragment_proofs, ).passed is False def test_naturalized_url_grammar_is_generic_and_keeps_http_https_distinct(): https = naturalized_url_rendering_proof( "https://trailweather.example.tw/tour" ) http = naturalized_url_rendering_proof( "http://trailweather.example.tw/tour" ) assert https is not None assert http is not None assert https.spoken_text == ( "安全網址是 trail weather 點 example 點 T W,路徑是 " "T O U R,網址輸入完畢" ) assert http.spoken_text == ( "網址是 trail weather 點 example 點 T W,路徑是 " "T O U R,網址輸入完畢" ) assert inverse_network_url_rendering(https) == ( "https://trailweather.example.tw/tour" ) assert inverse_network_url_rendering(http) == ( "http://trailweather.example.tw/tour" ) cross_scheme = compare_asr_text(http.spoken_text, https.spoken_text) assert cross_scheme.network_protected_spans_passed is False assert cross_scheme.passed is False @pytest.mark.parametrize( "raw", ( "https://coastwatch.example.tw/TIDE", "https://coastwatch.example.tw/Tide", "HTTPS://coastwatch.example.tw/tide", "https://Coastwatch.example.tw/tide", "https://coastwatch.example.tw/tides", "https://coastwatch.example.tw/tide?q=1", "https://coastwatch.example.tw:443/tide", "https://trailweather.example.tw/", "https://trailweather.example.tw?q=1", "https://trailweather.example.tw#status", "https://trailweather.example.tw:443", "https://museum.example.tw/path", "https://coastwatchweathertrailvoicesoundforestmuseum.example.tw/tide", ), ) def test_naturalized_url_unsupported_or_colliding_grammar_falls_back(raw): spoken = normalize_spoken_forms(raw) assert naturalized_url_rendering_proof(raw) is None assert not contains_naturalized_url_spoken_form(spoken) assert "網址輸入完畢" not in spoken if "coastwatch" in raw: assert "coastwatch" in spoken assert "coast watch" not in spoken if "trailweather" in raw: assert "trailweather" in spoken assert "trail weather" not in spoken @pytest.mark.parametrize("mutation", ("raw_range", "spoken", "rule", "reorder")) def test_inverse_naturalized_url_proof_rejects_forged_atoms(mutation): proof = naturalized_url_rendering_proof( "https://coastwatch.example.tw/tide" ) assert proof is not None atoms = list(proof.atoms) if mutation == "raw_range": atoms[1] = replace(atoms[1], raw_end=atoms[1].raw_end + 1) elif mutation == "spoken": atoms[1] = replace(atoms[1], spoken_text="weather") elif mutation == "rule": atoms[1] = replace(atoms[1], rule="lexical_host_label") else: atoms[1], atoms[2] = atoms[2], atoms[1] with pytest.raises(ValueError): inverse_network_url_rendering(replace(proof, atoms=tuple(atoms))) def test_naturalized_h07_planner_is_one_indivisible_proven_chunk(): raw = "潮汐預報可查詢 https://coastwatch.example.tw/tide。" normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert len(specs) == 1 spec = specs[0] assert spec.text == ( "潮汐預報可查詢,安全網址是 coast watch 點 example 點 " "T W,路徑是 T I D E,網址輸入完畢。" ) assert count_speech_units(spec.text) == 35 assert spec.network_conditioned assert spec.boundary_after == "none" assert spec.network_component_indices == ((0, 0),) assert len(spec.network_fragment_proofs) == 1 fragment = spec.network_fragment_proofs[0] assert fragment.parent_start == 0 assert fragment.parent_end == len(fragment.full_spoken_proof) assert fragment.raw_identifier == "https://coastwatch.example.tw/tide" assert fragment.url_rendering_proof is not None assert inverse_network_url_rendering(fragment.url_rendering_proof) == ( fragment.raw_identifier ) def test_naturalized_url_local_gate_revalidates_runtime_raw_render_provenance(): raw = "潮汐預報可查詢 https://coastwatch.example.tw/tide。" target = normalize_spoken_forms(raw) spec = plan_generation_chunks(raw, target)[0] fragment = spec.network_fragment_proofs[0] assert fragment.url_rendering_proof is not None forged_atom = replace( fragment.url_rendering_proof.atoms[1], spoken_text="weather", ) forged_render = replace( fragment.url_rendering_proof, atoms=( forged_atom, *fragment.url_rendering_proof.atoms[1:], ), ) for forged in ( replace(fragment, raw_identifier=""), replace(fragment, url_rendering_proof=None), replace(fragment, raw_identifier="", url_rendering_proof=None), replace(fragment, url_rendering_proof=forged_render), ): with pytest.raises(ValueError): canonicalize_asr_network_fragments(target, target, (forged,)) @pytest.mark.parametrize( "transcript", ( ( "潮汐预报可查询安全网址是coastwatch.example.tw" "路径是tide网址输入完毕" ), ( "潮汐預報可查詢安全網址是 coast watch 點 example.tw " "路徑是 TIDE 網址輸入完畢" ), "潮汐預報可查詢 https://coastwatch.example.tw/tide。", ), ) def test_naturalized_url_local_gate_accepts_only_proof_bound_asr_forms(transcript): raw = "潮汐預報可查詢 https://coastwatch.example.tw/tide。" target = normalize_spoken_forms(raw) spec = plan_generation_chunks(raw, target)[0] evidence = canonicalize_asr_network_fragments( transcript, target, spec.network_fragment_proofs, ) comparison = compare_asr_text( target, evidence.transcript_text, max_cer=0.20, max_prefix_cer=0.0, max_suffix_cer=0.0, ) assert evidence.protected_range_count == 1 assert evidence.passed is True assert comparison.cer == 0.0 assert comparison.passed is True @pytest.mark.parametrize( "transcript", ( ( "潮汐预报可查询安全网址是coastwatch.example.com" "路径是tide网址输入完毕" ), ( "潮汐预报可查询安全网址是coastwatch.example.tw" "路径是time网址输入完毕" ), ( "潮汐预报可查询安全网址是coastwatch.example.tw" "路径是tide" ), ( "潮汐预报可查询安全网址是coastwatch.example.tw" "路径是tiXde网址输入完毕" ), ), ) def test_naturalized_url_local_gate_rejects_mixed_asr_mutations(transcript): raw = "潮汐預報可查詢 https://coastwatch.example.tw/tide。" target = normalize_spoken_forms(raw) spec = plan_generation_chunks(raw, target)[0] evidence = canonicalize_asr_network_fragments( transcript, target, spec.network_fragment_proofs, ) assert evidence.protected_range_count == 1 assert evidence.passed is False @pytest.mark.parametrize( "mutate", ( lambda text: text.replace("安全網址是 ", ""), lambda text: text.replace("安全網址是", "網址是"), lambda text: text.replace("coast watch 點 ", ""), lambda text: text.replace(" D", ""), lambda text: text.replace(",網址輸入完畢", ""), lambda text: text.replace("example", "example X"), ), ) def test_naturalized_url_whole_gate_rejects_every_lexical_component_mutation( mutate, ): raw = "潮汐預報可查詢,https://coastwatch.example.tw/tide。" target = normalize_spoken_forms(raw) comparison = compare_asr_text( raw, mutate(target), max_cer=1.0, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert comparison.passed is False @pytest.mark.parametrize( ("mutate", "expected"), ( ( lambda text: text.replace(",路徑是", "路徑是").replace( ",網址輸入完畢", "網址輸入完畢", ), True, ), (lambda text: text.replace("前方甲乙,", "前方甲乙X,"), True), (lambda text: text.replace(",後方丙丁", ",X後方丙丁"), True), (lambda text: text.replace(",安全網址是", ",X安全網址是"), False), (lambda text: text.replace("網址輸入完畢,", "網址輸入完畢X,"), False), (lambda text: text.replace(",安全網址是", "X安全網址是"), False), (lambda text: text.replace("網址輸入完畢,", "網址輸入完畢X"), False), ), ) def test_naturalized_url_boundary_provenance_and_optional_prosodic_commas( mutate, expected, ): raw = ( "前方甲乙,https://coastwatch.example.tw/tide," "後方丙丁。" ) target = normalize_spoken_forms(raw) comparison = compare_asr_text( raw, mutate(target), max_cer=1.0, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is expected def test_naturalized_url_proof_cannot_borrow_plain_duplicate_text(): raw = "https://coastwatch.example.tw/tide" proof = naturalized_url_rendering_proof(raw) assert proof is not None duplicate = "coast watch 點 example 點 T W 路徑是 T I D E" prefix = f"先照字面念{duplicate},再核對," target = f"{prefix}{proof.spoken_text}。" fragment = NetworkFragmentProof( span_index=0, chunk_start=len(prefix), chunk_end=len(prefix) + len(proof.spoken_text), parent_start=0, parent_end=len(proof.spoken_text), full_spoken_proof=proof.spoken_text, raw_identifier=raw, url_rendering_proof=proof, ) transcript = target.replace( proof.spoken_text, proof.spoken_text.replace(" D", ""), ) local = canonicalize_asr_network_fragments( transcript, target, (fragment,), ) whole = compare_asr_text(target, transcript, max_cer=1.0) assert local.passed is False assert whole.network_protected_spans_passed is False assert whole.passed is False def test_url_naturalization_keeps_h06_email_frontend_and_atomic_proof(): raw = "若要更換導覽場次,請寄信到 tour.help@islandmuseum.tw。" assert normalize_spoken_forms(raw) == ( "若要更換導覽場次,請寄信到,tour 點 help 小老鼠 " "island museum 點 T W。" ) specs = plan_generation_chunks(raw, normalize_spoken_forms(raw)) assert tuple(count_speech_units(spec.text) for spec in specs) == (25,) assert len(specs[0].network_fragment_proofs) == 1 proof = specs[0].network_fragment_proofs[0] assert proof.parent_start == 0 assert proof.parent_end == len(proof.full_spoken_proof) assert all( proof.url_rendering_proof is None and not proof.raw_identifier for spec in specs for proof in spec.network_fragment_proofs ) def test_generation_network_planner_binds_duplicate_spoken_proof_to_raw_identifier(): spoken = "H T T P S 冒號 斜線 斜線 example 點 T W" raw = f"先念 {spoken},再查 https://example.tw。" normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert "".join(spec.text for spec in specs) == normalized assert [spec.network_conditioned for spec in specs] == [True] assert specs[0].text.count(spoken) == 2 assert specs[0].network_full_spoken_proofs == (spoken,) proof = specs[0].network_fragment_proofs[0] assert proof.chunk_start == specs[0].text.rindex(spoken) assert specs[0].text[proof.chunk_start : proof.chunk_end] == spoken assert all( normalized[spec.source_start : spec.source_end] == spec.text for spec in specs ) def test_generation_network_planner_is_range_first_for_english_and_long_prose(): cases = ( "Please visit https://example.tw。", "甲" * 35 + "請查 https://c.tw。", ) for raw in cases: normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert "".join(spec.text for spec in specs) == normalized assert all( count_speech_units(spec.text) <= (36 if spec.network_conditioned else 80) for spec in specs ) prose_specs = plan_generation_chunks(cases[1], normalize_spoken_forms(cases[1])) assert any( not spec.network_conditioned and count_speech_units(spec.text) > 36 for spec in prose_specs ) def test_generation_network_planner_splits_long_and_repeated_identifiers_by_proof(): cases = ( "請查 https://example.tw/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p。", "https://a.tw https://a.tw", ) for raw in cases: normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert "".join(spec.text for spec in specs) == normalized assert all(count_speech_units(spec.text) <= 36 for spec in specs) assert all(spec.network_conditioned for spec in specs) repeated = plan_generation_chunks(cases[1], normalize_spoken_forms(cases[1])) assert {index for spec in repeated for index in spec.network_span_indices} == {0, 1} @pytest.mark.parametrize( "raw", ( "a@b.co https://c.de", "https://a.co b@c.de", "a@b.co b@c.de", "https://a.co https://b.de", ), ) def test_generation_network_planner_maps_whitespace_adjacent_identifiers(raw): normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert "".join(spec.text for spec in specs) == normalized assert {index for spec in specs for index in spec.network_span_indices} == { 0, 1, } assert all(spec.network_conditioned for spec in specs) def test_split_preserves_long_strong_sentences_and_is_deterministic(): sentences = ( "甲" * 59 + "。", "乙" * 57 + "。", "丙" * 41 + "。", "丁" * 38 + "。", ) text = "".join(sentences) first = split_text_for_tts(text, max_chars=80, min_chunk_chars=12) second = split_text_for_tts(text, max_chars=80, min_chunk_chars=12) assert first == second == list(sentences) assert [count_speech_units(chunk) for chunk in first] == [59, 57, 41, 38] assert "".join(first) == text def test_structured_short_leading_clause_falls_back_to_legal_hard_split(): text = "折扣為百分之五,會員價新台幣一千元" + "甲" * 68 chunks = split_text_for_tts(text, max_chars=80, min_chunk_chars=12) assert "".join(chunks) == text assert [count_speech_units(chunk) for chunk in chunks] == [72, 12] def test_chunk_coalescing_respects_runtime_budget_without_losing_text(): sentence = "甲乙丙丁戊己庚辛壬癸子丑。" chunks = [sentence] * 21 coalesced = coalesce_text_chunks(chunks, max_chunks=20, max_units=80) assert len(coalesced) == 20 assert "".join(coalesced) == "".join(chunks) assert all(count_speech_units(chunk) <= 80 for chunk in coalesced) def test_split_never_bisects_a_network_identifier(): identifier = "trailweather.example.tw" text = "甲" * 18 + identifier + "乙" * 18 chunks = split_text_for_tts(text, max_chars=24, min_chunk_chars=12) assert "".join(chunks) == text assert sum(identifier in chunk for chunk in chunks) == 1 assert all(12 <= count_speech_units(chunk) <= 24 for chunk in chunks) def test_split_allows_one_genuinely_short_request(): assert split_text_for_tts("好喔。", max_chars=80, min_chunk_chars=12) == ["好喔。"] def test_onset_split_requires_a_natural_boundary(): with_boundary = "這是一段完整的開場,後面還有足夠長度的內容需要繼續合成。" chunks = split_leading_clause(with_boundary, search_chars=24, min_chunk_chars=8) assert "".join(chunks) == with_boundary assert chunks[0].endswith(",") without_boundary = "這是一段沒有中間標點且長度足夠的文字用來測試開頭切段" assert split_leading_clause(without_boundary, search_chars=24, min_chunk_chars=8) == [ without_boundary ] def test_variable_network_boundary_uses_short_fade_without_losing_chunks(): sample_rate = 1_000 chunks = [np.ones(1_000, dtype=np.float32) for _ in range(3)] faded = fade_variable_internal_edges( chunks, sample_rate, fade_ms_by_boundary=[5.0, 80.0], ) joined = join_audio_chunks_variable( faded, pauses=[0, 10], crossfade_samples_by_boundary=[5, 80], pre_faded_edges=True, ) assert joined.shape == (3_005,) assert np.isfinite(joined).all() assert np.max(np.abs(joined)) <= 1.0 assert np.count_nonzero(faded[0]) >= 995 assert np.count_nonzero(faded[1]) >= 915 with pytest.raises(ValueError, match="boundary count"): fade_variable_internal_edges(chunks, sample_rate, [5.0]) with pytest.raises(ValueError, match="boundary counts"): join_audio_chunks_variable(chunks, [0, 0], [5]) def test_variable_audio_boundaries_reject_empty_overlap_and_invalid_widths(): sample_rate = 1_000 empty_middle = [ np.ones(10, dtype=np.float32), np.zeros(0, dtype=np.float32), np.ones(10, dtype=np.float32), ] with pytest.raises(ValueError, match="non-empty"): fade_variable_internal_edges(empty_middle, sample_rate, [5.0, 5.0]) with pytest.raises(ValueError, match="non-empty"): join_audio_chunks_variable(empty_middle, [0, 0], [5, 5]) short_middle = [ np.ones(10, dtype=np.float32), np.ones(6, dtype=np.float32), np.ones(10, dtype=np.float32), ] with pytest.raises(ValueError, match="overlap"): fade_variable_internal_edges(short_middle, sample_rate, [4.0, 4.0]) with pytest.raises(ValueError, match="overlap"): join_audio_chunks_variable(short_middle, [0, 0], [4, 4]) with pytest.raises(ValueError, match="non-negative integers"): join_audio_chunks_variable(short_middle, [-1, 0], [0, 0]) with pytest.raises(ValueError, match="non-negative integers"): join_audio_chunks_variable(short_middle, [0, 0], [1.5, 0]) def test_prefaded_join_does_not_apply_a_second_edge_ramp(): sample_rate = 1_000 chunks = fade_internal_edges( [np.ones(100, dtype=np.float32), np.ones(100, dtype=np.float32)], sample_rate, fade_ms=80.0, ) joined = join_audio_chunks( chunks, pauses=[10], crossfade_samples=80, pre_faded_edges=True, ) # The second chunk begins after the first 100 samples plus the pause. Its # 20th ramp sample must remain linear, rather than being multiplied by an # identical second ramp in the assembler. assert joined[100 + 10 + 20] == pytest.approx(20.0 / 79.0, abs=1.0e-6) def test_prefaded_network_verifier_seam_preserves_exact_zero_silence(): sample_rate = 1_000 chunks = fade_variable_internal_edges( [ np.ones(100, dtype=np.float32), np.ones(100, dtype=np.float32), ], sample_rate, [5.0], ) joined = join_audio_chunks_variable( chunks, pauses=[400], crossfade_samples_by_boundary=[5], pre_faded_edges=True, ) assert joined.shape == (600,) assert np.array_equal(joined[100:500], np.zeros(400, dtype=np.float32)) assert joined[99] == 0.0 assert joined[500] == 0.0 assert joined[98] > 0.0 assert joined[501] > 0.0 def test_duration_units_and_target_pace_min_len(): assert count_speech_units("AI TTS 測試 1234") == 6 assert count_network_endpoint_duration_units("AI TTS 測試 1234") == 7 assert target_cps_min_len( "一二三四五六七八", target_cps=4.0, step_seconds=0.25, base_min_len=2, stop_consecutive=2, ) == 5 assert target_cps_steps("一二三四五六七八", target_cps=4.0, step_seconds=0.25) == 8 assert duration_hard_stop_steps(55, ratio=1.0, margin_steps=0) == 55 assert duration_hard_stop_steps(0, ratio=1.08, margin_steps=3) == 2000 def test_network_endpoint_duration_counter_is_separate_from_public_units(): text = " coastwatch 點 example 點 T W 斜線 tide。" assert count_speech_units(text) == 12 assert count_network_endpoint_duration_units(text) == 17 assert count_speech_units("tour123") == 2 assert count_network_endpoint_duration_units("tour123") == 4 def test_h07_safe_network_endpoint_uses_conservative_units_only_for_cap(): raw = "潮汐預報可查詢 https://coastwatch.example.tw/tide。" normalized = normalize_spoken_forms(raw) specs = plan_generation_chunks(raw, normalized) assert len(specs) == 1 row = specs[0] assert row.network_conditioned assert count_speech_units(row.text) == 35 duration_units = count_network_endpoint_duration_units(row.text) model_text, expected_steps, hard_stop_steps = endpoint_generation_plan( row.text, generation_cps=4.0, step_seconds=0.16, margin_steps=1, duration_units=duration_units, ) assert model_text == normalize_tts_text(row.text) assert duration_units == 39 assert expected_steps == 61 assert hard_stop_steps == 62 # Planner and public pace/work accounting remain on the frozen counter. assert tuple(count_speech_units(spec.text) for spec in specs) == (35,) assert target_pace_speed( 28_000, 10_000, row.text, target_cps=4.0, min_speed=0.80, ) == pytest.approx(0.8) @pytest.mark.parametrize("duration_units", [True, -1, 1.5, "17"]) def test_endpoint_generation_plan_rejects_invalid_explicit_duration_units( duration_units, ): with pytest.raises(ValueError, match="duration_units"): endpoint_generation_plan( "測試", generation_cps=4.0, step_seconds=0.16, duration_units=duration_units, ) def test_terminal_punctuation_supplies_an_endpoint_cue(): assert ensure_terminal_punctuation("內容已經說完") == "內容已經說完。" assert ensure_terminal_punctuation("內容已經說完!") == "內容已經說完!" assert ensure_terminal_punctuation("內容已經說完,") == "內容已經說完。" assert ensure_terminal_punctuation('He said "done"') == 'He said "done."' def test_endpoint_generation_cap_is_independent_of_output_pace(): model_text, expected_steps, hard_stop_steps = endpoint_generation_plan( "這次測試到這裡就完成", generation_cps=5.2, step_seconds=0.16, margin_steps=1, ) output_pace_steps = target_cps_steps(model_text, target_cps=4.0, step_seconds=0.16) assert model_text.endswith("。") assert expected_steps < output_pace_steps assert hard_stop_steps == expected_steps + 1 def test_short_text_can_keep_its_original_conditioning_sequence(): model_text, _, _ = endpoint_generation_plan( "你好", generation_cps=5.2, step_seconds=0.16, add_terminal_punctuation=False, ) assert model_text == "你好" def test_ascii_text_receives_a_wider_generation_window(): assert select_generation_cps("純中文測試", cjk_cps=5.2, ascii_cps=4.6) == 5.2 assert select_generation_cps("AI TTS 測試", cjk_cps=5.2, ascii_cps=4.6) == 4.6 def test_hybrid_network_text_keeps_the_ascii_generation_window(): normalized = normalize_spoken_forms("https://example.tw/path") assert any(character.isascii() and character.isalnum() for character in normalized) assert network_protected_spoken_spans(normalized) assert select_generation_cps(normalized, cjk_cps=5.2, ascii_cps=4.6) == 4.6 def test_short_text_uses_stronger_cfg_without_changing_long_text(): assert effective_generation_cfg("你好", 2.0) == 3.0 assert effective_generation_cfg("測試完成", 2.5) == 3.0 assert effective_generation_cfg("一二三四五六", 2.0) == 3.0 assert effective_generation_cfg("一二三四五六七", 2.0) == 2.0 assert effective_generation_cfg("你好", 3.5) == 3.5 def test_target_pace_speed_slows_completed_audio_without_extending_generation(): text = "一二三四五六七八" assert target_pace_speed(1800, 1000, text, target_cps=4.0) == 0.9 assert target_pace_speed(1000, 1000, text, target_cps=4.0) == 0.8 assert target_pace_speed(2500, 1000, text, target_cps=4.0) == 1.0 def test_active_pace_correction_uses_active_duration_formula(): text = "一二三四五六七八" assert active_pace_correction_speed( 1.8, text, target_cps=4.0, prior_speed=1.0, ) == pytest.approx(0.9) def test_active_pace_correction_never_speeds_up_slow_audio(): assert active_pace_correction_speed( 2.2, "一二三四五六七八", target_cps=4.0, prior_speed=1.0, ) == 1.0 def test_pcm16_output_preserves_production_level_without_peak_normalizing(): waveform = np.array([-0.62, -0.25, 0.0, 0.31, 0.50], dtype=np.float32) sample_rate, pcm = pcm16_audio_output(48_000, waveform) assert sample_rate == 48_000 assert pcm.dtype == np.int16 assert float(np.max(np.abs(pcm.astype(np.float64))) / 32768.0) == pytest.approx( 0.62, abs=1.0 / 32768.0, ) restored = decode_pcm16(pcm) np.testing.assert_allclose(restored, waveform, atol=1.0 / 32768.0) def test_pcm16_verification_roundtrip_is_exhaustively_idempotent(): codes = np.arange(-32768, 32768, dtype=np.int32).astype(np.int16) decoded = decode_pcm16(codes) reencoded = encode_pcm16(decoded) np.testing.assert_array_equal(reencoded, codes) np.testing.assert_array_equal( encode_pcm16(pcm16_verification_waveform(decoded)), codes, ) @pytest.mark.parametrize( ("sample_rate", "waveform"), [ (True, np.array([0.0], dtype=np.float32)), (0, np.array([0.0], dtype=np.float32)), (48_000, np.array([], dtype=np.float32)), (48_000, np.array([[0.0]], dtype=np.float32)), (48_000, np.array([0], dtype=np.int16)), (48_000, np.array([float("nan")], dtype=np.float32)), (48_000, np.array([1.01], dtype=np.float32)), ], ) def test_pcm16_output_rejects_malformed_or_unbounded_audio(sample_rate, waveform): with pytest.raises(ValueError): pcm16_audio_output(sample_rate, waveform) def test_active_pace_correction_respects_combined_stretch_cap(): text = "一二三四五六七八" second_rate = active_pace_correction_speed( 1.0, text, target_cps=4.0, prior_speed=0.9, min_total_speed=0.8, ) assert second_rate == pytest.approx(0.8 / 0.9) assert 0.9 * second_rate == pytest.approx(0.8) assert active_pace_correction_speed( 1.0, text, target_cps=4.0, prior_speed=0.8, min_total_speed=0.8, ) == 1.0 @pytest.mark.parametrize( ("active_duration", "text", "target_cps", "prior_speed", "minimum"), [ (0.0, "完整文字", 4.0, 1.0, 0.8), (-1.0, "完整文字", 4.0, 1.0, 0.8), (float("nan"), "完整文字", 4.0, 1.0, 0.8), (float("inf"), "完整文字", 4.0, 1.0, 0.8), (1.0, "", 4.0, 1.0, 0.8), (1.0, "完整文字", 0.0, 1.0, 0.8), (1.0, "完整文字", 4.0, 0.0, 0.8), (1.0, "完整文字", 4.0, float("nan"), 0.8), (1.0, "完整文字", 4.0, 1.0, 0.0), (1.0, "完整文字", 4.0, 1.0, float("nan")), ], ) def test_active_pace_correction_invalid_evidence_fails_safe( active_duration, text, target_cps, prior_speed, minimum, ): assert active_pace_correction_speed( active_duration, text, target_cps=target_cps, prior_speed=prior_speed, min_total_speed=minimum, ) == 1.0 def test_finish_audio_fades_endpoint_and_appends_silence(): output = finish_audio( np.ones(10, dtype=np.float32), 1000, fade_ms=4, trailing_silence_ms=3, ) assert output.shape == (13,) np.testing.assert_allclose(output[-4:], 0.0) assert output[-5] < output[-6] < output[-7] def test_five_millisecond_finish_fade_preserves_the_preceding_tail_and_pad(): output = finish_audio( np.ones(1_000, dtype=np.float32), 1_000, fade_ms=5, trailing_silence_ms=180, ) assert output.shape == (1_180,) np.testing.assert_allclose(output[:996], 1.0) assert output[999] == 0.0 np.testing.assert_allclose(output[1_000:], 0.0) def test_spoken_form_normalizer_expands_common_zh_tw_forms(): text = "日期 2026/07/15,成長 12.5%,距離 10 km,顯卡 RTX 5090。" assert normalize_spoken_forms(text) == ( "日期 二零二六年七月十五日,成長 百分之十二點五," "距離 十公里,顯卡 R T X 五零九零。" ) assert normalize_spoken_forms("2026年7月16日,RTX5090") == ( "二零二六年七月十六日,R T X 五零九零" ) def test_spoken_form_normalizer_expands_clock_time_and_spells_urls(): assert normalize_spoken_forms("時間是 15:30,午夜是 00:00。") == ( "時間是 十五點三十分,午夜是 零點整。" ) assert normalize_spoken_forms("無效時間 25:30") == "無效時間 25:30" assert normalize_spoken_forms("ASR 寫成 15點30分。") == ( "A S R 寫成 15點30分。" ) assert normalize_spoken_forms("簡體 15点30分。") == "簡體 15点30分。" assert normalize_spoken_forms("整點 15時00分。") == "整點 15時00分。" assert normalize_spoken_forms("缺少單位 15點30。") == "缺少單位 15點30。" assert normalize_spoken_forms("不完整 15點。") == "不完整 15點。" assert normalize_spoken_forms("無效 24點30分、15點60分。") == ( "無效 24點30分、15點60分。" ) assert normalize_spoken_forms("IP 192.168.1.1,網址 https://example.test:30/path") == ( "I P 192.168.1.1,網址,H T T P S 冒號 斜線 斜線 " "example 點 test 冒號 三零 斜線 path" ) assert normalize_spoken_forms( "音量15點05分貝,區間15點30分鐘,比分15點05分。" ) == "音量15點05分貝,區間15點30分鐘,比分15點05分。" def test_asr_comparison_unifies_traditional_and_simplified_chinese(): comparison = compare_asr_text("今天天氣真好。", "今天天气真好") assert comparison.target_text == comparison.transcript_text == "今天天气真好" assert comparison.cer == 0.0 def test_asr_comparison_canonicalizes_target_proven_bare_clock_time_only(): comparison = compare_asr_text("會議在十五點三十分開始。", "會議在15點30開始。") assert comparison.cer == 0.0 assert comparison.passed explicit = compare_asr_text("會議在十五點三十分開始。", "會議在15點30分開始。") assert explicit.cer == 0.0 assert explicit.passed colon_bare = compare_asr_text("會議在15:30開始。", "會議在15點30開始。") assert colon_bare.cer == 0.0 assert colon_bare.passed colon_explicit = compare_asr_text("會議在15:30開始。", "會議在15點30分開始。") assert colon_explicit.cer == 0.0 assert colon_explicit.passed for ambiguous in ("延遲15點30秒。", "重量15點30公斤。", "比分15點30。"): comparison = compare_asr_text(ambiguous, ambiguous) assert comparison.cer == 0.0 assert comparison.target_text == comparison.transcript_text simplified = compare_asr_text( "会议在十五点三十分开始。", "会议在15点30开始。", locale="zh-CN", ) assert simplified.cer == 0.0 assert simplified.passed assert comparison.passed is True assert normalize_spoken_forms("溫度 -3.5 °C,頻率 3.2 GHz。") == ( "溫度 負三點五攝氏度,頻率 三點二吉赫茲。" ) def test_breeze_asr_spaced_date_and_dot_clock_are_exact_target_proven_forms(): target = "維護時段是二零二六年八月九日晚上七點四十五分。" transcript = "維護時段是 2026 年 8 月 9 日晚上 7.45 分" comparison = compare_asr_text(target, transcript) assert comparison.passed assert comparison.cer == 0.0 assert comparison.prefix_cer == 0.0 assert comparison.suffix_cer == 0.0 assert comparison.extra_tail_units == 0 def test_breeze_asr_target_proven_dot_clock_may_omit_written_minute_unit(): target = "海龜調查預定在二零三三年四月二十七日凌晨四點五十五分出發。" transcript = "海龜調查預定在 2033 年 4 月 27 日凌晨 4.55 出發" comparison = compare_asr_text(target, transcript) assert comparison.passed assert comparison.cer == 0.0 assert comparison.prefix_cer == 0.0 assert comparison.suffix_cer == 0.0 @pytest.mark.parametrize( "transcript", ( "維護時段是 2026 年 8 月 8 日晚上 7.45 分", "維護時段是 2026 年 8 月 9 日晚上 7.44 分", "維護時段是 2027 年 8 月 9 日晚上 7.45 分", ), ) def test_breeze_asr_structured_date_clock_forms_keep_wrong_values_visible( transcript, ): target = "維護時段是二零二六年八月九日晚上七點四十五分。" comparison = compare_asr_text(target, transcript) assert comparison.edit_distance > 0 assert comparison.target_text != comparison.transcript_text def test_breeze_asr_lowercase_compact_model_code_requires_exact_target_proof(): target = "請先安裝韌體 v3.7.2,再核對模組代碼 AX-418。" transcript = "請先安裝韌體版本 3.7.2再核對模組代碼 ax418" comparison = compare_asr_text(target, transcript) assert comparison.passed assert comparison.cer == 0.0 assert comparison.prefix_cer == 0.0 assert comparison.suffix_cer == 0.0 @pytest.mark.parametrize( "transcript", ( "請先安裝韌體版本 3.7.2再核對模組代碼 ax419", "請先安裝韌體版本 3.7.2再核對模組代碼 bx418", "請先安裝韌體版本 3.7.2再核對模組代碼 ax4188", ), ) def test_breeze_asr_model_code_proof_rejects_wrong_or_extra_atoms(transcript): target = "請先安裝韌體 v3.7.2,再核對模組代碼 AX-418。" assert not compare_asr_text(target, transcript).passed def test_asr_numeric_runs_require_exact_target_proven_cardinal_values(): target = "原價是 NT$1,250,折扣後是 NT$999。" transcript = "原價是新台幣1250,折扣後是新台幣999。" assert normalize_asr_spoken_forms(transcript, target) == ( "原價是新台幣一千二百五十,折扣後是新台幣九百九十九。" ) assert compare_asr_text(target, transcript).passed wrong_value = "原價是新台幣1250,折扣後是新台幣996。" normalized_wrong = normalize_asr_spoken_forms(wrong_value, target) assert "新台幣996" in normalized_wrong assert "九百九十九" not in normalized_wrong assert not compare_asr_text(target, wrong_value).passed @pytest.mark.parametrize( "target", ( "租借望遠鏡另收 JPY 4,500。", "租借望遠鏡另收 日圓四千五百。", ), ) def test_asr_jpy_alias_and_amount_require_one_exact_target_proof(target): transcript = "租借望遠鏡另收日元4500" assert normalize_asr_spoken_forms(transcript, target) == ( "租借望遠鏡另收日圓四千五百" ) comparison = compare_asr_text(target, transcript) assert comparison.passed assert comparison.cer == 0.0 assert comparison.prefix_cer == 0.0 assert comparison.suffix_cer == 0.0 assert comparison.extra_tail_units == 0 def test_asr_jpy_alias_handles_exact_decimal_amounts(): target = "加購費用是 JPY 4,500.50。" transcript = "加購費用是日元4500.50。" assert normalize_asr_spoken_forms(transcript, target) == ( "加購費用是日圓四千五百點五零。" ) assert compare_asr_text(target, transcript).passed def test_asr_jpy_alias_is_symmetric_under_an_exact_explicit_target(): target = "租借望遠鏡另收日元四千五百。" transcript = "租借望遠鏡另收日圓4500。" assert normalize_asr_spoken_forms(transcript, target) == target assert compare_asr_text(target, transcript).passed @pytest.mark.parametrize( "transcript", ( "租借望遠鏡另收日圓4500元。", "租借望遠鏡另收日元4500元。", "租借望遠鏡另收日元四千五百元。", ), ) def test_asr_jpy_redundant_unit_requires_one_exact_target_proof(transcript): target = "租借望遠鏡另收 JPY 4,500。" assert normalize_asr_spoken_forms( transcript, target, ) == "租借望遠鏡另收日圓四千五百。" comparison = compare_asr_text(target, transcript) assert comparison.passed assert comparison.cer == 0.0 assert comparison.prefix_cer == 0.0 assert comparison.suffix_cer == 0.0 assert comparison.extra_tail_units == 0 @pytest.mark.parametrize( "transcript", ( "租借望遠鏡另收日元4000。", "租借望遠鏡另收日元四千。", "租借望遠鏡另收日元4000元。", "租借望遠鏡另收日元四千元。", "租借望遠鏡另收日元4500元整。", "租借望遠鏡另收日元4500元元。", "租借望遠鏡另收美元4500。", "租借望遠鏡另收美元4500元。", ), ) def test_asr_jpy_alias_rejects_wrong_truncated_extra_and_other_currency( transcript, ): target = "租借望遠鏡另收 JPY 4,500。" assert normalize_asr_spoken_forms(transcript, target) != ( normalize_spoken_forms(target) ) assert not compare_asr_text(target, transcript).passed def test_asr_jpy_alias_cannot_borrow_an_amount_from_another_range(): target = "第一筆 JPY 4,500,第二筆 JPY 4,000。" exact = "第一筆日元4500,第二筆日元4000。" swapped = "第一筆日元4000,第二筆日元4500。" duplicated = "第一筆日元4000,第二筆日元4000。" assert normalize_asr_spoken_forms(exact, target) == ( "第一筆日圓四千五百,第二筆日圓四千。" ) assert compare_asr_text(target, exact).passed for transcript in (swapped, duplicated): assert normalize_asr_spoken_forms( transcript, target, ) != normalize_spoken_forms(target) assert not compare_asr_text(target, transcript).passed def test_asr_jpy_redundant_unit_rejects_ambiguous_duplicate_proofs(): target = "甲 JPY 4,500 乙;甲 JPY 4,500 乙。" transcript = "甲日元4500元乙;甲日圓4500元乙。" assert normalize_asr_spoken_forms( transcript, target, ) != normalize_spoken_forms(target) assert not compare_asr_text(target, transcript).passed @pytest.mark.parametrize( "transcript", ( "丙日元4500元乙。", "甲日元4500元丙。", ), ) def test_asr_jpy_redundant_unit_requires_matching_semantic_neighbors( transcript, ): target = "甲 JPY 4,500 乙。" assert normalize_asr_spoken_forms( transcript, target, ) != normalize_spoken_forms(target) assert not compare_asr_text(target, transcript).passed def test_asr_jpy_alias_requires_currency_proof_and_is_not_global_homophone(): assert normalize_asr_spoken_forms( "費用日元4500元。", "費用四千五百。", ) == "費用日元四千五百元。" assert not compare_asr_text("費用四千五百。", "費用日元4500元。").passed assert normalize_asr_spoken_forms( "這個元形很完整。", "這個圓形很完整。", ) == "這個元形很完整。" assert not compare_asr_text("這個圓形很完整。", "這個元形很完整。").passed def test_v3f13_currency_transcript_passes_without_relaxing_semantic_gates(): target = "兩張船票合計 NT$2,680,租借望遠鏡另收 JPY 4,500。" transcript = "兩張船票合計新台幣2680租借望遠鏡另收日元4500" comparison = compare_asr_text(target, transcript) assert comparison.passed assert comparison.cer == 0.0 assert comparison.prefix_cer == 0.0 assert comparison.suffix_cer == 0.0 assert comparison.extra_tail_units == 0 def test_asr_numeric_runs_choose_context_matched_cardinal_or_digit_reading(): target = "序號一二五零,售價一千二百五十。" transcript = "序號1250,售價1250。" assert normalize_asr_spoken_forms(transcript, target) == target assert compare_asr_text(target, transcript).passed wrong_sequence = "序號1251,售價1250。" assert normalize_asr_spoken_forms(wrong_sequence, target) == ( "序號1251,售價一千二百五十。" ) assert not compare_asr_text(target, wrong_sequence).passed def test_asr_numeric_runs_cannot_borrow_an_exact_value_from_another_context(): target = "售價九百九十九,箱號九百九十六。" transcript = "售價996,箱號996。" assert normalize_asr_spoken_forms(transcript, target) == ( "售價996,箱號九百九十六。" ) assert not compare_asr_text(target, transcript).passed @pytest.mark.parametrize( ("target", "transcript"), [ ("型號 RTX 1250。", "數量1250。"), ("會議在十五點三十分開始。", "數量15。"), ("版本 v1.2.3 已發布。", "批次1已發布。"), ("日期是 2026/07/15。", "數量2026。"), ], ) def test_asr_numeric_runs_do_not_borrow_model_clock_version_or_date_proofs( target, transcript, ): assert normalize_asr_spoken_forms(transcript, target) == transcript assert not compare_asr_text(target, transcript).passed def test_structured_numeric_frontends_still_canonicalize_in_their_own_domains(): for target in ( "型號 RTX 1250。", "會議在 15:30 開始。", "版本 v1.2.3 已發布。", "日期是 2026/07/15。", ): assert compare_asr_text(target, target).passed @pytest.mark.parametrize( ("target", "incorrect_clock_reading"), [ ("音量15點05分貝。", "音量十五點五分貝。"), ("區間15點30分鐘。", "區間十五點三十分鐘。"), ("比分15點05分。", "比分十五點五分。"), ( "會議在十五點三十分開始,區間15點30分鐘。", "會議在十五點三十分開始,區間十五點三十分鐘。", ), ], ) def test_asr_comparison_does_not_let_ambiguous_numeric_target_prove_clock_time( target, incorrect_clock_reading, ): exact_numeric = compare_asr_text(target, target) assert exact_numeric.cer == 0.0 assert exact_numeric.passed miscanonicalized = compare_asr_text(target, incorrect_clock_reading) assert miscanonicalized.cer > 0.0 assert not miscanonicalized.passed def test_spoken_form_normalizer_canonicalizes_zh_measures_for_asr_comparison(): assert normalize_spoken_forms("成長 12.5%,距離 10公里,重量 1.5公斤。") == ( "成長 百分之十二點五,距離 十公里,重量 一點五公斤。" ) comparison = compare_asr_text("距離十公里", "距離10公里") assert comparison.passed assert comparison.edit_distance == 0 @pytest.mark.parametrize( ("target", "transcript"), [ ("取樣率為 48 kHz。", "取樣率為48000Hz。"), ("頻率為 48000 Hz。", "頻率為48kHz。"), ("頻率為 0.048 MHz。", "頻率為48kHz。"), ("頻率為 0.000048 GHz。", "頻率為48000Hz。"), ("精確頻率為 0.1 MHz。", "精確頻率為100kHz。"), ("頻率為48千赫茲。", "頻率為48000赫茲。"), ( "精確頻率為 0.123456789012345678901234567890123 MHz。", "精確頻率為123456.789012345678901234567890123Hz。", ), ], ) def test_asr_frequency_scales_require_exact_target_proven_base_value( target, transcript, ): # The frontend keeps the target's requested scale. Only ASR comparison # adopts that reading after exact Decimal conversion to base Hz. expected = normalize_spoken_forms(target) assert normalize_asr_spoken_forms(transcript, target).replace(" ", "") == ( expected.replace(" ", "") ) comparison = compare_asr_text(target, transcript) assert comparison.target_text == comparison.transcript_text assert comparison.edit_distance == 0 assert comparison.passed def test_asr_frequency_scale_canonicalization_handles_multiple_measurements(): target = "範圍48 kHz到96 kHz,備援為0.048 MHz。" transcript = "範圍48000Hz到96000Hz,備援為48000Hz。" assert normalize_asr_spoken_forms(transcript, target) == ( normalize_spoken_forms(target) ) assert compare_asr_text(target, transcript).passed one_wrong = "範圍48000Hz到95000Hz,備援為48000Hz。" normalized_wrong = normalize_asr_spoken_forms(one_wrong, target) assert "範圍四十八千赫茲到九萬五千赫茲" in normalized_wrong assert "備援為零點零四八兆赫茲" in normalized_wrong wrong_comparison = compare_asr_text(target, one_wrong) assert wrong_comparison.edit_distance > 0 assert wrong_comparison.target_text != wrong_comparison.transcript_text swapped = "範圍96000Hz到48000Hz,備援為48000Hz。" assert not compare_asr_text(target, swapped).passed decimal_first_target = "範圍0.048 MHz到96 kHz。" decimal_first_asr = "範圍48000Hz到96000Hz。" assert compare_asr_text(decimal_first_target, decimal_first_asr).passed def test_asr_frequency_scale_proof_normalizes_remaining_sentence_context(): target = "測試頻率是 48 kHz,移動速度是 10 km/h。" transcript = "測試頻率是48000Hz移動速度是10公里每小時" assert normalize_asr_spoken_forms(transcript, target) == ( "測試頻率是四十八千赫茲移動速度是十公里每小時" ) comparison = compare_asr_text(target, transcript) assert comparison.target_text == comparison.transcript_text assert comparison.edit_distance == 0 assert comparison.passed def test_asr_frequency_scale_proof_survives_app_normalized_target(): target = normalize_spoken_forms( "測試頻率是 48 kHz,移動速度是 10 km/h。" ) transcript = "測試頻率是48000Hz移動速度是10公里每小時" assert normalize_asr_spoken_forms(transcript, target) == ( "測試頻率是四十八千赫茲移動速度是十公里每小時" ) assert compare_asr_text(target, transcript).passed wrong_value = transcript.replace("48000Hz", "49000Hz") normalized_wrong = normalize_asr_spoken_forms(wrong_value, target) assert "四萬九千赫茲" in normalized_wrong assert "四十八千赫茲" not in normalized_wrong wrong_comparison = compare_asr_text(target, wrong_value) assert wrong_comparison.target_text != wrong_comparison.transcript_text assert wrong_comparison.edit_distance > 0 def test_asr_frequency_scale_proof_accepts_simplified_whisper_unit(): target = normalize_spoken_forms( "測試頻率是 48 kHz,移動速度是 10 km/h。" ) transcript = "测试频率是48000赫兹移动速度是10公里每小时" comparison = compare_asr_text(target, transcript) assert comparison.passed assert comparison.cer == 0.0 assert not compare_asr_text( target, transcript.replace("48000赫兹", "49000赫兹"), ).passed @pytest.mark.parametrize( ("target", "transcript"), [ ("頻率48 Hz。", "频率48赫兹。"), ("頻率48 kHz。", "频率48千赫兹。"), ("頻率48 MHz。", "频率48兆赫兹。"), ("頻率48 GHz。", "频率48吉赫兹。"), ("頻率四十八千赫兹。", "频率48000Hz。"), ], ) def test_asr_frequency_scale_proof_accepts_simplified_frequency_units( target, transcript, ): assert compare_asr_text(target, transcript).passed def test_simplified_frequency_units_keep_scale_and_ambiguity_guards(): assert not compare_asr_text("頻率48 kHz。", "频率48兆赫兹。").passed assert not compare_asr_text("频率一兆赫兹。", "频率1MHz。").passed assert not compare_asr_text( "型号 rtx 四十八千赫兹。", "型号 rtx 48000Hz。", ).passed @pytest.mark.parametrize( ("raw_target", "transcript"), [ ("頻率為 48000 Hz。", "頻率為48kHz。"), ("頻率為 0.048 MHz。", "頻率為48000Hz。"), ("頻率為 0.000048 GHz。", "頻率為48kHz。"), ], ) def test_asr_frequency_scale_proof_parses_canonical_cardinal_and_decimal_target( raw_target, transcript, ): normalized_target = normalize_spoken_forms(raw_target) assert compare_asr_text(normalized_target, transcript).passed def test_normalized_frequency_proofs_keep_multiple_and_structured_guards(): target = normalize_spoken_forms("範圍48 kHz到96 kHz。") assert compare_asr_text(target, "範圍48000Hz到96000Hz。").passed assert not compare_asr_text(target, "範圍96000Hz到48000Hz。").passed one_wrong = compare_asr_text(target, "範圍48000Hz到95000Hz。") assert one_wrong.target_text != one_wrong.transcript_text assert one_wrong.edit_distance > 0 model_target = normalize_spoken_forms("型號 RTX 48 kHz。") assert not compare_asr_text(model_target, "型號 R T X 48000Hz。").passed date_target = normalize_spoken_forms("頻率48 kHz,日期2026/07/15。") assert not compare_asr_text( date_target, "頻率48kHz,日期48000Hz。", ).passed @pytest.mark.parametrize( ("raw_target", "wrong_transcript"), [ ("頻率1000000000000 Hz。", "頻率1MHz。"), ("頻率10000000000000 Hz。", "頻率10MHz。"), ("頻率1000000000000000 Hz。", "頻率1GHz。"), ], ) def test_normalized_frequency_target_ambiguity_fails_closed( raw_target, wrong_transcript, ): normalized_target = normalize_spoken_forms(raw_target) assert not compare_asr_text(normalized_target, normalized_target).passed assert not compare_asr_text(normalized_target, wrong_transcript).passed @pytest.mark.parametrize("multiplier", [1, 7, 48, 999, 1000, 9999]) def test_raw_frequency_target_rejects_generic_megahertz_collision(multiplier): target = f"頻率{multiplier * 10**12} Hz。" wrong_transcript = f"頻率{multiplier} MHz。" assert not compare_asr_text(target, wrong_transcript).passed assert not compare_asr_text(wrong_transcript, target).passed @pytest.mark.parametrize("separator", ["/", "\\", ":", "_", "@", "#"]) def test_raw_frequency_contradiction_cannot_bypass_proof_boundaries(separator): target = "頻率1000000000000 Hz。" wrong_transcript = f"頻率{separator}1MHz。" assert not compare_asr_text(target, wrong_transcript).passed def test_raw_frequency_provenance_accepts_matching_ambiguous_spoken_form(): assert compare_asr_text("頻率1 MHz。", "頻率一兆赫茲。").passed assert compare_asr_text("頻率1000000000000 Hz。", "頻率一兆赫茲。").passed @pytest.mark.parametrize( "target", [ "型號 x 四十八千赫茲。", "型號 rtx 四十八千赫茲。", "型號 rtx5090 四十八千赫茲。", "型號 R T X 五零九零 四十八千赫茲。", "版本 model 四十八千赫茲。", "型號 R T X 四十八千赫茲。", "代碼A四十八千赫茲。", "版本1.四十八千赫茲。", r"路徑\四十八千赫茲。", ], ) def test_normalized_frequency_model_and_ascii_boundaries_do_not_prove_scale( target, ): assert not compare_asr_text(target, target.replace("四十八千赫茲", "48000Hz")).passed @pytest.mark.parametrize( ("target", "transcript"), [ ("頻率為49 kHz。", "頻率為48000Hz。"), ("功率為48 kW。", "功率為48000Hz。"), ("型號為48 kHz。", "頻率為48000Hz。"), ( "頻率48 kHz,日期2026/07/15。", "頻率48 kHz,日期48000Hz。", ), ("型號 RTX 48 kHz。", "型號 RTX 48000 Hz。"), ( "網址 https://example.test/48kHz。", "網址 https://example.test/48000Hz。", ), ], ) def test_asr_frequency_proofs_do_not_cross_value_dimension_or_context( target, transcript, ): assert normalize_asr_spoken_forms(transcript, target) != ( normalize_spoken_forms(target) ) assert not compare_asr_text(target, transcript).passed def test_asr_frequency_scale_proof_is_zh_only_and_does_not_change_frontend(): target = "Sample rate 48 kHz." transcript = "Sample rate 48000Hz." assert normalize_asr_spoken_forms( transcript, target, locale="en-US", ) == transcript assert normalize_spoken_forms("取樣率為48 kHz。") == ( "取樣率為四十八千赫茲。" ) def test_spoken_form_normalizer_expands_milliseconds_before_meter_prefixes(): assert normalize_spoken_forms( "延遲 100 ms,速度 8 m/s,距離 3 m。" ) == "延遲 一百毫秒,速度 八公尺每秒,距離 三公尺。" assert normalize_spoken_forms("等待100毫秒後再試。") == ( "等待一百毫秒後再試。" ) def test_millisecond_normalizer_does_not_consume_longer_unknown_m_words(): assert normalize_spoken_forms("延遲 100 msec,變數 msValue 與 ms。") == ( "延遲 100 msec,變數 msValue 與 ms。" ) def test_spoken_form_normalizer_expands_network_identifiers_and_versions(): assert contains_network_identifier("請看 https://example.tw/path") assert contains_network_identifier("寄到 user@example.tw") assert not contains_network_identifier("一般英文 example text") assert normalize_spoken_forms( "網址 https://api.example.com/v1/items?q=RTX-5090&n=2。" ) == ( "網址,H T T P S 冒號 斜線 斜線 api 點 example 點 C O M " "斜線 v 一 斜線 items 問號 q 等於 R T X 橫線 五零九零 " "和 n 等於 二。" ) assert normalize_spoken_forms("信箱 USER.name+tts@example.com。") == ( "信箱,U S E R 點 name 加號 tts 小老鼠 example 點 C O M。" ) assert normalize_spoken_forms( "版本 v1.2.3,候選 version 10.4.0-beta.1+build.5。" ) == ( "版本一點二點三,候選 版本十點四點零 預發布 B E T A 點 一 " "建置 B U I L D 點 五。" ) @pytest.mark.parametrize( ("target", "mutation"), ( ("a!b@example.tw", "ab@example.tw"), ("a$b@example.tw", "ab@example.tw"), ("a^b@example.tw", "ab@example.tw"), ("https://example.tw/a;b", "https://example.tw/ab"), ("https://example.tw/a'b", "https://example.tw/ab"), (r"https://example.tw/a\b", "https://example.tw/ab"), ("https://example.tw/a§b", "https://example.tw/ab"), ("https://example.tw/a/b", "https://example.tw/ab"), ), ) def test_network_protected_span_never_drops_supported_punctuation(target, mutation): comparison = compare_asr_text( normalize_spoken_forms(target), normalize_spoken_forms(mutation), max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed @pytest.mark.parametrize( "symbol", ("$", "!", "?", "=", "&", "#", "_", "+", "~", "^", "§", "/"), ) def test_expanded_url_rejects_raw_symbol_inserted_inside_path(symbol): target = normalize_spoken_forms("https://example.tw/path") transcript = target.replace( _SPOKEN_PATH, f"pa{symbol}th", ) comparison = compare_asr_text( target, transcript, max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed @pytest.mark.parametrize("symbol", ("$", "!", "?", "_", "+", "§")) def test_expanded_email_rejects_raw_symbol_inserted_inside_local_part(symbol): target = normalize_spoken_forms("museum@example.tw") transcript = target.replace( "museum", f"mu{symbol}seum", ) comparison = compare_asr_text( target, transcript, max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed @pytest.mark.parametrize( "transcript", ( "H T T P S 冒號 斜線 斜線 example 點 T W 斜線 a ? x = 2 & n = 3 # k", "H T T P S 冒號 斜線 斜線 example 點 T W 斜線 a?x=二&n=三#k", ), ) def test_network_query_accepts_equivalent_mixed_asr_renderings(transcript): target = normalize_spoken_forms("https://example.tw/a?x=2&n=3#k") comparison = compare_asr_text(target, transcript, max_cer=0.20) assert comparison.cer == 0.0 assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed @pytest.mark.parametrize( "transcript", ( "H T T P S 冒號 斜線 斜線 example 點 T W 斜線 a ? x = 9 & n = 3 # k", "H T T P S 冒號 斜線 斜線 example 點 T W 斜線 a ? x = 2 n = 3 # k", "H T T P S 冒號 斜線 斜線 example 點 C O M 斜線 a ? x = 2 & n = 3 # k", ), ) def test_network_query_mixed_asr_renderings_still_reject_wrong_content(transcript): target = normalize_spoken_forms("https://example.tw/a?x=2&n=3#k") comparison = compare_asr_text( target, transcript, max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed @pytest.mark.parametrize("symbol", tuple(".:?!,;")) def test_ambiguous_trailing_url_punctuation_fails_closed(symbol): with pytest.raises(ValueError, match="ambiguous punctuation"): normalize_spoken_forms(f"https://example.tw/a{symbol},回覆。") def test_asr_only_sentence_punctuation_after_url_is_target_proven(): sentence_punctuation = compare_asr_text( "請看 https://example.tw/path,回覆。", "請看 https://example.tw/path. 回覆。", max_cer=0.20, ) assert sentence_punctuation.network_protected_spans_passed is True assert sentence_punctuation.passed @pytest.mark.parametrize( "render", ("spoken", "spoken_without_commas", "raw"), ) def test_naturalized_url_whole_asr_exact_forms_are_target_proven(render): raw = "潮汐預報可查詢 https://coastwatch.example.tw/tide。" target = normalize_spoken_forms(raw) if render == "spoken": transcript = target elif render == "spoken_without_commas": transcript = target.replace(",路徑是", "路徑是").replace( ",網址輸入完畢", "網址輸入完畢", ) else: transcript = raw comparison = compare_asr_text(raw, transcript, max_cer=0.20) assert comparison.cer == 0.0 assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed @pytest.mark.parametrize( "transcript", ( "潮汐預報可查詢 H T T P S://coastwatch.example.com/tide。", "潮汐預報可查詢 H T T P S://coastwatch.example.tw/tides。", ), ) def test_network_asr_mixed_literal_forms_still_reject_wrong_content(transcript): target = normalize_spoken_forms( "潮汐預報可查詢 https://coastwatch.example.tw/tide。" ) comparison = compare_asr_text( target, transcript, max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed @pytest.mark.parametrize( "boundary", tuple( "、:()【】「」『』《》〈〉”’" "…⋯—–.。、〔〕〖〗〘〙〚〛﹁﹂"'" "︰﹐﹑﹒﹔﹕﹖﹗" ), ) def test_chinese_punctuation_terminates_url_protected_span(boundary): target = normalize_spoken_forms( f"前文 https://example.tw/path{boundary}後方文字保持完整。" ) comparison = compare_asr_text( target, target.replace("後方", "後芳"), max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.cer <= 0.20 assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed def test_quoted_email_local_part_is_explicitly_unsupported(): for text in ( '請寄到 "john..doe"@example.tw。', '請寄到 ""@example.tw。', '請寄到 "john"@[127.0.0.1]。', ): with pytest.raises(ValueError, match="quoted email local parts"): normalize_spoken_forms(text) def test_expanded_www_url_keeps_an_exact_protected_span(): target = normalize_spoken_forms( "前方說明內容保持不變,www.example.tw/path,後方文字也保持完整。" ) comparison = compare_asr_text( target, target.replace(_SPOKEN_PATH, "pat"), max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.cer <= 0.20 assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed @pytest.mark.parametrize("character", ("漢", "é", "β", "Ж", "١")) def test_expanded_iri_alphanumeric_units_remain_protected(character): target = normalize_spoken_forms( f"前方說明內容保持不變,https://example.tw/a{character}b,後方文字也保持完整。" ) comparison = compare_asr_text( target, target.replace(character, ""), max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.cer <= 0.20 assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is False assert not comparison.passed @pytest.mark.parametrize( "target", ( "https://example.tw/path and reply", "contact a@example.tw then reply", ), ) def test_expanded_network_boundary_does_not_capture_surrounding_english(target): normalized = normalize_spoken_forms(target) comparison = compare_asr_text( normalized, normalized.replace("reply", "replay"), max_cer=0.20, max_prefix_cer=1.0, max_suffix_cer=1.0, ) assert comparison.cer <= 0.20 assert comparison.network_protected_spans == 1 assert comparison.network_protected_spans_passed is True assert comparison.passed def test_spoken_form_normalizer_expands_currency_and_preserves_ip_and_invalid_date(): assert normalize_spoken_forms("售價 NT$1,299.50、USD 12.50、100 JPY。") == ( "售價 新台幣一千二百九十九點五零、美元十二點五零、日圓一百。" ) assert normalize_spoken_forms("IP 192.168.1.1") == "I P 192.168.1.1" assert normalize_spoken_forms("無效日期 2026年2月30日") == ( "無效日期 2026年2月30日" ) def test_spoken_form_normalizer_is_conservative_and_locale_aware(): text = "Use OpenAI for stable English on 2026/07/15." assert normalize_spoken_forms(text, locale="en-US") == text assert normalize_spoken_forms("OpenAI stable diffusion") == "OpenAI stable diffusion" assert normalize_spoken_forms("日期 2026/02/30") == "日期 2026/02/30" with np.testing.assert_raises(ValueError): normalize_spoken_forms("10 km", locale="fr-FR") def test_asr_comparison_accepts_normalized_semantic_match(): comparison = compare_asr_text( "成長 12.5%,使用 RTX 5090。", "成長百分之十二點五使用 R T X 五零九零", ) assert comparison.passed assert comparison.edit_distance == 0 assert comparison.extra_tail == "" def test_asr_comparison_rejects_prefix_suffix_and_extra_tail_errors(): missing_prefix = compare_asr_text("第一個字不能遺漏", "一個字不能遺漏") assert not missing_prefix.passed assert missing_prefix.prefix_deletions == 1 assert missing_prefix.prefix_cer > 0.0 missing_suffix = compare_asr_text("句尾內容必須完整", "句尾內容必須") assert not missing_suffix.passed assert missing_suffix.suffix_deletions == 2 assert missing_suffix.suffix_cer > 0.0 extra_tail = compare_asr_text("內容完成", "內容完成多講兩字") assert not extra_tail.passed assert extra_tail.extra_tail == "多讲两字" assert extra_tail.extra_tail_units == 4 def test_asr_comparison_fails_closed_for_empty_or_nonfinite_limits(): assert not compare_asr_text("", "任意輸出").passed assert not compare_asr_text("目標內容", "", max_cer=1.0).passed assert not compare_asr_text("目標內容", "目標內容", max_cer=math.nan).passed assert not compare_asr_text("目標內容", "目標內容", prefix_units=0).passed def test_candidate_scores_are_finite_safe_and_gate_unsafe_audio(): score = candidate_local_score( cer=0.02, speaker_similarity=0.8, boundary_speaker_drop=0.01, prefix_cer=0.0, suffix_cer=0.0, pace_penalty=0.03, style_penalty=0.02, max_cer=0.10, min_speaker_similarity=0.5, max_boundary_speaker_drop=0.03, ) assert math.isfinite(score) assert math.isclose(score, 0.081) base = dict(cer=0.02, speaker_similarity=0.8, boundary_speaker_drop=0.01) assert math.isinf(candidate_local_score(**base, asr_passed=False)) assert math.isinf(candidate_local_score(**base, truncated=True)) assert math.isinf(candidate_local_score(**base, extra_tail=True)) assert math.isinf(candidate_local_score(**{**base, "cer": math.nan})) assert math.isinf(candidate_local_score(**{**base, "boundary_speaker_drop": -0.01})) assert math.isinf(candidate_local_score(**base, max_cer=math.nan)) def test_transition_score_rejects_nan_and_invalid_similarity(): assert candidate_transition_score( speaker_similarity=0.9, f0_delta=0.1, rms_delta=0.2, ) == 0.4 assert math.isinf( candidate_transition_score( speaker_similarity=math.nan, f0_delta=0.1, rms_delta=0.2, ) ) assert math.isinf( candidate_transition_score( speaker_similarity=1.1, f0_delta=0.1, rms_delta=0.2, ) ) def test_sequence_selector_finds_stable_global_path(): selection = select_candidate_sequence( local_scores=[[0.0, 0.4], [0.0, 0.1], [0.2, 0.0]], transition_scores=[ [[2.0, 0.0], [0.0, 2.0]], [[2.0, 0.0], [0.0, 2.0]], ], ) assert selection is not None assert selection.candidate_indices == (0, 1, 0) assert math.isclose(selection.total_score, 0.3) def test_sequence_selector_is_deterministic_and_fail_closed(): tied = select_candidate_sequence([[0.0, 0.0]]) assert tied is not None assert tied.candidate_indices == (0,) skips_nan = select_candidate_sequence([[math.nan, 0.2]]) assert skips_nan is not None assert skips_nan.candidate_indices == (1,) assert select_candidate_sequence([[math.nan], [0.0]], [[[0.0]]]) is None assert select_candidate_sequence([[0.0], [0.0]], [[[math.nan]]]) is None assert select_candidate_sequence([[0.0], [0.0]], [[[0.0, 1.0]]]) is None assert select_candidate_sequence([]) is None