"""Tests for harness/vision.py — the readability judge's pure logic + fail-open behaviour. No moviepy / no live API: frame extraction and the vision call are injected, so we exercise parsing, aggregation, and the all-important fail-open guarantees deterministically. """ import pytest from harness import vision from harness.vision import ReadabilityReport, NOT_ASSESSED, assess_readability _FAKE_FRAMES = [(0.25, b"png0"), (0.5, b"png1"), (0.75, b"png2")] def _extract_ok(_path): return list(_FAKE_FRAMES) # ── JSON parsing ────────────────────────────────────────────────────────────── def test_parse_plain_json(): score, issues = vision._parse_score_json('{"score": 0.82, "issues": ["text too small"]}') assert score == 0.82 and issues == ["text too small"] def test_parse_fenced_json_with_prose(): raw = "Here is my verdict:\n```json\n{\"score\": 0.4, \"issues\": [\"bubble clipped\", \" \"]}\n```" score, issues = vision._parse_score_json(raw) assert score == 0.4 assert issues == ["bubble clipped"] # blank issue dropped def test_parse_clamps_score(): assert vision._parse_score_json('{"score": 1.7}')[0] == 1.0 assert vision._parse_score_json('{"score": -3}')[0] == 0.0 # ── assess_readability: happy path + aggregation ────────────────────────────── def test_assess_happy_path_injected(): rep = assess_readability( "x.mp4", extract=_extract_ok, assess=lambda frames: (0.73, ["one issue"]), ) assert rep.assessed and rep.score == 0.73 assert rep.issues == ["one issue"] assert rep.frames_assessed == 3 assert rep.error == "" def test_assess_passed_threshold_semantics(): rep = ReadabilityReport(score=0.5) assert rep.passed(0.4) is True assert rep.passed(0.6) is False # ── fail-open guarantees (a vision/extraction problem must never block) ─────── def test_assess_fails_open_on_extract_error(): def boom(_path): raise RuntimeError("no moviepy") rep = assess_readability("x.mp4", extract=boom, assess=lambda f: (0.9, [])) assert rep.score == NOT_ASSESSED assert not rep.assessed assert "frame_extract_failed" in rep.error assert rep.passed(0.99) is True # a non-assessment never fails the gate def test_assess_fails_open_on_vision_error(): def boom(_frames): raise RuntimeError("api 500") rep = assess_readability("x.mp4", extract=_extract_ok, assess=boom) assert rep.score == NOT_ASSESSED assert "vision_failed" in rep.error assert rep.frames_assessed == 3 # we still recorded that frames were extracted def test_assess_no_frames_is_not_assessed(): rep = assess_readability("x.mp4", extract=lambda p: [], assess=lambda f: (0.9, [])) assert rep.score == NOT_ASSESSED and "no frames" in rep.error def test_assess_no_api_key_with_default_assessor(monkeypatch): # Default (real) assessor but no key anywhere → not assessed, never raises. monkeypatch.delenv("GEMINI_API_KEY", raising=False) rep = assess_readability("x.mp4", api_key="", extract=_extract_ok) assert rep.score == NOT_ASSESSED assert "GEMINI_API_KEY" in rep.error def test_summary_is_human_readable(): assert "not assessed" in ReadabilityReport(error="boom").summary() assert "0.42" in ReadabilityReport(score=0.42, issues=["x"]).summary()