""" Tests for encoding.py (DeepSeek-V4.1 encoding). Adapted from dsv41-master/deepseek_harmony/tests/test_deepseek_v41.py for the self-contained dict-based API in this repo. """ import json from pathlib import Path from typing import Any import pytest import encoding as enc from encoding import ( IMAGE_PLACEHOLDER, SYSTEM_SP_TOKEN, encode_messages, parse_message_from_completion_text, render_message, merge_tool_messages, ) REASONING_EFFORT_TEMPLATE = ( SYSTEM_SP_TOKEN + "Reasoning Effort: {budget} " "(range 1-100, the higher the value, the more thorough the reasoning)\n\n" ) V41_TOOL_CALL_OUTPUT = ( ' reason summary\n\n' '<|DSML| calls>\n' '<|DSML| invoke name="lookup">\n' '<|DSML| parameter name="query" string="true">value' '\n' '<|DSML| parameter name="limit" string="false">2' '\n' '\n' '<|end▁of▁sentence|>' ) def make_tool() -> dict: return { "type": "function", "function": { "name": "lookup", "description": "Look up a value", "parameters": { "type": "object", "properties": { "query": {"type": "string"}, "limit": {"type": "integer"}, }, }, }, } def make_tool_call_messages() -> list: return [ {"role": "user", "content": "question"}, { "role": "assistant", "reasoning_content": " reason ", "content": "summary", "tool_calls": [ { "type": "function", "function": { "name": "lookup", "arguments": '{"query":"value","limit":2}', }, } ], }, ] # ============================================================ # Vision # ============================================================ def test_v41_renders_images() -> None: prompt, media = encode_messages( [ { "role": "user", "content": [ {"type": "text", "text": "inspect"}, {"type": "image_url", "image_url": {"url": "/unused/image.png"}}, ], } ], thinking_mode="chat", return_multi_modal_data=True, ) assert prompt == ( '<|begin▁of▁sentence|><|User|>inspect\n\n' f'{IMAGE_PLACEHOLDER}<|Assistant|>' ) assert media == {"images": [{"type": "image", "url": "/unused/image.png"}]} def test_v41_rejects_image_placeholder_in_text() -> None: with pytest.raises(ValueError): encode_messages( [{"role": "user", "content": f"hi {IMAGE_PLACEHOLDER}"}], thinking_mode="chat", ) # ============================================================ # Reasoning Effort # ============================================================ @pytest.mark.parametrize( ("effort", "budget"), [ (None, 75), ("low", 50), ("high", 75), ("max", 100), (1, 1), (42, 42), (100, 100), ], ) def test_v41_maps_reasoning_effort_to_1_100_budget( effort: Any, budget: int, ) -> None: prompt = encode_messages( [{"role": "user", "content": "question"}], thinking_mode="thinking", reasoning_effort=effort, ) assert prompt == ( '<|begin▁of▁sentence|>' f'{REASONING_EFFORT_TEMPLATE.format(budget=budget)}' '<|User|>question<|Assistant|>' ) def test_v41_only_adds_reasoning_effort_to_first_thinking_message() -> None: messages = [ {"role": "system", "content": "system"}, {"role": "user", "content": "question"}, ] later_message = render_message( 1, messages, thinking_mode="thinking", reasoning_effort=100 ) chat_message = render_message( 0, messages, thinking_mode="chat", reasoning_effort=100 ) assert "Reasoning Effort:" not in later_message assert "Reasoning Effort:" not in chat_message def test_v41_chat_mode_has_no_reasoning_effort_or_system_token() -> None: prompt = encode_messages( [{"role": "user", "content": "hello"}], thinking_mode="chat", reasoning_effort="max", ) assert prompt == '<|begin▁of▁sentence|><|User|>hello<|Assistant|>' @pytest.mark.parametrize("effort", [-1, 0, 101, "medium"]) def test_v41_rejects_out_of_range_or_unknown_reasoning_effort( effort: Any, ) -> None: with pytest.raises(AssertionError, match=r"int within \[1,100\]"): encode_messages( [{"role": "user", "content": "question"}], thinking_mode="thinking", reasoning_effort=effort, ) @pytest.mark.parametrize("effort", [True, False, 1.5]) def test_v41_rejects_non_string_non_integer_effort_types(effort: Any) -> None: # bool is not `type(...) is int`; float is invalid too with pytest.raises(AssertionError): encode_messages( [{"role": "user", "content": "question"}], thinking_mode="thinking", reasoning_effort=effort, ) # ============================================================ # System token # ============================================================ def test_v41_leading_system_message_uses_system_token() -> None: prompt = encode_messages( [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "hello"}, ], thinking_mode="chat", ) assert prompt == ( '<|begin▁of▁sentence|><|System|>You are a helpful assistant.' '<|User|>hello<|Assistant|>' ) def test_v41_mid_conversation_system_message() -> None: prompt = encode_messages( [ {"role": "system", "content": "sys"}, {"role": "user", "content": "q1"}, {"role": "assistant", "content": "a1", "reasoning_content": "r1"}, {"role": "system", "content": "mid sys"}, ], thinking_mode="thinking", reasoning_effort=88, ) # Mid-conversation system gets its own <|System|> token and triggers # the assistant generation header afterwards. assert prompt == ( '<|begin▁of▁sentence|>' f'{REASONING_EFFORT_TEMPLATE.format(budget=88)}' 'sys<|User|>q1<|Assistant|>a1<|end▁of▁sentence|>' '<|System|>mid sys<|Assistant|>' ) # ============================================================ # DSML tool tags # ============================================================ def test_v41_tool_instructions_use_spaced_dsml_tags_in_chat_mode() -> None: prompt = encode_messages( [ {"role": "system", "content": "system", "tools": [make_tool()]}, {"role": "user", "content": "question"}, ], thinking_mode="chat", ) assert ( '<|DSML| calls>\n' '<|DSML| invoke name="$TOOL_NAME">\n' '<|DSML| parameter name="$PARAMETER_NAME" ' 'string="true|false">$PARAMETER_VALUE\n' '...\n' '' ) in prompt assert '<|DSML|tool_calls>' not in prompt assert '<|DSML|invoke' not in prompt assert '<|DSML|parameter' not in prompt def test_v41_renders_spaced_dsml_with_v4_assistant_semantics() -> None: messages = make_tool_call_messages() prompt = render_message(1, messages, thinking_mode="thinking") assert prompt == V41_TOOL_CALL_OUTPUT def test_v41_parses_spaced_dsml_roundtrip() -> None: messages = make_tool_call_messages() parsed = parse_message_from_completion_text( V41_TOOL_CALL_OUTPUT, thinking_mode="thinking" ) assert parsed["role"] == "assistant" assert parsed["reasoning_content"] == " reason " assert parsed["content"] == "summary" assert parsed["tool_calls"] assert parsed["tool_calls"][0]["function"]["name"] == "lookup" assert json.loads(parsed["tool_calls"][0]["function"]["arguments"]) == { "query": "value", "limit": 2, } # Re-encoding the parsed message reproduces the original completion text assert encode_messages( [parsed], thinking_mode="thinking", context=messages[:1], ) == V41_TOOL_CALL_OUTPUT def test_v41_parse_rejects_unspaced_v4_dsml() -> None: v4_output = V41_TOOL_CALL_OUTPUT.replace("|DSML| calls", "|DSML|tool_calls") \ .replace("|DSML| invoke", "|DSML|invoke") \ .replace("|DSML| parameter", "|DSML|parameter") with pytest.raises(AssertionError): parse_message_from_completion_text(v4_output, thinking_mode="thinking") # ============================================================ # Multi-turn flow # ============================================================ def test_v41_drop_thinking_without_tools() -> None: prompt = encode_messages( [ {"role": "user", "content": "q1"}, {"role": "assistant", "content": "a1", "reasoning_content": "r1"}, {"role": "user", "content": "q2"}, ], thinking_mode="thinking", drop_thinking=True, ) # Earlier turn reasoning dropped, form; new turn opens assert '<|User|>q1<|Assistant|>a1<|end▁of▁sentence|>' in prompt assert 'r1' not in prompt assert prompt.endswith('<|User|>q2<|Assistant|>') # ============================================================ # Preprocessing # ============================================================ def test_merge_tool_messages_creates_tool_result_blocks() -> None: merged = merge_tool_messages([ {"role": "assistant", "content": "", "tool_calls": []}, {"role": "tool", "tool_call_id": "a", "content": "r1"}, {"role": "tool", "tool_call_id": "b", "content": "r2"}, ]) assert len(merged) == 2 assert merged[1]["role"] == "user" assert [b["type"] for b in merged[1]["content_blocks"]] == ["tool_result", "tool_result"] def test_v41_task_sp_token() -> None: prompt = encode_messages( [{"role": "user", "content": "classify me", "task": "query"}], thinking_mode="chat", ) assert prompt.endswith("classify me<|query|>") assert "<|Assistant|>" not in prompt # ============================================================ # Golden fixtures from encoding/tests # ============================================================ ENCODING_DIR = Path(__file__).resolve().parent ENCODING_FIXTURES_DIR = ENCODING_DIR / "tests" INFERENCE_EXAMPLES_DIR = ENCODING_DIR.parent / "inference" / "examples" FIXTURE_CASE_IDS = sorted( int(p.stem.split("_")[-1]) for p in ENCODING_FIXTURES_DIR.glob("test_input_*.json") ) @pytest.mark.parametrize("case_id", FIXTURE_CASE_IDS) def test_examples_encoding_golden_outputs(case_id: int) -> None: """Each tests/encoding input must encode to its checked-in golden output.""" input_file = ENCODING_FIXTURES_DIR / f"test_input_{case_id}.json" output_file = ENCODING_FIXTURES_DIR / f"test_output_{case_id}.txt" assert output_file.exists(), f"missing golden output: {output_file.name} (run tests/encoding/regen_outputs.py)" case = enc.load_cases(str(input_file))[0] prompt, _ = enc.encode_case(case, thinking_mode="chat") assert prompt == output_file.read_text(), ( f"{output_file.name} is stale; regenerate with tests/encoding/regen_outputs.py" ) def test_examples_v41_output_uses_v41_format_markers() -> None: """Sanity-check the V4.1 goldens actually exercise V4.1-specific format.""" # case 1: tool calls with spaced DSML tags out1 = (ENCODING_FIXTURES_DIR / "test_output_1.txt").read_text() assert '<|DSML| calls>' in out1 and '<|DSML| invoke name="get_weather">' in out1 assert '<|DSML|tool_calls>' not in out1 # case 5: numeric reasoning effort behind the system token out5 = (ENCODING_FIXTURES_DIR / "test_output_5.txt").read_text() assert out5.startswith( '<|begin▁of▁sentence|>' + REASONING_EFFORT_TEMPLATE.format(budget=100) ) assert out5.count(IMAGE_PLACEHOLDER) == 2 def test_examples_vl_txt_and_json_encode_identically() -> None: """The TXT (last block of example.txt) and JSON vision examples must encode identically.""" txt = (INFERENCE_EXAMPLES_DIR / "example.txt").read_text().rstrip("\n").split("\n\n")[-1] messages = [{"role": "user", "content": enc.parse_tagged_text(txt)}] p1, m1 = encode_messages(messages, thinking_mode="chat", return_multi_modal_data=True) case = enc.load_cases(str(INFERENCE_EXAMPLES_DIR / "example_harmony.json"))[0] p2, m2 = enc.encode_case(case, thinking_mode="chat") assert p1 == p2 assert m1["images"] == m2 assert len(m2) == 2 def test_examples_harmony_cases_encode() -> None: """All example_harmony.json cases encode without error.""" cases = enc.load_cases(str(INFERENCE_EXAMPLES_DIR / "example_harmony.json")) assert len(cases) == 4 # case 1 (vision) is covered by test_examples_vl_txt_and_json_encode_identically # cases are pure OpenAI format: mode/effort are passed at call time prompt = encode_messages( cases[1]["messages"], thinking_mode="thinking", reasoning_effort=75 ) assert REASONING_EFFORT_TEMPLATE.format(budget=75) in prompt # case 3: tools with spaced DSML tags prompt, _ = enc.encode_case(cases[2], thinking_mode="chat") assert '<|DSML| calls>' in prompt # case 4: mid-conversation system message triggers assistant header prompt, _ = enc.encode_case(cases[3], thinking_mode="chat") assert '<|System|>Mid-conversation instruction update' in prompt assert prompt.endswith('<|Assistant|>') if __name__ == "__main__": import sys sys.exit(pytest.main([__file__, "-v"]))