dev-strender's picture
feat: switch demo to the solar-mini4 pipeline (v44 = v41 + document chunking)
dd41633 verified
Raw History Blame Contribute Delete
6.45 kB
"""rule ๊ณ„์—ด ์Šคํ… -- `rules.py` ์˜ ์ˆœ์ˆ˜ ํ•จ์ˆ˜๋ฅผ ๋ ˆ์ง€์ŠคํŠธ๋ฆฌ ์Šคํ…์œผ๋กœ ๊ฐ์‹ผ๋‹ค.
`multi_step.py`์˜ `_apply_rule` if/elif ๋””์ŠคํŒจ์น˜์—์„œ ์ด๋™ (2026-08-10, ๋™์ž‘ ๋™์ผ).
v24 ํŒŒ์ดํ”„๋ผ์ธ์ด LLM ์Šคํ… ์‚ฌ์ด์‚ฌ์ด์— ๋ผ์›Œ ๋„ฃ๋Š” "code block ํ›„์ฒ˜๋ฆฌ"๊ฐ€ ๋ฐ”๋กœ
์ด ์Šคํ…๋“ค์ด๋‹ค. ๊ฐ rule_type ์ด ์ด์ œ ๋…๋ฆฝ ๋“ฑ๋ก ํ•ญ๋ชฉ์ด๋ผ, ์ƒˆ ํ›„์ฒ˜๋ฆฌ๋Š” ์—ฌ๊ธฐ
ํด๋ž˜์Šค๋ฅผ ์ถ”๊ฐ€ํ•˜๋Š” ๊ฒŒ ์•„๋‹ˆ๋ผ ์–ด๋””์„œ๋“  `@register_step` ์œผ๋กœ ๋“ฑ๋กํ•˜๋ฉด ๋œ๋‹ค.
"""
import logging
from solar_eval.models.sample import EvalSample
from solar_eval.pipelines.pipeline import BaseStep, StepContext
from solar_eval.pipelines.registry import register_step
from solar_eval.pipelines.rules import (
apply_change_rate_guard,
apply_comma_removal,
apply_compound_whitelist_restore,
apply_correction_filter,
apply_currency_compact,
apply_judge_decisions,
apply_pronoun_postprocess,
apply_unit_unicode,
apply_vocabulary,
apply_yul_ryul_normalize,
load_compound_whitelist,
)
from solar_eval.pipelines.rules_josa import apply_josa_fix
from solar_eval.pipelines.rules_revert import apply_diff_revert
logger = logging.getLogger(__name__)
@register_step("vocabulary_sub", "vocabulary_reapply")
class VocabularyStep(BaseStep):
"""์–ดํœ˜ ์น˜ํ™˜. CSV ๋Š” run ์ˆ˜๋ช… ์บ์‹œ(`ctx.shared`)์— ํ•œ ๋ฒˆ๋งŒ ๋กœ๋“œํ•œ๋‹ค.
์˜ˆ์ „์—๋Š” `MultiStepPipeline._vocabulary` ๋‹จ์ผ ์Šฌ๋กฏ์ด๋ผ ์„œ๋กœ ๋‹ค๋ฅธ CSV ๋ฅผ
์„ ์–ธํ•œ ๋‘ vocab ์Šคํ…์ด ์ฒซ ๋กœ๋“œ๋ฅผ ์ž˜๋ชป ๊ณต์œ ํ•  ์ˆ˜ ์žˆ์—ˆ๋‹ค. ์ง€๊ธˆ์€ config
(repo, ๊ฒฝ๋กœ, source) ๋ฅผ ์บ์‹œ ํ‚ค๋กœ ์จ์„œ ๊ฐ™์€ CSV ๋งŒ ๊ณต์œ ํ•œ๋‹ค --
prod/v24 ์ฒ˜๋Ÿผ ๊ฐ™์€ CSV ๋ฅผ ๋‘ ๋ฒˆ ์“ฐ๋Š” ํŒŒ์ดํ”„๋ผ์ธ์€ ์—ฌ์ „ํžˆ 1ํšŒ ๋กœ๋“œ.
"""
async def run(self, sample: EvalSample, ctx: StepContext) -> None:
config = self.config.get("config", {})
cache_key = (
"vocabulary",
config.get("repo", ""),
config.get("vocab_dataset", ""),
config.get("source", "huggingface"),
)
vocabulary = ctx.shared.get(cache_key)
text, vocabulary = apply_vocabulary(
ctx.text, config, vocabulary, self.env.dataset_loader, self.env.config_dir
)
ctx.shared[cache_key] = vocabulary
ctx.text = text
@register_step("post_process")
class PostProcessStep(BaseStep):
"""๊ฒฐ์ •๋ก ์  ํ›„์ฒ˜๋ฆฌ ๋ฌถ์Œ -- config ํ”Œ๋ž˜๊ทธ๋กœ ์ผœ๋Š” ๋Œ€๋ช…์‚ฌ/ํ†ตํ™”/์‰ผํ‘œ/๋‹จ์œ„ ์น˜ํ™˜."""
async def run(self, sample: EvalSample, ctx: StepContext) -> None:
config = self.config.get("config", {})
text = ctx.text
if config.get("pronoun_replacements"):
text = apply_pronoun_postprocess(text, config["pronoun_replacements"])
if config.get("currency_compact"):
text = apply_currency_compact(text)
if config.get("comma_removal"):
text = apply_comma_removal(text)
if config.get("unit_unicode"):
text = apply_unit_unicode(text)
ctx.text = text
@register_step("compound_whitelist_restore")
class CompoundWhitelistRestoreStep(BaseStep):
"""ํŒŒ์ดํ”„๋ผ์ธ์ด ๊ฐˆ๋ผ๋†“์€ ํ™”์ดํŠธ๋ฆฌ์ŠคํŠธ ๋ณตํ•ฉ์–ด๋ฅผ ์›๋ฌธ ํ‘œ๊ธฐ๋กœ ๋˜๋Œ๋ฆฐ๋‹ค (v19/v32 ๊ณ„๋ณด).
ํ™”์ดํŠธ๋ฆฌ์ŠคํŠธ๋Š” run ์ˆ˜๋ช… ์บ์‹œ(`ctx.shared`)์— ํ•œ ๋ฒˆ๋งŒ ๋กœ๋“œํ•œ๋‹ค โ€” ๊ฒฝ๋กœ๊ฐ€
๋‹ค๋ฅธ ๋‘ ์Šคํ…์ด ์บ์‹œ๋ฅผ ์ž˜๋ชป ๊ณต์œ ํ•˜์ง€ ์•Š๋„๋ก ๊ฒฝ๋กœ๋ฅผ ์บ์‹œ ํ‚ค์— ๋„ฃ๋Š”๋‹ค.
"""
async def run(self, sample: EvalSample, ctx: StepContext) -> None:
config = self.config.get("config", {})
cache_key = ("compound_whitelist", config.get("whitelist_path", ""))
whitelist = ctx.shared.get(cache_key)
if whitelist is None:
whitelist = load_compound_whitelist(config, self.env.config_dir)
ctx.shared[cache_key] = whitelist
ctx.text = apply_compound_whitelist_restore(ctx.text, ctx.original_text, whitelist)
@register_step("yul_ryul_normalize")
class YulRyulNormalizeStep(BaseStep):
"""์œจ/๋ฅ  ๋ฐ›์นจ ๊ทœ์น™(๋งž์ถค๋ฒ• ์ œ11ํ•ญ)์„ ๊ฒฐ์ •๋ก ์ ์œผ๋กœ ๊ฐ•์ œํ•œ๋‹ค.
์กฐ์‚ฌ ๊ฐ€๋“œ + `exceptions`(์ธ๋ช… ๋“ฑ) 2์ค‘ ๋ฐฉ์–ด๋กœ ์ ‘๋ฏธ์‚ฌ ์šฉ๋ฒ•์ผ ๋•Œ๋งŒ ๊ณ ์นœ๋‹ค โ€”
์ƒ์„ธ์™€ ์žฌ์ƒ ์‹ค์ธก์€ `rules.apply_yul_ryul_normalize` docstring ์ฐธ์กฐ.
"""
async def run(self, sample: EvalSample, ctx: StepContext) -> None:
config = self.config.get("config", {})
exceptions = tuple(config.get("exceptions") or ())
ctx.text = apply_yul_ryul_normalize(ctx.text, exceptions)
@register_step("correction_filter")
class CorrectionFilterStep(BaseStep):
async def run(self, sample: EvalSample, ctx: StepContext) -> None:
config = self.config.get("config", {})
ctx.text = apply_correction_filter(ctx.text, config, ctx.original_text)
@register_step("apply_judge_decisions")
class ApplyJudgeDecisionsStep(BaseStep):
"""`tool_calling_judge` ๊ฐ€ `sample.artifacts` ์— ๋‚จ๊ธด ๊ตฌ์กฐํ™” ๊ฒฐ์ •์„ ํ…์ŠคํŠธ์— ์ ์šฉ."""
async def run(self, sample: EvalSample, ctx: StepContext) -> None:
config = self.config.get("config", {})
ctx.text = apply_judge_decisions(ctx.text, config, sample)
@register_step("josa_fix")
class JosaFixStep(BaseStep):
"""๊ธฐํ˜ธยท์ˆซ์žยท๋‹จ์œ„ ๋’ค ์กฐ์‚ฌ ๊ฒฐ์ •๋ก  ๊ต์ • (2026-09-18, ์ƒ์„ธ๋Š” `rules_josa` docstring)."""
async def run(self, sample: EvalSample, ctx: StepContext) -> None:
ctx.text = apply_josa_fix(ctx.text)
@register_step("diff_revert")
class DiffRevertStep(BaseStep):
"""์›๋ฌธ ๋Œ€๋น„ diff ๋ฅผ ํด๋ž˜์Šค ํŒ์ •ํ•ด ๋˜๋Œ๋ฆฌ๋Š” ์ •๋ฐ€๋„ ๊ฐ€๋“œ (2026-09-18).
๊ธ€๋ฆฌํ”„ ๋ณ€ํ™˜ยท์‰ผํ‘œ ์‚ฝ์ž…ยท๊ธˆ์ง€ ์น˜ํ™˜ยท๊ฒฉ์‹ ์ „ํ™˜๋งŒ ๋˜๋Œ๋ฆฐ๋‹ค โ€” ์ƒ์„ธ์™€ ์‹ค์ธก ๊ทผ๊ฑฐ๋Š”
`rules_revert` docstring ์ฐธ์กฐ. ํŒŒ์ดํ”„๋ผ์ธ ๊ผฌ๋ฆฌ(post_process ๋’ค)์— ๋‘๋Š” ๊ฒƒ์„
์ „์ œ๋กœ ์„ค๊ณ„ํ–ˆ๋‹ค.
"""
async def run(self, sample: EvalSample, ctx: StepContext) -> None:
config = self.config.get("config", {})
ctx.text = apply_diff_revert(ctx.text, ctx.original_text, config)
@register_step("change_rate_guard")
class ChangeRateGuardStep(BaseStep):
"""์›๋ฌธ ๋Œ€๋น„ ๋ณ€๊ฒฝ๋ฅ ์ด ์ž„๊ณ„๋ฅผ ๋„˜์œผ๋ฉด ์›๋ฌธ์œผ๋กœ ๋˜๋Œ๋ฆฌ๋Š” ์•ˆ์ „๋ง (v17 ๊ณ„์—ด)."""
async def run(self, sample: EvalSample, ctx: StepContext) -> None:
config = self.config.get("config", {})
ctx.text = apply_change_rate_guard(ctx.text, ctx.original_text, config)