Spaces:
Running
Running
Download solar_eval/pipelines/steps/rule_steps.py from dev-strender/proofread-demo: direct link, hf CLI and curl.
- Browser
- Download file 6.45 kB
-
https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/pipelines/steps/rule_steps.py
- Command line
-
hf download hf://spaces/dev-strender/proofread-demo/solar_eval/pipelines/steps/rule_steps.py
-
curl -L -o rule_steps.py https://huggingface.co/spaces/dev-strender/proofread-demo/resolve/main/solar_eval/pipelines/steps/rule_steps.py
6.45 kB
| """rule ๊ณ์ด ์คํ -- `rules.py` ์ ์์ ํจ์๋ฅผ ๋ ์ง์คํธ๋ฆฌ ์คํ ์ผ๋ก ๊ฐ์ผ๋ค. | |
| `multi_step.py`์ `_apply_rule` if/elif ๋์คํจ์น์์ ์ด๋ (2026-08-10, ๋์ ๋์ผ). | |
| v24 ํ์ดํ๋ผ์ธ์ด LLM ์คํ ์ฌ์ด์ฌ์ด์ ๋ผ์ ๋ฃ๋ "code block ํ์ฒ๋ฆฌ"๊ฐ ๋ฐ๋ก | |
| ์ด ์คํ ๋ค์ด๋ค. ๊ฐ rule_type ์ด ์ด์ ๋ ๋ฆฝ ๋ฑ๋ก ํญ๋ชฉ์ด๋ผ, ์ ํ์ฒ๋ฆฌ๋ ์ฌ๊ธฐ | |
| ํด๋์ค๋ฅผ ์ถ๊ฐํ๋ ๊ฒ ์๋๋ผ ์ด๋์๋ `@register_step` ์ผ๋ก ๋ฑ๋กํ๋ฉด ๋๋ค. | |
| """ | |
| import logging | |
| from solar_eval.models.sample import EvalSample | |
| from solar_eval.pipelines.pipeline import BaseStep, StepContext | |
| from solar_eval.pipelines.registry import register_step | |
| from solar_eval.pipelines.rules import ( | |
| apply_change_rate_guard, | |
| apply_comma_removal, | |
| apply_compound_whitelist_restore, | |
| apply_correction_filter, | |
| apply_currency_compact, | |
| apply_judge_decisions, | |
| apply_pronoun_postprocess, | |
| apply_unit_unicode, | |
| apply_vocabulary, | |
| apply_yul_ryul_normalize, | |
| load_compound_whitelist, | |
| ) | |
| from solar_eval.pipelines.rules_josa import apply_josa_fix | |
| from solar_eval.pipelines.rules_revert import apply_diff_revert | |
| logger = logging.getLogger(__name__) | |
| class VocabularyStep(BaseStep): | |
| """์ดํ ์นํ. CSV ๋ run ์๋ช ์บ์(`ctx.shared`)์ ํ ๋ฒ๋ง ๋ก๋ํ๋ค. | |
| ์์ ์๋ `MultiStepPipeline._vocabulary` ๋จ์ผ ์ฌ๋กฏ์ด๋ผ ์๋ก ๋ค๋ฅธ CSV ๋ฅผ | |
| ์ ์ธํ ๋ vocab ์คํ ์ด ์ฒซ ๋ก๋๋ฅผ ์๋ชป ๊ณต์ ํ ์ ์์๋ค. ์ง๊ธ์ config | |
| (repo, ๊ฒฝ๋ก, source) ๋ฅผ ์บ์ ํค๋ก ์จ์ ๊ฐ์ CSV ๋ง ๊ณต์ ํ๋ค -- | |
| prod/v24 ์ฒ๋ผ ๊ฐ์ CSV ๋ฅผ ๋ ๋ฒ ์ฐ๋ ํ์ดํ๋ผ์ธ์ ์ฌ์ ํ 1ํ ๋ก๋. | |
| """ | |
| async def run(self, sample: EvalSample, ctx: StepContext) -> None: | |
| config = self.config.get("config", {}) | |
| cache_key = ( | |
| "vocabulary", | |
| config.get("repo", ""), | |
| config.get("vocab_dataset", ""), | |
| config.get("source", "huggingface"), | |
| ) | |
| vocabulary = ctx.shared.get(cache_key) | |
| text, vocabulary = apply_vocabulary( | |
| ctx.text, config, vocabulary, self.env.dataset_loader, self.env.config_dir | |
| ) | |
| ctx.shared[cache_key] = vocabulary | |
| ctx.text = text | |
| class PostProcessStep(BaseStep): | |
| """๊ฒฐ์ ๋ก ์ ํ์ฒ๋ฆฌ ๋ฌถ์ -- config ํ๋๊ทธ๋ก ์ผ๋ ๋๋ช ์ฌ/ํตํ/์ผํ/๋จ์ ์นํ.""" | |
| async def run(self, sample: EvalSample, ctx: StepContext) -> None: | |
| config = self.config.get("config", {}) | |
| text = ctx.text | |
| if config.get("pronoun_replacements"): | |
| text = apply_pronoun_postprocess(text, config["pronoun_replacements"]) | |
| if config.get("currency_compact"): | |
| text = apply_currency_compact(text) | |
| if config.get("comma_removal"): | |
| text = apply_comma_removal(text) | |
| if config.get("unit_unicode"): | |
| text = apply_unit_unicode(text) | |
| ctx.text = text | |
| class CompoundWhitelistRestoreStep(BaseStep): | |
| """ํ์ดํ๋ผ์ธ์ด ๊ฐ๋ผ๋์ ํ์ดํธ๋ฆฌ์คํธ ๋ณตํฉ์ด๋ฅผ ์๋ฌธ ํ๊ธฐ๋ก ๋๋๋ฆฐ๋ค (v19/v32 ๊ณ๋ณด). | |
| ํ์ดํธ๋ฆฌ์คํธ๋ run ์๋ช ์บ์(`ctx.shared`)์ ํ ๋ฒ๋ง ๋ก๋ํ๋ค โ ๊ฒฝ๋ก๊ฐ | |
| ๋ค๋ฅธ ๋ ์คํ ์ด ์บ์๋ฅผ ์๋ชป ๊ณต์ ํ์ง ์๋๋ก ๊ฒฝ๋ก๋ฅผ ์บ์ ํค์ ๋ฃ๋๋ค. | |
| """ | |
| async def run(self, sample: EvalSample, ctx: StepContext) -> None: | |
| config = self.config.get("config", {}) | |
| cache_key = ("compound_whitelist", config.get("whitelist_path", "")) | |
| whitelist = ctx.shared.get(cache_key) | |
| if whitelist is None: | |
| whitelist = load_compound_whitelist(config, self.env.config_dir) | |
| ctx.shared[cache_key] = whitelist | |
| ctx.text = apply_compound_whitelist_restore(ctx.text, ctx.original_text, whitelist) | |
| class YulRyulNormalizeStep(BaseStep): | |
| """์จ/๋ฅ ๋ฐ์นจ ๊ท์น(๋ง์ถค๋ฒ ์ 11ํญ)์ ๊ฒฐ์ ๋ก ์ ์ผ๋ก ๊ฐ์ ํ๋ค. | |
| ์กฐ์ฌ ๊ฐ๋ + `exceptions`(์ธ๋ช ๋ฑ) 2์ค ๋ฐฉ์ด๋ก ์ ๋ฏธ์ฌ ์ฉ๋ฒ์ผ ๋๋ง ๊ณ ์น๋ค โ | |
| ์์ธ์ ์ฌ์ ์ค์ธก์ `rules.apply_yul_ryul_normalize` docstring ์ฐธ์กฐ. | |
| """ | |
| async def run(self, sample: EvalSample, ctx: StepContext) -> None: | |
| config = self.config.get("config", {}) | |
| exceptions = tuple(config.get("exceptions") or ()) | |
| ctx.text = apply_yul_ryul_normalize(ctx.text, exceptions) | |
| class CorrectionFilterStep(BaseStep): | |
| async def run(self, sample: EvalSample, ctx: StepContext) -> None: | |
| config = self.config.get("config", {}) | |
| ctx.text = apply_correction_filter(ctx.text, config, ctx.original_text) | |
| class ApplyJudgeDecisionsStep(BaseStep): | |
| """`tool_calling_judge` ๊ฐ `sample.artifacts` ์ ๋จ๊ธด ๊ตฌ์กฐํ ๊ฒฐ์ ์ ํ ์คํธ์ ์ ์ฉ.""" | |
| async def run(self, sample: EvalSample, ctx: StepContext) -> None: | |
| config = self.config.get("config", {}) | |
| ctx.text = apply_judge_decisions(ctx.text, config, sample) | |
| class JosaFixStep(BaseStep): | |
| """๊ธฐํธยท์ซ์ยท๋จ์ ๋ค ์กฐ์ฌ ๊ฒฐ์ ๋ก ๊ต์ (2026-09-18, ์์ธ๋ `rules_josa` docstring).""" | |
| async def run(self, sample: EvalSample, ctx: StepContext) -> None: | |
| ctx.text = apply_josa_fix(ctx.text) | |
| class DiffRevertStep(BaseStep): | |
| """์๋ฌธ ๋๋น diff ๋ฅผ ํด๋์ค ํ์ ํด ๋๋๋ฆฌ๋ ์ ๋ฐ๋ ๊ฐ๋ (2026-09-18). | |
| ๊ธ๋ฆฌํ ๋ณํยท์ผํ ์ฝ์ ยท๊ธ์ง ์นํยท๊ฒฉ์ ์ ํ๋ง ๋๋๋ฆฐ๋ค โ ์์ธ์ ์ค์ธก ๊ทผ๊ฑฐ๋ | |
| `rules_revert` docstring ์ฐธ์กฐ. ํ์ดํ๋ผ์ธ ๊ผฌ๋ฆฌ(post_process ๋ค)์ ๋๋ ๊ฒ์ | |
| ์ ์ ๋ก ์ค๊ณํ๋ค. | |
| """ | |
| async def run(self, sample: EvalSample, ctx: StepContext) -> None: | |
| config = self.config.get("config", {}) | |
| ctx.text = apply_diff_revert(ctx.text, ctx.original_text, config) | |
| class ChangeRateGuardStep(BaseStep): | |
| """์๋ฌธ ๋๋น ๋ณ๊ฒฝ๋ฅ ์ด ์๊ณ๋ฅผ ๋์ผ๋ฉด ์๋ฌธ์ผ๋ก ๋๋๋ฆฌ๋ ์์ ๋ง (v17 ๊ณ์ด).""" | |
| async def run(self, sample: EvalSample, ctx: StepContext) -> None: | |
| config = self.config.get("config", {}) | |
| ctx.text = apply_change_rate_guard(ctx.text, ctx.original_text, config) | |