solar_eval/evaluators/registry.py

from typing import Any

from solar_eval.evaluators.base import BaseEvaluator
from solar_eval.evaluators.composite import CompositeEvaluator
from solar_eval.evaluators.lcs_diff import LCSDiffEvaluator
from solar_eval.evaluators.llm_judge import LLMJudgeEvaluator
from solar_eval.evaluators.rule_based import RuleBasedEvaluator


def create_evaluator(config: dict[str, Any]) -> BaseEvaluator:
    """Factory function to create evaluator from config dict."""
    eval_type = config["type"]

    if eval_type == "llm_judge":
        return LLMJudgeEvaluator(
            categories=config.get("categories", []),
            judge_prompts=config.get("judge_prompts"),
        )
    elif eval_type == "lcs_diff":
        return LCSDiffEvaluator(metrics=config.get("metrics"))
    elif eval_type == "rule_based":
        return RuleBasedEvaluator(rules=config.get("rules", []))
    elif eval_type == "composite":
        components = []
        for comp in config.get("components", []):
            sub_evaluator = create_evaluator(comp)
            components.append({"evaluator": sub_evaluator, "weight": comp.get("weight", 0.5)})
        return CompositeEvaluator(components=components)
    else:
        raise ValueError(f"Unknown evaluator type: {eval_type}")