init

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+.env
+.vscode/*

README.md

@@ -1,10 +1,43 @@
 ---
-title: SQL Eval
-emoji: 🐢
-colorFrom: green
-colorTo: indigo
-sdk: static
+title: SQL-Eval-Arena
+emoji: 🩺
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: 6.10.0
+app_file: app.py
 pinned: false
+license: apache-2.0
+short_description: 'Human labeling of SQL result correctness'
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# sql-eval-arena
+
+Human-labeling tool for judging whether an agent's SQL query + result
+(attached to each `ANS`/`SANS` turn in `results/chat_pp/all_raw_diag.json`)
+actually answers the user's final intent, including anything resolved via
+clarification.
+
+## Setup
+
+1. `pip install -r requirements.txt`
+2. Regenerate `data/sessions.json` whenever `all_raw_diag.json` changes:
+   `python3 ../results/chat_pp/build_sql_eval_data.py` (run from repo root)
+3. Create a `.env` with:
+   ```
+   GITHUB_TOKEN=...
+   GITHUB_REPO=your-org/sql-eval-human-labeling
+   ```
+   The target repo must already exist; submitted labels are written to its
+   `labels/{pid}_{sid}.json` files (one per completed session).
+4. `python3 app.py`
+
+## How assignment works
+
+Each "new session" click hands out one whole conversation session (all its
+judgable turns at once) to whoever clicked. A session counts as done the
+moment its label file is created in the GitHub repo -- from then on it's
+excluded from everyone's pool, so no two people label the same session.
+Assignment itself (which session goes to which click) is tracked in-memory
+per running process, so it only works correctly as a single app instance
+(don't scale this to multiple replicas without adding shared claim state).

app.py

@@ -0,0 +1,198 @@
+import random
+import datetime
+import gradio as gr
+
+from utils.filesys_utils import json_load
+from utils.common import list_completed_keys, upload_label
+
+
+ERROR_CATEGORIES = [
+    "대상군(코호트) 해석 오류",
+    "지표/계산식 해석 오류",
+    "SQL 구문/로직 오류",
+    "결과→설명 불일치",
+    "기타",
+]
+
+MAX_TURNS = 15  # actual max across the dataset is 13; a little headroom
+
+SESSIONS = json_load("data/sessions.json")
+SESSION_BY_KEY = {f"{s['pid']}_{s['sid']}": s for s in SESSIONS}
+
+# In-memory only, shared across all concurrent users of this single process.
+# Best-effort de-duplication of who's currently working on what; the durable
+# "is this session actually done" truth lives in the GitHub labels/ dir.
+CLAIMED: set = set()
+
+
+def render_dialogue_md(sess: dict) -> str:
+    role_label = {"user": "🧑 사용자", "assistant": "🤖 에이전트"}
+    judgable_set = set(sess["judgable_turns"])
+    lines = [f"### 참가자 `{sess['pid']}` / 세션 `{sess['sid']}` (전략: {sess['strategy']})", ""]
+    for i, t in enumerate(sess["dialogue"]):
+        marker = " **⬅ 라벨링 대상 (아래에서 판단)**" if i in judgable_set else ""
+        lines.append(f"**#{i} [{t['type']}] {role_label[t['role']]}**{marker}")
+        lines.append(t["content"])
+        lines.append("")
+    return "\n".join(lines)
+
+
+def render_turn_detail_md(sess: dict, idx: int) -> str:
+    t = sess["dialogue"][idx]
+    lines = [f"#### 판단 대상: 턴 #{idx} ({t['type']})"]
+    for j, s in enumerate(t.get("sql", [])):
+        lines.append(f"**쿼리 {j + 1}:**")
+        lines.append(f"```sql\n{s['query']}\n```")
+        lines.append(f"**실행 결과:** `{s['result']}`")
+    return "\n".join(lines)
+
+
+def _reset_turn_updates():
+    """One (visible=False, empty text, cleared radio, cleared checkboxes,
+    cleared textbox) group per pre-allocated turn slot."""
+    updates = []
+    for _ in range(MAX_TURNS):
+        updates += [gr.update(visible=False), "", gr.update(value=None), gr.update(value=[]), gr.update(value="")]
+    return updates
+
+
+def get_new_session():
+    completed = list_completed_keys()
+    available = [k for k in SESSION_BY_KEY if k not in completed and k not in CLAIMED]
+
+    if not available:
+        return (
+            [None, ""]
+            + _reset_turn_updates()
+            + [gr.update(visible=False), "🎉 모든 세션이 라벨링되었습니다! 감사합니다."]
+        )
+
+    key = random.choice(available)
+    CLAIMED.add(key)
+    sess = SESSION_BY_KEY[key]
+    judgable = sess["judgable_turns"]
+
+    updates = []
+    for i in range(MAX_TURNS):
+        if i < len(judgable):
+            updates += [
+                gr.update(visible=True),
+                render_turn_detail_md(sess, judgable[i]),
+                gr.update(value=None),
+                gr.update(value=[]),
+                gr.update(value=""),
+            ]
+        else:
+            updates += [gr.update(visible=False), "", gr.update(value=None), gr.update(value=[]), gr.update(value="")]
+
+    status = f"세션 `{key}` 배정됨 -- {len(judgable)}개 턴을 라벨링해주세요."
+    return [key, render_dialogue_md(sess)] + updates + [gr.update(visible=True), status]
+
+
+def submit_labels(key, *flat):
+    # session_key_state is a gr.State -- it needs its actual (unchanged) value
+    # passed through, not a bare gr.update(), which is only meaningful for
+    # ordinary display/interactive components.
+    def unchanged(msg):
+        return [key, gr.update()] + [gr.update()] * (MAX_TURNS * 5) + [gr.update(), msg]
+
+    if not key:
+        return unchanged("먼저 '새 세션 받기'를 눌러주세요.")
+
+    sess = SESSION_BY_KEY.get(key)
+    if sess is None:
+        return unchanged("세션 정보를 찾을 수 없습니다.")
+
+    judgable = sess["judgable_turns"]
+    n = len(judgable)
+    verdicts = list(flat[0::3])[:n]
+    cats_list = list(flat[1::3])[:n]
+    reasons = list(flat[2::3])[:n]
+
+    missing = [i + 1 for i, v in enumerate(verdicts) if not v]
+    if missing:
+        msg = f"❌ 아직 판단하지 않은 항목이 있습니다: #{', '.join(map(str, missing))}. 모두 체크해주세요."
+        return unchanged(msg)
+
+    turn_labels = [
+        {"turn_index": idx, "verdict": v, "error_categories": c or [], "reason": r or ""}
+        for idx, v, c, r in zip(judgable, verdicts, cats_list, reasons)
+    ]
+    payload = {
+        "pid": sess["pid"],
+        "sid": sess["sid"],
+        "strategy": sess.get("strategy"),
+        "source": sess.get("source"),
+        "labeled_at": datetime.datetime.now().isoformat(),
+        "turns": turn_labels,
+    }
+
+    ok, msg = upload_label(key, payload)
+    if not ok:
+        return unchanged(msg)
+
+    CLAIMED.discard(key)
+    # auto-advance to the next session on success
+    next_state = get_new_session()
+    next_state[-1] = f"{msg} 다음 세션을 배정했습니다.\n\n{next_state[-1]}"
+    return next_state
+
+
+css = """
+.dialog-box {
+    max-height: 500px;
+    overflow-y: auto;
+    border: 1px solid var(--border-color-primary);
+    border-radius: 8px;
+    padding: 12px;
+}
+"""
+
+with gr.Blocks(title="SQL Result Evaluation", css=css) as demo:
+    gr.Markdown("# 🩺 SQL 결과 적합성 라벨링")
+    gr.Markdown(
+        "아래 전체 대화를 끝까지 읽고, **'라벨링 대상'**으로 표시된 각 에이전트 응답이 "
+        "실제로 실행된 SQL과 그 결과를 바탕으로 사용자의 최종 의도"
+        "(중간에 명확화(clarification)가 있었다면 그것까지 반영한 의도)에 맞는 답을 했는지 판단해주세요."
+    )
+    gr.Markdown(
+        "* 한 세션에는 라벨링 대상이 여러 개 있을 수 있습니다 -- 세션의 모든 대상을 채워야 제출됩니다.\n"
+        "* 제출하면 그 세션은 완료 처리되어 다른 사람에게 다시 배정되지 않습니다.\n"
+        "* '틀림'을 선택한 경우 원인 카테고리를 체크하고, 가능하면 자유 서술로 이유도 남겨주세요."
+    )
+
+    session_key_state = gr.State(None)
+    btn_new = gr.Button("🎲 새 세션 받기")
+    status_msg = gr.Markdown("")
+    dialogue_md = gr.Markdown(elem_classes="dialog-box")
+
+    turn_groups, detail_mds, verdict_radios, category_checks, reason_boxes = [], [], [], [], []
+    for i in range(MAX_TURNS):
+        with gr.Group(visible=False) as grp:
+            detail = gr.Markdown()
+            verdict = gr.Radio(["맞음", "틀림"], label=f"판단 #{i + 1}: 이 응답은 사용자의 최종 의도에 맞나요?")
+            cats = gr.CheckboxGroup(ERROR_CATEGORIES, label="오류 유형 (틀렸을 때만 해당하는 항목 체크)")
+            reason = gr.Textbox(label="자유 서술 이유", lines=2)
+        turn_groups.append(grp)
+        detail_mds.append(detail)
+        verdict_radios.append(verdict)
+        category_checks.append(cats)
+        reason_boxes.append(reason)
+
+    with gr.Row(visible=False) as submit_row:
+        submit_btn = gr.Button("📤 제출하기")
+
+    all_outputs = [session_key_state, dialogue_md]
+    for grp, detail, verdict, cats, reason in zip(turn_groups, detail_mds, verdict_radios, category_checks, reason_boxes):
+        all_outputs += [grp, detail, verdict, cats, reason]
+    all_outputs += [submit_row, status_msg]
+
+    btn_new.click(fn=get_new_session, inputs=[], outputs=all_outputs)
+
+    submit_inputs = [session_key_state]
+    for verdict, cats, reason in zip(verdict_radios, category_checks, reason_boxes):
+        submit_inputs += [verdict, cats, reason]
+    submit_btn.click(fn=submit_labels, inputs=submit_inputs, outputs=all_outputs)
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,3 @@
+gradio==5.49.1
+dotenv==0.9.9
+PyGithub==2.8.1

utils/common.py

@@ -0,0 +1,48 @@
+import os
+import json
+from github import Github
+from dotenv import load_dotenv
+
+
+LABELS_DIR = "labels"
+
+
+def _get_repo():
+    load_dotenv(override=True)
+    repo_name = os.getenv("GITHUB_REPO")
+    if not repo_name:
+        raise RuntimeError("GITHUB_REPO env var is not set")
+    g = Github(os.getenv("GITHUB_TOKEN"))
+    return g.get_repo(repo_name)
+
+
+def list_completed_keys() -> set:
+    """Session keys ("{pid}_{sid}") that already have a submitted label file
+    in the GitHub repo's labels/ directory. Returns an empty set (rather than
+    raising) if the repo/labels dir isn't reachable yet, so the app still
+    works before it's configured or if the directory doesn't exist yet."""
+    try:
+        repo = _get_repo()
+        contents = repo.get_contents(LABELS_DIR, ref="main")
+        return {os.path.splitext(c.name)[0] for c in contents}
+    except Exception:
+        return set()
+
+
+def upload_label(key: str, payload: dict):
+    """Create labels/{key}.json in the GitHub repo. Uses create (not update)
+    so a second submission for an already-completed session fails loudly
+    instead of silently overwriting someone else's labels."""
+    path = f"{LABELS_DIR}/{key}.json"
+    content = json.dumps(payload, ensure_ascii=False, indent=2)
+    try:
+        repo = _get_repo()
+        try:
+            repo.get_contents(path, ref="main")
+            return False, "⚠️ 이미 다른 사람이 이 세션을 완료했습니다. 새 세션을 받아주세요."
+        except Exception:
+            pass
+        repo.create_file(path, f"feat: add labels for {key}", content, branch="main")
+        return True, "✅ 제출 완료! 감사합니다."
+    except Exception as e:
+        return False, f"❌ 업로드 실패: {e}"

utils/filesys_utils.py

@@ -0,0 +1,7 @@
+import json
+from typing import Any
+
+
+def json_load(path: str) -> Any:
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)