Upload script.py with huggingface_hub

script.py

@@ -85,13 +85,11 @@ SYSTEM = (
     "Then write a draft of the final answer. "
     "Subsequently, compare it with the format requirements again, "
     "and verify it's compliant with the deduced rules, and it is complete, i.e. has an answer for each element in QUERY. "
-    "If necessary, correct and refine.\n"
-    "Structure your response in exactly two sections with these headers:\n\n"
-    "**Reasoning:**\n"
-    "(your step-by-step analysis of linguistic rules, how to apply them, and answer format)\n\n"
-    "**Final Answers:**\n"
-    "(one answer per line, in query order -- bare answers only, no numbering, no quotes, "
-    "no extra text, according to the TASK TYPE)"
+    "If necessary, correct and refine."
+    "Finally, write a line that says exactly `FINAL ANSWERS:` "
+    "and, below it, write the answers to the items requested in QUERY (not those in CONTEXT),"
+    "one answer per line (separated by \n) in the order the items are asked for in the QUERY -- the "
+    "bare answer only, no numbering, no quotes, no extra text, according to the given TASK TYPE."
 )
 
 tok = load_tokenizer(MODEL_ID)
@@ -138,55 +136,6 @@ for r in test_rows:
     print(f"{len(outputs_queries_types)}/{len(test_rows)} done", flush=True)
 
 # Postprocess and store the answers.
-def _normalize_header(line: str) -> str:
-    header = line.strip().lower()
-    header = re.sub(r"^#{1,3}\s*", "", header)
-    header = re.sub(r"\*+", "", header)
-    return header.rstrip(":").strip()
-
-
-def _is_reasoning_header(line: str) -> bool:
-    header = _normalize_header(line)
-    return header == "reasoning" or header.endswith(" reasoning")
-
-
-def _is_final_answers_header(line: str) -> bool:
-    header = _normalize_header(line)
-    return header in ("final answer", "final answers")
-
-
-def split_response(text: str) -> tuple[str, str]:
-    """Return (explanation, raw final-answers section text)."""
-    lines = text.splitlines(keepends=True)
-    offset = 0
-    reasoning_content_start = None
-    explanation_end = None
-    last_final_content_start = None
-
-    reasoning_header_seen = False
-    final_header_indices: list[int] = []
-
-    for line in lines:
-        if _is_reasoning_header(line) and not reasoning_header_seen:
-            reasoning_header_seen = True
-            reasoning_content_start = offset + len(line)
-        elif _is_final_answers_header(line):
-            final_header_indices.append(offset)
-            if reasoning_content_start is not None and explanation_end is None:
-                explanation_end = offset
-            last_final_content_start = offset + len(line)
-        offset += len(line)
-
-    raw_final = text[last_final_content_start:].strip() if last_final_content_start is not None else ""
-
-    explanation = ""
-    if reasoning_content_start is not None and explanation_end is not None:
-        explanation = text[reasoning_content_start:explanation_end].strip()
-    if not explanation:
-        explanation = raw_final
-    return explanation, raw_final
-
-
 def expected_answer_count(query: str, task_type: str) -> int:
     if task_type == "match_letters":
         numbered = re.findall(r"^\s*\d+\.", query, re.MULTILINE)
@@ -232,33 +181,46 @@ def split_single_line_answer(text: str, expected: int, task_type: str) -> list[s
     return [text]
 
 
-def parse_answer_lines(text_after_marker: str, query: str, task_type: str) -> list[str]:
-    """Parse cleaned answer lines from the raw final-answers section."""
+def postprocess_answer(text, query, task_type):
+    """Keep only the lines after the last 'FINAL ANSWERS:' marker, one answer per line,
+    stopping at the first empty line. If eval_type is multiple and only one line as answer, split at whitespace."""
+    # Updated regex to be more flexible with surrounding characters
+    marker_match = list(re.finditer(r"(?im)^[^\w\n]*final answers?[^\w\n]*:?\s*$", text))
+    if marker_match:
+        text_after_marker = text[marker_match[-1].end():]
+        #print('FOUND FINAL ANSWER', text_after_marker)
+    else:
+        #print("No 'FINAL ANSWERS:' marker found")
+        return []
+
     answers = []
-    for line in text_after_marker.splitlines():
-        stripped_line = line.strip("`").strip()
+    answer_lines = text_after_marker.splitlines()
+    for i, line in enumerate(answer_lines):
+        stripped_line = line.strip('`').strip()
 
-        if stripped_line == "":
-            continue
+        # Stop processing if an empty line is encountered (not as first line)
+        if stripped_line=='':
+          continue
 
+        # Use a more precise regex to only remove numbering if it's a prefix to other text
+        # This ensures that lines which are just numbers (e.g., '1') are not stripped.
         match_numbered_prefix = re.match(r"^\s*\d+[.)]\s+(.*)", stripped_line)
         if match_numbered_prefix:
             cleaned_line = match_numbered_prefix.group(1).strip()
         else:
             cleaned_line = stripped_line
 
+        # Remove any bold markdown '**'
         cleaned_line = re.sub(r"\*\*", "", cleaned_line).strip()
 
-        if task_type == "match_letters":
+        # Specific handling for 'match_letters' task type to strip extra words
+        if task_type == 'match_letters':
             parts = [
                 part.strip("().[]")
                 for part in re.split(r"[\s,;]+", cleaned_line)
                 if part.strip()
             ]
-            if not (
-                len(parts) > 1
-                and all(re.fullmatch(r"[A-Za-z]", part) for part in parts)
-            ):
+            if not (len(parts) > 1 and all(re.fullmatch(r"[A-Za-z]", part) for part in parts)):
                 match_letter_word = re.match(
                     r"^\s*(?:\(([A-Za-z])\)|\[([A-Za-z])\]|([A-Za-z]))\.?:?\s*(.*)$",
                     cleaned_line,
@@ -271,34 +233,27 @@ def parse_answer_lines(text_after_marker: str, query: str, task_type: str) -> li
                     )
                     cleaned_line = letter.upper()
 
+        # Append the cleaned, non-empty line
         if cleaned_line:
             answers.append(cleaned_line)
 
+    #print('PARSED ANSWERS', answers)
+
+    # Compare against QUERY length: sometimes model forgets newlines
+    #print('QUERY', query)
     expected = expected_answer_count(query, task_type)
+    query_len = len(query.splitlines()) - 2
+    #print(query_len)
     if len(answers) == 1 and expected > 1:
         answers = split_single_line_answer(answers[0], expected, task_type)
     return answers
 
-
-def postprocess_answer(text, query, task_type):
-    """Extract explanation and parsed final answers from model output."""
-    explanation, raw_final = split_response(text)
-    if not raw_final:
-        return [], explanation
-    return parse_answer_lines(raw_final, query, task_type), explanation
-
 rows = []
 for answer, row_id, query, task_type in outputs_queries_types:
-    answers, explanation = postprocess_answer(answer, query, task_type)
-    rows.append(
-        {
-            "id": row_id,
-            "pred": json.dumps(answers, ensure_ascii=False),
-            "explanation": explanation,
-        }
-    )
+    answers = postprocess_answer(answer, query, task_type)
+    rows.append({"id": row_id, "pred": json.dumps(answers, ensure_ascii=False)})
 with open("submission.csv", "w", encoding="utf-8", newline="") as f:
-    writer = csv.DictWriter(f, fieldnames=["id", "pred", "explanation"])
+    writer = csv.DictWriter(f, fieldnames=["id", "pred"])
     writer.writeheader()
     writer.writerows(rows)
 print("wrote submission.csv", flush=True)