Upload eval_codeswitch_and_new_baselines.py with huggingface_hub

eval_codeswitch_and_new_baselines.py

@@ -31,6 +31,11 @@ def segment_words(t): return _WORD_PAT.findall(t)
 def count_graphemes(t): return len(regex.findall(r"\X", t))
 def filter_sp(tokens): return [t for t in tokens if t not in _SPECIAL]
 
+def normalize_decode(s):
+    s = s.replace("##", "")
+    s = " ".join(s.split())
+    return s
+
 def classify_script_detailed(t):
     """Classify as 'ar', 'az', or 'mi' (mixed)."""
     ar_chars = len(_AR_PAT.findall(t))
@@ -134,13 +139,12 @@ def evaluate_with_mixed(tok, name, source, algo, arch, vsz, texts):
 
             try:
                 dec = tok.decode(ids, script)
-                if dec.strip() == text.strip():
+                if normalize_decode(dec) == normalize_decode(text):
                     em_buckets[sc]["ok"] += 1
             except:
                 pass
         except:
             pass
-
     for sc in ("ar", "az", "mi"):
         setattr(m, f"fertility_{sc}", float(np.mean(buckets[sc])) if buckets[sc] else 0)
         setattr(m, f"cpt_{sc}", float(np.mean(cpt_buckets[sc])) if cpt_buckets[sc] else 0)
@@ -173,7 +177,7 @@ def evaluate_on_doda(tok, name, source, algo, arch, vsz, texts):
             all_c.append(cpt)
             try:
                 dec = tok.decode(ids, script)
-                if dec.strip() == text.strip():
+                if normalize_decode(dec) == normalize_decode(text):
                     em_ok += 1
             except:
                 pass
@@ -238,6 +242,8 @@ def main():
     externals_cs = [
         ("DarijaBERT-ar", "external_darija", "WordPiece", "shared", 80000,
          "SI2M-Lab/DarijaBERT", False),
+        ("DarijaBERT-mix", "external_darija", "WordPiece", "shared", 160000,
+         "SI2M-Lab/DarijaBERT-mix", False),
         ("Qwen2.5-Darija", "external_darija", "SentencePiece", "shared", 151643,
          "GemMaroc/Qwen2.5-7B-Instruct-darija", False),
     ]