Initial release: Aksharakuppy Manglish->Malayalam IME

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoints_ime/lexicon.json filter=lfs diff=lfs merge=lfs -text

.ipynb_checkpoints/README-checkpoint.md

@@ -0,0 +1,47 @@
+---
+license: apache-2.0
+language:
+- ml
+tags:
+- transliteration
+- malayalam
+- manglish
+- input-method
+- ime
+library_name: pytorch
+pipeline_tag: translation
+datasets:
+- ai4bharat/Aksharantar
+---
+
+# Aksharakuppy — Manglish → Malayalam Transliteration IME
+
+A character-level Transformer + frequency-dictionary hybrid that converts
+romanized Malayalam (Manglish) into Malayalam script, with Google-IME-style
+word suggestions.
+
+## Model
+- 4-layer encoder-decoder Transformer, ~16.6M params, character-level tokenization
+- Hybrid: ~677k-word frequency lexicon (trie prefix lookup) + neural fallback for out-of-vocabulary words
+- Trained from scratch on a custom corpus + AI4Bharat Aksharantar (Malayalam) + Google Input Tools corrections
+
+## Usage
+​```python
+from ime import IME
+ime = IME("checkpoints_ime")
+print(ime.suggest("dhoni"))    # ['ധോണി', ...]
+print(ime.suggest("keralam"))  # ['കേരളം', ...]
+​```
+
+## Web demo
+​```bash
+pip install torch fastapi uvicorn
+uvicorn server:app --host 0.0.0.0 --port 8000
+​```
+
+## Limitations
+- Ambiguous romanizations may return multiple valid words; the intended word is usually in the top 5.
+- Coverage reflects the training lexicon; rare or new words use the neural fallback.
+
+## License
+Apache-2.0. Training data includes AI4Bharat Aksharantar (manually collected: CC-BY; mined: CC0).

README.md

@@ -0,0 +1,47 @@
+---
+license: apache-2.0
+language:
+- ml
+tags:
+- transliteration
+- malayalam
+- manglish
+- input-method
+- ime
+library_name: pytorch
+pipeline_tag: translation
+datasets:
+- ai4bharat/Aksharantar
+---
+
+# Aksharakuppy — Manglish → Malayalam Transliteration IME
+
+A character-level Transformer + frequency-dictionary hybrid that converts
+romanized Malayalam (Manglish) into Malayalam script, with Google-IME-style
+word suggestions.
+
+## Model
+- 4-layer encoder-decoder Transformer, ~16.6M params, character-level tokenization
+- Hybrid: ~677k-word frequency lexicon (trie prefix lookup) + neural fallback for out-of-vocabulary words
+- Trained from scratch on a custom corpus + AI4Bharat Aksharantar (Malayalam) + Google Input Tools corrections
+
+## Usage
+​```python
+from ime import IME
+ime = IME("checkpoints_ime")
+print(ime.suggest("dhoni"))    # ['ധോണി', ...]
+print(ime.suggest("keralam"))  # ['കേരളം', ...]
+​```
+
+## Web demo
+​```bash
+pip install torch fastapi uvicorn
+uvicorn server:app --host 0.0.0.0 --port 8000
+​```
+
+## Limitations
+- Ambiguous romanizations may return multiple valid words; the intended word is usually in the top 5.
+- Coverage reflects the training lexicon; rare or new words use the neural fallback.
+
+## License
+Apache-2.0. Training data includes AI4Bharat Aksharantar (manually collected: CC-BY; mined: CC0).

checkpoints_ime/best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85f811c32ec277b2017abafb6afe44a11feefe3ca8600707baa61c3a077e6941
+size 199500041

checkpoints_ime/lexicon.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6285398a3e68b7ed1888fa3936026f61a91dfa6b820c5cd3b2e3c0ef8fbff5e2
+size 40547832

checkpoints_ime/vocab.json

@@ -0,0 +1 @@
+{"<pad>": 0, "<bos>": 1, "<eos>": 2, "<unk>": 3, "-": 4, "a": 5, "b": 6, "c": 7, "d": 8, "e": 9, "f": 10, "g": 11, "h": 12, "i": 13, "j": 14, "k": 15, "l": 16, "m": 17, "n": 18, "o": 19, "p": 20, "q": 21, "r": 22, "s": 23, "t": 24, "u": 25, "v": 26, "w": 27, "x": 28, "y": 29, "z": 30, "ം": 31, "ഃ": 32, "അ": 33, "ആ": 34, "ഇ": 35, "ഈ": 36, "ഉ": 37, "ഊ": 38, "ഋ": 39, "എ": 40, "ഏ": 41, "ഐ": 42, "ഒ": 43, "ഓ": 44, "ഔ": 45, "ക": 46, "ഖ": 47, "ഗ": 48, "ഘ": 49, "ങ": 50, "ച": 51, "ഛ": 52, "ജ": 53, "ഝ": 54, "ഞ": 55, "ട": 56, "ഠ": 57, "ഡ": 58, "ഢ": 59, "ണ": 60, "ത": 61, "ഥ": 62, "ദ": 63, "ധ": 64, "ന": 65, "പ": 66, "ഫ": 67, "ബ": 68, "ഭ": 69, "മ": 70, "യ": 71, "ര": 72, "റ": 73, "ല": 74, "ള": 75, "ഴ": 76, "വ": 77, "ശ": 78, "ഷ": 79, "സ": 80, "ഹ": 81, "ാ": 82, "ി": 83, "ീ": 84, "ു": 85, "ൂ": 86, "ൃ": 87, "െ": 88, "േ": 89, "ൈ": 90, "ൊ": 91, "ോ": 92, "ൌ": 93, "്": 94, "ൗ": 95, "ൺ": 96, "ൻ": 97, "ർ": 98, "ൽ": 99, "ൾ": 100, "ൿ": 101, "‌": 102, "‍": 103}

config.py

@@ -0,0 +1,53 @@
+import os
+import torch
+
+MODE = os.environ.get("TRANSLIT_MODE", "sentence")   # "sentence" | "ime"
+
+
+class Config:
+    # ---------------- common ----------------
+    d_model    = 384
+    nhead      = 6
+    num_layers = 4
+    dim_ff     = 1536
+    dropout    = 0.1
+
+    batch_size   = 1024
+    lr           = 7e-4
+    warmup_steps = 2000
+    weight_decay = 0.01
+    label_smooth = 0.1
+    grad_clip    = 1.0
+    num_workers  = 24
+    amp_dtype    = torch.bfloat16
+
+    max_src_len = 256
+    max_tgt_len = 320
+
+    dry_run_samples = 20_000
+    dry_run_steps   = 100
+
+    # ---------------- mode specific ----------------
+    if MODE == "ime":
+        train_path = "data/word_train.jsonl"
+        val_path   = "data/word_val.jsonl"
+        test_path  = "data/word_test.jsonl"
+        ckpt_dir   = "checkpoints_ime"
+        max_len    = 64
+        batch_size = 1024         # big dataset again -> big batch
+        epochs     = 10           # millions of words -> few epochs suffice
+        lr         = 7e-4
+        warmup_steps = 2000
+        dropout    = 0.1          # plenty of data -> normal dropout
+    else:
+        # sentence-level Malayalam -> Manglish (original)
+        train_path = "data/train.jsonl"
+        val_path   = "data/val.jsonl"
+        test_path  = "data/test.jsonl"
+        ckpt_dir   = "checkpoints"
+        max_len    = 512
+        epochs     = 8
+
+
+CFG = Config()
+print(f"[config] mode={MODE}  data={CFG.train_path}  ckpt={CFG.ckpt_dir}")

ime.py

@@ -0,0 +1,179 @@
+import json
+import torch
+import torch.nn.functional as F
+from tokenizer import CharTokenizer, PAD, BOS, EOS
+from model import TranslitModel
+from config import CFG
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+class Trie:
+    def __init__(self):
+        self.root = {}
+
+    def insert(self, word, payload):
+        node = self.root
+        for c in word:
+            node = node.setdefault(c, {})
+        node.setdefault("$", []).append(payload)
+
+    def prefix(self, pre, limit=400):
+        node = self.root
+        for c in pre:
+            if c not in node:
+                return []
+            node = node[c]
+        out, stack = [], [node]
+        while stack and len(out) < limit:
+            n = stack.pop()
+            if "$" in n:
+                out.extend(n["$"])
+            for k, v in n.items():
+                if k != "$":
+                    stack.append(v)
+        return out
+
+
+class IME:
+    def __init__(self, ckpt_dir="checkpoints_ime"):
+        # ---------- lexicon trie ----------
+        with open(f"{ckpt_dir}/lexicon.json", encoding="utf-8") as f:
+            lex = json.load(f)
+        self.trie = Trie()
+        for g, targets in lex.items():
+            for m, n in targets.items():
+                self.trie.insert(g, (g, m, n))
+
+        # ---------- neural fallback model ----------
+        self.tok = CharTokenizer.load(f"{ckpt_dir}/vocab.json")
+        self.model = TranslitModel(
+            len(self.tok), CFG.d_model, CFG.nhead,
+            CFG.num_layers, CFG.dim_ff,
+            dropout=0.0, max_len=64,
+        ).to(DEVICE)
+        sd = torch.load(f"{ckpt_dir}/best.pt", map_location=DEVICE)["model"]
+        sd = {k.replace("_orig_mod.", ""): v for k, v in sd.items()}
+        self.model.load_state_dict(sd)
+        self.model.eval()
+
+    # ---------- dictionary path (trie prefix lookup + noise filtering) ----------
+    def _dict_suggest(self, q, k):
+        hits = self.trie.prefix(q, limit=400)
+        exact = [h for h in hits if h[0] == q]
+        pref  = [h for h in hits if h[0] != q]
+
+        # exact-match noise filter: if a dominant exact form exists,
+        # drop rare "alternatives" (misaligned one-off pairs)
+        if exact:
+            top = max(h[2] for h in exact)
+            if top >= 5:
+                exact = [h for h in exact if h[2] >= max(2, top * 0.02)]
+        exact.sort(key=lambda h: -h[2])
+
+        # prefix completions: Aksharantar has millions of n=1 glued
+        # compounds (e.g. "entesuhruthaya"). Keep a completion only if it
+        # was seen more than once OR is not much longer than the query,
+        # then rank by frequency and shortness.
+        pref = [h for h in pref if h[2] >= 2 or len(h[0]) <= len(q) + 6]
+        pref.sort(key=lambda h: (-h[2], len(h[0])))
+
+        out, seen = [], set()
+        for g, m, n in exact + pref:
+            if m not in seen:
+                seen.add(m)
+                out.append(m)
+            if len(out) == k:
+                break
+        return out
+
+    # ---------- neural path (beam search) ----------
+    @torch.no_grad()
+    def _model_suggest(self, q, k=3, beam=8, max_len=48):
+        src = torch.tensor([self.tok.encode(q)], device=DEVICE)
+        beams = [(torch.tensor([[BOS]], device=DEVICE), 0.0, False)]
+        for _ in range(max_len):
+            if all(f for _, _, f in beams):
+                break
+            cands = []
+            for ids, lp, fin in beams:
+                if fin:
+                    cands.append((ids, lp, True))
+                    continue
+                logits = self.model(src, ids)[0, -1]
+                logp = F.log_softmax(logits.float(), -1)
+                tl, ti = logp.topk(beam)
+                for l, ix in zip(tl.tolist(), ti.tolist()):
+                    nids = torch.cat(
+                        [ids, torch.tensor([[ix]], device=DEVICE)], 1)
+                    cands.append((nids, lp + l, ix == EOS))
+            cands.sort(key=lambda c: c[1] / (c[0].size(1) ** 0.7),
+                       reverse=True)
+            beams = cands[:beam]
+
+        out, seen = [], set()
+        for ids, _, _ in beams:
+            s = self.tok.decode(ids[0].tolist())
+            if s and s not in seen:
+                seen.add(s)
+                out.append(s)
+            if len(out) == k:
+                break
+        return out
+
+    # ---------- public API ----------
+# ---------- public API ----------
+    def suggest(self, manglish_word, k=5):
+        q = manglish_word.strip()
+        if not q:
+            return []
+        if self._is_passthrough(q):
+            return self._number_suggestions(q)
+    
+        ql = q.lower()
+        dict_hits = self._dict_suggest(ql, k)
+    
+        # is there an exact-length dictionary match? (romanization == input)
+        has_exact = any(h[0] == ql for h in self.trie.prefix(ql, limit=50))
+    
+        if not has_exact:
+            # no bare-form entry -> the model's direct transliteration is
+            # usually the clean word the user wants (e.g. dhoni -> ധോണി)
+            model_hits = self._model_suggest(ql, k=2)
+            merged = []
+            for s in model_hits + dict_hits:
+                if s not in merged:
+                    merged.append(s)
+            return merged[:k]
+    
+        return dict_hits[:k]
+
+    @staticmethod
+    def _is_passthrough(q):
+        # token is all digits / punctuation / has no latin letters to transliterate
+        return not any(c.isalpha() and ord(c) < 128 for c in q)
+
+    # Western -> Malayalam digit map
+    _ML_DIGITS = str.maketrans("0123456789", "൦൧൨൩൪൫൬൭൮൯")
+
+    def _number_suggestions(self, q):
+        out = [q]                                  # keep as-is (10)
+        if any(c.isdigit() for c in q):
+            ml = q.translate(self._ML_DIGITS)      # malayalam numerals (൧൦)
+            if ml != q:
+                out.append(ml)
+        return out
+
+
+if __name__ == "__main__":
+    ime = IME()
+    tests = [
+        "thi", "thila",                          # prefix -> dict completions
+        "amma", "ente", "adukkala", "veedu",     # common words
+        "krithyamaayi",
+        "njan", "nammal", "keralam",             # now covered by aksharantar
+        "thiruvananthapuram",
+        "blockchain", "kunjava",                 # OOV / rare -> model fallback
+    ]
+    for w in tests:
+        print(f"{w:22s} -> {ime.suggest(w)}")

model.py

@@ -0,0 +1,65 @@
+import math, torch
+import torch.nn as nn
+
+
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model, max_len=512):
+        super().__init__()
+        pe = torch.zeros(max_len, d_model)
+        pos = torch.arange(max_len).unsqueeze(1).float()
+        div = torch.exp(torch.arange(0, d_model, 2).float()
+                        * (-math.log(10000.0) / d_model))
+        pe[:, 0::2] = torch.sin(pos * div)
+        pe[:, 1::2] = torch.cos(pos * div)
+        self.register_buffer("pe", pe.unsqueeze(0))
+
+    def forward(self, x):
+        return x + self.pe[:, : x.size(1)]
+
+
+class TranslitModel(nn.Module):
+    def __init__(self, vocab_size, d_model=384, nhead=6,
+                 num_layers=4, dim_ff=1536, dropout=0.1, max_len=512):
+        super().__init__()
+        self.d_model = d_model
+
+        self.embed = nn.Embedding(vocab_size, d_model, padding_idx=0)
+        self.pos = PositionalEncoding(d_model, max_len)
+
+        self.transformer = nn.Transformer(
+            d_model=d_model,
+            nhead=nhead,
+            num_encoder_layers=num_layers,
+            num_decoder_layers=num_layers,
+            dim_feedforward=dim_ff,
+            dropout=dropout,
+            batch_first=True,
+            norm_first=True,
+        )
+
+        self.out = nn.Linear(d_model, vocab_size)
+        self.out.weight = self.embed.weight  # weight tying
+
+        # ---- proper init for tied embedding (fixes huge initial loss) ----
+        nn.init.normal_(self.embed.weight, mean=0.0, std=d_model ** -0.5)
+        nn.init.zeros_(self.out.bias)
+        with torch.no_grad():
+            self.embed.weight[0].fill_(0)  # keep padding row at zero
+
+    def forward(self, src, tgt_in):
+        src_pad = src == 0
+        tgt_pad = tgt_in == 0
+        causal = nn.Transformer.generate_square_subsequent_mask(
+            tgt_in.size(1), device=src.device)
+
+        s = self.pos(self.embed(src) * math.sqrt(self.d_model))
+        t = self.pos(self.embed(tgt_in) * math.sqrt(self.d_model))
+
+        h = self.transformer(
+            s, t,
+            tgt_mask=causal,
+            src_key_padding_mask=src_pad,
+            tgt_key_padding_mask=tgt_pad,
+            memory_key_padding_mask=src_pad,
+        )
+        return self.out(h)

server.py

@@ -0,0 +1,123 @@
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse
+from ime import IME
+
+app = FastAPI()
+ime = IME()
+
+
+@app.get("/suggest")
+def suggest(q: str, k: int = 5):
+    return {"input": q, "suggestions": ime.suggest(q, k=k)}
+
+
+PAGE = """
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Manglish → Malayalam IME</title>
+<style>
+ :root{
+   --bg:#0f1115; --panel:#171a21; --panel2:#1e222b;
+   --text:#e6e8ec; --muted:#8b93a1; --accent:#4f8cff;
+   --accent-soft:#22314f; --border:#2a2f3a; --chip:#232833;
+ }
+ *{box-sizing:border-box}
+ body{
+   font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif;
+   background:var(--bg); color:var(--text);
+   margin:0; min-height:100vh;
+   display:flex; align-items:flex-start; justify-content:center;
+ }
+ .wrap{width:100%; max-width:680px; padding:56px 20px}
+ h1{font-size:22px; font-weight:600; margin:0 0 4px}
+ .sub{color:var(--muted); font-size:14px; margin:0 0 28px}
+ #out{
+   font-size:26px; line-height:1.5; min-height:52px;
+   background:var(--panel); border:1px solid var(--border);
+   border-radius:12px; padding:14px 16px; margin-bottom:16px;
+   word-break:break-word;
+ }
+ #out:empty::before{content:"Your Malayalam text appears here…"; color:var(--muted); font-size:17px}
+ input{
+   width:100%; font-size:22px; padding:14px 16px;
+   background:var(--panel2); color:var(--text);
+   border:1px solid var(--border); border-radius:12px; outline:none;
+   transition:border-color .15s;
+ }
+ input:focus{border-color:var(--accent)}
+ input::placeholder{color:var(--muted)}
+ #sug{margin-top:12px; min-height:52px; display:flex; flex-wrap:wrap; gap:8px}
+ .s{
+   padding:9px 16px; background:var(--chip); color:var(--text);
+   border:1px solid var(--border); border-radius:10px;
+   font-size:22px; cursor:pointer; user-select:none;
+   transition:background .12s, border-color .12s;
+ }
+ .s:hover{background:#2b313d}
+ .s:first-child{background:var(--accent-soft); border-color:var(--accent)}
+ .hint{color:var(--muted); font-size:13px; margin-top:18px; line-height:1.6}
+ .hint b{color:var(--text); font-weight:600}
+</style>
+</head>
+<body>
+<div class="wrap">
+  <h1>Manglish → Malayalam IME</h1>
+  <p class="sub">Type Malayalam using English letters.</p>
+
+  <div id="out"></div>
+  <input id="inp" placeholder="type manglish… e.g. ente veedu" autocomplete="off" autofocus>
+  <div id="sug"></div>
+
+  <p class="hint">
+    <b>space</b> or <b>enter</b> = accept first suggestion ·
+    <b>click</b> a chip to pick ·
+    <b>backspace</b> on empty input deletes last word
+  </p>
+</div>
+
+<script>
+const inp=document.getElementById('inp'),
+      sug=document.getElementById('sug'),
+      out=document.getElementById('out');
+let timer=null,current=[],ctr=0;
+
+inp.addEventListener('input',()=>{clearTimeout(timer);timer=setTimeout(fetchSug,110)});
+
+async function fetchSug(){
+  const q=inp.value.trim();
+  if(!q){sug.innerHTML='';current=[];return}
+  const my=++ctr;
+  try{
+    const r=await fetch('/suggest?q='+encodeURIComponent(q));
+    const j=await r.json();
+    if(my!==ctr)return;               // ignore stale responses
+    current=j.suggestions;
+    sug.innerHTML=current.map(s=>`<span class="s">${s}</span>`).join('');
+    [...sug.children].forEach((el,i)=>el.onclick=()=>pick(current[i]));
+  }catch(e){/* ignore */}
+}
+function pick(s){
+  out.textContent+=s+' ';
+  inp.value='';sug.innerHTML='';current=[];inp.focus();
+}
+inp.addEventListener('keydown',e=>{
+  if((e.key===' '||e.key==='Enter')&&current.length){
+    e.preventDefault();pick(current[0]);
+  }else if(e.key==='Backspace'&&!inp.value){
+    e.preventDefault();
+    out.textContent=out.textContent.trimEnd().split(' ').slice(0,-1).join(' ');
+    if(out.textContent)out.textContent+=' ';
+  }
+});
+</script>
+</body>
+</html>
+"""
+
+
+@app.get("/", response_class=HTMLResponse)
+def index():
+    return PAGE

tokenizer.py

@@ -0,0 +1,41 @@
+import json
+
+PAD, BOS, EOS, UNK = 0, 1, 2, 3
+SPECIALS = ["<pad>", "<bos>", "<eos>", "<unk>"]
+
+class CharTokenizer:
+    def __init__(self, vocab=None):
+        self.vocab = vocab or {}
+        self.inv = {i: c for c, i in self.vocab.items()}
+
+    @classmethod
+    def build(cls, texts, min_freq=20):
+        from collections import Counter
+        cnt = Counter()
+        for t in texts:
+            cnt.update(t)
+        vocab = {s: i for i, s in enumerate(SPECIALS)}
+        for ch, freq in sorted(cnt.items()):
+            if freq >= min_freq:
+                vocab[ch] = len(vocab)
+        return cls(vocab)
+
+    def encode(self, text, add_special=True):
+        ids = [self.vocab.get(c, UNK) for c in text]
+        return [BOS] + ids + [EOS] if add_special else ids
+
+    def decode(self, ids):
+        return "".join(self.inv.get(i, "") for i in ids
+                       if i not in (PAD, BOS, EOS, UNK))
+
+    def save(self, path):
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(self.vocab, f, ensure_ascii=False)
+
+    @classmethod
+    def load(cls, path):
+        with open(path, encoding="utf-8") as f:
+            return cls(json.load(f))
+
+    def __len__(self):
+        return len(self.vocab)