initial code and data

code/fasttext_embeddings.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env -S uv run
+# /// script
+# dependencies = ["fasttext", "matplotlib","numpy"]
+# ///
+
+# uv run fasttext_embeddings.py train ./corpus.txt --recursive --out ../models/kasturi_fasttext_v4
+# uv run fasttext_embeddings.py test ../models/kasturi_fasttext_v4.bin --word ಕನ್ನಡ
+# uv run fasttext_embeddings.py test ../models/kasturi_fasttext_v4.bin --word ರಾಜಧಾನಿ
+# uv run fasttext_embeddings.py test ../models/kasturi_fasttext_v4.bin --sentence "ಬೆಂಗಳೂರು ಕರ್ನಾಟಕದ ರಾಜಧಾನಿ"
+# uv run fasttext_embeddings.py compare ../models/kasturi_fasttext_v4.bin
+
+import argparse
+from pathlib import Path
+
+import fasttext
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib import font_manager
+
+
+def setup_kannada_font():
+    candidates = [
+        "Noto Sans Kannada",
+        "Noto Serif Kannada",
+        "Lohit Kannada",
+    ]
+
+    available = {f.name for f in font_manager.fontManager.ttflist}
+
+    for font in candidates:
+        if font in available:
+            plt.rcParams["font.family"] = font
+            return
+
+    print("Warning: Kannada font not found. Install fonts-noto-core.")
+
+
+def cosine_similarity(v1, v2):
+    denom = np.linalg.norm(v1) * np.linalg.norm(v2)
+    if denom == 0:
+        return 0.0
+    return float(np.dot(v1, v2) / denom)
+
+
+def compare_words_command(args):
+    setup_kannada_font()
+
+    model = fasttext.load_model(args.model)
+
+    words = args.words or [
+        "ಹಲಸು",
+        "ಹಣ್ಣು",
+        "ರೈತ",
+        "ನದಿ",
+        "ಮಳೆ",
+        "ತೆರಿಗೆ",
+        "ಅರಸ",
+        "ಅರಸಿ",
+        "ಹೆಣ್ಣು",
+        "ಮಾನವ",
+    ]
+
+    matrix = []
+    for w1 in words:
+        row = []
+        for w2 in words:
+            row.append(
+                cosine_similarity(
+                    model.get_word_vector(w1),
+                    model.get_word_vector(w2),
+                )
+            )
+        matrix.append(row)
+
+    matrix = np.array(matrix)
+
+    fig, ax = plt.subplots(figsize=(12, 8))
+    im = ax.imshow(matrix, vmin=0, vmax=1)
+
+    ax.set_xticks(np.arange(len(words)))
+    ax.set_yticks(np.arange(len(words)))
+    ax.set_xticklabels(words, fontsize=12)
+    ax.set_yticklabels(words, fontsize=12)
+
+    plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor")
+
+    for i in range(len(words)):
+        for j in range(len(words)):
+            ax.text(j, i, f"{matrix[i, j]:.3f}", ha="center", va="center", fontsize=9)
+
+    ax.set_title("Kannada Word Similarity Matrix", fontsize=14)
+    fig.colorbar(im, ax=ax)
+    fig.tight_layout()
+
+    plt.savefig(args.output, dpi=180, bbox_inches="tight")
+    print(f"Saved matrix graph to {args.output}")
+
+
+def train_command(args):
+    corpus_path = Path(args.corpus)
+
+    try:
+        model = fasttext.train_unsupervised(
+            input=str(corpus_path),
+            model="skipgram",
+            dim=args.dim,
+            epoch=args.epoch,
+            minCount=args.min_count,
+            wordNgrams=args.word_ngrams,
+            minn=args.minn,
+            maxn=args.maxn,
+            thread=args.threads,
+        )
+        print("training complete")
+        model.save_model(f"{args.out}.bin")
+        print(f"Saved {args.out}.bin")
+    except Exception as e:
+        print(f"Exception: {e}")
+
+
+def test_command(args):
+    model = fasttext.load_model(args.model)
+
+    if args.word:
+        for score, word in model.get_nearest_neighbors(args.word, k=args.topk):
+            print(f"{word:<30} {score:.4f}")
+
+    if args.sentence:
+        vec = model.get_sentence_vector(args.sentence)
+        print(f"Embedding dimensions: {len(vec)}")
+        print(vec[:20])
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    t = sub.add_parser("train")
+    t.add_argument("corpus", default="./corpus.txt")
+    t.add_argument("--recursive", action="store_true")
+    t.add_argument("--out", default="../models/kannada_fasttext")
+    t.add_argument("--dim", type=int, default=300)
+    t.add_argument("--epoch", type=int, default=20)
+    t.add_argument("--min-count", type=int, default=3)
+    t.add_argument("--word-ngrams", type=int, default=2)
+    t.add_argument("--minn", type=int, default=3)
+    t.add_argument("--maxn", type=int, default=8)
+    t.add_argument("--threads", type=int, default=16)
+
+    x = sub.add_parser("test")
+    x.add_argument("model")
+    x.add_argument("--word")
+    x.add_argument("--sentence")
+    x.add_argument("--topk", type=int, default=10)
+
+    c = sub.add_parser("compare")
+    c.add_argument("model")
+    c.add_argument("--output", default="../models/compare_matrix.png")
+    c.add_argument("--words", nargs="+")
+
+    args = parser.parse_args()
+
+    if args.command == "train":
+        train_command(args)
+    elif args.command == "test":
+        test_command(args)
+    elif args.command == "compare":
+        compare_words_command(args)
+
+
+if __name__ == "__main__":
+    main()

code/prepare_corpus.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env -S uv run
+# /// script
+# dependencies = []
+# ///
+#
+
+# uv run prepare_corpus.py prepare ../data ./corpus.txt --recursive
+
+
+import argparse
+import re
+import unicodedata
+from pathlib import Path
+
+
+def clean_text(text: str) -> str:
+    # Unicode normalization
+    text = unicodedata.normalize("NFC", text)
+
+    # Remove control characters except newline/tab
+    text = "".join(
+        ch for ch in text if unicodedata.category(ch)[0] != "C" or ch in "\n\t"
+    )
+
+    # Normalize line endings
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+
+    # Remove zero-width characters (common OCR artifacts)
+    text = re.sub(r"[\u200B-\u200D\uFEFF]", "", text)
+
+    # Normalize spaces
+    text = re.sub(r"[ \t]+", " ", text)
+
+    # Remove common OCR symbols
+    text = re.sub(r"[■□▪▫◆◇★☆◦•●]+", " ", text)
+    text = re.sub(r"[_=~`]+", " ", text)
+
+    # Remove separator lines
+    text = re.sub(r"(?m)^[-=*_]{3,}\s*$", "", text)
+
+    # Remove standalone page numbers
+    text = re.sub(r"(?m)^\s*\d+\s*$", "", text)
+
+    # Remove lines containing only Roman numerals (optional)
+    text = re.sub(r"(?m)^\s*[IVXLCDM]+\s*$", "", text)
+
+    # Trim spaces around newlines
+    text = re.sub(r" *\n *", "\n", text)
+
+    # Collapse multiple blank lines
+    text = re.sub(r"\n{3,}", "\n\n", text)
+
+    return text.strip()
+
+
+def collect_text_files(folder: Path, recursive: bool):
+    pattern = "**/*.txt" if recursive else "*.txt"
+    return sorted(folder.glob(pattern))
+
+
+def build_corpus(files, output_file: Path):
+    print(f"output_file {output_file}")
+
+    with output_file.open("w", encoding="utf-8") as out:
+        for file in files:
+            print(file)
+
+            try:
+                text = file.read_text(
+                    encoding="utf-8",
+                    errors="ignore",
+                )
+
+                text = clean_text(text)
+
+                out.write(text)
+                out.write("\n")
+
+            except Exception as e:
+                print(f"Skipping {file}: {e}")
+
+    print("DONE build_corpus")
+
+
+def prepare_command(args):
+    files = collect_text_files(Path(args.folder), args.recursive)
+    if not files:
+        raise SystemExit("No .txt files found")
+    build_corpus(files, Path(args.output))
+    print(f"Corpus written to {args.output}")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("prepare")
+    p.add_argument("folder", default="../data/")
+    p.add_argument("output", default="./corpus.txt")
+    p.add_argument("--recursive", action="store_true")
+
+    args = parser.parse_args()
+
+    if args.command == "prepare":
+        prepare_command(args)
+
+
+if __name__ == "__main__":
+    main()

code/word2vec_embeddings.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env -S uv run
+# /// script
+# dependencies = ["gensim"]
+# ///
+
+# uv run word2vec_embeddings.py train ./corpus.txt --out ../models/kannada_kasturi_word2vec.model --model skipgram
+# uv run word2vec_embeddings.py test ../models/kannada_kasturi_word2vec.model --word ಕನ್ನಡ
+# uv run word2vec_embeddings.py test ../models/kannada_kasturi_word2vec.model --word ರಾಜಧಾನಿ
+
+import argparse
+from pathlib import Path
+
+from gensim.models import Word2Vec
+from gensim.utils import simple_preprocess
+
+
+def read_sentences(corpus_path: Path):
+    with corpus_path.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                # For Kannada, whitespace tokenization is often better than simple_preprocess
+                yield line.split()
+
+
+def train_word2vec(args):
+    sentences = list(read_sentences(Path(args.input)))
+
+    model = Word2Vec(
+        sentences=sentences,
+        vector_size=args.dim,
+        window=args.window,
+        min_count=args.min_count,
+        workers=args.threads,
+        epochs=args.epoch,
+        sg=1 if args.model == "skipgram" else 0,
+    )
+
+    model.save(args.out)
+    model.wv.save_word2vec_format(args.out + ".vec")
+
+    print(f"Saved model: {args.out}")
+    print(f"Saved vectors: {args.out}.vec")
+
+
+def test_word(args):
+    model = Word2Vec.load(args.model_path)
+
+    print(model.wv[args.word])
+
+    print("\nNearest:")
+    for word, score in model.wv.most_similar(args.word, topn=10):
+        print(word, score)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    train = sub.add_parser("train")
+    train.add_argument("input", default="./corpus.txt")
+    train.add_argument("--out", default="../models/kannada_kasturi_word2vec.model")
+    train.add_argument("--model", choices=["skipgram", "cbow"], default="skipgram")
+    train.add_argument("--dim", type=int, default=300)
+    train.add_argument("--window", type=int, default=5)
+    train.add_argument("--epoch", type=int, default=20)
+    train.add_argument("--min-count", type=int, default=2)
+    train.add_argument("--threads", type=int, default=16)
+
+    test = sub.add_parser("test")
+    test.add_argument("model_path")
+    test.add_argument("--word", required=True)
+
+    args = parser.parse_args()
+
+    if args.cmd == "train":
+        train_word2vec(args)
+    elif args.cmd == "test":
+        test_word(args)
+
+
+if __name__ == "__main__":
+    main()