Upload folder using huggingface_hub

Browse files

Files changed (5) hide show

checkpoint.pt +2 -2
gen_tokenizer.py +89 -1824
infer_gguf.py +299 -75
model_tiny.py +121 -6
tokenizer.json +0 -0

checkpoint.pt CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8cc0d5d0927b27c8c3921d96f21029f47857b105acb3b68eed4d521ac2f20ec9
-size 2385856

 version https://git-lfs.github.com/spec/v1
+oid sha256:7341bcfb74b09c7c8c8bd1946f53f6e44823a574d9b2e9f0040707a43c6c05cd
+size 3906061

gen_tokenizer.py CHANGED Viewed

@@ -1,1833 +1,98 @@
 import json
-# === GPT-2 byte-to-unicode mapping ===
-def bytes_to_unicode():
-    bs = list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
-    cs = bs[:]
-    n = 0
-    for b in range(256):
-        if b not in bs:
-            bs.append(b)
-            cs.append(256 + n)
-            n += 1
-    return {b: chr(c) for b, c in zip(bs, cs)}
-byte2char = bytes_to_unicode()
-# Reverse mapping: char -> byte
-char2byte = {c: b for b, c in byte2char.items()}
-# === Vocab structure ===
-# IDs 0-50: 51 special tokens
-# IDs 51-306: 256 byte-level chars
-# IDs 307-4095: ~3789 curated English words + subwords
 special_tokens = [
-    ("<unk>", 0),
-    ("<s>", 1),
-    ("</s>", 2),
-    ("<pad>", 3),
-    ("<|system|>", 4),
-    ("<|user|>", 5),
-    ("<|assistant|>", 6),
-    ("<think>", 7),
-    ("</think>", 8),
-    ("[INST]", 9),
-    ("[/INST]", 10),
-    ("<|begin_of_thought|>", 11),
-    ("<|end_of_thought|>", 12),
-    ("<|reflect|>", 13),
-    ("<|revise|>", 14),
-    ("<|verify|>", 15),
-    ("<|code|>", 16),
-    ("<|text|>", 17),
-    ("<|math|>", 18),
-    ("<|think|>", 19),
-    ("<|answer|>", 20),
-    ("<|step|>", 21),
-    ("<|reason|>", 22),
-    ("<|check|>", 23),
-    ("<|output|>", 24),
-    ("<|plan|>", 25),
-    ("<|solve|>", 26),
-    ("<|analyze|>", 27),
-    ("<|conclude|>", 28),
-    ("<|approach|>", 29),
-    ("<|alternative|>", 30),
-    ("<|summary|>", 31),
-    ("<|question|>", 32),
-    ("<|hint|>", 33),
-    ("<|example|>", 34),
-    ("<|correct|>", 35),
-    ("<|incorrect|>", 36),
-    ("<|feedback|>", 37),
-    ("<|start|>", 38),
-    ("<|end|>", 39),
-    ("<|sep|>", 40),
-    ("<|cls|>", 41),
-    ("<|tool|>", 42),
-    ("<|function|>", 43),
-    ("<|result|>", 44),
-    ("<|input|>", 45),
-    ("<|detect|>", 46),
-    ("<|context|>", 47),
-    ("<|proof|>", 48),
-    ("<|lemma|>", 49),
-    ("<|theorem|>", 50),
-]
-# IDs 51-306: 256 byte-level characters
-byte_tokens = []
-for b in range(256):
-    byte_tokens.append((byte2char[b], 51 + b))
-# IDs 307+: common words (variable count, no filler)
-# Most frequent English words + programming terms
-common_words = [
-    "the", "be", "to", "of", "and", "a", "in", "that", "have", "I",
-    "it", "for", "not", "on", "with", "he", "as", "you", "do", "at",
-    "this", "but", "his", "by", "from", "they", "we", "say", "her", "she",
-    "or", "an", "will", "my", "one", "all", "would", "there", "their", "what",
-    "so", "up", "out", "if", "about", "who", "get", "which", "go", "me",
-    "when", "make", "can", "like", "time", "no", "just", "him", "know", "take",
-    "people", "into", "year", "your", "good", "some", "could", "them", "see", "other",
-    "than", "then", "now", "look", "only", "come", "its", "over", "think", "also",
-    "back", "after", "use", "two", "how", "our", "work", "first", "well", "way",
-    "even", "new", "want", "because", "any", "these", "give", "day", "most", "us",
-    "is", "was", "are", "were", "been", "has", "had", "did", "does", "am",
-    "being", "having", "doing", "saying", "going", "getting", "making", "knowing", "taking", "thinking",
-    "come", "coming", "came", "go", "goes", "gone", "going", "went",
-    "see", "saw", "seen", "seeing", "say", "said", "says", "saying",
-    "get", "got", "gotten", "getting", "make", "made", "makes", "making",
-    "know", "knew", "known", "knows", "think", "thought", "thinks", "thinking",
-    "take", "took", "taken", "takes", "taking", "give", "gave", "given", "gives", "giving",
-    "find", "found", "finds", "finding", "tell", "told", "tells", "telling",
-    "ask", "asked", "asks", "asking", "show", "showed", "shown", "shows", "showing",
-    "try", "tried", "tries", "trying", "leave", "left", "leaves", "leaving",
-    "call", "called", "calls", "calling", "keep", "kept", "keeps", "keeping",
-    "let", "lets", "letting", "begin", "began", "begun", "begins", "beginning",
-    "seem", "seemed", "seems", "seeming", "help", "helped", "helps", "helping",
-    "turn", "turned", "turns", "turning", "start", "started", "starts", "starting",
-    "bring", "brought", "brings", "bringing", "happen", "happened", "happens", "happening",
-    "write", "wrote", "written", "writes", "writing", "provide", "provided", "provides", "providing",
-    "consider", "considered", "considers", "considering", "appear", "appeared", "appears", "appearing",
-    "follow", "followed", "follows", "following", "change", "changed", "changes", "changing",
-    "form", "formed", "forms", "forming", "need", "needed", "needs", "needing",
-    "set", "sets", "setting", "put", "puts", "putting", "run", "runs", "running",
-    "move", "moved", "moves", "moving", "stand", "stood", "stands", "standing",
-    "win", "won", "wins", "winning", "play", "played", "plays", "playing",
-    "point", "points", "pointed", "pointing", "large", "small", "big", "little",
-    "long", "short", "high", "low", "old", "young", "great", "important",
-    "different", "same", "other", "many", "much", "more", "most", "few",
-    "own", "very", "such", "still", "just", "also", "even", "too",
-    "here", "there", "where", "when", "why", "how", "what", "which",
-    "while", "though", "although", "until", "since", "before", "after", "during",
-    "without", "within", "between", "through", "across", "around", "above", "below",
-    "under", "over", "again", "ever", "never", "always", "often", "sometimes",
-    "together", "alone", "already", "yet", "still", "almost", "quite", "rather",
-    "well", "bad", "better", "worse", "best", "worst", "more", "less",
-    "every", "each", "both", "either", "neither", "all", "any", "none",
-    "thing", "things", "way", "ways", "time", "times", "year", "years",
-    "day", "days", "week", "weeks", "month", "months", "part", "parts",
-    "place", "places", "case", "cases", "point", "points", "world", "worlds",
-    "number", "numbers", "group", "groups", "system", "systems", "program", "programs",
-    "data", "information", "problem", "problems", "solution", "solutions",
-    "method", "methods", "result", "results", "process", "processes",
-    "function", "functions", "value", "values", "type", "types",
-    "state", "states", "model", "models", "level", "levels",
-    "line", "lines", "file", "files", "code", "codes",
-    "set", "sets", "list", "lists", "array", "arrays",
-    "object", "objects", "class", "classes", "property", "properties",
-    "input", "inputs", "output", "outputs", "return", "returns",
-    "define", "defined", "defines", "defining", "declare", "declared",
-    "import", "imports", "export", "exports", "include", "includes",
-    "public", "private", "protected", "static", "final", "const",
-    "void", "int", "float", "double", "char", "bool", "string",
-    "true", "false", "null", "None", "nil", "undefined",
-    "if", "else", "elif", "then", "switch", "case", "default", "break",
-    "for", "while", "do", "each", "in", "of", "to", "by",
-    "try", "catch", "finally", "throw", "raise", "except",
-    "return", "yield", "await", "async", "defer",
-    "and", "or", "not", "is", "as", "with", "without",
-    "lambda", "map", "filter", "reduce", "sort",
-    "new", "delete", "free", "alloc", "realloc",
-    "print", "printf", "println", "log", "debug", "error",
-    "len", "size", "length", "count", "sum", "max", "min",
-    "abs", "pow", "sqrt", "floor", "ceil", "round",
-    "sin", "cos", "tan", "atan", "log", "exp",
-    "zero", "one", "two", "three", "four", "five",
-    "six", "seven", "eight", "nine", "ten",
-    "first", "second", "third", "last", "next", "previous",
-    "current", "initial", "final", "primary", "secondary",
-    "main", "primary", "secondary", "basic", "advanced",
-    "simple", "complex", "single", "double", "multiple",
-    "add", "sub", "mul", "div", "mod", "inc", "dec",
-    "push", "pop", "shift", "unshift", "insert", "remove",
-    "append", "prepend", "concat", "join", "split", "slice",
-    "open", "close", "read", "write", "load", "save",
-    "create", "update", "delete", "insert", "select", "merge",
-    "begin", "end", "start", "stop", "pause", "resume",
-    "enable", "disable", "allow", "deny", "grant", "revoke",
-    "user", "users", "name", "names", "id", "ids",
-    "key", "keys", "value", "values", "field", "fields",
-    "table", "tables", "row", "rows", "column", "columns",
-    "index", "indexes", "indices", "query", "queries",
-    "count", "avg", "total", "sum", "min", "max",
-    "page", "pages", "home", "login", "logout", "signup",
-    "error", "errors", "warning", "warnings", "info",
-    "success", "failure", "status", "message", "messages",
-    "request", "requests", "response", "responses",
-    "client", "server", "api", "endpoint", "route",
-    "http", "https", "url", "uri", "port", "host",
-    "config", "configuration", "setting", "settings",
-    "option", "options", "param", "params", "parameter", "parameters",
-    "arg", "args", "argument", "arguments", "kwargs",
-    "path", "dir", "directory", "dirs", "folder", "folders",
-    "item", "items", "element", "elements", "entry", "entries",
-    "note", "notes", "text", "texts", "content", "contents",
-    "source", "sources", "target", "targets", "ref", "refs",
-    "struct", "structs", "union", "unions", "enum", "enums",
-    "impl", "implement", "implementation", "interface",
-    "abstract", "virtual", "override", "overload",
-    "base", "derived", "parent", "child", "children",
-    "root", "leaf", "node", "nodes", "edge", "edges",
-    "tree", "graph", "list", "queue", "stack", "heap",
-    "map", "dict", "dictionary", "hash", "hashmap",
-    "link", "links", "linked", "pointer", "pointers",
-    "thread", "threads", "process", "processes", "task", "tasks",
-    "sync", "async", "lock", "mutex", "semaphore",
-    "buffer", "buffers", "cache", "cached", "pool", "pools",
-    "memory", "disk", "network", "socket", "sockets",
-    "stream", "streams", "packet", "packets", "frame",
-    "meta", "metadata", "header", "headers", "body", "payload",
-    "token", "tokens", "session", "cookie", "cookies",
-    "auth", "login", "logout", "register", "password",
-    "hash", "salt", "encrypt", "decrypt", "encode", "decode",
-    "cert", "certificate", "key", "public", "private",
-    "train", "training", "trained", "test", "testing", "tested",
-    "valid", "validate", "validation", "eval", "evaluate",
-    "model", "models", "layer", "layers", "weight", "weights",
-    "bias", "biases", "loss", "losses", "grad", "grads",
-    "lr", "learning_rate", "optimizer", "adam", "sgd",
-    "batch", "batches", "epoch", "epochs", "step", "steps",
-    "dataset", "dataloader", "tensor", "tensors",
-    "gpu", "cpu", "tpu", "device", "devices", "memory",
-    "math", "physics", "chemistry", "biology", "science",
-    "compute", "calculate", "computation", "calculation",
-    "equation", "formula", "expression", "theorem",
-    "proof", "prove", "lemma", "axiom", "corollary",
-    "function", "graph", "derivative", "integral", "limit",
-    "sequence", "series", "matrix", "vector", "tensor",
-    "set", "subset", "union", "intersection", "complement",
-    "space", "group", "ring", "field", "module", "algebra",
-    "analysis", "topology", "geometry", "statistics",
-    "probability", "distribution", "random", "sample",
-    "mean", "median", "mode", "variance", "std", "deviation",
-    "linear", "nonlinear", "convex", "concave", "smooth",
-    "algorithm", "algorithm", "complexity", "runtime",
-    "tree", "graph", "sort", "search", "traverse",
-    "recursive", "iterative", "dynamic", "greedy",
-    "optimization", "constraint", "feasible", "optimal",
-    "problem", "solution", "input", "output", "example",
-    "question", "answer", "hint", "step", "reason",
-    "think", "analyze", "approach", "solve", "verify",
-    "check", "conclude", "summarize", "explain", "describe",
-    "correct", "incorrect", "right", "wrong", "positive", "negative",
-    "yes", "no", "maybe", "always", "never", "sometimes",
-    ":", ";", ".", ",", "!", "?", "'", "\"", "(", ")", "[", "]", "{", "}",
-    "<", ">", "=", "+", "-", "*", "/", "%", "&", "|", "^", "~",
-    "@", "#", "$", "_", "`", "\\",
-    "==", "!=", "<=", ">=", "&&", "||", "++", "--",
-    "+=", "-=", "*=", "/=", "->", "=>", "::", "..",
-    "...", "/*", "*/", "//", "<!--", "-->",
-    "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
-    "10", "11", "12", "13", "14", "15", "16", "17", "18", "19",
-    "20", "30", "40", "50", "60", "70", "80", "90", "100",
-    "-1", "-2", "0x", "0b", "0o",
-    # GPT-2 style Ġ-prefixed versions for words preceded by space
-    # The ByteLevel pre_tokenizer adds Ġ (U+0120) prefix after space
-    "Ġthe", "Ġto", "Ġof", "Ġand", "Ġa", "Ġin", "Ġthat", "Ġis", "Ġwas", "Ġfor",
-    "Ġon", "Ġwith", "Ġas", "Ġby", "Ġat", "Ġfrom", "Ġor", "Ġan", "Ġwill", "Ġwould",
-    "Ġnot", "Ġbut", "Ġare", "Ġwere", "Ġbeen", "Ġhave", "Ġhas", "Ġhad", "Ġdo", "Ġdoes",
-    "Ġdid", "Ġcan", "Ġcould", "Ġshould", "Ġmay", "Ġmight", "Ġshall", "Ġmust",
-    "Ġif", "Ġelse", "Ġwhen", "Ġwhile", "Ġbecause", "Ġso", "Ġthen", "Ġthan",
-    "Ġalso", "Ġeven", "Ġonly", "Ġjust", "Ġvery", "Ġtoo", "Ġstill", "Ġalready",
-    "Ġhere", "Ġthere", "Ġwhere", "Ġwhen", "Ġwhy", "Ġhow", "Ġwhat", "Ġwhich",
-    "Ġthis", "Ġthat", "Ġthese", "Ġthose", "Ġit", "Ġits", "Ġthey", "Ġthem",
-    "Ġwe", "Ġus", "Ġour", "Ġyou", "Ġyour", "Ġhe", "Ġhim", "Ġhis",
-    "Ġshe", "Ġher", "Ġhers", "Ġone", "Ġno", "Ġall", "Ġany", "Ġsome",
-    "Ġeach", "Ġevery", "Ġboth", "Ġneither", "Ġeither", "Ġmore", "Ġmost", "Ġfew",
-    "Ġother", "Ġanother", "Ġsuch", "Ġsame", "Ġdifferent", "Ġown",
-    "Ġlike", "Ġwell", "Ġgood", "Ġbad", "Ġbetter", "Ġbest",
-    "Ġnew", "Ġold", "Ġbig", "Ġsmall", "Ġlong", "Ġshort",
-    "Ġhigh", "Ġlow", "Ġlarge", "Ġlittle", "Ġgreat", "Ġimportant",
-    "Ġup", "Ġdown", "Ġin", "Ġout", "Ġon", "Ġoff", "Ġover", "Ġunder",
-    "Ġagain", "Ġback", "Ġabout", "Ġaround", "Ġbetween", "Ġthrough",
-    "Ġbefore", "Ġafter", "Ġduring", "Ġuntil", "Ġsince",
-    "Ġfirst", "Ġlast", "Ġnext", "Ġprevious", "Ġfinal",
-    "Ġget", "Ġgot", "Ġmake", "Ġmade", "Ġtake", "Ġtook", "Ġgive", "Ġgave",
-    "Ġuse", "Ġused", "Ġusing", "Ġneed", "Ġneeds", "Ġneeded",
-    "Ġwant", "Ġwants", "Ġwanted", "Ġlet", "Ġlets", "Ġlets",
-    "Ġwork", "Ġworks", "Ġworked", "Ġworking", "Ġhelp", "Ġhelps", "Ġhelped",
-    "Ġcall", "Ġcalls", "Ġcalled", "Ġcalling", "Ġset", "Ġsets", "Ġsetting",
-    "Ġput", "Ġputs", "Ġputting", "Ġrun", "Ġruns", "Ġran", "Ġrunning",
-    "Ġkeep", "Ġkeeps", "Ġkept", "Ġfind", "Ġfinds", "Ġfound", "Ġshow", "Ġshows",
-    "Ġtry", "Ġtries", "Ġtried", "Ġtrying", "Ġstart", "��starts", "Ġstarted",
-    "Ġchange", "Ġchanges", "Ġchanged", "Ġchanging", "Ġfollow", "Ġfollows",
-    "Ġknow", "Ġknows", "Ġknown", "Ġthink", "Ġthinks", "Ġthought",
-    "Ġsay", "Ġsays", "Ġsaid", "Ġsee", "Ġsees", "Ġsaw", "Ġseen",
-    "Ġcome", "Ġcomes", "Ġcame", "Ġgo", "Ġgoes", "Ġwent", "Ġgone",
-    "Ġbring", "Ġbrings", "Ġbrought", "Ġtell", "Ġtells", "Ġtold",
-    "Ġlet", "Ġlets", "Ġleave", "Ġleaves", "Ġleft", "Ġhappen", "Ġhappens",
-    "Ġprovide", "Ġprovides", "Ġprovided", "Ġconsider", "Ġconsiders",
-    "Ġappear", "Ġappears", "Ġappeared", "Ġform", "Ġforms", "Ġformed",
-    "Ġseem", "Ġseems", "Ġseemed", "Ġpoint", "Ġpoints", "Ġpointed",
-    "Ġturn", "Ġturns", "Ġturned", "Ġturning", "Ġplay", "Ġplays",
-    "Ġthings", "Ġthing", "Ġtime", "Ġtimes", "Ġyear", "Ġyears",
-    "Ġpeople", "Ġplace", "Ġplaces", "Ġpart", "Ġparts", "Ġworld",
-    "Ġnumber", "Ġnumbers", "Ġsystem", "Ġsystems", "Ġgroup", "Ġgroups",
-    "Ġline", "Ġlines", "Ġfile", "Ġfiles", "Ġcode", "Ġdata",
-    "Ġfunction", "Ġfunctions", "Ġvalue", "Ġvalues", "Ġtype", "Ġtypes",
-    "Ġclass", "Ġclasses", "Ġobject", "Ġobjects", "Ġmethod", "Ġmethods",
-    "Ġresult", "Ġresults", "Ġprocess", "Ġprocesses", "Ġstate", "Ġstates",
-    "Ġmodel", "Ġmodels", "Ġlevel", "Ġlevels", "Ġcase", "Ġcases",
-    "Ġexample", "Ġexamples", "Ġinput", "Ġinputs", "Ġoutput", "Ġoutputs",
-    "Ġerror", "Ġerrors", "Ġstatus", "Ġmessage", "Ġmessages",
-    "Ġrequest", "Ġrequests", "Ġresponse", "Ġresponses",
-    "Ġreturn", "Ġreturns", "Ġimport", "Ġimports", "Ġexport", "Ġexports",
-    "Ġdefine", "Ġdefines", "Ġdefined", "Ġdeclare", "Ġdeclares",
-    "Ġinclude", "Ġincludes", "Ġconfig", "Ġsetting", "Ġsettings",
-    "Ġoption", "Ġoptions", "Ġparam", "Ġparams", "Ġargs",
-    "Ġtrue", "Ġfalse", "Ġnull", "ĠNone", "Ġundefined",
-    "Ġlen", "Ġsize", "Ġcount", "Ġsum", "Ġmax", "Ġmin",
-    "Ġprint", "Ġlog", "Ġdebug", "Ġinfo", "Ġwarn",
-    "Ġtrain", "Ġtest", "Ġeval", "Ġvalid", "Ġval",
-    "Ġpage", "Ġhome", "Ġname", "Ġkey", "Ġkeys",
-    "Ġpath", "Ġdir", "Ġroot", "Ġitem", "Ġitems",
-    "Ġsource", "Ġtarget", "Ġbase", "Ġmain", "Ġprimary",
-    "Ġadd", "Ġremove", "Ġcreate", "Ġdelete", "Ġupdate",
-    "Ġopen", "Ġclose", "Ġread", "Ġwrite", "Ġload", "Ġsave",
-    "Ġpush", "Ġpop", "Ġinsert", "Ġappend", "Ġsplit",
-    "Ġbegin", "Ġend", "Ġstart", "Ġstop", "Ġenable", "Ġdisable",
-    "Ġuser", "Ġusers", "Ġadmin", "Ġmanager",
-    "Ġstring", "Ġint", "Ġfloat", "Ġdouble", "Ġbool", "Ġvoid",
-    "Ġlist", "Ġdict", "Ġset", "Ġtuple", "Ġarray",
-    "Ġapi", "Ġurl", "Ġuri", "Ġendpoint", "Ġroute",
-    "Ġhttp", "Ġhttps", "Ġclient", "Ġserver", "Ġsocket",
-    "Ġmath", "Ġscience", "Ġdata", "Ġanalysis", "Ġtheory",
-    "Ġproof", "Ġtheorem", "Ġlemma", "Ġequation", "Ġformula",
-    "ing", "ed", "ly", "tion", "sion", "ment", "ness", "ity",
-    "able", "ible", "al", "ial", "ical", "ous", "eous", "ious",
-    "ive", "ative", "ful", "less", "like", "wise", "ward",
-    "un", "re", "in", "im", "ir", "il", "dis", "mis", "non",
-    "pre", "pro", "per", "trans", "inter", "intra", "extra",
-    "sub", "super", "sur", "semi", "multi", "mono", "bi", "tri",
-    "anti", "counter", "over", "under", "out", "up", "down",
-    "co", "con", "com", "col", "cor", "de", "di", "dif",
-    "ex", "extra", "fore", "macro", "micro", "mid", "mis",
-    "out", "over", "post", "pre", "pro", "re", "semi", "sub",
-    "super", "tele", "trans", "ultra", "un", "under", "up",
-    "-ing", "-ed", "-ly", "-tion", "-sion", "-ment", "-ness", "-ity",
-    "-able", "-ible", "-al", "-ous", "-ive", "-ful", "-less",
-    "un-", "re-", "pre-", "non-", "anti-", "counter-",
-    "self-", "all-", "well-", "so-", "to-", "in-",
-    "'t", "'s", "'m", "'re", "'ve", "'ll", "'d",
-    "n't", "don't", "can't", "won't", "isn't", "aren't",
-    "wasn't", "weren't", "hasn't", "haven't", "hadn't",
-    "doesn't", "didn't", "couldn't", "shouldn't", "wouldn't",
-    "mustn't", "needn't", "mightn't",
-    # Multi-char punctuation/symbols as single tokens
-    "->", "=>", "<-", "<=", ">=", "==", "!=",
-    "::", "..", "...", "/*", "*/", "//", "#",
-    "\n", "\t",
-    "  ", "   ", "    ", "     ",
-    "Ċ",  # byte-level newline
-    "ĠĠ", "ĠĠĠ", "ĠĠĠĠ",  # multiple spaces
-    # Additional English words for 4096 vocab
-    "about", "above", "across", "action", "actually", "address", "agree", "allow", "almost",
-    "along", "already", "though", "although", "always", "American", "among", "amount",
-    "animal", "another", "answer", "anything", "appear", "approach", "area", "argue",
-    "arm", "article", "artist", "ask", "author", "available", "avoid", "away", "ball",
-    "bank", "bar", "base", "battle", "beauty", "become", "became", "becoming", "bed",
-    "behavior", "behind", "believe", "benefit", "best", "beyond", "bit", "black",
-    "blood", "board", "body", "book", "born", "boss", "bother", "bottle", "bottom",
-    "box", "boy", "brain", "break", "bridge", "brief", "bright", "bring", "broad",
-    "brother", "budget", "build", "building", "burn", "business", "buy", "campaign",
-    "capital", "car", "care", "career", "carry", "catch", "category", "cause",
-    "central", "century", "certain", "chair", "chairman", "challenge", "chance",
-    "character", "charge", "check", "choice", "choose", "chosen", "church", "citizen",
-    "city", "civil", "claim", "clear", "clearly", "close", "club", "coach", "cold",
-    "collection", "college", "color", "come", "comfortable", "comment", "committee",
-    "common", "community", "company", "compare", "competition", "complete", "completely",
-    "condition", "conference", "Congress", "connect", "conscious", "consider", "contain",
-    "content", "continue", "contract", "control", "conversation", "cost", "could",
-    "country", "couple", "course", "court", "cover", "create", "crime", "cultural",
-    "culture", "cup", "current", "customer", "cut", "dark", "daughter", "deal",
-    "death", "debate", "decade", "decide", "decision", "deep", "defense", "degree",
-    "democrat", "democratic", "describe", "design", "despite", "detail", "determine",
-    "develop", "development", "device", "die", "difference", "difficult", "dinner",
-    "direction", "director", "discover", "discuss", "discussion", "disease", "dog",
-    "door", "doubt", "down", "draw", "dream", "drive", "driver", "drop", "drug",
-    "D", "early", "east", "eat", "economic", "economy", "edge", "edition", "editor",
-    "education", "effect", "effort", "eight", "either", "election", "else", "employee",
-    "encourage", "enemy", "energy", "enjoy", "enough", "enter", "entire", "environment",
-    "environmental", "especially", "establish", "evening", "event", "ever", "everybody",
-    "everyone", "everything", "evidence", "exactly", "examine", "example", "executive",
-    "exist", "expect", "experience", "explain", "explanation", "extremely", "eye",
-    "face", "fact", "factor", "fail", "fall", "family", "far", "fast", "father",
-    "fear", "feature", "federal", "feel", "feeling", "field", "fight", "figure",
-    "fill", "film", "final", "financial", "fine", "finish", "firm", "fish", "five",
-    "floor", "fly", "focus", "follow", "food", "foot", "force", "foreign", "forget",
-    "form", "former", "forward", "four", "free", "freedom", "friendly", "front",
-    "full", "fund", "future", "game", "garden", "gas", "general", "generation",
-    "gentleman", "girl", "glad", "glass", "goal", "god", "gold", "government",
-    "governor", "great", "green", "ground", "group", "grow", "growth", "guess",
-    "gun", "guy", "half", "hand", "handle", "hang", "happen", "happy", "hard",
-    "head", "health", "hear", "heart", "heat", "heavy", "hell", "help", "here",
-    "herself", "hide", "history", "hit", "hold", "home", "honest", "hope", "hospital",
-    "hotel", "house", "huge", "human", "hundred", "husband", "idea", "identify",
-    "image", "imagine", "impact", "implement", "imply", "important", "improve",
-    "include", "including", "increase", "indeed", "indicate", "individual", "industry",
-    "influence", "inform", "information", "inside", "instead", "institution",
-    "interest", "international", "interview", "introduce", "investment", "involve",
-    "issue", "item", "itself", "job", "join", "journal", "journey", "judge",
-    "jump", "justice", "keep", "kill", "kind", "kitchen", "knowledge", "land",
-    "language", "large", "last", "late", "later", "latter", "laugh", "launch",
-    "law", "lawyer", "lay", "lead", "leader", "leading", "learn", "least", "leave",
-    "left", "legal", "less", "let", "letter", "level", "lie", "life", "lift",
-    "light", "likely", "limit", "line", "link", "list", "listen", "little", "live",
-    "load", "local", "long", "look", "lord", "lose", "loss", "lost", "lot", "love",
-    "low", "luck", "lunch", "machine", "main", "maintain", "major", "majority",
-    "manage", "management", "manager", "manner", "manufacturer", "many", "map",
-    "mark", "market", "marriage", "master", "material", "matter", "may", "maybe",
-    "mean", "meaning", "measure", "media", "medical", "meet", "meeting", "member",
-    "memory", "mention", "message", "method", "middle", "might", "military", "million",
-    "mind", "minute", "miss", "mission", "mistake", "mix", "modern", "mom", "moment",
-    "money", "month", "moral", "morning", "mother", "motion", "move", "movement",
-    "movie", "music", "narrative", "nation", "national", "native", "natural", "nature",
-    "near", "nearly", "necessarily", "necessary", "neck", "need", "negative", "neighbor",
-    "neither", "network", "never", "nevertheless", "night", "none", "nor", "normal",
-    "north", "note", "nothing", "notice", "notion", "now", "nowhere", "nuclear",
-    "number", "occur", "ocean", "offer", "office", "officer", "official", "often",
-    "oil", "OK", "old", "once", "online", "open", "operate", "operation", "opinion",
-    "opportunity", "opposition", "option", "order", "organization", "original", "other",
-    "otherwise", "ought", "outside", "overcome", "owner", "page", "pain", "paint",
-    "pair", "paper", "parent", "park", "parliament", "part", "participant", "particular",
-    "particularly", "partner", "party", "pass", "passage", "past", "path", "patient",
-    "pattern", "pay", "peace", "pension", "people", "per", "percent", "perfect",
-    "perform", "performance", "perhaps", "period", "permit", "person", "personal",
-    "perspective", "phone", "physical", "pick", "picture", "piece", "place", "plan",
-    "plant", "play", "player", "please", "pleasure", "plus", "pocket", "point",
-    "police", "policy", "political", "politician", "politics", "pool", "poor",
-    "popular", "population", "position", "positive", "possibility", "possible",
-    "potentially", "power", "practice", "prepare", "presence", "present", "president",
-    "pressure", "pretty", "prevent", "previous", "price", "primary", "principle",
-    "prison", "private", "privilege", "probably", "problem", "procedure", "produce",
-    "product", "production", "professional", "professor", "profile", "profit",
-    "program", "project", "promise", "promote", "proper", "property", "proposal",
-    "propose", "protect", "protection", "prove", "provide", "public", "publication",
-    "publish", "pull", "purpose", "pursue", "push", "quality", "quarter", "question",
-    "quick", "quickly", "quiet", "quite", "race", "radio", "raise", "range", "rate",
-    "rather", "reach", "react", "reaction", "read", "reader", "reading", "ready",
-    "real", "reality", "realize", "really", "reason", "reasonable", "receive",
-    "recent", "recently", "recognize", "recommend", "record", "recover", "red",
-    "reduce", "reflect", "reform", "region", "relate", "relationship", "relative",
-    "relatively", "release", "relevant", "relief", "religion", "religious", "rely",
-    "remain", "remember", "remind", "remove", "repeat", "replace", "report", "reporter",
-    "represent", "representation", "republican", "reputation", "request", "require",
-    "research", "resource", "respond", "response", "responsibility", "responsible",
-    "rest", "restaurant", "result", "retain", "retire", "return", "reveal",
-    "review", "revolution", "rich", "ride", "right", "ring", "rise", "risk", "river",
-    "road", "rock", "role", "roll", "room", "rule", "run", "safe", "safety",
-    "sale", "same", "sample", "save", "scale", "scene", "schedule", "school",
-    "science", "scientist", "score", "screen", "sea", "search", "season", "seat",
-    "second", "secret", "section", "security", "seed", "seek", "select", "self",
-    "sell", "senate", "senator", "send", "sense", "serious", "serve", "service",
-    "session", "settle", "seven", "sexual", "shadow", "shape", "share", "sharp",
-    "sheet", "ship", "shock", "shoe", "shoot", "shop", "shot", "shoulder", "show",
-    "shut", "sick", "side", "sight", "sign", "signal", "significance", "significant",
-    "silence", "similar", "simple", "simply", "since", "sing", "single", "sister",
-    "sit", "site", "situation", "six", "size", "skill", "skin", "small", "smile",
-    "society", "soft", "soldier", "solid", "solution", "somebody", "somehow",
-    "someone", "something", "sometimes", "somewhat", "son", "song", "soon", "sort",
-    "sound", "source", "south", "space", "speak", "speaker", "special", "specific",
-    "speech", "speed", "spend", "spin", "spirit", "spiritual", "split", "spokesman",
-    "sport", "spot", "spread", "spring", "staff", "stage", "stand", "standard",
-    "star", "start", "state", "statement", "station", "status", "stay", "step",
-    "stick", "still", "stock", "stop", "store", "story", "straight", "strange",
-    "strategic", "strategy", "street", "strength", "stress", "stretch", "strike",
-    "strong", "structure", "struggle", "student", "study", "subject", "succeed",
-    "success", "successful", "suddenly", "suffer", "sufficient", "suggest", "suggestion",
-    "summer", "supply", "support", "suppose", "sure", "surface", "surgery", "surprise",
-    "survey", "survive", "suspect", "sustain", "symbol", "system", "table", "talent",
-    "talk", "tape", "target", "task", "taste", "tax", "teach", "teacher", "teaching",
-    "team", "tear", "technical", "technique", "technology", "telephone", "television",
-    "tell", "temperature", "tend", "term", "test", "testify", "testing", "text",
-    "thank", "themselves", "therefore", "they", "thick", "thin", "thing", "think",
-    "thinking", "third", "thirty", "threat", "threaten", "three", "throw", "thus",
-    "ticket", "tight", "till", "time", "tiny", "tip", "title", "today", "together",
-    "tomorrow", "tone", "tonight", "tool", "top", "total", "totally", "touch",
-    "tough", "tour", "toward", "town", "track", "trade", "tradition", "traditional",
-    "traffic", "train", "training", "transfer", "transform", "travel", "treat",
-    "treatment", "tree", "trial", "trip", "troop", "trouble", "truck", "true",
-    "truly", "trust", "truth", "try", "tube", "turn", "twice", "type", "typical",
-    "uncle", "under", "understand", "understanding", "unfortunately", "union",
-    "unique", "unit", "United", "universe", "university", "unless", "unlike",
-    "unlikely", "unusual", "upper", "urban", "urge", "use", "used", "useful",
-    "user", "usual", "usually", "value", "variety", "various", "vehicle", "version",
-    "very", "veteran", "victim", "victory", "video", "view", "village", "violence",
-    "visit", "voice", "volume", "vote", "voter", "wage", "wait", "walk", "wall",
-    "want", "war", "warm", "warn", "warning", "wash", "watch", "water", "wave",
-    "way", "weak", "weapon", "wear", "weather", "web", "wedding", "weekend",
-    "weight", "welcome", "welfare", "well", "west", "western", "whatever", "wheel",
-    "whenever", "whereas", "whether", "which", "while", "white", "whole", "whom",
-    "whose", "wide", "widely", "wife", "wild", "will", "win", "wind", "window",
-    "wine", "wing", "winner", "winter", "wire", "wish", "woman", "wonder", "wonderful",
-    "wood", "word", "worker", "working", "works", "world", "worry", "worth", "would",
-    "write", "writer", "writing", "wrong", "yard", "yeah", "year", "yet", "youth",
-    "zone", "ability", "abroad", "absent", "absolute", "absorb", "abstract", "abuse",
-    "academic", "accept", "access", "accident", "accompany", "accomplish", "account",
-    "accurate", "accuse", "achieve", "acknowledge", "acquire", "adapt", "addition",
-    "adjust", "administration", "admit", "adopt", "advance", "advantage", "adventure",
-    "advertise", "advice", "advise", "advocate", "affair", "affect", "afford",
-    "agency", "agenda", "agent", "aggression", "aggressive", "aid", "aim", "air",
-    "airport", "alarm", "alcohol", "alert", "alive", "alliance", "allocate", "ally",
-    "alone", "alter", "alternative", "amaze", "ambition", "amendment", "amid",
-    "amongst", "analysis", "analyst", "angle", "angry", "anniversary", "announce",
-    "annual", "anticipate", "anxiety", "anxious", "apart", "apartment", "apologize",
-    "apparent", "appeal", "appearance", "appetite", "apple", "applicant", "application",
-    "apply", "appoint", "appreciate", "appropriate", "approval", "approve", "architecture",
-    "archive", "argue", "argument", "arrange", "arrangement", "arrest", "arrival",
-    "arrive", "arrow", "articulate", "artificial", "aside", "aspect", "assault",
-    "assemble", "assembly", "assert", "assess", "assessment", "asset", "assign",
-    "assist", "assistance", "associate", "association", "assume", "assumption",
-    "atmosphere", "attach", "attack", "attempt", "attend", "attention", "attitude",
-    "attorney", "attract", "attraction", "attractive", "attribute", "audience",
-    "auto", "automatic", "automatically", "autonomy", "available", "avenue", "average",
-    "award", "aware", "awareness", "awful", "background", "bacteria", "balance",
-    "bare", "barely", "barrier", "basic", "basis", "basket", "bath", "battery",
-    "battle", "bay", "beach", "bean", "bear", "beat", "beautiful", "bedroom",
-    "beer", "beginning", "behalf", "behave", "behavior", "being", "belief", "believable",
-    "bell", "belong", "bench", "bend", "beneath", "beneficial", "beside", "bet",
-    "betray", "bible", "bicycle", "bid", "bike", "bill", "bind", "biological",
-    "biology", "birth", "biscuit", "bishop", "bite", "bitter", "blade", "blame",
-    "blank", "blast", "bleed", "blend", "bless", "blind", "block", "blow", "blue",
-    "blur", "board", "boast", "boat", "bomb", "bond", "bone", "bonus", "boom",
-    "boost", "border", "bore", "borrow", "bottom", "bound", "boundary", "bowl",
-    "brain", "branch", "brand", "brave", "bread", "breadth", "breast", "breath",
-    "breathe", "breathing", "breed", "brick", "bride", "bridge", "briefly", "brilliant",
-    "broadcast", "broken", "bronze", "brow", "brown", "brush", "bubble", "bucket",
-    "buddy", "buffalo", "bunch", "burden", "burglar", "burn", "burst", "bury",
-    "bus", "butter", "button", "cabin", "cabinet", "cable", "cake", "calculate",
-    "calculation", "calendar", "calm", "camera", "camp", "campus", "canal", "cancel",
-    "candidate", "candle", "cap", "capable", "capacity", "captain", "capture",
-    "carbon", "card", "careful", "carefully", "carrier", "carry", "cart", "carve",
-    "cast", "castle", "casualty", "catalog", "catalogue", "catch", "cattle",
-    "celebrate", "celebration", "cell", "cellular", "census", "centimeter",
-    "ceremony", "certainly", "certificate", "chain", "chair", "chairman",
-    "chamber", "champion", "championship", "channel", "chapter", "characteristic",
-    "charge", "charity", "chart", "chase", "cheap", "cheat", "cheek", "cheese",
-    "chemical", "chemistry", "chest", "chicken", "chief", "childhood", "chip",
-    "chocolate", "chorus", "christian", "Christmas", "chronic", "chunk", "circle",
-    "circuit", "circumstance", "cite", "citizen", "civilian", "claim",
-    "clarify", "clarity", "clash", "classic", "classical", "classification",
-    "classroom", "clause", "clean", "clear", "clever", "click", "client",
-    "cliff", "climate", "climb", "clinic", "clinical", "clock", "clone",
-    "closed", "closely", "closer", "closet", "closing", "cloth", "clothe",
-    "clothes", "clothing", "cloud", "club", "cluster", "coal", "coalition",
-    "coast", "coat", "code", "coffee", "cognitive", "coin", "cold",
-    "collapse", "collar", "colleague", "collect", "collection", "collective",
-    "colonial", "colony", "color", "column", "combat", "combine", "combined",
-    "comedy", "comfort", "command", "commander", "comment", "commerce",
-    "commercial", "commission", "commit", "commitment", "commodity", "communicate",
-    "communication", "communist", "compact", "companion", "comparison", "compel",
-    "compensate", "compensation", "compete", "competition", "competitive",
-    "competitor", "complain", "complaint", "complement", "complex", "complexity",
-    "complicate", "complicated", "comply", "component", "compose", "composition",
-    "compound", "comprehensive", "comprise", "compromise", "compulsory", "compute",
-    "computer", "conceal", "concede", "conceive", "concentrate", "concentration",
-    "concept", "conception", "concern", "concerning", "concert", "conclude",
-    "conclusion", "concrete", "condemn", "conduct", "conference", "confess",
-    "confession", "confidence", "confident", "confidential", "confine", "confirm",
-    "conflict", "confront", "confusion", "congratulate", "congress", "connect",
-    "connection", "conscious", "consciousness", "consecutive", "consensus",
-    "consent", "consequence", "consequently", "conservation", "conservative",
-    "considerable", "considerably", "consist", "consistent", "consistently",
-    "constant", "constantly", "constitute", "constitution", "constitutional",
-    "construct", "construction", "consult", "consultant", "consume", "consumer",
-    "consumption", "contact", "contemporary", "contend", "contest", "context",
-    "continent", "continually", "continuity", "continuous", "continuously",
-    "contradiction", "contrary", "contribute", "contribution", "contributor",
-    "controversial", "controversy", "convenience", "convenient", "convention",
-    "conventional", "conversation", "conversion", "convert", "convey", "convict",
-    "conviction", "convince", "cook", "cookie", "cool", "cooperate", "cooperation",
-    "coordinate", "coordination", "cope", "copper", "copy", "copyright", "core",
-    "corn", "corner", "corporate", "corporation", "correct", "correction",
-    "correctly", "correlate", "correlation", "correspond", "correspondent",
-    "corridor", "corruption", "costly", "cotton", "council", "counsel",
-    "counselor", "count", "counter", "counterpart", "county", "coup",
-    "courage", "cousin", "cover", "coverage", "crack", "craft", "crash",
-    "creative", "creativity", "creator", "creature", "credibility", "credit",
-    "creep", "crew", "crime", "criminal", "crisis", "criterion", "critic",
-    "critical", "criticism", "criticize", "crop", "cross", "crowd", "crown",
-    "crucial", "crude", "cruel", "cruise", "crush", "cry", "crystal", "cube",
-    "cuisine", "cultivate", "curious", "currency", "current", "curriculum",
-    "curtain", "curve", "custody", "custom", "customary", "customer", "cutting",
-    "cycle", "dad", "damage", "damn", "dance", "danger", "dangerous",
-    "dare", "data", "database", "dawn", "dead", "deadline", "deadly",
-    "deaf", "deal", "dealer", "dear", "debate", "debt", "decade",
-    "decay", "deceive", "decent", "decide", "decision", "decisive", "deck",
-    "declaration", "declare", "decline", "decorate", "decrease", "decree",
-    "dedicate", "deem", "defeat", "defend", "defendant", "defender", "defense",
-    "defensive", "deficit", "define", "definite", "definitely", "definition",
-    "defy", "degree", "delay", "delegate", "delegation", "delete", "deliberate",
-    "deliberately", "delicate", "delicious", "delight", "deliver", "delivery",
-    "demand", "democracy", "democrat", "democratic", "demographic", "demonstrate",
-    "demonstration", "denial", "denote", "deny", "depart", "department", "departure",
-    "depend", "dependence", "dependent", "depict", "deposit", "depress", "depression",
-    "deprive", "depth", "deputy", "derive", "descend", "describe", "description",
-    "desert", "deserve", "design", "designate", "designer", "desirable",
-    "desire", "desk", "desperate", "desperately", "despite", "destination",
-    "destroy", "destruction", "detail", "detailed", "detain", "detect",
-    "detection", "detective", "detention", "deteriorate", "determination",
-    "determine", "determined", "develop", "development", "developmental",
-    "devote", "devote", "diabetes", "diagnose", "diagnosis", "dialogue",
-    "diameter", "diamond", "diary", "dictate", "diet", "differ", "difference",
-    "differentiate", "differently", "difficulty", "dig", "digest", "digital",
-    "dignity", "dilemma", "dimension", "diminish", "dinner", "dioxide",
-    "dip", "diplomat", "diplomatic", "direct", "direction", "directly",
-    "director", "dirty", "disability", "disable", "disadvantage",
-    "disagree", "disappear", "disappoint", "disappointment", "disaster",
-    "disastrous", "disc", "discharge", "discipline", "disclose", "discount",
-    "discourse", "discover", "discovery", "discrepancy", "discretion",
-    "discrimination", "discuss", "discussion", "disease", "dismiss", "disorder",
-    "dispatch", "display", "disposal", "dispose", "dispute", "disrupt",
-    "dissolve", "distance", "distant", "distinct", "distinction", "distinctive",
-    "distinguish", "distort", "distract", "distress", "distribute", "distribution",
-    "distributor", "district", "disturb", "dive", "diverse", "diversity",
-    "divide", "division", "divorce", "dock", "doctor", "doctrine", "document",
-    "documentary", "dollar", "domain", "dome", "domestic", "dominant",
-    "dominate", "donation", "donor", "dose", "dot", "double", "doubt",
-    "doubtful", "downtown", "draft", "drag", "drain", "drama", "dramatic",
-    "dramatically", "drastic", "draw", "drawing", "drink", "drive",
-    "driver", "drop", "drought", "drown", "drum", "drunk", "dry",
-    "dual", "dubious", "duck", "due", "dull", "dump", "durable", "duration",
-    "dust", "duty", "dynamic", "dynamics", "eager", "eagle", "ear",
-    "earning", "earth", "ease", "easily", "eastern", "echo", "eclipse",
-    "ecological", "ecology", "economics", "economist", "economy",
-    "ecosystem", "edit", "edition", "editor", "editorial", "educate",
-    "education", "educational", "educator", "effective", "effectively",
-    "effectiveness", "efficiency", "efficient", "efficiently", "effort",
-    "elaborate", "elbow", "elderly", "elect", "election", "electoral",
-    "electric", "electrical", "electricity", "electronic", "electronics",
-    "elegant", "element", "elementary", "eliminate", "elimination", "elite",
-    "elsewhere", "email", "embargo", "embark", "embarrass", "embassy",
-    "embed", "embody", "embrace", "emerge", "emergence", "emergency",
-    "emission", "emotion", "emotional", "emphasis", "emphasize", "empire",
-    "empirical", "employ", "employee", "employer", "employment", "empower",
-    "enable", "enact", "encompass", "encounter", "encourage", "encouragement",
-    "endanger", "endeavor", "endorse", "endorsement", "endure", "enforce",
-    "enforcement", "engage", "engagement", "engine", "engineering", "enhance",
-    "enjoy", "enjoyment", "enlarge", "enormous", "enrich", "enroll",
-    "ensemble", "ensure", "enter", "enterprise", "entertain", "entertainment",
-    "enthusiasm", "enthusiast", "enthusiastic", "entirely", "entitle",
-    "entity", "entrepreneur", "entry", "envelope", "environment",
-    "environmental", "epidemic", "episode", "equal", "equality", "equation",
-    "equip", "equipment", "equivalent", "era", "erect", "error", "erupt",
-    "escalate", "escape", "especially", "essay", "essence", "essential",
-    "essentially", "establish", "establishment", "estate", "estimate",
-    "estimation", "eternal", "ethical", "ethics", "ethnic", "evacuate",
-    "evaluate", "evaluation", "evenly", "event", "eventually", "ever",
-    "everyday", "evidence", "evident", "evil", "evoke", "evolution",
-    "evolutionary", "evolve", "exact", "exaggerate", "examination",
-    "examine", "example", "exceed", "excellence", "excellent", "exception",
-    "exceptional", "excess", "excessive", "exchange", "excite", "excitement",
-    "exciting", "exclude", "exclusion", "exclusive", "exclusively",
-    "excuse", "execute", "execution", "executive", "exemplify",
-    "exercise", "exert", "exhaust", "exhibit", "exhibition", "exile",
-    "exist", "existence", "exit", "expand", "expansion", "expect",
-    "expectation", "expedition", "expel", "expenditure", "expense",
-    "expensive", "expert", "expertise", "explain", "explanation",
-    "explicit", "explicitly", "explode", "exploit", "exploitation",
-    "exploration", "explore", "explosion", "explosive", "export",
-    "expose", "exposure", "express", "expression", "extend", "extension",
-    "extensive", "extensively", "extent", "external", "extinct",
-    "extinction", "extra", "extract", "extraordinary", "extreme",
-    "extremely", "eye", "fabric", "fabulous", "facade", "face",
-    "facial", "facilitate", "facility", "fact", "faction", "faculty",
-    "fade", "fail", "failure", "fair", "fairly", "fairness", "faith",
-    "faithful", "fake", "fame", "familiar", "famine", "fan", "fancy",
-    "fantasy", "fare", "fascinate", "fascinating", "fashion", "fat",
-    "fate", "fatigue", "fault", "favor", "favorable", "favorite",
-    "fax", "fear", "feasible", "feast", "feather", "federal",
-    "federation", "fee", "feed", "feedback", "feel", "feeling",
-    "fellow", "fellowship", "female", "fence", "fertile", "fertilizer",
-    "festival", "fetch", "fever", "fiber", "fiction", "field", "fierce",
-    "fifteen", "fifty", "fig", "fight", "fighter", "figure", "file",
-    "fill", "filter", "final", "finally", "finance", "financial",
-    "financially", "financing", "finding", "finger", "finished",
-    "fire", "firm", "firmly", "fiscal", "fish", "fisherman", "fishing",
-    "fitness", "fix", "fixture", "flag", "flame", "flash", "flat",
-    "flavor", "flee", "fleet", "flesh", "flexibility", "flexible",
-    "flight", "float", "flock", "flood", "floor", "flour", "flow",
-    "flower", "fluid", "flush", "fly", "focus", "folk", "football",
-    "footnote", "footstep", "forbid", "forbidden", "forecast", "forehead",
-    "foreign", "foreigner", "forest", "forever", "forge", "forget",
-    "forgive", "fork", "form", "formal", "format", "formation", "former",
-    "formula", "formulate", "fort", "forth", "fortunate", "fortune",
-    "forum", "forward", "fossil", "foster", "found",
-    "foundation", "founder", "fountain", "fraction", "fracture", "fragile",
-    "fragment", "frame", "framework", "franchise", "frank", "frankly",
-    "fraud", "free", "freedom", "freely", "freeze", "freight",
-    "frequency", "frequent", "frequently", "fresh", "freshman", "friction",
-    "friendly", "friendship", "frighten", "frog", "front",
-    "frontier", "frost", "frown", "frozen", "fruit", "frustrate",
-    "frustration", "fuel", "fulfill", "full", "fun", "function",
-    "functional", "fund", "fundamental", "funding", "funeral", "funny",
-    "fur", "furious", "furniture", "further", "furthermore", "fury",
-    "fusion", "future", "gain", "galaxy", "gallery", "gallon", "gambling",
-    "gap", "garage", "garbage", "garden", "garlic", "garment", "gas",
-    "gasoline", "gate", "gather", "gathering", "gauge", "gaze", "gear",
-    "gender", "gene", "general", "generally", "generate", "generation",
-    "generator", "generous", "genetic", "genetics", "genius", "genocide",
-    "genre", "gentle", "gentleman", "gently", "genuine", "genuinely",
-    "gesture", "giant", "gift", "gigantic", "glimpse", "global",
-    "globalization", "globe", "glory", "glove", "glow", "glucose",
-    "goal", "goddess", "gold", "golden", "golf", "goodness",
-    "goods", "gorgeous", "gospel", "gossip", "govern", "governance",
-    "government", "governor", "grab", "grace", "grade", "gradually",
-    "graduate", "graduation", "grain", "gram", "grammar", "grand",
-    "grandfather", "grandmother", "grant", "graph", "graphic", "grasp",
-    "grass", "grateful", "grave", "gravity", "gray", "greatly",
-    "green", "greenhouse", "greet", "grief", "grin", "grind", "grip",
-    "grocery", "gross", "ground", "groundwater", "growth", "guarantee",
-    "guard", "guardian", "guess", "guest", "guidance", "guide",
-    "guideline", "guilty", "guitar", "gulf", "gun", "gut", "guy",
-    "gym", "habit", "habitat", "hair", "half", "hall", "halt", "hammer",
-    "hand", "handful", "handle", "handling", "handwriting", "handy",
-    "hang", "happen", "happiness", "harassment", "harbor", "hardly",
-    "hardware", "harm", "harmful", "harmony", "harvest", "hat",
-    "hate", "haul", "hay", "hazard", "head", "headache", "headline",
-    "headquarters", "heal", "health", "healthcare", "healthy",
-    "heap", "hearing", "heart", "heat", "heating", "heaven",
-    "heavily", "heavy", "hedge", "heel", "height", "helicopter",
-    "hell", "helmet", "helpful", "herb", "heritage", "hero",
-    "heroin", "herself", "hesitate", "hidden", "hide", "hierarchy",
-    "highlight", "highly", "highway", "hike", "hill", "himself",
-    "hip", "hire", "historian", "historic", "historical", "history",
-    "hit", "hobby", "hold", "holder", "holding", "hole",
-    "holiday", "hollow", "holy", "homeland", "homeless", "homework",
-    "honest", "honesty", "honey", "honor", "hook", "hope",
-    "hopeful", "hopefully", "horizon", "horizontal", "hormone",
-    "horn", "horrible", "horror", "horse", "hospitality", "host",
-    "hostage", "hostile", "hot", "hotline", "hour", "housing",
-    "hover", "human", "humane", "humanitarian", "humanity",
-    "humor", "hundred", "hunger", "hungry", "hunt", "hunter",
-    "hunting", "hurt", "husband", "hut", "hydrogen", "hygiene",
-    "hypothesis", "ice", "icon", "idea", "ideal", "identical",
-    "identification", "identify", "identity", "ideology", "ignorance",
-    "ignore", "ill", "illegal", "illness", "illusion", "illustrate",
-    "illustration", "image", "imaginary", "imagination", "imagine",
-    "imitate", "immediate", "immediately", "immense", "immigrant",
-    "immigration", "immune", "immunity", "impact", "implement",
-    "implementation", "implication", "implicit", "imply", "import",
-    "importance", "impose", "impossible", "impress", "impression",
-    "impressive", "imprison", "improbable", "improve", "improvement",
-    "impulse", "inability", "inappropriate", "incentive", "incidence",
-    "incident", "inclination", "incline", "include", "including",
-    "inclusion", "inclusive", "income", "incorporate", "incorrect",
-    "increase", "increasingly", "incredible", "incur", "indeed",
-    "independence", "independent", "independently", "index",
-    "indicate", "indication", "indicator", "indictment",
-    "indigenous", "indirect", "indispensable", "individual",
-    "individuality", "indoor", "induce", "indulge", "industrial",
-    "industrialize", "industry", "inequality", "inevitable",
-    "inevitably", "infant", "infect", "infection", "infer",
-    "inference", "inferior", "infinite", "infinity", "inflation",
-    "inflict", "influence", "influential", "info", "inform",
-    "informal", "information", "infrastructure", "ingredient",
-    "inhabit", "inhabitant", "inherent", "inherit", "inhibit",
-    "initial", "initially", "initiate", "initiative", "inject",
-    "injection", "injure", "injury", "inmate", "inner",
-    "innocent", "innovation", "innovative", "input", "inquiry",
-    "insect", "insert", "insertion", "inside", "insight",
-    "insist", "inspect", "inspection", "inspector", "inspiration",
-    "inspire", "install", "installation", "installment", "instance",
-    "instant", "instantly", "instead", "instinct",
-    "institute", "institution", "institutional", "instruct",
-    "instruction", "instructor", "instrument", "instrumental",
-    "insufficient", "insult", "insurance", "intact", "integral",
-    "integrate", "integration", "integrity", "intellectual",
-    "intelligence", "intelligent", "intend", "intense",
-    "intensity", "intensive", "intent", "intention", "intentional",
-    "interact", "interaction", "interactive", "interest",
-    "interested", "interesting", "interface", "interfere",
-    "interference", "interim", "interior", "intermediate",
-    "internal", "international", "internet", "interpret",
-    "interpretation", "interrupt", "interruption",
-    "intersection", "interval", "intervene", "intervention",
-    "interview", "intimate", "intrigue", "intrinsic", "introduce",
-    "introduction", "intuition", "intuitive", "invade",
-    "invasion", "invent", "invention", "inventory", "invest",
-    "investigate", "investigation", "investigator", "investment",
-    "investor", "invisible", "invitation", "invite", "involve",
-    "involvement", "iron", "irony", "irrelevant", "irrigation",
-    "island", "isolate", "isolation", "issue", "item",
-    "itself", "ivory", "jail", "jam", "jet", "jewel",
-    "jewelry", "job", "join", "joint", "jointly", "joke",
-    "journal", "journalism", "journalist", "journey", "joy",
-    "judge", "judgment", "judicial", "juice", "jump", "junction",
-    "jungle", "junior", "jurisdiction", "jury", "justice",
-    "justification", "justify", "juvenile", "keen", "keeper",
-    "kettle", "key", "keyboard", "kick", "kid", "kidnap",
-    "kidney", "kin", "kindness", "king", "kingdom", "kiss",
-    "kit", "kitchen", "knee", "kneel", "knife", "knock",
-    "knot", "label", "labor", "laboratory", "lace", "lack",
-    "ladder", "lady", "lake", "lamp", "land", "landing",
-    "landlord", "landmark", "landscape", "lane", "lap", "largely",
-    "laser", "late", "latte", "latter", "laugh", "laughter",
-    "launch", "laundry", "lavatory", "law", "lawn", "lawmaker",
-    "lawn", "lawsuit", "lawyer", "layout", "leading", "leaf",
-    "league", "leak", "lean", "leap", "learner", "learning",
-    "lease", "leather", "leave", "lecture",
-    "legacy", "legend", "legendary", "legislation", "legislative",
-    "legislature", "legitimate", "leisure", "lemon", "lend",
-    "length", "lens", "lesson", "lest", "lethal", "letter",
-    "lettuce", "liberal", "liberation", "liberty", "library",
-    "license", "lid", "lie", "lifelong", "lifestyle", "lifetime",
-    "lift", "light", "lighting", "lightly", "likelihood",
-    "likewise", "limb", "limit", "limitation", "limited",
-    "limitless", "limp", "line", "linear", "linen", "liner",
-    "linger", "linguistic", "lining", "link", "lion",
-    "lip", "liquid", "liquor", "list", "listen", "listener",
-    "listing", "liter", "literally", "literary", "literature",
-    "litigation", "liver", "living", "load", "loan",
-    "lobby", "local", "locate", "location", "lock", "lodge",
-    "log", "logic", "logical", "logo", "lonely", "longitudinal",
-    "lookout", "loop", "loose", "loosen", "lord", "lose",
-    "loss", "loud", "lounge", "lovely", "lover",
-    "low", "lower", "loyal", "loyalty", "luck", "lucky",
-    "luggage", "lump", "lunch", "lung", "luxury",
-    "lyric", "machine", "machinery", "mad", "magazine", "magic",
-    "magical", "magnetic", "magnificent", "magnitude", "maid",
-    "mail", "mainland", "mainly", "mainstream", "maintain",
-    "maintenance", "majesty", "majority", "maker",
-    "makeup", "male", "mall", "mama", "mammal", "manage",
-    "manageable", "management", "manager", "mandate",
-    "mandatory", "maneuver", "manifest", "manipulate",
-    "manipulation", "mankind", "manuscript", "maple",
-    "marathon", "marble", "march", "margin", "marginal",
-    "marine", "mark", "marker", "market", "marketing",
-    "marketplace", "marriage", "married", "marry", "mask",
-    "mass", "massacre", "massive", "master", "masterpiece",
-    "match", "mate", "material", "maternal", "math",
-    "mathematical", "mathematics", "matter", "mature",
-    "maturity", "maximize", "maximum", "mayor", "meadow",
-    "meaning", "meaningful", "means", "meantime",
-    "measurable", "measure", "measurement", "mechanic",
-    "mechanical", "mechanism", "medal", "media",
-    "mediate", "mediation", "medicaid", "medical",
-    "medication", "medicine", "medieval", "meditation",
-    "medium", "meet", "melody", "melt", "member",
-    "membership", "memo", "memoir", "memorandum",
-    "memorial", "memorize", "menace",
-    "mental", "mentally", "mention", "mentor", "menu",
-    "merchandise", "merchant", "mercy", "mere", "merely",
-    "merge", "merger", "merit", "merry", "mess",
-    "messenger", "metal", "metaphor", "method",
-    "methodology", "metric", "metropolitan",
-    "microphone", "microscope", "midday", "middle",
-    "midnight", "midst", "migrant", "migrate", "migration",
-    "mild", "mile", "milestone", "militant", "military",
-    "militia", "mill", "millennium", "millimeter",
-    "mineral", "mingle", "miniature", "minimal",
-    "minimize", "minimum", "mining", "minister",
-    "ministry", "minor", "minority", "minute",
-    "miracle", "mirror", "miserable", "misery",
-    "misleading", "missile", "missing", "mission",
-    "missionary", "mist", "mistake", "mistaken",
-    "mistress", "misunderstand", "misunderstanding",
-    "mixture", "moan", "mobile", "mobility",
-    "mobilize", "mode", "moderate", "moderately",
-    "moderation", "modern", "modest", "modification",
-    "modify", "module", "moisture", "molecule",
-    "molest", "moment", "momentum", "monarchy",
-    "monastery", "monitor", "monk", "monopoly",
-    "monster", "monument", "mood", "moon",
-    "moral", "morale", "morality", "moreover",
-    "mortal", "mortality", "mortgage", "mosaic",
-    "mosque", "mostly", "mother", "motion",
-    "motivate", "motivation", "motive", "motor",
-    "motorcycle", "mount", "mountain", "mounting",
-    "mourn", "mouse", "mouth", "movement",
-    "movie", "mud", "multiple", "multiplication",
-    "multiply", "multitude", "municipal",
-    "municipality", "murder", "murderer", "murmur",
-    "muscle", "muscular", "museum", "mushroom",
-    "musical", "musician", "muslim", "mutual",
-    "mutually", "mysterious", "mystery", "myth",
-    "mythology", "nail", "naked", "narrative",
-    "narrow", "nasty", "nation", "national",
-    "nationalism", "nationalist", "nationality",
-    "nationwide", "native", "natural", "naturally",
-    "nature", "naval", "navigation", "navy",
-    "nearby", "neat", "necessarily", "necessary",
-    "necessity", "neck", "necklace", "needle",
-    "negative", "neglect", "negotiate",
-    "negotiation", "negotiator", "neighbor",
-    "neighborhood", "neither", "nerve", "nervous",
-    "nest", "net", "network", "neutral", "nevertheless",
-    "niche", "nickel", "niece", "night", "nightmare",
-    "nitrogen", "noble", "nobody", "nod", "noise",
-    "noisy", "nominal", "nominate", "nomination",
-    "nominee", "nonprofit", "nonsense", "norm",
-    "normal", "normally", "normative", "north",
-    "northeast", "northern", "northwest", "notable",
-    "notably", "notation", "notebook", "nothing",
-    "notice", "noticeable", "notification",
-    "notion", "notorious", "novel", "novelist",
-    "novelty", "nowhere", "nuclear", "nuance",
-    "nucleus", "nuisance", "number", "numerical",
-    "numerous", "nurse", "nursery", "nursing",
-    "nutrient", "nutrition", "nutritional",
-    "nutritious", "nylon", "oak", "obedience",
-    "obedient", "obese", "obesity", "obey",
-    "objection", "objective", "obligation",
-    "oblige", "obscure", "observation", "observe",
-    "observer", "obsession", "obstacle",
-    "obtain", "obvious", "obviously", "occasion",
-    "occasional", "occasionally", "occupation",
-    "occupy", "occur", "occurrence",
-    "offend", "offense", "offensive", "offer",
-    "offering", "officer", "official", "officially",
-    "offspring", "olive", "omission",
-    "omit", "ongoing", "onion", "onset",
-    "opening", "openly", "opera", "operate",
-    "operating", "operation", "operational",
-    "operator", "opinion", "opponent",
-    "opportunity", "oppose", "opposite",
-    "opposition", "opt", "optical", "optimism",
-    "optimist", "optimistic", "optimum",
-    "option", "optional", "oral", "orbit",
-    "orchestra", "ordeal", "orderly",
-    "ordinarily", "ordinary", "organ",
-    "organic", "organization", "organizational",
-    "organize", "organized", "organizer",
-    "orientation", "origin", "original",
-    "originally", "originate", "ornament",
-    "orthodox", "other", "otherwise",
-    "ought", "ounce", "outbreak",
-    "outcome", "outdoor", "outer", "outfit",
-    "outing", "outlet", "outline", "outlook",
-    "output", "outrage", "outright",
-    "outset", "outside", "outsider", "outstanding",
-    "outward", "oval", "oven", "overall",
-    "overcome", "overlook", "overnight",
-    "override", "overseas", "oversee",
-    "overturn", "overwhelm", "overwhelming",
-    "owe", "own", "owner", "ownership",
-    "oxygen", "ozone", "pace", "pack",
-    "package", "packaging", "packet", "pad",
-    "paddle", "page", "pain", "painful",
-    "paint", "painter", "painting", "pair",
-    "palace", "pale", "palm", "pan",
-    "panel", "panic", "paper", "parade",
-    "paradigm", "paradise", "paradox",
-    "paragraph", "parallel", "parameter",
-    "parcel", "pardon", "parent",
-    "parental", "parish", "park",
-    "parking", "parliament", "parliamentary",
-    "partial", "partially", "participant",
-    "participate", "participation",
-    "particle", "particular", "particularly",
-    "partly", "partner", "partnership",
-    "passage", "passenger", "passing",
-    "passion", "passionate", "passive",
-    "passport", "password", "past",
-    "paste", "pastor", "patch", "patent",
-    "path", "pathway", "patience",
-    "patient", "patrol", "patron",
-    "pattern", "pause", "pave", "pavement",
-    "paw", "payment", "peace", "peaceful",
-    "peak", "peasant", "peculiar", "pedestrian",
-    "peer", "penalty", "pencil", "penetrate",
-    "peninsula", "pension", "people",
-    "pepper", "perceive", "percentage",
-    "perception", "perfect", "perfectly",
-    "perform", "performance", "performer",
-    "perfume", "perhaps", "period",
-    "periodic", "peripheral", "permanent",
-    "permanently", "permission", "permit",
-    "persist", "persistence", "persistent",
-    "persona", "personal", "personality",
-    "personally", "personnel",
-    "perspective", "persuade", "pet",
-    "petition", "petroleum", "phase",
-    "phenomenon", "philosopher", "philosophical",
-    "philosophy", "phone",
-    "photograph", "photographer", "photographic",
-    "photography", "phrase", "physical",
-    "physically", "physician", "physics",
-    "piano", "pick", "picture",
-    "piece", "pierce", "pig", "pile",
-    "pillar", "pillow", "pilot",
-    "pin", "pine", "pink", "pioneer",
-    "pipe", "pit", "pitch", "pizza",
-    "placement", "plain", "plaintiff",
-    "plantation", "plate", "platform",
-    "plausible", "playback", "playful",
-    "playground", "plead", "pleasant",
-    "pledge", "plenty", "plot",
-    "plug", "plunge", "plural", "plus",
-    "pocket", "poem", "poet", "poetic",
-    "poetry", "poison", "poisonous",
-    "polar", "pole", "police",
-    "policeman", "policy", "polish",
-    "polite", "political", "politically",
-    "politician", "politics",
-    "poll", "pollution", "pond",
-    "pony", "pool", "pop",
-    "pope", "popular", "popularity",
-    "population", "porch", "port",
-    "portable", "porter", "portion",
-    "portrait", "portray", "pose",
-    "position", "positive", "positively",
-    "possess", "possession", "possessive",
-    "possibility", "possible", "possibly",
-    "postage", "postal", "poster",
-    "potato", "potent", "potential",
-    "potentially", "pottery",
-    "poverty", "powder", "powerful",
-    "practically", "practice",
-    "practitioner", "praise", "pray",
-    "prayer", "preach", "precede",
-    "precedent", "precious", "precise",
-    "precisely", "precision", "predict",
-    "predictable", "prediction",
-    "predominantly", "preface", "prefer",
-    "preference", "pregnancy", "pregnant",
-    "prejudice", "preliminary",
-    "premier", "premise", "premium",
-    "preparation", "prepare", "prepared",
-    "prescription", "presence",
-    "presentation", "presently",
-    "preservation", "preserve",
-    "presidency", "president", "presidential",
-    "pressing", "pressure",
-    "presumably", "presume", "pretend",
-    "pretty", "prevail", "prevalence",
-    "prevalent", "prevention",
-    "preview", "previous", "previously",
-    "prey", "pricing", "pride",
-    "priest", "primarily", "primary",
-    "prime", "primitive", "prince",
-    "princess", "principal", "principle",
-    "print", "printer", "printing",
-    "prior", "priority", "prison",
-    "prisoner", "privacy",
-    "privilege", "privileged",
-    "prize", "proactive", "probable",
-    "probably", "probe", "problem",
-    "problematic", "procedural",
-    "procedure", "proceed", "proceeding",
-    "proceeds", "processor",
-    "proclaim", "produce", "producer",
-    "productive", "productivity",
-    "profession", "professional",
-    "professor", "proficiency",
-    "profile", "profit",
-    "profitable", "profound",
-    "program", "programming",
-    "progressive", "prohibit",
-    "prohibition", "project",
-    "projection", "prominent",
-    "promise", "promising",
-    "promote", "promotion",
-    "prompt", "proof",
-    "propaganda", "proper", "properly",
-    "property", "prophet",
-    "proportion", "proposal",
-    "propose", "proposed",
-    "prosecution", "prosecutor",
-    "prospect", "prosperity",
-    "protect", "protection",
-    "protective", "protein",
-    "protest", "protester",
-    "protocol", "proud",
-    "prove", "proverb",
-    "provide", "provided",
-    "province", "provincial",
-    "provision", "provoke",
-    "proxy", "prudent",
-    "psychiatric", "psychiatry",
-    "psychological", "psychologist",
-    "psychology", "pub",
-    "publication", "publicity",
-    "publicly", "publish",
-    "publisher", "publishing",
-    "pulse", "pump",
-    "punch", "punish",
-    "punishment", "pupil",
-    "purchase", "purchaser",
-    "pure", "purely",
-    "purify", "purple",
-    "purpose", "purse",
-    "pursue", "pursuit",
-    "puzzle", "qualification",
-    "qualified", "qualify",
-    "qualitative", "quantify",
-    "quantitative", "quarterly",
-    "queen", "quest",
-    "questionable", "questionnaire",
-    "queue", "quit",
-    "quiz", "quota",
-    "quotation", "quote",
-    "rabbit", "racial",
-    "racism", "racist",
-    "radiation", "radical",
-    "rage", "raid",
-    "rail", "railroad",
-    "railway", "rain",
-    "rainbow", "rally",
-    "random", "range",
-    "rank", "ranking",
-    "rape", "rapid",
-    "rapidly", "rare",
-    "rarely", "rat",
-    "rate", "rating",
-    "ratio", "rational",
-    "raw", "ray",
-    "react", "reaction",
-    "readily", "reading",
-    "realistic", "reality",
-    "realization", "realize",
-    "realm", "rear",
-    "reason", "reasonable",
-    "reasonably", "reasoning",
-    "reassure", "rebel",
-    "rebellion", "rebuild",
-    "recall", "receipt",
-    "receiver", "recent",
-    "recently", "reception",
-    "recession", "recipe",
-    "recipient", "reckon",
-    "recognition", "recognize",
-    "recommend", "recommendation",
-    "reconcile", "reconstruction",
-    "recording", "recover",
-    "recovery", "recreation",
-    "recruit", "recruitment",
-    "reduction", "redundant",
-    "reef", "refer",
-    "referee", "reference",
-    "referendum", "referral",
-    "reflection", "reform",
-    "refrain", "refresh",
-    "refuge", "refugee",
-    "refund", "refusal",
-    "refuse", "regain",
-    "regard", "regarding",
-    "regardless", "regime",
-    "regiment", "regional",
-    "register", "registration",
-    "regret", "regular",
-    "regularly", "regulate",
-    "regulation", "regulator",
-    "regulatory", "rehabilitation",
-    "reign", "reinforce",
-    "reinforcement", "reject",
-    "rejection", "relate",
-    "relation", "relationship",
-    "relative", "relatively",
-    "relax", "relaxation",
-    "release", "relevant",
-    "reliability", "reliable",
-    "relief", "relieve",
-    "religion", "religious",
-    "reluctant", "reluctantly",
-    "remainder", "remains",
-    "remark", "remarkable",
-    "remarkably", "remedy",
-    "reminder", "remnant",
-    "remote", "removal",
-    "remove", "renaissance",
-    "render", "renew",
-    "renewable", "renewal",
-    "rent", "rental",
-    "repair", "repay",
-    "repeat", "repeatedly",
-    "repent", "repetition",
-    "replace", "replacement",
-    "reportedly", "reporter",
-    "representation", "representative",
-    "repression", "reprint",
-    "reproduce", "reproduction",
-    "republic", "republican",
-    "reputation", "request",
-    "require", "requirement",
-    "rescue", "resemble",
-    "resentment", "reservation",
-    "reserve", "reservoir",
-    "reside", "residence",
-    "residence", "resident",
-    "residential", "residual",
-    "resign", "resignation",
-    "resist", "resistance",
-    "resistant", "resolution",
-    "resolve", "resort",
-    "resource", "respect",
-    "respectable", "respectful",
-    "respective", "respectively",
-    "respond", "respondent",
-    "response", "responsibility",
-    "responsible", "responsive",
-    "restoration", "restore",
-    "restrain", "restraint",
-    "restrict", "restriction",
-    "restrictive", "restructuring",
-    "resume", "retail",
-    "retailer", "retain",
-    "retaliation", "retire",
-    "retirement", "retreat",
-    "retrieval", "retrieve",
-    "revelation", "revenge",
-    "revenue", "reverse",
-    "review", "revise",
-    "revision", "revival",
-    "revive", "revolution",
-    "revolutionary", "reward",
-    "rhetoric", "rhythm",
-    "rib", "ribbon",
-    "rid", "ride",
-    "ridge", "ridiculous",
-    "rifle", "rigid",
-    "riot", "rip",
-    "ripple", "ritual",
-    "rival", "rivalry",
-    "roar", "robbery",
-    "robe", "robot",
-    "robust", "rocket",
-    "rod", "roll",
-    "roller", "romance",
-    "romantic", "rookie",
-    "rope", "rose",
-    "rotate", "rotation",
-    "rotten", "rough",
-    "roughly", "round",
-    "route", "routine",
-    "row", "royal",
-    "royalty", "rub",
-    "rubber", "rug",
-    "ruin", "ruler",
-    "ruling", "rumor",
-    "runner", "running",
-    "rural", "rush",
-    "sacred", "sacrifice",
-    "sad", "saddle",
-    "sadly", "sadness",
-    "sake", "salad",
-    "salary", "sale",
-    "salmon", "salon",
-    "salt", "salute",
-    "salvation", "sample",
-    "sanction", "sanctuary",
-    "sand", "satellite",
-    "satisfaction", "satisfactory",
-    "satisfy", "sauce",
-    "saving", "savings",
-    "scale", "scandal",
-    "scare", "scared",
-    "scary", "scatter",
-    "scenario", "scent",
-    "schedule", "scheme",
-    "scholar", "scholarship",
-    "schooling", "scientific",
-    "scientist", "scope",
-    "score", "scorn",
-    "scrap", "scream",
-    "screen", "screening",
-    "screw", "script",
-    "scrutiny", "seal",
-    "search", "season",
-    "seasonal", "seating",
-    "secular", "secure",
-    "security", "seed",
-    "seek", "segment",
-    "seize", "seizure",
-    "select", "selection",
-    "selective", "self",
-    "seller", "senate",
-    "senator", "senior",
-    "sensation", "sensational",
-    "sensitivity", "sensor",
-    "sentence", "sentiment",
-    "separate", "separation",
-    "sequence", "sequencing",
-    "serial", "series",
-    "session", "settle",
-    "settlement", "settler",
-    "setup", "severe",
-    "severely", "severity",
-    "sew", "sewage",
-    "shade", "shadow",
-    "shaft", "shake",
-    "shall", "shallow",
-    "shame", "shape",
-    "shareholder", "sharing",
-    "shark", "shed",
-    "sheer", "sheet",
-    "shelf", "shell",
-    "shelter", "shield",
-    "shift", "shine",
-    "shipment", "shipping",
-    "shirt", "shock",
-    "shopping", "shortage",
-    "shortly", "shot",
-    "shoulder", "shout",
-    "shove", "shower",
-    "shrimp", "shrink",
-    "shrub", "shrug",
-    "shutter", "sibling",
-    "sickness", "sideways",
-    "siege", "sigh",
-    "sight", "sign",
-    "signal", "signature",
-    "significance", "significant",
-    "significantly", "silence",
-    "silent", "silicon",
-    "silk", "silly",
-    "silver", "similar",
-    "similarity", "similarly",
-    "simmer", "simplicity",
-    "simplify", "simply",
-    "simulation", "simultaneously",
-    "sin", "sincere",
-    "sincerely", "singer",
-    "single", "singular",
-    "sink", "sip",
-    "sister", "situation",
-    "sizable", "size",
-    "sketch", "ski",
-    "skilled", "skillful",
-    "skim", "skin",
-    "skip", "skirt",
-    "skull", "slap",
-    "slash", "slave",
-    "slavery", "sleep",
-    "sleeve", "slice",
-    "slide", "slight",
-    "slightly", "slim",
-    "slip", "slogan",
-    "slope", "slot",
-    "slow", "slowly",
-    "smart", "smell",
-    "smile", "smoke",
-    "smooth", "smoothly",
-    "snap", "sneak",
-    "snapshot", "snow",
-    "soak", "soap",
-    "soar", "soccer",
-    "social", "socialism",
-    "socialist", "societal",
-    "society", "sociological",
-    "sociology", "soda",
-    "software", "soil",
-    "solar", "soldier",
-    "sole", "solely",
-    "solemn", "solicitor",
-    "solidarity", "solitary",
-    "solo", "soluble",
-    "solution", "solve",
-    "somebody", "somehow",
-    "someone", "sometime",
-    "somewhat", "song",
-    "sophisticated", "sore",
-    "sorrow", "sort",
-    "soul", "sound",
-    "soup", "sour",
-    "source", "southeast",
-    "southern", "southwest",
-    "sovereign", "sovereignty",
-    "sow", "spacecraft",
-    "spacing", "span",
-    "spare", "spark",
-    "speak", "speaker",
-    "spear", "special",
-    "specialist", "specialize",
-    "specialty", "species",
-    "specific", "specifically",
-    "specification", "specify",
-    "specimen", "spectacle",
-    "spectacular", "spectator",
-    "spectrum", "speculate",
-    "speculation", "speech",
-    "spell", "spelling",
-    "spend", "sphere",
-    "spill", "spin",
-    "spine", "spiral",
-    "spirit", "spiritual",
-    "spite", "splash",
-    "split", "spokesman",
-    "spokesperson", "spokeswoman",
-    "sponsor", "sponsorship",
-    "spontaneous", "spoon",
-    "sport", "spot",
-    "spouse", "spread",
-    "spring", "sprint",
-    "spur", "spy",
-    "squad", "squadron",
-    "square", "squeeze",
-    "stability", "stabilize",
-    "stable", "stadium",
-    "staff", "stage",
-    "stake", "stakeholder",
-    "stall", "stance",
-    "stand", "standard",
-    "standing", "staple",
-    "stare", "stark",
-    "startup", "starvation",
-    "starve", "statement",
-    "statue", "status",
-    "statute", "statutory",
-    "steady", "steal",
-    "steam", "steel",
-    "steep", "steer",
-    "stem", "stereotype",
-    "sterling", "stern",
-    "steward", "stick",
-    "sticky", "stiff",
-    "stimulate", "stimulus",
-    "stir", "stitch",
-    "stock", "stomach",
-    "stoppage", "storage",
-    "storm", "story",
-    "strain", "strand",
-    "strap", "strategic",
-    "strategically", "strategist",
-    "strategy", "straw",
-    "stream", "street",
-    "strength", "strengthen",
-    "stress", "stretch",
-    "strict", "strictly",
-    "stride", "strike",
-    "striker", "string",
-    "strip", "stripe",
-    "strive", "stroke",
-    "stronghold", "strongly",
-    "structural", "structure",
-    "struggle", "stubborn",
-    "studio", "study",
-    "stuff", "stumble",
-    "stun", "stunning",
-    "stupid", "style",
-    "subject", "subjective",
-    "sublime", "submission",
-    "submit", "subordinate",
-    "subsequent", "subsequently",
-    "subsidy", "substance",
-    "substantial", "substantially",
-    "substantive", "substitute",
-    "substitution", "subtle",
-    "subtlety", "subtract",
-    "suburb", "suburban",
-    "subversion", "subvert",
-    "succeed", "success",
-    "successful", "successfully",
-    "succession", "successive",
-    "successor", "suck",
-    "sudden", "suddenly",
-    "sue", "suffer",
-    "suffering", "sufficient",
-    "sufficiently", "sugar",
-    "suicide", "suit",
-    "suitable", "suite",
-    "sulfur", "sum",
-    "summarize", "summary",
-    "summit", "sunlight",
-    "sunny", "sunrise",
-    "sunset", "sunshine",
-    "superb", "superficial",
-    "superintendent", "superior",
-    "superiority", "supermarket",
-    "supervise", "supervision",
-    "supervisor", "supplement",
-    "supplementary", "supplier",
-    "supply", "support",
-    "supporter", "supportive",
-    "suppose", "supposedly",
-    "suppress", "suppression",
-    "supreme", "surcharge",
-    "surface", "surge",
-    "surgeon", "surgery",
-    "surgical", "surname",
-    "surpass", "surplus",
-    "surprise", "surprised",
-    "surprising", "surprisingly",
-    "surrender", "surround",
-    "surrounding", "surveillance",
-    "survey", "survival",
-    "survive", "survivor",
-    "susceptible", "suspect",
-    "suspend", "suspense",
-    "suspension", "suspicion",
-    "suspicious", "sustain",
-    "sustainable", "sustained",
-    "swap", "swear",
-    "sweep", "sweet",
-    "swell", "swift",
-    "swim", "swimming",
-    "swing", "switch",
-    "sword", "symbol",
-    "symbolic", "symmetry",
-    "sympathetic", "sympathy",
-    "symphony", "symptom",
-    "syndrome", "synthesis",
-    "synthetic", "system",
-    "systematic", "tackle",
-    "tactical", "tactics",
-    "tag", "tail",
-    "takeover", "tale",
-    "talent", "talented",
-    "tank", "tap",
-    "tape", "target",
-    "tariff", "task",
-    "taste", "tax",
-    "taxation", "taxpayer",
-    "teaching", "tear",
-    "tease", "technical",
-    "technically", "technician",
-    "technique", "technological",
-    "technology", "teenage",
-    "teenager", "telecommunications",
-    "telegraph", "telephone",
-    "telescope", "television",
-    "temper", "temperature",
-    "temple", "temporarily",
-    "temporary", "tempt",
-    "temptation", "tenant",
-    "tendency", "tender",
-    "tennis", "tension",
-    "tent", "tenure",
-    "terminal", "terminate",
-    "termination", "term",
-    "terrain", "terrible",
-    "terribly", "terrific",
-    "territorial", "territory",
-    "terror", "terrorism",
-    "terrorist", "testament",
-    "testify", "testimony",
-    "textbook", "textile",
-    "texture", "thankful",
-    "theater", "theatre",
-    "theft", "theological",
-    "theology", "theoretical",
-    "theorist", "theory",
-    "therapist", "therapy",
-    "thereafter", "thereby",
-    "thermal", "thesis",
-    "thickness", "thief",
-    "thigh", "thin",
-    "thinking", "thirst",
-    "thirsty", "thorn",
-    "thorough", "thoroughly",
-    "thoughtful", "thoughtless",
-    "thriller", "thrive",
-    "throat", "throne",
-    "thrust", "thumb",
-    "thunder", "tide",
-    "timber", "timely",
-    "timing", "tissue",
-    "title", "toe",
-    "tolerance", "tolerant",
-    "tolerate", "toll",
-    "tomato", "ton",
-    "tone", "tongue",
-    "tool", "tooth",
-    "topic", "topical",
-    "torch", "torture",
-    "total", "totally",
-    "touch", "tourism",
-    "tourist", "tournament",
-    "tow", "towel",
-    "tower", "toxic",
-    "trace", "track",
-    "tractor", "trade",
-    "trademark", "trader",
-    "trading", "tradition",
-    "traditional", "traditionally",
-    "traffic", "tragedy",
-    "tragic", "trail",
-    "trainer", "training",
-    "trait", "transaction",
-    "transcript", "transfer",
-    "transform", "transformation",
-    "transit", "transition",
-    "translate", "translation",
-    "translator", "transmission",
-    "transmit", "transparency",
-    "transparent", "transplant",
-    "transport", "transportation",
-    "trap", "trash",
-    "trauma", "traumatic",
-    "traveler", "treasure",
-    "treat", "treatment",
-    "treaty", "tremendous",
-    "trend", "trial",
-    "triangle", "tribal",
-    "tribe", "tribunal",
-    "trigger", "trim",
-    "triumph", "troop",
-    "trophy", "tropical",
-    "trouble", "troublesome",
-    "trunk", "trust",
-    "trustee", "truth",
-    "tube", "tuition",
-    "tumor", "tune",
-    "tunnel", "turmoil",
-    "turnover", "turtle",
-    "tutor", "tutorial",
-    "twist", "typical",
-    "typically", "tyranny",
-    "ugly", "ultimate",
-    "ultimately", "ultimatum",
-    "umbrella", "unable",
-    "unacceptable", "unanimous",
-    "unaware", "uncertainty",
-    "uncomfortable", "unconscious",
-    "unconstitutional", "undergo",
-    "undergraduate", "underground",
-    "underline", "underlying",
-    "undermine", "underneath",
-    "underscore", "undertake",
-    "undertaking", "underwear",
-    "undoubtedly", "unemployment",
-    "unexpected", "unexpectedly",
-    "unfair", "unfold",
-    "unfortunate", "unfortunately",
-    "unhappy", "unhealthy",
-    "unified", "uniform",
-    "unilateral", "unintended",
-    "union", "unique",
-    "unity", "universal",
-    "universally", "universe",
-    "unknown", "unlawful",
-    "unlike", "unlikely",
-    "unnecessary", "unpleasant",
-    "unprecedented", "unrest",
-    "unveil", "upcoming",
-    "update", "upgrade",
-    "uphold", "upset",
-    "upturn", "uranium",
-    "urban", "urge",
-    "urgency", "urgent",
-    "usage", "utilize",
-    "utmost", "utter",
-    "utterly", "vacancy",
-    "vacation", "vaccination",
-    "vaccine", "vacuum",
-    "valid", "validate",
-    "validity", "valley",
-    "valuable", "valuation",
-    "valve", "variable",
-    "variation", "varied",
-    "vary", "vast",
-    "vector", "vegetable",
-    "vegetation", "vehicle",
-    "veil", "vein",
-    "velocity", "vendor",
-    "venture", "venue",
-    "verbal", "verdict",
-    "verify", "versatile",
-    "verse", "version",
-    "versus", "vertical",
-    "vessel", "veteran",
-    "veterinary", "viable",
-    "vibrant", "vibration",
-    "vice", "victim",
-    "victorious", "victory",
-    "video", "view",
-    "viewer", "viewpoint",
-    "vigorous", "village",
-    "violate", "violation",
-    "violence", "violent",
-    "virgin", "virtual",
-    "virtually", "virtue",
-    "virus", "visa",
-    "visible", "vision",
-    "visual", "vital",
-    "vitamin", "vivid",
-    "vocabulary", "vocal",
-    "vocational", "voice",
-    "volatile", "volcano",
-    "volume", "voluntary",
-    "volunteer", "vote",
-    "voter", "voting",
-    "vulnerability", "vulnerable",
-    "wage", "wagon",
-    "waist", "wallet",
-    "wander", "ward",
-    "warehouse", "warfare",
-    "warmth", "warn",
-    "warning", "warrant",
-    "warranty", "warrior",
-    "wary", "waterfall",
-    "waterproof", "watershed",
-    "wave", "wavelength",
-    "wax", "weakness",
-    "wealth", "wealthy",
-    "weapon", "wear",
-    "weary", "weather",
-    "weave", "web",
-    "website", "wedding",
-    "weed", "weekday",
-    "weekend", "weekly",
-    "weigh", "weight",
-    "weird", "welcome",
-    "welfare", "wellness",
-    "whatsoever", "wheel",
-    "whenever", "whereas",
-    "whereby", "wherein",
-    "whichever", "whisper",
-    "white", "whoever",
-    "wholesale", "wholly",
-    "widespread", "widow",
-    "width", "willing",
-    "willingness", "wisdom",
-    "withdraw", "withdrawal",
-    "wither", "withhold",
-    "within", "without",
-    "witness", "wolf",
-    "wonder", "wooden",
-    "wool", "workforce",
-    "workout", "workplace",
-    "workshop", "worship",
-    "worst", "worthwhile",
-    "worthy", "wound",
-    "wrap", "wrist",
-    "writer", "writing",
-    "wrongly", "yacht",
-    "yell", "young",
-    "youngster", "zone",
 ]
-# Exclude byte-level chars (they already have IDs 51-306)
-byte_chars = {byte2char[b] for b in range(256)}
-# Deduplicate while preserving order
-seen = set()
-deduped = []
-for w in common_words:
-    if w not in seen and w not in byte_chars:
-        seen.add(w)
-        deduped.append(w)
-# Truncate to fit total vocab <= 4096 (IDs 307-4095 = 3789 slots)
-max_words = 4096 - 307
-if len(deduped) > max_words:
-    deduped = deduped[:max_words]
-common_words = deduped
-next_id = 307
-word_tokens = []
-for w in common_words:
-    word_tokens.append((w, next_id))
-    next_id += 1
-# Build vocab dict
-vocab = {}
-for name, idx in special_tokens:
-    if name in vocab:
-        print(f"DUPLICATE: {name!r} @ {vocab[name]} and {idx}")
-    vocab[name] = idx
-for char, idx in byte_tokens:
-    if char in vocab:
-        print(f"DUPLICATE BYTE: {char!r} (U+{ord(char):04X}) @ {vocab[char]} and {idx}")
-    vocab[char] = idx
-for name, idx in word_tokens:
-    if name in vocab:
-        print(f"DUPLICATE WORD: {name!r} @ {vocab[name]} and {idx}")
-    vocab[name] = idx
-# Build added_tokens
-added_tokens = []
-for name, idx in special_tokens:
-    added_tokens.append({
-        "id": idx,
-        "content": name,
-        "single_word": False,
-        "lstrip": False,
-        "rstrip": False,
-        "normalized": False,
-        "special": True,
-    })
-# Build tokenizer.json
-tokenizer_json = {
-    "version": "1.0",
-    "truncation": None,
-    "padding": None,
-    "added_tokens": added_tokens,
-    "normalizer": None,
-    "pre_tokenizer": {
-        "type": "ByteLevel",
-        "add_prefix_space": False,
-        "trim_offsets": True,
-    },
-    "post_processor": None,
-    "decoder": {
-        "type": "ByteLevel",
-        "add_prefix_space": False,
-        "trim_offsets": True,
-    },
-    "model": {
-        "type": "BPE",
-        "dropout": None,
-        "unk_token": "<unk>",
-        "byte_fallback": True,
-        "vocab": vocab,
-        "merges": [],
-    },
-}
 if __name__ == "__main__":
-    with open("tokenizer/tokenizer.json", "w", encoding="utf-8") as f:
-        json.dump(tokenizer_json, f, ensure_ascii=False, separators=(",", ":"))
-    with open("tokenizer/special_tokens_map.json", "w", encoding="utf-8") as f:
-        json.dump({
-            "bos_token": "<s>",
-            "eos_token": "</s>",
-            "pad_token": "<pad>",
-            "unk_token": "<unk>",
-        }, f, ensure_ascii=False, indent=2)
-    import os
-    size = os.path.getsize("tokenizer/tokenizer.json")
-    print(f"tokenizer.json: {size:,} bytes")
-    print(f"Vocab entries: {len(vocab)}")
-    print(f"Added tokens: {len(added_tokens)}")
-    print(f"Merges: 0")
-    print(f"Byte fallback: True")
-    assert len(vocab) <= 4096, f"Vocab size overflow: {len(vocab)}"
-    print(f"Free slots: {4096 - len(vocab)}")
-    from transformers import PreTrainedTokenizerFast
-    from tokenizers import Tokenizer as Tk
-    tok_obj = Tk.from_file("tokenizer/tokenizer.json")
-    tok = PreTrainedTokenizerFast(tokenizer_object=tok_obj)
-    tok.add_special_tokens({"pad_token": "<pad>", "bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>"})
-    test = "Hello, how are you?"
-    enc = tok.encode(test)
-    dec = tok.decode(enc)
-    print(f"OK: {enc} -> {dec!r}")
-    test2 = "def f(): return 42"
-    enc2 = tok.encode(test2)
-    dec2 = tok.decode(enc2)
-    print(f"OK: {enc2} -> {dec2!r}")
-    enc3 = tok.encode("<|system|>Hi<|user|>there")
-    print(f"Special: {enc3} -> {tok.decode(enc3)!r}")
-    test4 = "café résumé"
-    enc4 = tok.encode(test4)
-    dec4 = tok.decode(enc4)
-    print(f"Unicode: {enc4} -> {dec4!r}")

+"""Generate BPE tokenizer trained on textbook corpus.
+Vocab layout: 0-50 special tokens, 51-306 byte-level chars, 307-4095 BPE merges.
+"""
 import json
+import os
+from tokenizers import Tokenizer, models, pre_tokenizers, trainers, decoders
+TOKENIZER_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "tokenizer")
+BOOKS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data", "books")
+os.makedirs(TOKENIZER_DIR, exist_ok=True)
 special_tokens = [
+    "<unk>", "<s>", "</s>", "<pad>",
+    "<|system|>", "<|user|>", "<|assistant|>",
+    "<think>", "</think>",
+    "[INST]", "[/INST]",
+    "<|begin_of_thought|>", "<|end_of_thought|>",
+    "<|reflect|>", "<|revise|>", "<|verify|>",
+    "<|code|>", "<|text|>", "<|math|>",
+    "<|think|>", "<|answer|>", "<|step|>", "<|reason|>",
+    "<|check|>", "<|output|>", "<|plan|>", "<|solve|>",
+    "<|analyze|>", "<|conclude|>", "<|approach|>", "<|alternative|>",
+    "<|summary|>", "<|question|>", "<|hint|>", "<|example|>",
+    "<|correct|>", "<|incorrect|>", "<|feedback|>",
+    "<|start|>", "<|end|>", "<|sep|>", "<|cls|>",
+    "<|tool|>", "<|function|>", "<|result|>", "<|input|>",
+    "<|detect|>", "<|context|>", "<|proof|>", "<|lemma|>", "<|theorem|>",
 ]
+special_map = {t: i for i, t in enumerate(special_tokens)}
+NUM_SPECIAL = len(special_tokens)
+def find_book_files():
+    if not os.path.isdir(BOOKS_DIR):
+        print(f"Warning: {BOOKS_DIR} not found, using empty corpus")
+        return []
+    files = []
+    for fname in os.listdir(BOOKS_DIR):
+        if fname.endswith(".txt"):
+            fpath = os.path.join(BOOKS_DIR, fname)
+            if os.path.getsize(fpath) > 100:
+                files.append(fpath)
+    files.sort()
+    print(f"Found {len(files)} book files in {BOOKS_DIR}")
+    return files
+def build_tokenizer():
+    # Create BPE tokenizer with ByteLevel pre-tokenizer
+    tokenizer = Tokenizer(models.BPE(unk_token="<unk>"))
+    tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
+    tokenizer.decoder = decoders.ByteLevel(add_prefix_space=False)
+    tokenizer.add_special_tokens(special_tokens)
+    book_files = find_book_files()
+    print(f"Training BPE with vocab_size=4096, {len(book_files)} files...")
+    trainer = trainers.BpeTrainer(
+        vocab_size=4096,
+        min_frequency=2,
+        special_tokens=special_tokens,
+        show_progress=True,
+        initial_alphabet=[],
+    )
+    tokenizer.train(book_files, trainer)
+    print("BPE training done.")
+    output_path = os.path.join(TOKENIZER_DIR, "tokenizer.json")
+    tokenizer.save(output_path)
+    print(f"Saved to {output_path}")
+    # Post-process: ensure byte_fallback=True
+    with open(output_path) as f:
+        data = json.load(f)
+    data["model"]["byte_fallback"] = True
+    data["model"]["dropout"] = None
+    with open(output_path, "w") as f:
+        json.dump(data, f, ensure_ascii=False)
+    # Verify
+    with open(output_path) as f:
+        data = json.load(f)
+    vocab = data["model"]["vocab"]
+    merges = data["model"]["merges"]
+    sorted_vocab = sorted(vocab.items(), key=lambda x: x[1])
+    print(f"Vocab size: {len(vocab)}")
+    print(f"Merges: {len(merges)}")
+    print(f"First tokens: {sorted_vocab[:7]}")
+    print(f"Tokens 50-55: {sorted_vocab[50:55]}")
+    print(f"Last tokens: {sorted_vocab[-5:]}")
+    if merges:
+        print(f"First merges: {merges[:5]}")
 if __name__ == "__main__":
+    build_tokenizer()

infer_gguf.py CHANGED Viewed

@@ -1,47 +1,287 @@
-"""
-Inference with GGUF INT4 model or QLoRA checkpoint.
-Usage:
-  python3 scripts/infer_gguf.py --gguf outputs/tiny-sft/tiny.gguf
-  python3 scripts/infer_gguf.py --checkpoint model.pt
-"""
-import os, sys, argparse
-import gguf
 import torch
 import torch.nn.functional as F
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
-from scripts.model_tiny import TinyModel, apply_qlora
-def quantize_to_q4(tensor):
-    t = tensor.float()
-    max_val = t.abs().max()
-    if max_val < 1e-8:
-        return t
-    scale = max_val / 7.0
-    q = (t / scale).round().clamp(-7, 7).char()
-    dq = q.float() * scale
-    return dq
-def load_gguf_model(gguf_path):
     print(f" Loading GGUF: {gguf_path}")
-    reader = gguf.GGUFReader(gguf_path)
-    model = TinyModel(
-        vocab_size=1757, hidden=128, intermediate=640,
-        num_layers=3, num_heads=8, num_kv_heads=4,
-        max_seq_len=2048, tie_weights=True,
-    )
     model.eval()
     state = {}
     for tensor in reader.tensors:
         name = tensor.name
         data = torch.from_numpy(tensor.data.copy())
         if name == "token_embd.weight":
             state["token_embed.weight"] = data
         elif name == "output_norm.weight":
@@ -73,56 +313,39 @@ def load_gguf_model(gguf_path):
     model.load_state_dict(state, strict=False)
     print(f" Loaded: {len(state)} tensors")
-    for name, param in model.named_parameters():
-        if "weight" in name and "norm" not in name and "embed" not in name:
-            param.data.copy_(quantize_to_q4(param.data))
     return model
-def load_checkpoint_model(ckpt_path):
     print(f" Loading checkpoint: {ckpt_path}")
     state = torch.load(ckpt_path, map_location="cpu", weights_only=True)
-    has_qlora = any("qweight" in k for k in state)
-    print(f" Detected: {'QLoRA' if has_qlora else 'full'} checkpoint")
-    model = TinyModel(
-        vocab_size=1757, hidden=128, intermediate=640,
-        num_layers=3, num_heads=8, num_kv_heads=4,
-        max_seq_len=2048, tie_weights=True,
-    )
-    if has_qlora:
-        model = apply_qlora(model,
-            target_modules=["q_proj","k_proj","v_proj","o_proj","gate","up","down"],
-            r=8, alpha=16, dropout=0.0, freeze_embeds=True)
-    missing, unexpected = model.load_state_dict(state, strict=False)
-    if missing:
-        print(f" Missing keys: {missing}")
-    if unexpected:
-        print(f" Unexpected keys: {unexpected}")
     model.eval()
-    return model
-def load_tokenizer():
-    from tokenizers import Tokenizer as Tk
-    tok = Tk.from_file(os.path.join(os.path.dirname(__file__), "..", "tokenizer", "tokenizer.json"))
-    return tok
-def _build_chat(system, user, tokenizer):
-    parts = [f"<|system|>\n{system}", f"<|user|>\n{user}", "<|assistant|>\n"]
-    text = "\n".join(parts)
-    return tokenizer.encode(text).ids
 def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--gguf", default=None)
-    parser.add_argument("--checkpoint", default=None)
     parser.add_argument("--prompt", default="What is 2+2?")
     parser.add_argument("--system", default="You are a helpful AI assistant.")
     parser.add_argument("--max-tokens", type=int, default=128)
@@ -135,16 +358,18 @@ def main():
         sys.exit(1)
     model = None
     if args.gguf:
         if not os.path.exists(args.gguf):
             print(f"GGUF not found: {args.gguf}")
             sys.exit(1)
-        model = load_gguf_model(args.gguf)
     elif args.checkpoint:
         if not os.path.exists(args.checkpoint):
             print(f"Checkpoint not found: {args.checkpoint}")
             sys.exit(1)
-        model = load_checkpoint_model(args.checkpoint)
     else:
         print("Specify --gguf or --checkpoint")
         sys.exit(1)
@@ -153,17 +378,16 @@ def main():
     model.to(device)
     print(f" Device: {device}")
-    tok = load_tokenizer()
-    input_ids = _build_chat(args.system, args.prompt, tok)
     input_ids = torch.tensor([input_ids], device=device)
     print("\n" + tok.decode(input_ids[0].tolist()), end="", flush=True)
     def stream(id_):
-        t = tok.decode([id_])
-        print(t, end="", flush=True)
-    out = model.generate(
         input_ids,
         max_new_tokens=args.max_tokens,
         temperature=args.temperature,

+import os, sys, argparse, math as _math
 import torch
 import torch.nn.functional as F
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+HERE = os.path.dirname(os.path.abspath(__file__))
+PROJ = os.path.dirname(HERE)
+# ─── V2 model (backward compat: GGUF + QLoRA checkpoints) ───────────────
+class ALiBiAttention(torch.nn.Module):
+    def __init__(self, hidden: int, num_heads: int, num_kv_heads: int):
+        super().__init__()
+        self.num_heads = num_heads
+        self.num_kv_heads = num_kv_heads
+        self.head_dim = hidden // num_heads
+        self.num_groups = num_heads // num_kv_heads
+        self.q_proj = torch.nn.Linear(hidden, hidden, bias=False)
+        self.k_proj = torch.nn.Linear(hidden, num_kv_heads * self.head_dim, bias=False)
+        self.v_proj = torch.nn.Linear(hidden, num_kv_heads * self.head_dim, bias=False)
+        self.o_proj = torch.nn.Linear(hidden, hidden, bias=False)
+    @staticmethod
+    def _get_alibi_slopes(num_heads):
+        n = 2 ** _math.ceil(_math.log2(num_heads))
+        return torch.tensor([2.0 ** (-(i + 1)) for i in range(n)][:num_heads])
+    def forward(self, x):
+        B, T, C = x.shape
+        q = self.q_proj(x).view(B, T, self.num_heads, self.head_dim).transpose(1, 2)
+        k = self.k_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2)
+        v = self.v_proj(x).view(B, T, self.num_kv_heads, self.head_dim).transpose(1, 2)
+        k = k.repeat_interleave(self.num_groups, dim=1)
+        v = v.repeat_interleave(self.num_groups, dim=1)
+        scores = (q @ k.transpose(-2, -1)) * (self.head_dim ** -0.5)
+        slopes = self._get_alibi_slopes(self.num_heads).to(x.device, x.dtype)
+        pos = torch.arange(T, device=x.device)
+        alibi = (pos.view(1, T) - pos.view(T, 1)).abs().neg().unsqueeze(0).unsqueeze(0)
+        scores = scores + alibi * slopes.view(-1, 1, 1)
+        causal = torch.triu(torch.full((T, T), float('-inf'), device=x.device, dtype=x.dtype), diagonal=1)
+        scores = scores + causal
+        attn = F.softmax(scores, dim=-1, dtype=torch.float32).to(x.dtype)
+        out = (attn @ v).transpose(1, 2).contiguous().view(B, T, C)
+        return self.o_proj(out)
+class RMSNormV2(torch.nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6):
+        super().__init__()
+        self.weight = torch.nn.Parameter(torch.ones(dim))
+        self.eps = eps
+    def forward(self, x):
+        norm = x.float().pow(2).mean(-1, keepdim=True).add(self.eps).rsqrt()
+        return (x.float() * norm).type_as(x) * self.weight
+class SwiGLU(torch.nn.Module):
+    def __init__(self, hidden: int, intermediate: int):
+        super().__init__()
+        self.gate = torch.nn.Linear(hidden, intermediate, bias=False)
+        self.up = torch.nn.Linear(hidden, intermediate, bias=False)
+        self.down = torch.nn.Linear(intermediate, hidden, bias=False)
+    def forward(self, x):
+        return self.down(F.silu(self.gate(x)) * self.up(x))
+class TransformerBlockV2(torch.nn.Module):
+    def __init__(self, hidden: int, intermediate: int, num_heads: int, num_kv_heads: int):
+        super().__init__()
+        self.ln1 = RMSNormV2(hidden)
+        self.attn = ALiBiAttention(hidden, num_heads, num_kv_heads)
+        self.ln2 = RMSNormV2(hidden)
+        self.mlp = SwiGLU(hidden, intermediate)
+    def forward(self, x):
+        x = x + self.attn(self.ln1(x))
+        x = x + self.mlp(self.ln2(x))
+        return x
+class TinyModelV2(torch.nn.Module):
+    def __init__(self, vocab_size=1757, hidden=128, intermediate=640,
+                 num_layers=3, num_heads=8, num_kv_heads=4, max_seq_len=2048,
+                 tie_weights=True):
+        super().__init__()
+        self.max_seq_len = max_seq_len
+        self.token_embed = torch.nn.Embedding(vocab_size, hidden)
+        self.blocks = torch.nn.ModuleList([
+            TransformerBlockV2(hidden, intermediate, num_heads, num_kv_heads)
+            for _ in range(num_layers)
+        ])
+        self.ln_f = RMSNormV2(hidden)
+        self.lm_head = torch.nn.Linear(hidden, vocab_size, bias=False)
+        if tie_weights:
+            self.lm_head.weight = self.token_embed.weight
+    def forward(self, input_ids):
+        x = self.token_embed(input_ids)
+        for block in self.blocks:
+            x = block(x)
+        x = self.ln_f(x)
+        return self.lm_head(x)
+    @torch.no_grad()
+    def generate(self, input_ids, max_new_tokens=128, temperature=0.7, top_p=0.9, stream_callback=None):
+        self.eval()
+        for _ in range(max_new_tokens):
+            if input_ids.size(1) > self.max_seq_len:
+                input_ids = input_ids[:, -self.max_seq_len:]
+            logits = self.forward(input_ids)
+            logits = logits[:, -1, :]
+            if temperature > 0:
+                logits = logits / temperature
+            if top_p < 1.0:
+                sorted_logits, sorted_idx = logits.sort(dim=-1, descending=True)
+                cum_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1)
+                cutoff = cum_probs > top_p
+                cutoff[..., 1:] = cutoff[..., :-1].clone()
+                cutoff[..., 0] = False
+                logits[~cutoff] = float('-inf')
+            probs = F.softmax(logits, dim=-1)
+            if temperature > 0:
+                next_token = torch.multinomial(probs, 1)
+            else:
+                next_token = probs.argmax(dim=-1, keepdim=True)
+            input_ids = torch.cat([input_ids, next_token], dim=1)
+            if stream_callback:
+                stream_callback(next_token.item())
+            if next_token.item() == 2:
+                break
+        return input_ids
+# ─── QLoRA (NF4 + LoRA) for V2 ─────────────────────────────────────────
+NF4_LEVELS = torch.tensor([
+    -1.0, -0.6961928009986877, -0.5250730514526367, -0.39491748809814453,
+    -0.28444138169288635, -0.18477343022823334, -0.09105003625154495, 0.0,
+    0.07958029955625534, 0.16093020141124725, 0.24611230194568634,
+    0.33791524171829224, 0.44070982933044434, 0.5626170039176941,
+    0.7229568362236023, 1.0,
+])
+def unpack_nf4(qweight, shape):
+    n = qweight.numel() * 2
+    lo = (qweight & 0x0F).view(-1)
+    hi = ((qweight >> 4) & 0x0F).view(-1)
+    indices = torch.stack([lo, hi], dim=1).reshape(n)
+    return indices[:shape[0] * shape[1]].reshape(shape)
+def pack_nf4(indices):
+    n = indices.numel()
+    even = indices[0::2]
+    odd = indices[1::2]
+    packed = (odd << 4) | even
+    return packed.to(torch.uint8)
+class QLoRALinear(torch.nn.Module):
+    def __init__(self, in_features: int, out_features: int, r: int = 8, alpha: float = 16, dropout: float = 0.0):
+        super().__init__()
+        self.in_features = in_features
+        self.out_features = out_features
+        self.r = r
+        self.scaling = alpha / r
+        self.dropout = torch.nn.Dropout(dropout)
+        self.register_buffer("qweight", torch.zeros((in_features * out_features + 1) // 2, dtype=torch.uint8))
+        self.register_buffer("scales", torch.zeros(out_features))
+        self.register_buffer("bias", torch.zeros(out_features))
+        self.lora_A = torch.nn.Parameter(torch.zeros(r, in_features))
+        self.lora_B = torch.nn.Parameter(torch.zeros(out_features, r))
+        torch.nn.init.kaiming_uniform_(self.lora_A, a=_math.sqrt(5))
+    def dequantize(self):
+        indices = unpack_nf4(self.qweight, (self.out_features, self.in_features))
+        return NF4_LEVELS.to(self.qweight.device)[indices.long()] * self.scales[:, None]
+    def quantize_from(self, weight, bias=None):
+        w = weight.float()
+        scales = w.abs().max(dim=1, keepdim=True).values.clamp(min=1e-8)
+        scaled = (w / scales).clamp(-1, 1)
+        idx = (scaled[:, :, None] - NF4_LEVELS[None, None, :].to(w.device)).abs().argmin(dim=-1)
+        self.qweight.copy_(pack_nf4(idx.reshape(-1)))
+        self.scales.copy_(scales.squeeze(1))
+        if bias is not None:
+            self.bias.copy_(bias.float())
+    def forward(self, x):
+        base = F.linear(x, self.dequantize(), self.bias)
+        return base + self.dropout(x) @ self.lora_A.T @ self.lora_B.T * self.scaling
+def apply_qlora_v2(model, target_modules=None, r=8, alpha=16, dropout=0.0):
+    if target_modules is None:
+        target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate", "up", "down"]
+    qlora_params = 0
+    for name, module in model.named_modules():
+        if not isinstance(module, torch.nn.Linear):
+            continue
+        key = name.split(".")[-1]
+        if key not in target_modules:
+            continue
+        parent = model
+        parts = name.split(".")
+        for p in parts[:-1]:
+            parent = getattr(parent, p)
+        qlora = QLoRALinear(module.in_features, module.out_features, r=r, alpha=alpha, dropout=dropout)
+        qlora.quantize_from(module.weight, module.bias)
+        setattr(parent, parts[-1], qlora)
+        qlora_params += 2 * r * module.in_features + module.out_features * r
+    n = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    print(f" QLoRA applied: {qlora_params:,} LoRA params | trainable: {n:,}")
+# ─── V3 model (from model_tiny) ─────────────────────────────────────────
+def _load_v3_model():
+    from scripts.model_tiny import TinyModel as TinyModelV3
+    return TinyModelV3
+# ─── Tokenizer ──────────────────────────────────────────────────────────
+def load_tokenizer(ckpt_type=None):
+    from tokenizers import Tokenizer as Tk
+    paths = [os.path.join(PROJ, "tokenizer", "tokenizer.json"),
+             os.path.join(PROJ, "tokenizer.json")]
+    if ckpt_type and ckpt_type.startswith("v2"):
+        v2_path = os.path.join(os.path.dirname(PROJ), "lumia-v1", "tokenizer", "tokenizer.json")
+        if os.path.exists(v2_path):
+            paths.insert(0, v2_path)
+    for p in paths:
+        if os.path.exists(p):
+            tok = Tk.from_file(p)
+            print(f" Tokenizer: {p} ({tok.get_vocab_size()} vocab)")
+            return tok
+    print(" Tokenizer not found")
+    sys.exit(1)
+def build_chat(system, user, tokenizer):
+    parts = [f"<|system|>\n{system}", f"<|user|>\n{user}", "<|assistant|>\n"]
+    text = "\n".join(parts)
+    return tokenizer.encode(text).ids
+# ─── Detect checkpoint type ─────────────────────────────────────────────
+def _detect_ckpt_type(state):
+    keys = list(state.keys())
+    if any("rpw" in k or "vcr" in k for k in keys):
+        return "v3"
+    if any("qweight" in k for k in keys):
+        return "v2_qlora"
+    if any("gate" in k for k in keys) or any("mlp.gate" in k for k in keys):
+        return "v2_fp32"
+    if any(k.startswith("blocks.") for k in keys) and any("ln1.weight" in k for k in keys):
+        blk_keys = [k.split(".")[1] for k in keys if k.startswith("blocks.")]
+        max_blk = max(int(b) for b in blk_keys) if blk_keys else 0
+        if max_blk >= 3:
+            return "v3"
+    return "v2_fp32"
+# ─── Loaders ────────────────────────────────────────────────────────────
+def load_gguf(gguf_path):
     print(f" Loading GGUF: {gguf_path}")
+    import gguf as _gguf
+    reader = _gguf.GGUFReader(gguf_path)
+    model = TinyModelV2(vocab_size=1757, hidden=128, intermediate=640,
+                        num_layers=3, num_heads=8, num_kv_heads=4,
+                        max_seq_len=2048, tie_weights=True)
     model.eval()
     state = {}
     for tensor in reader.tensors:
         name = tensor.name
         data = torch.from_numpy(tensor.data.copy())
         if name == "token_embd.weight":
             state["token_embed.weight"] = data
         elif name == "output_norm.weight":
     model.load_state_dict(state, strict=False)
     print(f" Loaded: {len(state)} tensors")
     return model
+def load_checkpoint(ckpt_path):
     print(f" Loading checkpoint: {ckpt_path}")
     state = torch.load(ckpt_path, map_location="cpu", weights_only=True)
+    ckpt_type = _detect_ckpt_type(state)
+    print(f" Detected: {ckpt_type}")
+    if ckpt_type.startswith("v2"):
+        has_qlora = ckpt_type == "v2_qlora"
+        model = TinyModelV2(vocab_size=1757, hidden=128, intermediate=640,
+                            num_layers=3, num_heads=8, num_kv_heads=4,
+                            max_seq_len=2048, tie_weights=True)
+        if has_qlora:
+            apply_qlora_v2(model)
+        model.load_state_dict(state, strict=False)
+        model.eval()
+        return model, ckpt_type
+    TV3 = _load_v3_model()
+    model = TV3()
+    model.load_state_dict(state)
     model.eval()
+    return model, ckpt_type
+# ─── Main ───────────────────────────────────────────────────────────────
 def main():
+    parser = argparse.ArgumentParser(description="Infer Lumia V2 (GGUF/QLoRA) or V3 (best.pt)")
+    parser.add_argument("--gguf", default=None, help="V2 GGUF path")
+    parser.add_argument("--checkpoint", default="best.pt", help="Checkpoint path (default: best.pt)")
     parser.add_argument("--prompt", default="What is 2+2?")
     parser.add_argument("--system", default="You are a helpful AI assistant.")
     parser.add_argument("--max-tokens", type=int, default=128)
         sys.exit(1)
     model = None
+    ckpt_type = None
     if args.gguf:
         if not os.path.exists(args.gguf):
             print(f"GGUF not found: {args.gguf}")
             sys.exit(1)
+        model = load_gguf(args.gguf)
+        ckpt_type = "gguf"
     elif args.checkpoint:
         if not os.path.exists(args.checkpoint):
             print(f"Checkpoint not found: {args.checkpoint}")
             sys.exit(1)
+        model, ckpt_type = load_checkpoint(args.checkpoint)
     else:
         print("Specify --gguf or --checkpoint")
         sys.exit(1)
     model.to(device)
     print(f" Device: {device}")
+    tok = load_tokenizer(ckpt_type)
+    input_ids = build_chat(args.system, args.prompt, tok)
     input_ids = torch.tensor([input_ids], device=device)
     print("\n" + tok.decode(input_ids[0].tolist()), end="", flush=True)
     def stream(id_):
+        print(tok.decode([id_]), end="", flush=True)
+    model.generate(
         input_ids,
         max_new_tokens=args.max_tokens,
         temperature=args.temperature,

model_tiny.py CHANGED Viewed

@@ -175,8 +175,9 @@ class TinyModel(nn.Module):
                 input_ids = input_ids[:, -self.max_seq_len:]
             with torch.no_grad():
                 logits, _ = self.forward(input_ids)
-            logits = logits[:, -1, :] / temperature
             if top_p < 1.0:
                 sorted_logits, sorted_idx = logits.sort(dim=-1, descending=True)
                 cum_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1)
@@ -184,14 +185,14 @@ class TinyModel(nn.Module):
                 cutoff[..., 1:] = cutoff[..., :-1].clone()
                 cutoff[..., 0] = False
                 logits[~cutoff] = float('-inf')
             probs = F.softmax(logits, dim=-1)
-            next_token = torch.multinomial(probs, num_samples=1)
             input_ids = torch.cat([input_ids, next_token], dim=1)
             if stream_callback:
                 stream_callback(next_token.item())
             if next_token.item() == 2:
                 break
         return input_ids
@@ -244,6 +245,120 @@ def restore_from_v2(v2_ckpt_path: str | None = None, strict: bool = False) -> Ti
     return model
 # ─── Helpers ───────────────────────────────────────────────────────────────
 def count_params(model):

                 input_ids = input_ids[:, -self.max_seq_len:]
             with torch.no_grad():
                 logits, _ = self.forward(input_ids)
+            logits = logits[:, -1, :]
+            if temperature > 0:
+                logits = logits / temperature
             if top_p < 1.0:
                 sorted_logits, sorted_idx = logits.sort(dim=-1, descending=True)
                 cum_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1)
                 cutoff[..., 1:] = cutoff[..., :-1].clone()
                 cutoff[..., 0] = False
                 logits[~cutoff] = float('-inf')
             probs = F.softmax(logits, dim=-1)
+            if temperature > 0:
+                next_token = torch.multinomial(probs, num_samples=1)
+            else:
+                next_token = probs.argmax(dim=-1, keepdim=True)
             input_ids = torch.cat([input_ids, next_token], dim=1)
             if stream_callback:
                 stream_callback(next_token.item())
             if next_token.item() == 2:
                 break
         return input_ids
     return model
+# ─── QLoRA (4-bit NF4 + LoRA) ──────────────────────────────────────────────
+NF4_LEVELS = torch.tensor([
+    -1.0, -0.6961928009986877, -0.5250730514526367, -0.39491748809814453,
+    -0.28444138169288635, -0.18477343022823334, -0.09105003625154495, 0.0,
+    0.07958029955625534, 0.16093020141124725, 0.24611230194568634,
+    0.33791524171829224, 0.44070982933044434, 0.5626170039176941,
+    0.7229568362236023, 1.0,
+], dtype=torch.float32)
+def _quantize_nf4_row(row: torch.Tensor) -> tuple:
+    absmax = row.abs().max().clamp(min=1e-12)
+    scaled = row / absmax
+    idx = (scaled[:, None] - NF4_LEVELS[None, :].to(row.device)).abs().argmin(dim=-1)
+    return idx.to(torch.uint8), absmax
+def pack_nf4(indices: torch.Tensor) -> torch.Tensor:
+    n = indices.numel()
+    if n % 2 != 0:
+        indices = torch.cat([indices, indices.new_zeros(1)])
+    packed = (indices[0::2].to(torch.uint8) | (indices[1::2].to(torch.uint8) << 4))
+    return packed
+def unpack_nf4(packed: torch.Tensor, shape) -> torch.Tensor:
+    n = shape[0] * shape[1]
+    low = (packed & 0x0F).to(torch.long)
+    high = ((packed >> 4) & 0x0F).to(torch.long)
+    indices = torch.stack([low, high], dim=-1).reshape(n)
+    return indices[:shape[0] * shape[1]].reshape(shape)
+def dequantize_nf4(packed_weight: torch.Tensor, scales: torch.Tensor, shape) -> torch.Tensor:
+    indices = unpack_nf4(packed_weight, shape)
+    return NF4_LEVELS[indices] * scales[:, None]
+class QLoRALinear(nn.Module):
+    def __init__(self, in_features: int, out_features: int, r: int = 8, alpha: float = 16, dropout: float = 0.0):
+        super().__init__()
+        self.in_features = in_features
+        self.out_features = out_features
+        self.r = r
+        self.alpha = alpha
+        self.scaling = alpha / r
+        self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
+        n_elements = in_features * out_features
+        n_packed = (n_elements + 1) // 2
+        self.register_buffer("qweight", torch.zeros(n_packed, dtype=torch.uint8))
+        self.register_buffer("scales", torch.zeros(out_features, dtype=torch.float32))
+        self.register_buffer("bias", torch.zeros(out_features, dtype=torch.float32))
+        self._has_bias = False
+        self._shape = (out_features, in_features)
+        self.lora_A = nn.Parameter(torch.zeros(r, in_features))
+        self.lora_B = nn.Parameter(torch.zeros(out_features, r))
+        nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
+    def _dequantized_weight(self) -> torch.Tensor:
+        dev = self.qweight.device
+        indices = unpack_nf4(self.qweight, self._shape)
+        return NF4_LEVELS.to(dev)[indices] * self.scales.to(dev)[:, None]
+    def quantize_from(self, weight: torch.Tensor, bias: torch.Tensor | None = None):
+        w = weight.float().detach()
+        rows = []
+        scales = []
+        for i in range(w.shape[0]):
+            idx, s = _quantize_nf4_row(w[i])
+            rows.append(idx)
+            scales.append(s.item())
+        all_idx = torch.stack(rows)
+        self.qweight.copy_(pack_nf4(all_idx.reshape(-1)))
+        self.scales.copy_(torch.tensor(scales, dtype=torch.float32))
+        if bias is not None:
+            self.bias.copy_(bias.float().detach())
+            self._has_bias = True
+    def forward(self, x):
+        w = self._dequantized_weight()
+        b = self.bias if self._has_bias else None
+        base = F.linear(x, w, b)
+        return base + self.dropout(x) @ self.lora_A.T @ self.lora_B.T * self.scaling
+def apply_qlora(model, target_modules=None, r=8, alpha=16, dropout=0.0, freeze_embeds=True):
+    if target_modules is None:
+        target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "down", "up"]
+    qlora_params = 0
+    for name, module in model.named_modules():
+        if not isinstance(module, nn.Linear):
+            continue
+        key = name.split(".")[-1]
+        if key not in target_modules:
+            continue
+        parent = model
+        parts = name.split(".")
+        for p in parts[:-1]:
+            parent = getattr(parent, p)
+        qlora = QLoRALinear(module.in_features, module.out_features, r=r, alpha=alpha, dropout=dropout)
+        qlora.quantize_from(module.weight, module.bias)
+        setattr(parent, parts[-1], qlora)
+        qlora_params += 2 * r * module.in_features + module.out_features * r
+    for name, param in model.named_parameters():
+        if "lora_" not in name:
+            param.requires_grad = False
+    n = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    print(f"QLoRA applied: {qlora_params:,} LoRA params | trainable: {n:,}")
+    return model
 # ─── Helpers ───────────────────────────────────────────────────────────────
 def count_params(model):

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff