Upload folder using huggingface_hub

config.json

@@ -1,10 +1,10 @@
 {
   "architectures": ["TinyModel"],
   "model_type": "tiny",
-  "vocab_size": 1757,
+  "vocab_size": 4096,
   "hidden_size": 128,
-  "intermediate_size": 640,
-  "num_hidden_layers": 3,
+  "code_dim": 96,
+  "num_hidden_layers": 6,
   "num_attention_heads": 8,
   "num_key_value_heads": 4,
   "max_position_embeddings": 2048,

gen_tokenizer.py

@@ -0,0 +1,1833 @@
+import json
+
+# === GPT-2 byte-to-unicode mapping ===
+def bytes_to_unicode():
+    bs = list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
+    cs = bs[:]
+    n = 0
+    for b in range(256):
+        if b not in bs:
+            bs.append(b)
+            cs.append(256 + n)
+            n += 1
+    return {b: chr(c) for b, c in zip(bs, cs)}
+
+byte2char = bytes_to_unicode()
+# Reverse mapping: char -> byte
+char2byte = {c: b for b, c in byte2char.items()}
+
+# === Vocab structure ===
+# IDs 0-50: 51 special tokens
+# IDs 51-306: 256 byte-level chars
+# IDs 307-4095: ~3789 curated English words + subwords
+special_tokens = [
+    ("<unk>", 0),
+    ("<s>", 1),
+    ("</s>", 2),
+    ("<pad>", 3),
+    ("<|system|>", 4),
+    ("<|user|>", 5),
+    ("<|assistant|>", 6),
+    ("<think>", 7),
+    ("</think>", 8),
+    ("[INST]", 9),
+    ("[/INST]", 10),
+    ("<|begin_of_thought|>", 11),
+    ("<|end_of_thought|>", 12),
+    ("<|reflect|>", 13),
+    ("<|revise|>", 14),
+    ("<|verify|>", 15),
+    ("<|code|>", 16),
+    ("<|text|>", 17),
+    ("<|math|>", 18),
+    ("<|think|>", 19),
+    ("<|answer|>", 20),
+    ("<|step|>", 21),
+    ("<|reason|>", 22),
+    ("<|check|>", 23),
+    ("<|output|>", 24),
+    ("<|plan|>", 25),
+    ("<|solve|>", 26),
+    ("<|analyze|>", 27),
+    ("<|conclude|>", 28),
+    ("<|approach|>", 29),
+    ("<|alternative|>", 30),
+    ("<|summary|>", 31),
+    ("<|question|>", 32),
+    ("<|hint|>", 33),
+    ("<|example|>", 34),
+    ("<|correct|>", 35),
+    ("<|incorrect|>", 36),
+    ("<|feedback|>", 37),
+    ("<|start|>", 38),
+    ("<|end|>", 39),
+    ("<|sep|>", 40),
+    ("<|cls|>", 41),
+    ("<|tool|>", 42),
+    ("<|function|>", 43),
+    ("<|result|>", 44),
+    ("<|input|>", 45),
+    ("<|detect|>", 46),
+    ("<|context|>", 47),
+    ("<|proof|>", 48),
+    ("<|lemma|>", 49),
+    ("<|theorem|>", 50),
+]
+
+# IDs 51-306: 256 byte-level characters
+byte_tokens = []
+for b in range(256):
+    byte_tokens.append((byte2char[b], 51 + b))
+
+# IDs 307+: common words (variable count, no filler)
+# Most frequent English words + programming terms
+common_words = [
+    "the", "be", "to", "of", "and", "a", "in", "that", "have", "I",
+    "it", "for", "not", "on", "with", "he", "as", "you", "do", "at",
+    "this", "but", "his", "by", "from", "they", "we", "say", "her", "she",
+    "or", "an", "will", "my", "one", "all", "would", "there", "their", "what",
+    "so", "up", "out", "if", "about", "who", "get", "which", "go", "me",
+    "when", "make", "can", "like", "time", "no", "just", "him", "know", "take",
+    "people", "into", "year", "your", "good", "some", "could", "them", "see", "other",
+    "than", "then", "now", "look", "only", "come", "its", "over", "think", "also",
+    "back", "after", "use", "two", "how", "our", "work", "first", "well", "way",
+    "even", "new", "want", "because", "any", "these", "give", "day", "most", "us",
+    "is", "was", "are", "were", "been", "has", "had", "did", "does", "am",
+    "being", "having", "doing", "saying", "going", "getting", "making", "knowing", "taking", "thinking",
+    "come", "coming", "came", "go", "goes", "gone", "going", "went",
+    "see", "saw", "seen", "seeing", "say", "said", "says", "saying",
+    "get", "got", "gotten", "getting", "make", "made", "makes", "making",
+    "know", "knew", "known", "knows", "think", "thought", "thinks", "thinking",
+    "take", "took", "taken", "takes", "taking", "give", "gave", "given", "gives", "giving",
+    "find", "found", "finds", "finding", "tell", "told", "tells", "telling",
+    "ask", "asked", "asks", "asking", "show", "showed", "shown", "shows", "showing",
+    "try", "tried", "tries", "trying", "leave", "left", "leaves", "leaving",
+    "call", "called", "calls", "calling", "keep", "kept", "keeps", "keeping",
+    "let", "lets", "letting", "begin", "began", "begun", "begins", "beginning",
+    "seem", "seemed", "seems", "seeming", "help", "helped", "helps", "helping",
+    "turn", "turned", "turns", "turning", "start", "started", "starts", "starting",
+    "bring", "brought", "brings", "bringing", "happen", "happened", "happens", "happening",
+    "write", "wrote", "written", "writes", "writing", "provide", "provided", "provides", "providing",
+    "consider", "considered", "considers", "considering", "appear", "appeared", "appears", "appearing",
+    "follow", "followed", "follows", "following", "change", "changed", "changes", "changing",
+    "form", "formed", "forms", "forming", "need", "needed", "needs", "needing",
+    "set", "sets", "setting", "put", "puts", "putting", "run", "runs", "running",
+    "move", "moved", "moves", "moving", "stand", "stood", "stands", "standing",
+    "win", "won", "wins", "winning", "play", "played", "plays", "playing",
+    "point", "points", "pointed", "pointing", "large", "small", "big", "little",
+    "long", "short", "high", "low", "old", "young", "great", "important",
+    "different", "same", "other", "many", "much", "more", "most", "few",
+    "own", "very", "such", "still", "just", "also", "even", "too",
+    "here", "there", "where", "when", "why", "how", "what", "which",
+    "while", "though", "although", "until", "since", "before", "after", "during",
+    "without", "within", "between", "through", "across", "around", "above", "below",
+    "under", "over", "again", "ever", "never", "always", "often", "sometimes",
+    "together", "alone", "already", "yet", "still", "almost", "quite", "rather",
+    "well", "bad", "better", "worse", "best", "worst", "more", "less",
+    "every", "each", "both", "either", "neither", "all", "any", "none",
+    "thing", "things", "way", "ways", "time", "times", "year", "years",
+    "day", "days", "week", "weeks", "month", "months", "part", "parts",
+    "place", "places", "case", "cases", "point", "points", "world", "worlds",
+    "number", "numbers", "group", "groups", "system", "systems", "program", "programs",
+    "data", "information", "problem", "problems", "solution", "solutions",
+    "method", "methods", "result", "results", "process", "processes",
+    "function", "functions", "value", "values", "type", "types",
+    "state", "states", "model", "models", "level", "levels",
+    "line", "lines", "file", "files", "code", "codes",
+    "set", "sets", "list", "lists", "array", "arrays",
+    "object", "objects", "class", "classes", "property", "properties",
+    "input", "inputs", "output", "outputs", "return", "returns",
+    "define", "defined", "defines", "defining", "declare", "declared",
+    "import", "imports", "export", "exports", "include", "includes",
+    "public", "private", "protected", "static", "final", "const",
+    "void", "int", "float", "double", "char", "bool", "string",
+    "true", "false", "null", "None", "nil", "undefined",
+    "if", "else", "elif", "then", "switch", "case", "default", "break",
+    "for", "while", "do", "each", "in", "of", "to", "by",
+    "try", "catch", "finally", "throw", "raise", "except",
+    "return", "yield", "await", "async", "defer",
+    "and", "or", "not", "is", "as", "with", "without",
+    "lambda", "map", "filter", "reduce", "sort",
+    "new", "delete", "free", "alloc", "realloc",
+    "print", "printf", "println", "log", "debug", "error",
+    "len", "size", "length", "count", "sum", "max", "min",
+    "abs", "pow", "sqrt", "floor", "ceil", "round",
+    "sin", "cos", "tan", "atan", "log", "exp",
+    "zero", "one", "two", "three", "four", "five",
+    "six", "seven", "eight", "nine", "ten",
+    "first", "second", "third", "last", "next", "previous",
+    "current", "initial", "final", "primary", "secondary",
+    "main", "primary", "secondary", "basic", "advanced",
+    "simple", "complex", "single", "double", "multiple",
+    "add", "sub", "mul", "div", "mod", "inc", "dec",
+    "push", "pop", "shift", "unshift", "insert", "remove",
+    "append", "prepend", "concat", "join", "split", "slice",
+    "open", "close", "read", "write", "load", "save",
+    "create", "update", "delete", "insert", "select", "merge",
+    "begin", "end", "start", "stop", "pause", "resume",
+    "enable", "disable", "allow", "deny", "grant", "revoke",
+    "user", "users", "name", "names", "id", "ids",
+    "key", "keys", "value", "values", "field", "fields",
+    "table", "tables", "row", "rows", "column", "columns",
+    "index", "indexes", "indices", "query", "queries",
+    "count", "avg", "total", "sum", "min", "max",
+    "page", "pages", "home", "login", "logout", "signup",
+    "error", "errors", "warning", "warnings", "info",
+    "success", "failure", "status", "message", "messages",
+    "request", "requests", "response", "responses",
+    "client", "server", "api", "endpoint", "route",
+    "http", "https", "url", "uri", "port", "host",
+    "config", "configuration", "setting", "settings",
+    "option", "options", "param", "params", "parameter", "parameters",
+    "arg", "args", "argument", "arguments", "kwargs",
+    "path", "dir", "directory", "dirs", "folder", "folders",
+    "item", "items", "element", "elements", "entry", "entries",
+    "note", "notes", "text", "texts", "content", "contents",
+    "source", "sources", "target", "targets", "ref", "refs",
+    "struct", "structs", "union", "unions", "enum", "enums",
+    "impl", "implement", "implementation", "interface",
+    "abstract", "virtual", "override", "overload",
+    "base", "derived", "parent", "child", "children",
+    "root", "leaf", "node", "nodes", "edge", "edges",
+    "tree", "graph", "list", "queue", "stack", "heap",
+    "map", "dict", "dictionary", "hash", "hashmap",
+    "link", "links", "linked", "pointer", "pointers",
+    "thread", "threads", "process", "processes", "task", "tasks",
+    "sync", "async", "lock", "mutex", "semaphore",
+    "buffer", "buffers", "cache", "cached", "pool", "pools",
+    "memory", "disk", "network", "socket", "sockets",
+    "stream", "streams", "packet", "packets", "frame",
+    "meta", "metadata", "header", "headers", "body", "payload",
+    "token", "tokens", "session", "cookie", "cookies",
+    "auth", "login", "logout", "register", "password",
+    "hash", "salt", "encrypt", "decrypt", "encode", "decode",
+    "cert", "certificate", "key", "public", "private",
+    "train", "training", "trained", "test", "testing", "tested",
+    "valid", "validate", "validation", "eval", "evaluate",
+    "model", "models", "layer", "layers", "weight", "weights",
+    "bias", "biases", "loss", "losses", "grad", "grads",
+    "lr", "learning_rate", "optimizer", "adam", "sgd",
+    "batch", "batches", "epoch", "epochs", "step", "steps",
+    "dataset", "dataloader", "tensor", "tensors",
+    "gpu", "cpu", "tpu", "device", "devices", "memory",
+    "math", "physics", "chemistry", "biology", "science",
+    "compute", "calculate", "computation", "calculation",
+    "equation", "formula", "expression", "theorem",
+    "proof", "prove", "lemma", "axiom", "corollary",
+    "function", "graph", "derivative", "integral", "limit",
+    "sequence", "series", "matrix", "vector", "tensor",
+    "set", "subset", "union", "intersection", "complement",
+    "space", "group", "ring", "field", "module", "algebra",
+    "analysis", "topology", "geometry", "statistics",
+    "probability", "distribution", "random", "sample",
+    "mean", "median", "mode", "variance", "std", "deviation",
+    "linear", "nonlinear", "convex", "concave", "smooth",
+    "algorithm", "algorithm", "complexity", "runtime",
+    "tree", "graph", "sort", "search", "traverse",
+    "recursive", "iterative", "dynamic", "greedy",
+    "optimization", "constraint", "feasible", "optimal",
+    "problem", "solution", "input", "output", "example",
+    "question", "answer", "hint", "step", "reason",
+    "think", "analyze", "approach", "solve", "verify",
+    "check", "conclude", "summarize", "explain", "describe",
+    "correct", "incorrect", "right", "wrong", "positive", "negative",
+    "yes", "no", "maybe", "always", "never", "sometimes",
+    ":", ";", ".", ",", "!", "?", "'", "\"", "(", ")", "[", "]", "{", "}",
+    "<", ">", "=", "+", "-", "*", "/", "%", "&", "|", "^", "~",
+    "@", "#", "$", "_", "`", "\\",
+    "==", "!=", "<=", ">=", "&&", "||", "++", "--",
+    "+=", "-=", "*=", "/=", "->", "=>", "::", "..",
+    "...", "/*", "*/", "//", "<!--", "-->",
+    "0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
+    "10", "11", "12", "13", "14", "15", "16", "17", "18", "19",
+    "20", "30", "40", "50", "60", "70", "80", "90", "100",
+    "-1", "-2", "0x", "0b", "0o",
+    # GPT-2 style Ġ-prefixed versions for words preceded by space
+    # The ByteLevel pre_tokenizer adds Ġ (U+0120) prefix after space
+    "Ġthe", "Ġto", "Ġof", "Ġand", "Ġa", "Ġin", "Ġthat", "Ġis", "Ġwas", "Ġfor",
+    "Ġon", "Ġwith", "Ġas", "Ġby", "Ġat", "Ġfrom", "Ġor", "Ġan", "Ġwill", "Ġwould",
+    "Ġnot", "Ġbut", "Ġare", "Ġwere", "Ġbeen", "Ġhave", "Ġhas", "Ġhad", "Ġdo", "Ġdoes",
+    "Ġdid", "Ġcan", "Ġcould", "Ġshould", "Ġmay", "Ġmight", "Ġshall", "Ġmust",
+    "Ġif", "Ġelse", "Ġwhen", "Ġwhile", "Ġbecause", "Ġso", "Ġthen", "Ġthan",
+    "Ġalso", "Ġeven", "Ġonly", "Ġjust", "Ġvery", "Ġtoo", "Ġstill", "Ġalready",
+    "Ġhere", "Ġthere", "Ġwhere", "Ġwhen", "Ġwhy", "Ġhow", "Ġwhat", "Ġwhich",
+    "Ġthis", "Ġthat", "Ġthese", "Ġthose", "Ġit", "Ġits", "Ġthey", "Ġthem",
+    "Ġwe", "Ġus", "Ġour", "Ġyou", "Ġyour", "Ġhe", "Ġhim", "Ġhis",
+    "Ġshe", "Ġher", "Ġhers", "Ġone", "Ġno", "Ġall", "Ġany", "Ġsome",
+    "Ġeach", "Ġevery", "Ġboth", "Ġneither", "Ġeither", "Ġmore", "Ġmost", "Ġfew",
+    "Ġother", "Ġanother", "Ġsuch", "Ġsame", "Ġdifferent", "Ġown",
+    "Ġlike", "Ġwell", "Ġgood", "Ġbad", "Ġbetter", "Ġbest",
+    "Ġnew", "Ġold", "Ġbig", "Ġsmall", "Ġlong", "Ġshort",
+    "Ġhigh", "Ġlow", "Ġlarge", "Ġlittle", "Ġgreat", "Ġimportant",
+    "Ġup", "Ġdown", "Ġin", "Ġout", "Ġon", "Ġoff", "Ġover", "Ġunder",
+    "Ġagain", "Ġback", "Ġabout", "Ġaround", "Ġbetween", "Ġthrough",
+    "Ġbefore", "Ġafter", "Ġduring", "Ġuntil", "Ġsince",
+    "Ġfirst", "Ġlast", "Ġnext", "Ġprevious", "Ġfinal",
+    "Ġget", "Ġgot", "Ġmake", "Ġmade", "Ġtake", "Ġtook", "Ġgive", "Ġgave",
+    "Ġuse", "Ġused", "Ġusing", "Ġneed", "Ġneeds", "Ġneeded",
+    "Ġwant", "Ġwants", "Ġwanted", "Ġlet", "Ġlets", "Ġlets",
+    "Ġwork", "Ġworks", "Ġworked", "Ġworking", "Ġhelp", "Ġhelps", "Ġhelped",
+    "Ġcall", "Ġcalls", "Ġcalled", "Ġcalling", "Ġset", "Ġsets", "Ġsetting",
+    "Ġput", "Ġputs", "Ġputting", "Ġrun", "Ġruns", "Ġran", "Ġrunning",
+    "Ġkeep", "Ġkeeps", "Ġkept", "Ġfind", "Ġfinds", "Ġfound", "Ġshow", "Ġshows",
+    "Ġtry", "Ġtries", "Ġtried", "Ġtrying", "Ġstart", "Ġstarts", "Ġstarted",
+    "Ġchange", "Ġchanges", "Ġchanged", "Ġchanging", "Ġfollow", "Ġfollows",
+    "Ġknow", "Ġknows", "Ġknown", "Ġthink", "Ġthinks", "Ġthought",
+    "Ġsay", "Ġsays", "Ġsaid", "Ġsee", "Ġsees", "Ġsaw", "Ġseen",
+    "Ġcome", "Ġcomes", "Ġcame", "Ġgo", "Ġgoes", "Ġwent", "Ġgone",
+    "Ġbring", "Ġbrings", "Ġbrought", "Ġtell", "Ġtells", "Ġtold",
+    "Ġlet", "Ġlets", "Ġleave", "Ġleaves", "Ġleft", "Ġhappen", "Ġhappens",
+    "��provide", "Ġprovides", "Ġprovided", "Ġconsider", "Ġconsiders",
+    "Ġappear", "Ġappears", "Ġappeared", "Ġform", "Ġforms", "Ġformed",
+    "Ġseem", "Ġseems", "Ġseemed", "Ġpoint", "Ġpoints", "Ġpointed",
+    "Ġturn", "Ġturns", "Ġturned", "Ġturning", "Ġplay", "Ġplays",
+    "Ġthings", "Ġthing", "Ġtime", "Ġtimes", "Ġyear", "Ġyears",
+    "Ġpeople", "Ġplace", "Ġplaces", "Ġpart", "Ġparts", "Ġworld",
+    "Ġnumber", "Ġnumbers", "Ġsystem", "Ġsystems", "Ġgroup", "Ġgroups",
+    "Ġline", "Ġlines", "Ġfile", "Ġfiles", "Ġcode", "Ġdata",
+    "Ġfunction", "Ġfunctions", "Ġvalue", "Ġvalues", "Ġtype", "Ġtypes",
+    "Ġclass", "Ġclasses", "Ġobject", "Ġobjects", "Ġmethod", "Ġmethods",
+    "Ġresult", "Ġresults", "Ġprocess", "Ġprocesses", "Ġstate", "Ġstates",
+    "Ġmodel", "Ġmodels", "Ġlevel", "Ġlevels", "Ġcase", "Ġcases",
+    "Ġexample", "Ġexamples", "Ġinput", "Ġinputs", "Ġoutput", "Ġoutputs",
+    "Ġerror", "Ġerrors", "Ġstatus", "Ġmessage", "Ġmessages",
+    "Ġrequest", "Ġrequests", "Ġresponse", "Ġresponses",
+    "Ġreturn", "Ġreturns", "Ġimport", "Ġimports", "Ġexport", "Ġexports",
+    "Ġdefine", "Ġdefines", "Ġdefined", "Ġdeclare", "Ġdeclares",
+    "Ġinclude", "Ġincludes", "Ġconfig", "Ġsetting", "Ġsettings",
+    "Ġoption", "Ġoptions", "Ġparam", "Ġparams", "Ġargs",
+    "Ġtrue", "Ġfalse", "Ġnull", "ĠNone", "Ġundefined",
+    "Ġlen", "Ġsize", "Ġcount", "Ġsum", "Ġmax", "Ġmin",
+    "Ġprint", "Ġlog", "Ġdebug", "Ġinfo", "Ġwarn",
+    "Ġtrain", "Ġtest", "Ġeval", "Ġvalid", "Ġval",
+    "Ġpage", "Ġhome", "Ġname", "Ġkey", "Ġkeys",
+    "Ġpath", "Ġdir", "Ġroot", "Ġitem", "Ġitems",
+    "Ġsource", "Ġtarget", "Ġbase", "Ġmain", "Ġprimary",
+    "Ġadd", "Ġremove", "Ġcreate", "Ġdelete", "Ġupdate",
+    "Ġopen", "Ġclose", "Ġread", "Ġwrite", "Ġload", "Ġsave",
+    "Ġpush", "Ġpop", "Ġinsert", "Ġappend", "Ġsplit",
+    "Ġbegin", "Ġend", "Ġstart", "Ġstop", "Ġenable", "Ġdisable",
+    "Ġuser", "Ġusers", "Ġadmin", "Ġmanager",
+    "Ġstring", "Ġint", "Ġfloat", "Ġdouble", "Ġbool", "Ġvoid",
+    "Ġlist", "Ġdict", "Ġset", "Ġtuple", "Ġarray",
+    "Ġapi", "Ġurl", "Ġuri", "Ġendpoint", "Ġroute",
+    "Ġhttp", "Ġhttps", "Ġclient", "Ġserver", "Ġsocket",
+    "Ġmath", "Ġscience", "Ġdata", "Ġanalysis", "Ġtheory",
+    "Ġproof", "Ġtheorem", "Ġlemma", "Ġequation", "Ġformula",
+    "ing", "ed", "ly", "tion", "sion", "ment", "ness", "ity",
+    "able", "ible", "al", "ial", "ical", "ous", "eous", "ious",
+    "ive", "ative", "ful", "less", "like", "wise", "ward",
+    "un", "re", "in", "im", "ir", "il", "dis", "mis", "non",
+    "pre", "pro", "per", "trans", "inter", "intra", "extra",
+    "sub", "super", "sur", "semi", "multi", "mono", "bi", "tri",
+    "anti", "counter", "over", "under", "out", "up", "down",
+    "co", "con", "com", "col", "cor", "de", "di", "dif",
+    "ex", "extra", "fore", "macro", "micro", "mid", "mis",
+    "out", "over", "post", "pre", "pro", "re", "semi", "sub",
+    "super", "tele", "trans", "ultra", "un", "under", "up",
+    "-ing", "-ed", "-ly", "-tion", "-sion", "-ment", "-ness", "-ity",
+    "-able", "-ible", "-al", "-ous", "-ive", "-ful", "-less",
+    "un-", "re-", "pre-", "non-", "anti-", "counter-",
+    "self-", "all-", "well-", "so-", "to-", "in-",
+    "'t", "'s", "'m", "'re", "'ve", "'ll", "'d",
+    "n't", "don't", "can't", "won't", "isn't", "aren't",
+    "wasn't", "weren't", "hasn't", "haven't", "hadn't",
+    "doesn't", "didn't", "couldn't", "shouldn't", "wouldn't",
+    "mustn't", "needn't", "mightn't",
+    # Multi-char punctuation/symbols as single tokens
+    "->", "=>", "<-", "<=", ">=", "==", "!=",
+    "::", "..", "...", "/*", "*/", "//", "#",
+    "\n", "\t",
+    "  ", "   ", "    ", "     ",
+    "Ċ",  # byte-level newline
+    "ĠĠ", "ĠĠĠ", "ĠĠĠĠ",  # multiple spaces
+    # Additional English words for 4096 vocab
+    "about", "above", "across", "action", "actually", "address", "agree", "allow", "almost",
+    "along", "already", "though", "although", "always", "American", "among", "amount",
+    "animal", "another", "answer", "anything", "appear", "approach", "area", "argue",
+    "arm", "article", "artist", "ask", "author", "available", "avoid", "away", "ball",
+    "bank", "bar", "base", "battle", "beauty", "become", "became", "becoming", "bed",
+    "behavior", "behind", "believe", "benefit", "best", "beyond", "bit", "black",
+    "blood", "board", "body", "book", "born", "boss", "bother", "bottle", "bottom",
+    "box", "boy", "brain", "break", "bridge", "brief", "bright", "bring", "broad",
+    "brother", "budget", "build", "building", "burn", "business", "buy", "campaign",
+    "capital", "car", "care", "career", "carry", "catch", "category", "cause",
+    "central", "century", "certain", "chair", "chairman", "challenge", "chance",
+    "character", "charge", "check", "choice", "choose", "chosen", "church", "citizen",
+    "city", "civil", "claim", "clear", "clearly", "close", "club", "coach", "cold",
+    "collection", "college", "color", "come", "comfortable", "comment", "committee",
+    "common", "community", "company", "compare", "competition", "complete", "completely",
+    "condition", "conference", "Congress", "connect", "conscious", "consider", "contain",
+    "content", "continue", "contract", "control", "conversation", "cost", "could",
+    "country", "couple", "course", "court", "cover", "create", "crime", "cultural",
+    "culture", "cup", "current", "customer", "cut", "dark", "daughter", "deal",
+    "death", "debate", "decade", "decide", "decision", "deep", "defense", "degree",
+    "democrat", "democratic", "describe", "design", "despite", "detail", "determine",
+    "develop", "development", "device", "die", "difference", "difficult", "dinner",
+    "direction", "director", "discover", "discuss", "discussion", "disease", "dog",
+    "door", "doubt", "down", "draw", "dream", "drive", "driver", "drop", "drug",
+    "D", "early", "east", "eat", "economic", "economy", "edge", "edition", "editor",
+    "education", "effect", "effort", "eight", "either", "election", "else", "employee",
+    "encourage", "enemy", "energy", "enjoy", "enough", "enter", "entire", "environment",
+    "environmental", "especially", "establish", "evening", "event", "ever", "everybody",
+    "everyone", "everything", "evidence", "exactly", "examine", "example", "executive",
+    "exist", "expect", "experience", "explain", "explanation", "extremely", "eye",
+    "face", "fact", "factor", "fail", "fall", "family", "far", "fast", "father",
+    "fear", "feature", "federal", "feel", "feeling", "field", "fight", "figure",
+    "fill", "film", "final", "financial", "fine", "finish", "firm", "fish", "five",
+    "floor", "fly", "focus", "follow", "food", "foot", "force", "foreign", "forget",
+    "form", "former", "forward", "four", "free", "freedom", "friendly", "front",
+    "full", "fund", "future", "game", "garden", "gas", "general", "generation",
+    "gentleman", "girl", "glad", "glass", "goal", "god", "gold", "government",
+    "governor", "great", "green", "ground", "group", "grow", "growth", "guess",
+    "gun", "guy", "half", "hand", "handle", "hang", "happen", "happy", "hard",
+    "head", "health", "hear", "heart", "heat", "heavy", "hell", "help", "here",
+    "herself", "hide", "history", "hit", "hold", "home", "honest", "hope", "hospital",
+    "hotel", "house", "huge", "human", "hundred", "husband", "idea", "identify",
+    "image", "imagine", "impact", "implement", "imply", "important", "improve",
+    "include", "including", "increase", "indeed", "indicate", "individual", "industry",
+    "influence", "inform", "information", "inside", "instead", "institution",
+    "interest", "international", "interview", "introduce", "investment", "involve",
+    "issue", "item", "itself", "job", "join", "journal", "journey", "judge",
+    "jump", "justice", "keep", "kill", "kind", "kitchen", "knowledge", "land",
+    "language", "large", "last", "late", "later", "latter", "laugh", "launch",
+    "law", "lawyer", "lay", "lead", "leader", "leading", "learn", "least", "leave",
+    "left", "legal", "less", "let", "letter", "level", "lie", "life", "lift",
+    "light", "likely", "limit", "line", "link", "list", "listen", "little", "live",
+    "load", "local", "long", "look", "lord", "lose", "loss", "lost", "lot", "love",
+    "low", "luck", "lunch", "machine", "main", "maintain", "major", "majority",
+    "manage", "management", "manager", "manner", "manufacturer", "many", "map",
+    "mark", "market", "marriage", "master", "material", "matter", "may", "maybe",
+    "mean", "meaning", "measure", "media", "medical", "meet", "meeting", "member",
+    "memory", "mention", "message", "method", "middle", "might", "military", "million",
+    "mind", "minute", "miss", "mission", "mistake", "mix", "modern", "mom", "moment",
+    "money", "month", "moral", "morning", "mother", "motion", "move", "movement",
+    "movie", "music", "narrative", "nation", "national", "native", "natural", "nature",
+    "near", "nearly", "necessarily", "necessary", "neck", "need", "negative", "neighbor",
+    "neither", "network", "never", "nevertheless", "night", "none", "nor", "normal",
+    "north", "note", "nothing", "notice", "notion", "now", "nowhere", "nuclear",
+    "number", "occur", "ocean", "offer", "office", "officer", "official", "often",
+    "oil", "OK", "old", "once", "online", "open", "operate", "operation", "opinion",
+    "opportunity", "opposition", "option", "order", "organization", "original", "other",
+    "otherwise", "ought", "outside", "overcome", "owner", "page", "pain", "paint",
+    "pair", "paper", "parent", "park", "parliament", "part", "participant", "particular",
+    "particularly", "partner", "party", "pass", "passage", "past", "path", "patient",
+    "pattern", "pay", "peace", "pension", "people", "per", "percent", "perfect",
+    "perform", "performance", "perhaps", "period", "permit", "person", "personal",
+    "perspective", "phone", "physical", "pick", "picture", "piece", "place", "plan",
+    "plant", "play", "player", "please", "pleasure", "plus", "pocket", "point",
+    "police", "policy", "political", "politician", "politics", "pool", "poor",
+    "popular", "population", "position", "positive", "possibility", "possible",
+    "potentially", "power", "practice", "prepare", "presence", "present", "president",
+    "pressure", "pretty", "prevent", "previous", "price", "primary", "principle",
+    "prison", "private", "privilege", "probably", "problem", "procedure", "produce",
+    "product", "production", "professional", "professor", "profile", "profit",
+    "program", "project", "promise", "promote", "proper", "property", "proposal",
+    "propose", "protect", "protection", "prove", "provide", "public", "publication",
+    "publish", "pull", "purpose", "pursue", "push", "quality", "quarter", "question",
+    "quick", "quickly", "quiet", "quite", "race", "radio", "raise", "range", "rate",
+    "rather", "reach", "react", "reaction", "read", "reader", "reading", "ready",
+    "real", "reality", "realize", "really", "reason", "reasonable", "receive",
+    "recent", "recently", "recognize", "recommend", "record", "recover", "red",
+    "reduce", "reflect", "reform", "region", "relate", "relationship", "relative",
+    "relatively", "release", "relevant", "relief", "religion", "religious", "rely",
+    "remain", "remember", "remind", "remove", "repeat", "replace", "report", "reporter",
+    "represent", "representation", "republican", "reputation", "request", "require",
+    "research", "resource", "respond", "response", "responsibility", "responsible",
+    "rest", "restaurant", "result", "retain", "retire", "return", "reveal",
+    "review", "revolution", "rich", "ride", "right", "ring", "rise", "risk", "river",
+    "road", "rock", "role", "roll", "room", "rule", "run", "safe", "safety",
+    "sale", "same", "sample", "save", "scale", "scene", "schedule", "school",
+    "science", "scientist", "score", "screen", "sea", "search", "season", "seat",
+    "second", "secret", "section", "security", "seed", "seek", "select", "self",
+    "sell", "senate", "senator", "send", "sense", "serious", "serve", "service",
+    "session", "settle", "seven", "sexual", "shadow", "shape", "share", "sharp",
+    "sheet", "ship", "shock", "shoe", "shoot", "shop", "shot", "shoulder", "show",
+    "shut", "sick", "side", "sight", "sign", "signal", "significance", "significant",
+    "silence", "similar", "simple", "simply", "since", "sing", "single", "sister",
+    "sit", "site", "situation", "six", "size", "skill", "skin", "small", "smile",
+    "society", "soft", "soldier", "solid", "solution", "somebody", "somehow",
+    "someone", "something", "sometimes", "somewhat", "son", "song", "soon", "sort",
+    "sound", "source", "south", "space", "speak", "speaker", "special", "specific",
+    "speech", "speed", "spend", "spin", "spirit", "spiritual", "split", "spokesman",
+    "sport", "spot", "spread", "spring", "staff", "stage", "stand", "standard",
+    "star", "start", "state", "statement", "station", "status", "stay", "step",
+    "stick", "still", "stock", "stop", "store", "story", "straight", "strange",
+    "strategic", "strategy", "street", "strength", "stress", "stretch", "strike",
+    "strong", "structure", "struggle", "student", "study", "subject", "succeed",
+    "success", "successful", "suddenly", "suffer", "sufficient", "suggest", "suggestion",
+    "summer", "supply", "support", "suppose", "sure", "surface", "surgery", "surprise",
+    "survey", "survive", "suspect", "sustain", "symbol", "system", "table", "talent",
+    "talk", "tape", "target", "task", "taste", "tax", "teach", "teacher", "teaching",
+    "team", "tear", "technical", "technique", "technology", "telephone", "television",
+    "tell", "temperature", "tend", "term", "test", "testify", "testing", "text",
+    "thank", "themselves", "therefore", "they", "thick", "thin", "thing", "think",
+    "thinking", "third", "thirty", "threat", "threaten", "three", "throw", "thus",
+    "ticket", "tight", "till", "time", "tiny", "tip", "title", "today", "together",
+    "tomorrow", "tone", "tonight", "tool", "top", "total", "totally", "touch",
+    "tough", "tour", "toward", "town", "track", "trade", "tradition", "traditional",
+    "traffic", "train", "training", "transfer", "transform", "travel", "treat",
+    "treatment", "tree", "trial", "trip", "troop", "trouble", "truck", "true",
+    "truly", "trust", "truth", "try", "tube", "turn", "twice", "type", "typical",
+    "uncle", "under", "understand", "understanding", "unfortunately", "union",
+    "unique", "unit", "United", "universe", "university", "unless", "unlike",
+    "unlikely", "unusual", "upper", "urban", "urge", "use", "used", "useful",
+    "user", "usual", "usually", "value", "variety", "various", "vehicle", "version",
+    "very", "veteran", "victim", "victory", "video", "view", "village", "violence",
+    "visit", "voice", "volume", "vote", "voter", "wage", "wait", "walk", "wall",
+    "want", "war", "warm", "warn", "warning", "wash", "watch", "water", "wave",
+    "way", "weak", "weapon", "wear", "weather", "web", "wedding", "weekend",
+    "weight", "welcome", "welfare", "well", "west", "western", "whatever", "wheel",
+    "whenever", "whereas", "whether", "which", "while", "white", "whole", "whom",
+    "whose", "wide", "widely", "wife", "wild", "will", "win", "wind", "window",
+    "wine", "wing", "winner", "winter", "wire", "wish", "woman", "wonder", "wonderful",
+    "wood", "word", "worker", "working", "works", "world", "worry", "worth", "would",
+    "write", "writer", "writing", "wrong", "yard", "yeah", "year", "yet", "youth",
+    "zone", "ability", "abroad", "absent", "absolute", "absorb", "abstract", "abuse",
+    "academic", "accept", "access", "accident", "accompany", "accomplish", "account",
+    "accurate", "accuse", "achieve", "acknowledge", "acquire", "adapt", "addition",
+    "adjust", "administration", "admit", "adopt", "advance", "advantage", "adventure",
+    "advertise", "advice", "advise", "advocate", "affair", "affect", "afford",
+    "agency", "agenda", "agent", "aggression", "aggressive", "aid", "aim", "air",
+    "airport", "alarm", "alcohol", "alert", "alive", "alliance", "allocate", "ally",
+    "alone", "alter", "alternative", "amaze", "ambition", "amendment", "amid",
+    "amongst", "analysis", "analyst", "angle", "angry", "anniversary", "announce",
+    "annual", "anticipate", "anxiety", "anxious", "apart", "apartment", "apologize",
+    "apparent", "appeal", "appearance", "appetite", "apple", "applicant", "application",
+    "apply", "appoint", "appreciate", "appropriate", "approval", "approve", "architecture",
+    "archive", "argue", "argument", "arrange", "arrangement", "arrest", "arrival",
+    "arrive", "arrow", "articulate", "artificial", "aside", "aspect", "assault",
+    "assemble", "assembly", "assert", "assess", "assessment", "asset", "assign",
+    "assist", "assistance", "associate", "association", "assume", "assumption",
+    "atmosphere", "attach", "attack", "attempt", "attend", "attention", "attitude",
+    "attorney", "attract", "attraction", "attractive", "attribute", "audience",
+    "auto", "automatic", "automatically", "autonomy", "available", "avenue", "average",
+    "award", "aware", "awareness", "awful", "background", "bacteria", "balance",
+    "bare", "barely", "barrier", "basic", "basis", "basket", "bath", "battery",
+    "battle", "bay", "beach", "bean", "bear", "beat", "beautiful", "bedroom",
+    "beer", "beginning", "behalf", "behave", "behavior", "being", "belief", "believable",
+    "bell", "belong", "bench", "bend", "beneath", "beneficial", "beside", "bet",
+    "betray", "bible", "bicycle", "bid", "bike", "bill", "bind", "biological",
+    "biology", "birth", "biscuit", "bishop", "bite", "bitter", "blade", "blame",
+    "blank", "blast", "bleed", "blend", "bless", "blind", "block", "blow", "blue",
+    "blur", "board", "boast", "boat", "bomb", "bond", "bone", "bonus", "boom",
+    "boost", "border", "bore", "borrow", "bottom", "bound", "boundary", "bowl",
+    "brain", "branch", "brand", "brave", "bread", "breadth", "breast", "breath",
+    "breathe", "breathing", "breed", "brick", "bride", "bridge", "briefly", "brilliant",
+    "broadcast", "broken", "bronze", "brow", "brown", "brush", "bubble", "bucket",
+    "buddy", "buffalo", "bunch", "burden", "burglar", "burn", "burst", "bury",
+    "bus", "butter", "button", "cabin", "cabinet", "cable", "cake", "calculate",
+    "calculation", "calendar", "calm", "camera", "camp", "campus", "canal", "cancel",
+    "candidate", "candle", "cap", "capable", "capacity", "captain", "capture",
+    "carbon", "card", "careful", "carefully", "carrier", "carry", "cart", "carve",
+    "cast", "castle", "casualty", "catalog", "catalogue", "catch", "cattle",
+    "celebrate", "celebration", "cell", "cellular", "census", "centimeter",
+    "ceremony", "certainly", "certificate", "chain", "chair", "chairman",
+    "chamber", "champion", "championship", "channel", "chapter", "characteristic",
+    "charge", "charity", "chart", "chase", "cheap", "cheat", "cheek", "cheese",
+    "chemical", "chemistry", "chest", "chicken", "chief", "childhood", "chip",
+    "chocolate", "chorus", "christian", "Christmas", "chronic", "chunk", "circle",
+    "circuit", "circumstance", "cite", "citizen", "civilian", "claim",
+    "clarify", "clarity", "clash", "classic", "classical", "classification",
+    "classroom", "clause", "clean", "clear", "clever", "click", "client",
+    "cliff", "climate", "climb", "clinic", "clinical", "clock", "clone",
+    "closed", "closely", "closer", "closet", "closing", "cloth", "clothe",
+    "clothes", "clothing", "cloud", "club", "cluster", "coal", "coalition",
+    "coast", "coat", "code", "coffee", "cognitive", "coin", "cold",
+    "collapse", "collar", "colleague", "collect", "collection", "collective",
+    "colonial", "colony", "color", "column", "combat", "combine", "combined",
+    "comedy", "comfort", "command", "commander", "comment", "commerce",
+    "commercial", "commission", "commit", "commitment", "commodity", "communicate",
+    "communication", "communist", "compact", "companion", "comparison", "compel",
+    "compensate", "compensation", "compete", "competition", "competitive",
+    "competitor", "complain", "complaint", "complement", "complex", "complexity",
+    "complicate", "complicated", "comply", "component", "compose", "composition",
+    "compound", "comprehensive", "comprise", "compromise", "compulsory", "compute",
+    "computer", "conceal", "concede", "conceive", "concentrate", "concentration",
+    "concept", "conception", "concern", "concerning", "concert", "conclude",
+    "conclusion", "concrete", "condemn", "conduct", "conference", "confess",
+    "confession", "confidence", "confident", "confidential", "confine", "confirm",
+    "conflict", "confront", "confusion", "congratulate", "congress", "connect",
+    "connection", "conscious", "consciousness", "consecutive", "consensus",
+    "consent", "consequence", "consequently", "conservation", "conservative",
+    "considerable", "considerably", "consist", "consistent", "consistently",
+    "constant", "constantly", "constitute", "constitution", "constitutional",
+    "construct", "construction", "consult", "consultant", "consume", "consumer",
+    "consumption", "contact", "contemporary", "contend", "contest", "context",
+    "continent", "continually", "continuity", "continuous", "continuously",
+    "contradiction", "contrary", "contribute", "contribution", "contributor",
+    "controversial", "controversy", "convenience", "convenient", "convention",
+    "conventional", "conversation", "conversion", "convert", "convey", "convict",
+    "conviction", "convince", "cook", "cookie", "cool", "cooperate", "cooperation",
+    "coordinate", "coordination", "cope", "copper", "copy", "copyright", "core",
+    "corn", "corner", "corporate", "corporation", "correct", "correction",
+    "correctly", "correlate", "correlation", "correspond", "correspondent",
+    "corridor", "corruption", "costly", "cotton", "council", "counsel",
+    "counselor", "count", "counter", "counterpart", "county", "coup",
+    "courage", "cousin", "cover", "coverage", "crack", "craft", "crash",
+    "creative", "creativity", "creator", "creature", "credibility", "credit",
+    "creep", "crew", "crime", "criminal", "crisis", "criterion", "critic",
+    "critical", "criticism", "criticize", "crop", "cross", "crowd", "crown",
+    "crucial", "crude", "cruel", "cruise", "crush", "cry", "crystal", "cube",
+    "cuisine", "cultivate", "curious", "currency", "current", "curriculum",
+    "curtain", "curve", "custody", "custom", "customary", "customer", "cutting",
+    "cycle", "dad", "damage", "damn", "dance", "danger", "dangerous",
+    "dare", "data", "database", "dawn", "dead", "deadline", "deadly",
+    "deaf", "deal", "dealer", "dear", "debate", "debt", "decade",
+    "decay", "deceive", "decent", "decide", "decision", "decisive", "deck",
+    "declaration", "declare", "decline", "decorate", "decrease", "decree",
+    "dedicate", "deem", "defeat", "defend", "defendant", "defender", "defense",
+    "defensive", "deficit", "define", "definite", "definitely", "definition",
+    "defy", "degree", "delay", "delegate", "delegation", "delete", "deliberate",
+    "deliberately", "delicate", "delicious", "delight", "deliver", "delivery",
+    "demand", "democracy", "democrat", "democratic", "demographic", "demonstrate",
+    "demonstration", "denial", "denote", "deny", "depart", "department", "departure",
+    "depend", "dependence", "dependent", "depict", "deposit", "depress", "depression",
+    "deprive", "depth", "deputy", "derive", "descend", "describe", "description",
+    "desert", "deserve", "design", "designate", "designer", "desirable",
+    "desire", "desk", "desperate", "desperately", "despite", "destination",
+    "destroy", "destruction", "detail", "detailed", "detain", "detect",
+    "detection", "detective", "detention", "deteriorate", "determination",
+    "determine", "determined", "develop", "development", "developmental",
+    "devote", "devote", "diabetes", "diagnose", "diagnosis", "dialogue",
+    "diameter", "diamond", "diary", "dictate", "diet", "differ", "difference",
+    "differentiate", "differently", "difficulty", "dig", "digest", "digital",
+    "dignity", "dilemma", "dimension", "diminish", "dinner", "dioxide",
+    "dip", "diplomat", "diplomatic", "direct", "direction", "directly",
+    "director", "dirty", "disability", "disable", "disadvantage",
+    "disagree", "disappear", "disappoint", "disappointment", "disaster",
+    "disastrous", "disc", "discharge", "discipline", "disclose", "discount",
+    "discourse", "discover", "discovery", "discrepancy", "discretion",
+    "discrimination", "discuss", "discussion", "disease", "dismiss", "disorder",
+    "dispatch", "display", "disposal", "dispose", "dispute", "disrupt",
+    "dissolve", "distance", "distant", "distinct", "distinction", "distinctive",
+    "distinguish", "distort", "distract", "distress", "distribute", "distribution",
+    "distributor", "district", "disturb", "dive", "diverse", "diversity",
+    "divide", "division", "divorce", "dock", "doctor", "doctrine", "document",
+    "documentary", "dollar", "domain", "dome", "domestic", "dominant",
+    "dominate", "donation", "donor", "dose", "dot", "double", "doubt",
+    "doubtful", "downtown", "draft", "drag", "drain", "drama", "dramatic",
+    "dramatically", "drastic", "draw", "drawing", "drink", "drive",
+    "driver", "drop", "drought", "drown", "drum", "drunk", "dry",
+    "dual", "dubious", "duck", "due", "dull", "dump", "durable", "duration",
+    "dust", "duty", "dynamic", "dynamics", "eager", "eagle", "ear",
+    "earning", "earth", "ease", "easily", "eastern", "echo", "eclipse",
+    "ecological", "ecology", "economics", "economist", "economy",
+    "ecosystem", "edit", "edition", "editor", "editorial", "educate",
+    "education", "educational", "educator", "effective", "effectively",
+    "effectiveness", "efficiency", "efficient", "efficiently", "effort",
+    "elaborate", "elbow", "elderly", "elect", "election", "electoral",
+    "electric", "electrical", "electricity", "electronic", "electronics",
+    "elegant", "element", "elementary", "eliminate", "elimination", "elite",
+    "elsewhere", "email", "embargo", "embark", "embarrass", "embassy",
+    "embed", "embody", "embrace", "emerge", "emergence", "emergency",
+    "emission", "emotion", "emotional", "emphasis", "emphasize", "empire",
+    "empirical", "employ", "employee", "employer", "employment", "empower",
+    "enable", "enact", "encompass", "encounter", "encourage", "encouragement",
+    "endanger", "endeavor", "endorse", "endorsement", "endure", "enforce",
+    "enforcement", "engage", "engagement", "engine", "engineering", "enhance",
+    "enjoy", "enjoyment", "enlarge", "enormous", "enrich", "enroll",
+    "ensemble", "ensure", "enter", "enterprise", "entertain", "entertainment",
+    "enthusiasm", "enthusiast", "enthusiastic", "entirely", "entitle",
+    "entity", "entrepreneur", "entry", "envelope", "environment",
+    "environmental", "epidemic", "episode", "equal", "equality", "equation",
+    "equip", "equipment", "equivalent", "era", "erect", "error", "erupt",
+    "escalate", "escape", "especially", "essay", "essence", "essential",
+    "essentially", "establish", "establishment", "estate", "estimate",
+    "estimation", "eternal", "ethical", "ethics", "ethnic", "evacuate",
+    "evaluate", "evaluation", "evenly", "event", "eventually", "ever",
+    "everyday", "evidence", "evident", "evil", "evoke", "evolution",
+    "evolutionary", "evolve", "exact", "exaggerate", "examination",
+    "examine", "example", "exceed", "excellence", "excellent", "exception",
+    "exceptional", "excess", "excessive", "exchange", "excite", "excitement",
+    "exciting", "exclude", "exclusion", "exclusive", "exclusively",
+    "excuse", "execute", "execution", "executive", "exemplify",
+    "exercise", "exert", "exhaust", "exhibit", "exhibition", "exile",
+    "exist", "existence", "exit", "expand", "expansion", "expect",
+    "expectation", "expedition", "expel", "expenditure", "expense",
+    "expensive", "expert", "expertise", "explain", "explanation",
+    "explicit", "explicitly", "explode", "exploit", "exploitation",
+    "exploration", "explore", "explosion", "explosive", "export",
+    "expose", "exposure", "express", "expression", "extend", "extension",
+    "extensive", "extensively", "extent", "external", "extinct",
+    "extinction", "extra", "extract", "extraordinary", "extreme",
+    "extremely", "eye", "fabric", "fabulous", "facade", "face",
+    "facial", "facilitate", "facility", "fact", "faction", "faculty",
+    "fade", "fail", "failure", "fair", "fairly", "fairness", "faith",
+    "faithful", "fake", "fame", "familiar", "famine", "fan", "fancy",
+    "fantasy", "fare", "fascinate", "fascinating", "fashion", "fat",
+    "fate", "fatigue", "fault", "favor", "favorable", "favorite",
+    "fax", "fear", "feasible", "feast", "feather", "federal",
+    "federation", "fee", "feed", "feedback", "feel", "feeling",
+    "fellow", "fellowship", "female", "fence", "fertile", "fertilizer",
+    "festival", "fetch", "fever", "fiber", "fiction", "field", "fierce",
+    "fifteen", "fifty", "fig", "fight", "fighter", "figure", "file",
+    "fill", "filter", "final", "finally", "finance", "financial",
+    "financially", "financing", "finding", "finger", "finished",
+    "fire", "firm", "firmly", "fiscal", "fish", "fisherman", "fishing",
+    "fitness", "fix", "fixture", "flag", "flame", "flash", "flat",
+    "flavor", "flee", "fleet", "flesh", "flexibility", "flexible",
+    "flight", "float", "flock", "flood", "floor", "flour", "flow",
+    "flower", "fluid", "flush", "fly", "focus", "folk", "football",
+    "footnote", "footstep", "forbid", "forbidden", "forecast", "forehead",
+    "foreign", "foreigner", "forest", "forever", "forge", "forget",
+    "forgive", "fork", "form", "formal", "format", "formation", "former",
+    "formula", "formulate", "fort", "forth", "fortunate", "fortune",
+    "forum", "forward", "fossil", "foster", "found",
+    "foundation", "founder", "fountain", "fraction", "fracture", "fragile",
+    "fragment", "frame", "framework", "franchise", "frank", "frankly",
+    "fraud", "free", "freedom", "freely", "freeze", "freight",
+    "frequency", "frequent", "frequently", "fresh", "freshman", "friction",
+    "friendly", "friendship", "frighten", "frog", "front",
+    "frontier", "frost", "frown", "frozen", "fruit", "frustrate",
+    "frustration", "fuel", "fulfill", "full", "fun", "function",
+    "functional", "fund", "fundamental", "funding", "funeral", "funny",
+    "fur", "furious", "furniture", "further", "furthermore", "fury",
+    "fusion", "future", "gain", "galaxy", "gallery", "gallon", "gambling",
+    "gap", "garage", "garbage", "garden", "garlic", "garment", "gas",
+    "gasoline", "gate", "gather", "gathering", "gauge", "gaze", "gear",
+    "gender", "gene", "general", "generally", "generate", "generation",
+    "generator", "generous", "genetic", "genetics", "genius", "genocide",
+    "genre", "gentle", "gentleman", "gently", "genuine", "genuinely",
+    "gesture", "giant", "gift", "gigantic", "glimpse", "global",
+    "globalization", "globe", "glory", "glove", "glow", "glucose",
+    "goal", "goddess", "gold", "golden", "golf", "goodness",
+    "goods", "gorgeous", "gospel", "gossip", "govern", "governance",
+    "government", "governor", "grab", "grace", "grade", "gradually",
+    "graduate", "graduation", "grain", "gram", "grammar", "grand",
+    "grandfather", "grandmother", "grant", "graph", "graphic", "grasp",
+    "grass", "grateful", "grave", "gravity", "gray", "greatly",
+    "green", "greenhouse", "greet", "grief", "grin", "grind", "grip",
+    "grocery", "gross", "ground", "groundwater", "growth", "guarantee",
+    "guard", "guardian", "guess", "guest", "guidance", "guide",
+    "guideline", "guilty", "guitar", "gulf", "gun", "gut", "guy",
+    "gym", "habit", "habitat", "hair", "half", "hall", "halt", "hammer",
+    "hand", "handful", "handle", "handling", "handwriting", "handy",
+    "hang", "happen", "happiness", "harassment", "harbor", "hardly",
+    "hardware", "harm", "harmful", "harmony", "harvest", "hat",
+    "hate", "haul", "hay", "hazard", "head", "headache", "headline",
+    "headquarters", "heal", "health", "healthcare", "healthy",
+    "heap", "hearing", "heart", "heat", "heating", "heaven",
+    "heavily", "heavy", "hedge", "heel", "height", "helicopter",
+    "hell", "helmet", "helpful", "herb", "heritage", "hero",
+    "heroin", "herself", "hesitate", "hidden", "hide", "hierarchy",
+    "highlight", "highly", "highway", "hike", "hill", "himself",
+    "hip", "hire", "historian", "historic", "historical", "history",
+    "hit", "hobby", "hold", "holder", "holding", "hole",
+    "holiday", "hollow", "holy", "homeland", "homeless", "homework",
+    "honest", "honesty", "honey", "honor", "hook", "hope",
+    "hopeful", "hopefully", "horizon", "horizontal", "hormone",
+    "horn", "horrible", "horror", "horse", "hospitality", "host",
+    "hostage", "hostile", "hot", "hotline", "hour", "housing",
+    "hover", "human", "humane", "humanitarian", "humanity",
+    "humor", "hundred", "hunger", "hungry", "hunt", "hunter",
+    "hunting", "hurt", "husband", "hut", "hydrogen", "hygiene",
+    "hypothesis", "ice", "icon", "idea", "ideal", "identical",
+    "identification", "identify", "identity", "ideology", "ignorance",
+    "ignore", "ill", "illegal", "illness", "illusion", "illustrate",
+    "illustration", "image", "imaginary", "imagination", "imagine",
+    "imitate", "immediate", "immediately", "immense", "immigrant",
+    "immigration", "immune", "immunity", "impact", "implement",
+    "implementation", "implication", "implicit", "imply", "import",
+    "importance", "impose", "impossible", "impress", "impression",
+    "impressive", "imprison", "improbable", "improve", "improvement",
+    "impulse", "inability", "inappropriate", "incentive", "incidence",
+    "incident", "inclination", "incline", "include", "including",
+    "inclusion", "inclusive", "income", "incorporate", "incorrect",
+    "increase", "increasingly", "incredible", "incur", "indeed",
+    "independence", "independent", "independently", "index",
+    "indicate", "indication", "indicator", "indictment",
+    "indigenous", "indirect", "indispensable", "individual",
+    "individuality", "indoor", "induce", "indulge", "industrial",
+    "industrialize", "industry", "inequality", "inevitable",
+    "inevitably", "infant", "infect", "infection", "infer",
+    "inference", "inferior", "infinite", "infinity", "inflation",
+    "inflict", "influence", "influential", "info", "inform",
+    "informal", "information", "infrastructure", "ingredient",
+    "inhabit", "inhabitant", "inherent", "inherit", "inhibit",
+    "initial", "initially", "initiate", "initiative", "inject",
+    "injection", "injure", "injury", "inmate", "inner",
+    "innocent", "innovation", "innovative", "input", "inquiry",
+    "insect", "insert", "insertion", "inside", "insight",
+    "insist", "inspect", "inspection", "inspector", "inspiration",
+    "inspire", "install", "installation", "installment", "instance",
+    "instant", "instantly", "instead", "instinct",
+    "institute", "institution", "institutional", "instruct",
+    "instruction", "instructor", "instrument", "instrumental",
+    "insufficient", "insult", "insurance", "intact", "integral",
+    "integrate", "integration", "integrity", "intellectual",
+    "intelligence", "intelligent", "intend", "intense",
+    "intensity", "intensive", "intent", "intention", "intentional",
+    "interact", "interaction", "interactive", "interest",
+    "interested", "interesting", "interface", "interfere",
+    "interference", "interim", "interior", "intermediate",
+    "internal", "international", "internet", "interpret",
+    "interpretation", "interrupt", "interruption",
+    "intersection", "interval", "intervene", "intervention",
+    "interview", "intimate", "intrigue", "intrinsic", "introduce",
+    "introduction", "intuition", "intuitive", "invade",
+    "invasion", "invent", "invention", "inventory", "invest",
+    "investigate", "investigation", "investigator", "investment",
+    "investor", "invisible", "invitation", "invite", "involve",
+    "involvement", "iron", "irony", "irrelevant", "irrigation",
+    "island", "isolate", "isolation", "issue", "item",
+    "itself", "ivory", "jail", "jam", "jet", "jewel",
+    "jewelry", "job", "join", "joint", "jointly", "joke",
+    "journal", "journalism", "journalist", "journey", "joy",
+    "judge", "judgment", "judicial", "juice", "jump", "junction",
+    "jungle", "junior", "jurisdiction", "jury", "justice",
+    "justification", "justify", "juvenile", "keen", "keeper",
+    "kettle", "key", "keyboard", "kick", "kid", "kidnap",
+    "kidney", "kin", "kindness", "king", "kingdom", "kiss",
+    "kit", "kitchen", "knee", "kneel", "knife", "knock",
+    "knot", "label", "labor", "laboratory", "lace", "lack",
+    "ladder", "lady", "lake", "lamp", "land", "landing",
+    "landlord", "landmark", "landscape", "lane", "lap", "largely",
+    "laser", "late", "latte", "latter", "laugh", "laughter",
+    "launch", "laundry", "lavatory", "law", "lawn", "lawmaker",
+    "lawn", "lawsuit", "lawyer", "layout", "leading", "leaf",
+    "league", "leak", "lean", "leap", "learner", "learning",
+    "lease", "leather", "leave", "lecture",
+    "legacy", "legend", "legendary", "legislation", "legislative",
+    "legislature", "legitimate", "leisure", "lemon", "lend",
+    "length", "lens", "lesson", "lest", "lethal", "letter",
+    "lettuce", "liberal", "liberation", "liberty", "library",
+    "license", "lid", "lie", "lifelong", "lifestyle", "lifetime",
+    "lift", "light", "lighting", "lightly", "likelihood",
+    "likewise", "limb", "limit", "limitation", "limited",
+    "limitless", "limp", "line", "linear", "linen", "liner",
+    "linger", "linguistic", "lining", "link", "lion",
+    "lip", "liquid", "liquor", "list", "listen", "listener",
+    "listing", "liter", "literally", "literary", "literature",
+    "litigation", "liver", "living", "load", "loan",
+    "lobby", "local", "locate", "location", "lock", "lodge",
+    "log", "logic", "logical", "logo", "lonely", "longitudinal",
+    "lookout", "loop", "loose", "loosen", "lord", "lose",
+    "loss", "loud", "lounge", "lovely", "lover",
+    "low", "lower", "loyal", "loyalty", "luck", "lucky",
+    "luggage", "lump", "lunch", "lung", "luxury",
+    "lyric", "machine", "machinery", "mad", "magazine", "magic",
+    "magical", "magnetic", "magnificent", "magnitude", "maid",
+    "mail", "mainland", "mainly", "mainstream", "maintain",
+    "maintenance", "majesty", "majority", "maker",
+    "makeup", "male", "mall", "mama", "mammal", "manage",
+    "manageable", "management", "manager", "mandate",
+    "mandatory", "maneuver", "manifest", "manipulate",
+    "manipulation", "mankind", "manuscript", "maple",
+    "marathon", "marble", "march", "margin", "marginal",
+    "marine", "mark", "marker", "market", "marketing",
+    "marketplace", "marriage", "married", "marry", "mask",
+    "mass", "massacre", "massive", "master", "masterpiece",
+    "match", "mate", "material", "maternal", "math",
+    "mathematical", "mathematics", "matter", "mature",
+    "maturity", "maximize", "maximum", "mayor", "meadow",
+    "meaning", "meaningful", "means", "meantime",
+    "measurable", "measure", "measurement", "mechanic",
+    "mechanical", "mechanism", "medal", "media",
+    "mediate", "mediation", "medicaid", "medical",
+    "medication", "medicine", "medieval", "meditation",
+    "medium", "meet", "melody", "melt", "member",
+    "membership", "memo", "memoir", "memorandum",
+    "memorial", "memorize", "menace",
+    "mental", "mentally", "mention", "mentor", "menu",
+    "merchandise", "merchant", "mercy", "mere", "merely",
+    "merge", "merger", "merit", "merry", "mess",
+    "messenger", "metal", "metaphor", "method",
+    "methodology", "metric", "metropolitan",
+    "microphone", "microscope", "midday", "middle",
+    "midnight", "midst", "migrant", "migrate", "migration",
+    "mild", "mile", "milestone", "militant", "military",
+    "militia", "mill", "millennium", "millimeter",
+    "mineral", "mingle", "miniature", "minimal",
+    "minimize", "minimum", "mining", "minister",
+    "ministry", "minor", "minority", "minute",
+    "miracle", "mirror", "miserable", "misery",
+    "misleading", "missile", "missing", "mission",
+    "missionary", "mist", "mistake", "mistaken",
+    "mistress", "misunderstand", "misunderstanding",
+    "mixture", "moan", "mobile", "mobility",
+    "mobilize", "mode", "moderate", "moderately",
+    "moderation", "modern", "modest", "modification",
+    "modify", "module", "moisture", "molecule",
+    "molest", "moment", "momentum", "monarchy",
+    "monastery", "monitor", "monk", "monopoly",
+    "monster", "monument", "mood", "moon",
+    "moral", "morale", "morality", "moreover",
+    "mortal", "mortality", "mortgage", "mosaic",
+    "mosque", "mostly", "mother", "motion",
+    "motivate", "motivation", "motive", "motor",
+    "motorcycle", "mount", "mountain", "mounting",
+    "mourn", "mouse", "mouth", "movement",
+    "movie", "mud", "multiple", "multiplication",
+    "multiply", "multitude", "municipal",
+    "municipality", "murder", "murderer", "murmur",
+    "muscle", "muscular", "museum", "mushroom",
+    "musical", "musician", "muslim", "mutual",
+    "mutually", "mysterious", "mystery", "myth",
+    "mythology", "nail", "naked", "narrative",
+    "narrow", "nasty", "nation", "national",
+    "nationalism", "nationalist", "nationality",
+    "nationwide", "native", "natural", "naturally",
+    "nature", "naval", "navigation", "navy",
+    "nearby", "neat", "necessarily", "necessary",
+    "necessity", "neck", "necklace", "needle",
+    "negative", "neglect", "negotiate",
+    "negotiation", "negotiator", "neighbor",
+    "neighborhood", "neither", "nerve", "nervous",
+    "nest", "net", "network", "neutral", "nevertheless",
+    "niche", "nickel", "niece", "night", "nightmare",
+    "nitrogen", "noble", "nobody", "nod", "noise",
+    "noisy", "nominal", "nominate", "nomination",
+    "nominee", "nonprofit", "nonsense", "norm",
+    "normal", "normally", "normative", "north",
+    "northeast", "northern", "northwest", "notable",
+    "notably", "notation", "notebook", "nothing",
+    "notice", "noticeable", "notification",
+    "notion", "notorious", "novel", "novelist",
+    "novelty", "nowhere", "nuclear", "nuance",
+    "nucleus", "nuisance", "number", "numerical",
+    "numerous", "nurse", "nursery", "nursing",
+    "nutrient", "nutrition", "nutritional",
+    "nutritious", "nylon", "oak", "obedience",
+    "obedient", "obese", "obesity", "obey",
+    "objection", "objective", "obligation",
+    "oblige", "obscure", "observation", "observe",
+    "observer", "obsession", "obstacle",
+    "obtain", "obvious", "obviously", "occasion",
+    "occasional", "occasionally", "occupation",
+    "occupy", "occur", "occurrence",
+    "offend", "offense", "offensive", "offer",
+    "offering", "officer", "official", "officially",
+    "offspring", "olive", "omission",
+    "omit", "ongoing", "onion", "onset",
+    "opening", "openly", "opera", "operate",
+    "operating", "operation", "operational",
+    "operator", "opinion", "opponent",
+    "opportunity", "oppose", "opposite",
+    "opposition", "opt", "optical", "optimism",
+    "optimist", "optimistic", "optimum",
+    "option", "optional", "oral", "orbit",
+    "orchestra", "ordeal", "orderly",
+    "ordinarily", "ordinary", "organ",
+    "organic", "organization", "organizational",
+    "organize", "organized", "organizer",
+    "orientation", "origin", "original",
+    "originally", "originate", "ornament",
+    "orthodox", "other", "otherwise",
+    "ought", "ounce", "outbreak",
+    "outcome", "outdoor", "outer", "outfit",
+    "outing", "outlet", "outline", "outlook",
+    "output", "outrage", "outright",
+    "outset", "outside", "outsider", "outstanding",
+    "outward", "oval", "oven", "overall",
+    "overcome", "overlook", "overnight",
+    "override", "overseas", "oversee",
+    "overturn", "overwhelm", "overwhelming",
+    "owe", "own", "owner", "ownership",
+    "oxygen", "ozone", "pace", "pack",
+    "package", "packaging", "packet", "pad",
+    "paddle", "page", "pain", "painful",
+    "paint", "painter", "painting", "pair",
+    "palace", "pale", "palm", "pan",
+    "panel", "panic", "paper", "parade",
+    "paradigm", "paradise", "paradox",
+    "paragraph", "parallel", "parameter",
+    "parcel", "pardon", "parent",
+    "parental", "parish", "park",
+    "parking", "parliament", "parliamentary",
+    "partial", "partially", "participant",
+    "participate", "participation",
+    "particle", "particular", "particularly",
+    "partly", "partner", "partnership",
+    "passage", "passenger", "passing",
+    "passion", "passionate", "passive",
+    "passport", "password", "past",
+    "paste", "pastor", "patch", "patent",
+    "path", "pathway", "patience",
+    "patient", "patrol", "patron",
+    "pattern", "pause", "pave", "pavement",
+    "paw", "payment", "peace", "peaceful",
+    "peak", "peasant", "peculiar", "pedestrian",
+    "peer", "penalty", "pencil", "penetrate",
+    "peninsula", "pension", "people",
+    "pepper", "perceive", "percentage",
+    "perception", "perfect", "perfectly",
+    "perform", "performance", "performer",
+    "perfume", "perhaps", "period",
+    "periodic", "peripheral", "permanent",
+    "permanently", "permission", "permit",
+    "persist", "persistence", "persistent",
+    "persona", "personal", "personality",
+    "personally", "personnel",
+    "perspective", "persuade", "pet",
+    "petition", "petroleum", "phase",
+    "phenomenon", "philosopher", "philosophical",
+    "philosophy", "phone",
+    "photograph", "photographer", "photographic",
+    "photography", "phrase", "physical",
+    "physically", "physician", "physics",
+    "piano", "pick", "picture",
+    "piece", "pierce", "pig", "pile",
+    "pillar", "pillow", "pilot",
+    "pin", "pine", "pink", "pioneer",
+    "pipe", "pit", "pitch", "pizza",
+    "placement", "plain", "plaintiff",
+    "plantation", "plate", "platform",
+    "plausible", "playback", "playful",
+    "playground", "plead", "pleasant",
+    "pledge", "plenty", "plot",
+    "plug", "plunge", "plural", "plus",
+    "pocket", "poem", "poet", "poetic",
+    "poetry", "poison", "poisonous",
+    "polar", "pole", "police",
+    "policeman", "policy", "polish",
+    "polite", "political", "politically",
+    "politician", "politics",
+    "poll", "pollution", "pond",
+    "pony", "pool", "pop",
+    "pope", "popular", "popularity",
+    "population", "porch", "port",
+    "portable", "porter", "portion",
+    "portrait", "portray", "pose",
+    "position", "positive", "positively",
+    "possess", "possession", "possessive",
+    "possibility", "possible", "possibly",
+    "postage", "postal", "poster",
+    "potato", "potent", "potential",
+    "potentially", "pottery",
+    "poverty", "powder", "powerful",
+    "practically", "practice",
+    "practitioner", "praise", "pray",
+    "prayer", "preach", "precede",
+    "precedent", "precious", "precise",
+    "precisely", "precision", "predict",
+    "predictable", "prediction",
+    "predominantly", "preface", "prefer",
+    "preference", "pregnancy", "pregnant",
+    "prejudice", "preliminary",
+    "premier", "premise", "premium",
+    "preparation", "prepare", "prepared",
+    "prescription", "presence",
+    "presentation", "presently",
+    "preservation", "preserve",
+    "presidency", "president", "presidential",
+    "pressing", "pressure",
+    "presumably", "presume", "pretend",
+    "pretty", "prevail", "prevalence",
+    "prevalent", "prevention",
+    "preview", "previous", "previously",
+    "prey", "pricing", "pride",
+    "priest", "primarily", "primary",
+    "prime", "primitive", "prince",
+    "princess", "principal", "principle",
+    "print", "printer", "printing",
+    "prior", "priority", "prison",
+    "prisoner", "privacy",
+    "privilege", "privileged",
+    "prize", "proactive", "probable",
+    "probably", "probe", "problem",
+    "problematic", "procedural",
+    "procedure", "proceed", "proceeding",
+    "proceeds", "processor",
+    "proclaim", "produce", "producer",
+    "productive", "productivity",
+    "profession", "professional",
+    "professor", "proficiency",
+    "profile", "profit",
+    "profitable", "profound",
+    "program", "programming",
+    "progressive", "prohibit",
+    "prohibition", "project",
+    "projection", "prominent",
+    "promise", "promising",
+    "promote", "promotion",
+    "prompt", "proof",
+    "propaganda", "proper", "properly",
+    "property", "prophet",
+    "proportion", "proposal",
+    "propose", "proposed",
+    "prosecution", "prosecutor",
+    "prospect", "prosperity",
+    "protect", "protection",
+    "protective", "protein",
+    "protest", "protester",
+    "protocol", "proud",
+    "prove", "proverb",
+    "provide", "provided",
+    "province", "provincial",
+    "provision", "provoke",
+    "proxy", "prudent",
+    "psychiatric", "psychiatry",
+    "psychological", "psychologist",
+    "psychology", "pub",
+    "publication", "publicity",
+    "publicly", "publish",
+    "publisher", "publishing",
+    "pulse", "pump",
+    "punch", "punish",
+    "punishment", "pupil",
+    "purchase", "purchaser",
+    "pure", "purely",
+    "purify", "purple",
+    "purpose", "purse",
+    "pursue", "pursuit",
+    "puzzle", "qualification",
+    "qualified", "qualify",
+    "qualitative", "quantify",
+    "quantitative", "quarterly",
+    "queen", "quest",
+    "questionable", "questionnaire",
+    "queue", "quit",
+    "quiz", "quota",
+    "quotation", "quote",
+    "rabbit", "racial",
+    "racism", "racist",
+    "radiation", "radical",
+    "rage", "raid",
+    "rail", "railroad",
+    "railway", "rain",
+    "rainbow", "rally",
+    "random", "range",
+    "rank", "ranking",
+    "rape", "rapid",
+    "rapidly", "rare",
+    "rarely", "rat",
+    "rate", "rating",
+    "ratio", "rational",
+    "raw", "ray",
+    "react", "reaction",
+    "readily", "reading",
+    "realistic", "reality",
+    "realization", "realize",
+    "realm", "rear",
+    "reason", "reasonable",
+    "reasonably", "reasoning",
+    "reassure", "rebel",
+    "rebellion", "rebuild",
+    "recall", "receipt",
+    "receiver", "recent",
+    "recently", "reception",
+    "recession", "recipe",
+    "recipient", "reckon",
+    "recognition", "recognize",
+    "recommend", "recommendation",
+    "reconcile", "reconstruction",
+    "recording", "recover",
+    "recovery", "recreation",
+    "recruit", "recruitment",
+    "reduction", "redundant",
+    "reef", "refer",
+    "referee", "reference",
+    "referendum", "referral",
+    "reflection", "reform",
+    "refrain", "refresh",
+    "refuge", "refugee",
+    "refund", "refusal",
+    "refuse", "regain",
+    "regard", "regarding",
+    "regardless", "regime",
+    "regiment", "regional",
+    "register", "registration",
+    "regret", "regular",
+    "regularly", "regulate",
+    "regulation", "regulator",
+    "regulatory", "rehabilitation",
+    "reign", "reinforce",
+    "reinforcement", "reject",
+    "rejection", "relate",
+    "relation", "relationship",
+    "relative", "relatively",
+    "relax", "relaxation",
+    "release", "relevant",
+    "reliability", "reliable",
+    "relief", "relieve",
+    "religion", "religious",
+    "reluctant", "reluctantly",
+    "remainder", "remains",
+    "remark", "remarkable",
+    "remarkably", "remedy",
+    "reminder", "remnant",
+    "remote", "removal",
+    "remove", "renaissance",
+    "render", "renew",
+    "renewable", "renewal",
+    "rent", "rental",
+    "repair", "repay",
+    "repeat", "repeatedly",
+    "repent", "repetition",
+    "replace", "replacement",
+    "reportedly", "reporter",
+    "representation", "representative",
+    "repression", "reprint",
+    "reproduce", "reproduction",
+    "republic", "republican",
+    "reputation", "request",
+    "require", "requirement",
+    "rescue", "resemble",
+    "resentment", "reservation",
+    "reserve", "reservoir",
+    "reside", "residence",
+    "residence", "resident",
+    "residential", "residual",
+    "resign", "resignation",
+    "resist", "resistance",
+    "resistant", "resolution",
+    "resolve", "resort",
+    "resource", "respect",
+    "respectable", "respectful",
+    "respective", "respectively",
+    "respond", "respondent",
+    "response", "responsibility",
+    "responsible", "responsive",
+    "restoration", "restore",
+    "restrain", "restraint",
+    "restrict", "restriction",
+    "restrictive", "restructuring",
+    "resume", "retail",
+    "retailer", "retain",
+    "retaliation", "retire",
+    "retirement", "retreat",
+    "retrieval", "retrieve",
+    "revelation", "revenge",
+    "revenue", "reverse",
+    "review", "revise",
+    "revision", "revival",
+    "revive", "revolution",
+    "revolutionary", "reward",
+    "rhetoric", "rhythm",
+    "rib", "ribbon",
+    "rid", "ride",
+    "ridge", "ridiculous",
+    "rifle", "rigid",
+    "riot", "rip",
+    "ripple", "ritual",
+    "rival", "rivalry",
+    "roar", "robbery",
+    "robe", "robot",
+    "robust", "rocket",
+    "rod", "roll",
+    "roller", "romance",
+    "romantic", "rookie",
+    "rope", "rose",
+    "rotate", "rotation",
+    "rotten", "rough",
+    "roughly", "round",
+    "route", "routine",
+    "row", "royal",
+    "royalty", "rub",
+    "rubber", "rug",
+    "ruin", "ruler",
+    "ruling", "rumor",
+    "runner", "running",
+    "rural", "rush",
+    "sacred", "sacrifice",
+    "sad", "saddle",
+    "sadly", "sadness",
+    "sake", "salad",
+    "salary", "sale",
+    "salmon", "salon",
+    "salt", "salute",
+    "salvation", "sample",
+    "sanction", "sanctuary",
+    "sand", "satellite",
+    "satisfaction", "satisfactory",
+    "satisfy", "sauce",
+    "saving", "savings",
+    "scale", "scandal",
+    "scare", "scared",
+    "scary", "scatter",
+    "scenario", "scent",
+    "schedule", "scheme",
+    "scholar", "scholarship",
+    "schooling", "scientific",
+    "scientist", "scope",
+    "score", "scorn",
+    "scrap", "scream",
+    "screen", "screening",
+    "screw", "script",
+    "scrutiny", "seal",
+    "search", "season",
+    "seasonal", "seating",
+    "secular", "secure",
+    "security", "seed",
+    "seek", "segment",
+    "seize", "seizure",
+    "select", "selection",
+    "selective", "self",
+    "seller", "senate",
+    "senator", "senior",
+    "sensation", "sensational",
+    "sensitivity", "sensor",
+    "sentence", "sentiment",
+    "separate", "separation",
+    "sequence", "sequencing",
+    "serial", "series",
+    "session", "settle",
+    "settlement", "settler",
+    "setup", "severe",
+    "severely", "severity",
+    "sew", "sewage",
+    "shade", "shadow",
+    "shaft", "shake",
+    "shall", "shallow",
+    "shame", "shape",
+    "shareholder", "sharing",
+    "shark", "shed",
+    "sheer", "sheet",
+    "shelf", "shell",
+    "shelter", "shield",
+    "shift", "shine",
+    "shipment", "shipping",
+    "shirt", "shock",
+    "shopping", "shortage",
+    "shortly", "shot",
+    "shoulder", "shout",
+    "shove", "shower",
+    "shrimp", "shrink",
+    "shrub", "shrug",
+    "shutter", "sibling",
+    "sickness", "sideways",
+    "siege", "sigh",
+    "sight", "sign",
+    "signal", "signature",
+    "significance", "significant",
+    "significantly", "silence",
+    "silent", "silicon",
+    "silk", "silly",
+    "silver", "similar",
+    "similarity", "similarly",
+    "simmer", "simplicity",
+    "simplify", "simply",
+    "simulation", "simultaneously",
+    "sin", "sincere",
+    "sincerely", "singer",
+    "single", "singular",
+    "sink", "sip",
+    "sister", "situation",
+    "sizable", "size",
+    "sketch", "ski",
+    "skilled", "skillful",
+    "skim", "skin",
+    "skip", "skirt",
+    "skull", "slap",
+    "slash", "slave",
+    "slavery", "sleep",
+    "sleeve", "slice",
+    "slide", "slight",
+    "slightly", "slim",
+    "slip", "slogan",
+    "slope", "slot",
+    "slow", "slowly",
+    "smart", "smell",
+    "smile", "smoke",
+    "smooth", "smoothly",
+    "snap", "sneak",
+    "snapshot", "snow",
+    "soak", "soap",
+    "soar", "soccer",
+    "social", "socialism",
+    "socialist", "societal",
+    "society", "sociological",
+    "sociology", "soda",
+    "software", "soil",
+    "solar", "soldier",
+    "sole", "solely",
+    "solemn", "solicitor",
+    "solidarity", "solitary",
+    "solo", "soluble",
+    "solution", "solve",
+    "somebody", "somehow",
+    "someone", "sometime",
+    "somewhat", "song",
+    "sophisticated", "sore",
+    "sorrow", "sort",
+    "soul", "sound",
+    "soup", "sour",
+    "source", "southeast",
+    "southern", "southwest",
+    "sovereign", "sovereignty",
+    "sow", "spacecraft",
+    "spacing", "span",
+    "spare", "spark",
+    "speak", "speaker",
+    "spear", "special",
+    "specialist", "specialize",
+    "specialty", "species",
+    "specific", "specifically",
+    "specification", "specify",
+    "specimen", "spectacle",
+    "spectacular", "spectator",
+    "spectrum", "speculate",
+    "speculation", "speech",
+    "spell", "spelling",
+    "spend", "sphere",
+    "spill", "spin",
+    "spine", "spiral",
+    "spirit", "spiritual",
+    "spite", "splash",
+    "split", "spokesman",
+    "spokesperson", "spokeswoman",
+    "sponsor", "sponsorship",
+    "spontaneous", "spoon",
+    "sport", "spot",
+    "spouse", "spread",
+    "spring", "sprint",
+    "spur", "spy",
+    "squad", "squadron",
+    "square", "squeeze",
+    "stability", "stabilize",
+    "stable", "stadium",
+    "staff", "stage",
+    "stake", "stakeholder",
+    "stall", "stance",
+    "stand", "standard",
+    "standing", "staple",
+    "stare", "stark",
+    "startup", "starvation",
+    "starve", "statement",
+    "statue", "status",
+    "statute", "statutory",
+    "steady", "steal",
+    "steam", "steel",
+    "steep", "steer",
+    "stem", "stereotype",
+    "sterling", "stern",
+    "steward", "stick",
+    "sticky", "stiff",
+    "stimulate", "stimulus",
+    "stir", "stitch",
+    "stock", "stomach",
+    "stoppage", "storage",
+    "storm", "story",
+    "strain", "strand",
+    "strap", "strategic",
+    "strategically", "strategist",
+    "strategy", "straw",
+    "stream", "street",
+    "strength", "strengthen",
+    "stress", "stretch",
+    "strict", "strictly",
+    "stride", "strike",
+    "striker", "string",
+    "strip", "stripe",
+    "strive", "stroke",
+    "stronghold", "strongly",
+    "structural", "structure",
+    "struggle", "stubborn",
+    "studio", "study",
+    "stuff", "stumble",
+    "stun", "stunning",
+    "stupid", "style",
+    "subject", "subjective",
+    "sublime", "submission",
+    "submit", "subordinate",
+    "subsequent", "subsequently",
+    "subsidy", "substance",
+    "substantial", "substantially",
+    "substantive", "substitute",
+    "substitution", "subtle",
+    "subtlety", "subtract",
+    "suburb", "suburban",
+    "subversion", "subvert",
+    "succeed", "success",
+    "successful", "successfully",
+    "succession", "successive",
+    "successor", "suck",
+    "sudden", "suddenly",
+    "sue", "suffer",
+    "suffering", "sufficient",
+    "sufficiently", "sugar",
+    "suicide", "suit",
+    "suitable", "suite",
+    "sulfur", "sum",
+    "summarize", "summary",
+    "summit", "sunlight",
+    "sunny", "sunrise",
+    "sunset", "sunshine",
+    "superb", "superficial",
+    "superintendent", "superior",
+    "superiority", "supermarket",
+    "supervise", "supervision",
+    "supervisor", "supplement",
+    "supplementary", "supplier",
+    "supply", "support",
+    "supporter", "supportive",
+    "suppose", "supposedly",
+    "suppress", "suppression",
+    "supreme", "surcharge",
+    "surface", "surge",
+    "surgeon", "surgery",
+    "surgical", "surname",
+    "surpass", "surplus",
+    "surprise", "surprised",
+    "surprising", "surprisingly",
+    "surrender", "surround",
+    "surrounding", "surveillance",
+    "survey", "survival",
+    "survive", "survivor",
+    "susceptible", "suspect",
+    "suspend", "suspense",
+    "suspension", "suspicion",
+    "suspicious", "sustain",
+    "sustainable", "sustained",
+    "swap", "swear",
+    "sweep", "sweet",
+    "swell", "swift",
+    "swim", "swimming",
+    "swing", "switch",
+    "sword", "symbol",
+    "symbolic", "symmetry",
+    "sympathetic", "sympathy",
+    "symphony", "symptom",
+    "syndrome", "synthesis",
+    "synthetic", "system",
+    "systematic", "tackle",
+    "tactical", "tactics",
+    "tag", "tail",
+    "takeover", "tale",
+    "talent", "talented",
+    "tank", "tap",
+    "tape", "target",
+    "tariff", "task",
+    "taste", "tax",
+    "taxation", "taxpayer",
+    "teaching", "tear",
+    "tease", "technical",
+    "technically", "technician",
+    "technique", "technological",
+    "technology", "teenage",
+    "teenager", "telecommunications",
+    "telegraph", "telephone",
+    "telescope", "television",
+    "temper", "temperature",
+    "temple", "temporarily",
+    "temporary", "tempt",
+    "temptation", "tenant",
+    "tendency", "tender",
+    "tennis", "tension",
+    "tent", "tenure",
+    "terminal", "terminate",
+    "termination", "term",
+    "terrain", "terrible",
+    "terribly", "terrific",
+    "territorial", "territory",
+    "terror", "terrorism",
+    "terrorist", "testament",
+    "testify", "testimony",
+    "textbook", "textile",
+    "texture", "thankful",
+    "theater", "theatre",
+    "theft", "theological",
+    "theology", "theoretical",
+    "theorist", "theory",
+    "therapist", "therapy",
+    "thereafter", "thereby",
+    "thermal", "thesis",
+    "thickness", "thief",
+    "thigh", "thin",
+    "thinking", "thirst",
+    "thirsty", "thorn",
+    "thorough", "thoroughly",
+    "thoughtful", "thoughtless",
+    "thriller", "thrive",
+    "throat", "throne",
+    "thrust", "thumb",
+    "thunder", "tide",
+    "timber", "timely",
+    "timing", "tissue",
+    "title", "toe",
+    "tolerance", "tolerant",
+    "tolerate", "toll",
+    "tomato", "ton",
+    "tone", "tongue",
+    "tool", "tooth",
+    "topic", "topical",
+    "torch", "torture",
+    "total", "totally",
+    "touch", "tourism",
+    "tourist", "tournament",
+    "tow", "towel",
+    "tower", "toxic",
+    "trace", "track",
+    "tractor", "trade",
+    "trademark", "trader",
+    "trading", "tradition",
+    "traditional", "traditionally",
+    "traffic", "tragedy",
+    "tragic", "trail",
+    "trainer", "training",
+    "trait", "transaction",
+    "transcript", "transfer",
+    "transform", "transformation",
+    "transit", "transition",
+    "translate", "translation",
+    "translator", "transmission",
+    "transmit", "transparency",
+    "transparent", "transplant",
+    "transport", "transportation",
+    "trap", "trash",
+    "trauma", "traumatic",
+    "traveler", "treasure",
+    "treat", "treatment",
+    "treaty", "tremendous",
+    "trend", "trial",
+    "triangle", "tribal",
+    "tribe", "tribunal",
+    "trigger", "trim",
+    "triumph", "troop",
+    "trophy", "tropical",
+    "trouble", "troublesome",
+    "trunk", "trust",
+    "trustee", "truth",
+    "tube", "tuition",
+    "tumor", "tune",
+    "tunnel", "turmoil",
+    "turnover", "turtle",
+    "tutor", "tutorial",
+    "twist", "typical",
+    "typically", "tyranny",
+    "ugly", "ultimate",
+    "ultimately", "ultimatum",
+    "umbrella", "unable",
+    "unacceptable", "unanimous",
+    "unaware", "uncertainty",
+    "uncomfortable", "unconscious",
+    "unconstitutional", "undergo",
+    "undergraduate", "underground",
+    "underline", "underlying",
+    "undermine", "underneath",
+    "underscore", "undertake",
+    "undertaking", "underwear",
+    "undoubtedly", "unemployment",
+    "unexpected", "unexpectedly",
+    "unfair", "unfold",
+    "unfortunate", "unfortunately",
+    "unhappy", "unhealthy",
+    "unified", "uniform",
+    "unilateral", "unintended",
+    "union", "unique",
+    "unity", "universal",
+    "universally", "universe",
+    "unknown", "unlawful",
+    "unlike", "unlikely",
+    "unnecessary", "unpleasant",
+    "unprecedented", "unrest",
+    "unveil", "upcoming",
+    "update", "upgrade",
+    "uphold", "upset",
+    "upturn", "uranium",
+    "urban", "urge",
+    "urgency", "urgent",
+    "usage", "utilize",
+    "utmost", "utter",
+    "utterly", "vacancy",
+    "vacation", "vaccination",
+    "vaccine", "vacuum",
+    "valid", "validate",
+    "validity", "valley",
+    "valuable", "valuation",
+    "valve", "variable",
+    "variation", "varied",
+    "vary", "vast",
+    "vector", "vegetable",
+    "vegetation", "vehicle",
+    "veil", "vein",
+    "velocity", "vendor",
+    "venture", "venue",
+    "verbal", "verdict",
+    "verify", "versatile",
+    "verse", "version",
+    "versus", "vertical",
+    "vessel", "veteran",
+    "veterinary", "viable",
+    "vibrant", "vibration",
+    "vice", "victim",
+    "victorious", "victory",
+    "video", "view",
+    "viewer", "viewpoint",
+    "vigorous", "village",
+    "violate", "violation",
+    "violence", "violent",
+    "virgin", "virtual",
+    "virtually", "virtue",
+    "virus", "visa",
+    "visible", "vision",
+    "visual", "vital",
+    "vitamin", "vivid",
+    "vocabulary", "vocal",
+    "vocational", "voice",
+    "volatile", "volcano",
+    "volume", "voluntary",
+    "volunteer", "vote",
+    "voter", "voting",
+    "vulnerability", "vulnerable",
+    "wage", "wagon",
+    "waist", "wallet",
+    "wander", "ward",
+    "warehouse", "warfare",
+    "warmth", "warn",
+    "warning", "warrant",
+    "warranty", "warrior",
+    "wary", "waterfall",
+    "waterproof", "watershed",
+    "wave", "wavelength",
+    "wax", "weakness",
+    "wealth", "wealthy",
+    "weapon", "wear",
+    "weary", "weather",
+    "weave", "web",
+    "website", "wedding",
+    "weed", "weekday",
+    "weekend", "weekly",
+    "weigh", "weight",
+    "weird", "welcome",
+    "welfare", "wellness",
+    "whatsoever", "wheel",
+    "whenever", "whereas",
+    "whereby", "wherein",
+    "whichever", "whisper",
+    "white", "whoever",
+    "wholesale", "wholly",
+    "widespread", "widow",
+    "width", "willing",
+    "willingness", "wisdom",
+    "withdraw", "withdrawal",
+    "wither", "withhold",
+    "within", "without",
+    "witness", "wolf",
+    "wonder", "wooden",
+    "wool", "workforce",
+    "workout", "workplace",
+    "workshop", "worship",
+    "worst", "worthwhile",
+    "worthy", "wound",
+    "wrap", "wrist",
+    "writer", "writing",
+    "wrongly", "yacht",
+    "yell", "young",
+    "youngster", "zone",
+]
+
+# Exclude byte-level chars (they already have IDs 51-306)
+byte_chars = {byte2char[b] for b in range(256)}
+
+# Deduplicate while preserving order
+seen = set()
+deduped = []
+for w in common_words:
+    if w not in seen and w not in byte_chars:
+        seen.add(w)
+        deduped.append(w)
+
+# Truncate to fit total vocab <= 4096 (IDs 307-4095 = 3789 slots)
+max_words = 4096 - 307
+if len(deduped) > max_words:
+    deduped = deduped[:max_words]
+common_words = deduped
+
+next_id = 307
+word_tokens = []
+for w in common_words:
+    word_tokens.append((w, next_id))
+    next_id += 1
+
+# Build vocab dict
+vocab = {}
+for name, idx in special_tokens:
+    if name in vocab:
+        print(f"DUPLICATE: {name!r} @ {vocab[name]} and {idx}")
+    vocab[name] = idx
+for char, idx in byte_tokens:
+    if char in vocab:
+        print(f"DUPLICATE BYTE: {char!r} (U+{ord(char):04X}) @ {vocab[char]} and {idx}")
+    vocab[char] = idx
+for name, idx in word_tokens:
+    if name in vocab:
+        print(f"DUPLICATE WORD: {name!r} @ {vocab[name]} and {idx}")
+    vocab[name] = idx
+
+# Build added_tokens
+added_tokens = []
+for name, idx in special_tokens:
+    added_tokens.append({
+        "id": idx,
+        "content": name,
+        "single_word": False,
+        "lstrip": False,
+        "rstrip": False,
+        "normalized": False,
+        "special": True,
+    })
+
+# Build tokenizer.json
+tokenizer_json = {
+    "version": "1.0",
+    "truncation": None,
+    "padding": None,
+    "added_tokens": added_tokens,
+    "normalizer": None,
+    "pre_tokenizer": {
+        "type": "ByteLevel",
+        "add_prefix_space": False,
+        "trim_offsets": True,
+    },
+    "post_processor": None,
+    "decoder": {
+        "type": "ByteLevel",
+        "add_prefix_space": False,
+        "trim_offsets": True,
+    },
+    "model": {
+        "type": "BPE",
+        "dropout": None,
+        "unk_token": "<unk>",
+        "byte_fallback": True,
+        "vocab": vocab,
+        "merges": [],
+    },
+}
+
+if __name__ == "__main__":
+    with open("tokenizer/tokenizer.json", "w", encoding="utf-8") as f:
+        json.dump(tokenizer_json, f, ensure_ascii=False, separators=(",", ":"))
+
+    with open("tokenizer/special_tokens_map.json", "w", encoding="utf-8") as f:
+        json.dump({
+            "bos_token": "<s>",
+            "eos_token": "</s>",
+            "pad_token": "<pad>",
+            "unk_token": "<unk>",
+        }, f, ensure_ascii=False, indent=2)
+
+    import os
+    size = os.path.getsize("tokenizer/tokenizer.json")
+    print(f"tokenizer.json: {size:,} bytes")
+    print(f"Vocab entries: {len(vocab)}")
+    print(f"Added tokens: {len(added_tokens)}")
+    print(f"Merges: 0")
+    print(f"Byte fallback: True")
+    assert len(vocab) <= 4096, f"Vocab size overflow: {len(vocab)}"
+    print(f"Free slots: {4096 - len(vocab)}")
+
+    from transformers import PreTrainedTokenizerFast
+    from tokenizers import Tokenizer as Tk
+    tok_obj = Tk.from_file("tokenizer/tokenizer.json")
+    tok = PreTrainedTokenizerFast(tokenizer_object=tok_obj)
+    tok.add_special_tokens({"pad_token": "<pad>", "bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>"})
+    test = "Hello, how are you?"
+    enc = tok.encode(test)
+    dec = tok.decode(enc)
+    print(f"OK: {enc} -> {dec!r}")
+
+    test2 = "def f(): return 42"
+    enc2 = tok.encode(test2)
+    dec2 = tok.decode(enc2)
+    print(f"OK: {enc2} -> {dec2!r}")
+
+    enc3 = tok.encode("<|system|>Hi<|user|>there")
+    print(f"Special: {enc3} -> {tok.decode(enc3)!r}")
+
+    test4 = "café résumé"
+    enc4 = tok.encode(test4)
+    dec4 = tok.decode(enc4)
+    print(f"Unicode: {enc4} -> {dec4!r}")

infer_gguf.py

@@ -0,0 +1,177 @@
+"""
+Inference with GGUF INT4 model or QLoRA checkpoint.
+
+Usage:
+  python3 scripts/infer_gguf.py --gguf outputs/tiny-sft/tiny.gguf
+  python3 scripts/infer_gguf.py --checkpoint model.pt
+"""
+
+import os, sys, argparse
+import gguf
+import torch
+import torch.nn.functional as F
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+from scripts.model_tiny import TinyModel, apply_qlora
+
+
+def quantize_to_q4(tensor):
+    t = tensor.float()
+    max_val = t.abs().max()
+    if max_val < 1e-8:
+        return t
+    scale = max_val / 7.0
+    q = (t / scale).round().clamp(-7, 7).char()
+    dq = q.float() * scale
+    return dq
+
+
+def load_gguf_model(gguf_path):
+    print(f" Loading GGUF: {gguf_path}")
+    reader = gguf.GGUFReader(gguf_path)
+
+    model = TinyModel(
+        vocab_size=1757, hidden=128, intermediate=640,
+        num_layers=3, num_heads=8, num_kv_heads=4,
+        max_seq_len=2048, tie_weights=True,
+    )
+    model.eval()
+
+    state = {}
+    for tensor in reader.tensors:
+        name = tensor.name
+        data = torch.from_numpy(tensor.data.copy())
+
+        if name == "token_embd.weight":
+            state["token_embed.weight"] = data
+        elif name == "output_norm.weight":
+            state["ln_f.weight"] = data
+        elif name == "output.weight":
+            state["lm_head.weight"] = data
+        elif name.startswith("blk."):
+            parts = name.split(".")
+            blk = int(parts[1])
+            sub = parts[2]
+            if sub == "attn_norm":
+                state[f"blocks.{blk}.ln1.{parts[3]}"] = data
+            elif sub == "ffn_norm":
+                state[f"blocks.{blk}.ln2.{parts[3]}"] = data
+            elif sub == "attn_q":
+                state[f"blocks.{blk}.attn.q_proj.weight"] = data
+            elif sub == "attn_k":
+                state[f"blocks.{blk}.attn.k_proj.weight"] = data
+            elif sub == "attn_v":
+                state[f"blocks.{blk}.attn.v_proj.weight"] = data
+            elif sub == "attn_output":
+                state[f"blocks.{blk}.attn.o_proj.weight"] = data
+            elif sub == "ffn_gate":
+                state[f"blocks.{blk}.mlp.gate.weight"] = data
+            elif sub == "ffn_up":
+                state[f"blocks.{blk}.mlp.up.weight"] = data
+            elif sub == "ffn_down":
+                state[f"blocks.{blk}.mlp.down.weight"] = data
+
+    model.load_state_dict(state, strict=False)
+    print(f" Loaded: {len(state)} tensors")
+
+    for name, param in model.named_parameters():
+        if "weight" in name and "norm" not in name and "embed" not in name:
+            param.data.copy_(quantize_to_q4(param.data))
+
+    return model
+
+
+def load_checkpoint_model(ckpt_path):
+    print(f" Loading checkpoint: {ckpt_path}")
+    state = torch.load(ckpt_path, map_location="cpu", weights_only=True)
+    has_qlora = any("qweight" in k for k in state)
+    print(f" Detected: {'QLoRA' if has_qlora else 'full'} checkpoint")
+
+    model = TinyModel(
+        vocab_size=1757, hidden=128, intermediate=640,
+        num_layers=3, num_heads=8, num_kv_heads=4,
+        max_seq_len=2048, tie_weights=True,
+    )
+
+    if has_qlora:
+        model = apply_qlora(model,
+            target_modules=["q_proj","k_proj","v_proj","o_proj","gate","up","down"],
+            r=8, alpha=16, dropout=0.0, freeze_embeds=True)
+
+    missing, unexpected = model.load_state_dict(state, strict=False)
+    if missing:
+        print(f" Missing keys: {missing}")
+    if unexpected:
+        print(f" Unexpected keys: {unexpected}")
+    model.eval()
+    return model
+
+
+def load_tokenizer():
+    from tokenizers import Tokenizer as Tk
+    tok = Tk.from_file(os.path.join(os.path.dirname(__file__), "..", "tokenizer", "tokenizer.json"))
+    return tok
+
+
+def _build_chat(system, user, tokenizer):
+    parts = [f"<|system|>\n{system}", f"<|user|>\n{user}", "<|assistant|>\n"]
+    text = "\n".join(parts)
+    return tokenizer.encode(text).ids
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--gguf", default=None)
+    parser.add_argument("--checkpoint", default=None)
+    parser.add_argument("--prompt", default="What is 2+2?")
+    parser.add_argument("--system", default="You are a helpful AI assistant.")
+    parser.add_argument("--max-tokens", type=int, default=128)
+    parser.add_argument("--temperature", type=float, default=0.7)
+    parser.add_argument("--top-p", type=float, default=0.9)
+    args = parser.parse_args()
+
+    if args.gguf and args.checkpoint:
+        print("Specify only one of --gguf or --checkpoint")
+        sys.exit(1)
+
+    model = None
+    if args.gguf:
+        if not os.path.exists(args.gguf):
+            print(f"GGUF not found: {args.gguf}")
+            sys.exit(1)
+        model = load_gguf_model(args.gguf)
+    elif args.checkpoint:
+        if not os.path.exists(args.checkpoint):
+            print(f"Checkpoint not found: {args.checkpoint}")
+            sys.exit(1)
+        model = load_checkpoint_model(args.checkpoint)
+    else:
+        print("Specify --gguf or --checkpoint")
+        sys.exit(1)
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model.to(device)
+    print(f" Device: {device}")
+
+    tok = load_tokenizer()
+    input_ids = _build_chat(args.system, args.prompt, tok)
+    input_ids = torch.tensor([input_ids], device=device)
+
+    print("\n" + tok.decode(input_ids[0].tolist()), end="", flush=True)
+
+    def stream(id_):
+        t = tok.decode([id_])
+        print(t, end="", flush=True)
+
+    out = model.generate(
+        input_ids,
+        max_new_tokens=args.max_tokens,
+        temperature=args.temperature,
+        top_p=args.top_p,
+        stream_callback=stream,
+    )
+    print()
+
+
+if __name__ == "__main__":
+    main()

model_tiny.py

@@ -4,23 +4,31 @@ import torch.nn as nn
 import torch.nn.functional as F
 
 
-def get_alibi_slopes(num_heads: int) -> torch.Tensor:
-    n = 2 ** math.ceil(math.log2(num_heads))
-    m = torch.tensor([2 ** (-(i + 1)) for i in range(n)])
-    m = m[:num_heads]
-    return m
+# ─── RPW: Relative Positional Warp (learned Fourier additive bias) ──────────
 
-
-def build_alibi_bias(num_heads: int, seq_len: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
-    slopes = get_alibi_slopes(num_heads).to(device=device, dtype=dtype)
-    pos = torch.arange(seq_len, device=device, dtype=dtype)
-    mask = pos.view(1, seq_len) - pos.view(seq_len, 1)
-    mask = mask.abs().neg().unsqueeze(0).unsqueeze(0)
-    bias = mask * slopes.view(-1, 1, 1)
-    return bias
-
-
-class ALiBiAttention(nn.Module):
+class RPW(nn.Module):
+    def __init__(self, num_heads: int, num_freqs: int = 16, max_seq_len: int = 2048):
+        super().__init__()
+        self.num_heads = num_heads
+        self.num_freqs = num_freqs
+        freqs = 1.0 / (10000.0 ** (torch.arange(num_freqs, dtype=torch.float32) / num_freqs))
+        self.register_buffer("_freqs", freqs)
+        self.W_phi = nn.Parameter(torch.zeros(num_freqs * 2, num_heads))
+        nn.init.normal_(self.W_phi, std=0.02)
+    def _make_bias(self, seq_len: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
+        pos = torch.arange(seq_len, device=device, dtype=torch.float32)
+        delta = pos.view(1, seq_len) - pos.view(seq_len, 1)
+        angles = delta.unsqueeze(-1) * self._freqs.to(device=device)
+        phi = torch.cat([angles.sin(), angles.cos()], dim=-1)
+        bias = phi @ self.W_phi
+        bias = bias.permute(2, 0, 1).unsqueeze(0)
+        causal = torch.triu(torch.full((seq_len, seq_len), float('-inf'), device=device, dtype=torch.float32), diagonal=1)
+        return (bias + causal.unsqueeze(0).unsqueeze(0)).to(dtype=dtype)
+
+
+# ─── GQA with RPW ──────────────────────────────────────────────────────────
+
+class GQAAttention(nn.Module):
     def __init__(self, hidden: int, num_heads: int, num_kv_heads: int):
         super().__init__()
         self.hidden = hidden
@@ -33,16 +41,7 @@ class ALiBiAttention(nn.Module):
         self.k_proj = nn.Linear(hidden, num_kv_heads * self.head_dim, bias=False)
         self.v_proj = nn.Linear(hidden, num_kv_heads * self.head_dim, bias=False)
         self.o_proj = nn.Linear(hidden, hidden, bias=False)
-        self._alibi_bias = None
-        self._alibi_seq_len = 0
-
-    def _get_alibi(self, seq_len: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
-        if self._alibi_bias is None or seq_len > self._alibi_seq_len:
-            bias = build_alibi_bias(self.num_heads, seq_len, device, dtype)
-            causal = torch.triu(torch.full((seq_len, seq_len), float('-inf'), device=device, dtype=dtype), diagonal=1)
-            self._alibi_bias = bias + causal
-            self._alibi_seq_len = seq_len
-        return self._alibi_bias[:, :, :seq_len, :seq_len]
+        self.rpw = RPW(num_heads, num_freqs=16)
 
     def forward(self, x):
         B, T, _ = x.shape
@@ -55,13 +54,41 @@ class ALiBiAttention(nn.Module):
             v = v.repeat_interleave(self.num_groups, dim=1)
 
         scores = (q @ k.transpose(-2, -1)) / math.sqrt(self.head_dim)
-        scores = scores + self._get_alibi(T, x.device, scores.dtype)
+        scores = scores + self.rpw._make_bias(T, x.device, scores.dtype)
 
         attn = F.softmax(scores, dim=-1, dtype=torch.float32).to(x.dtype)
         out = (attn @ v).transpose(1, 2).contiguous().view(B, T, self.hidden)
         return self.o_proj(out)
 
 
+# ─── GPP: Gated Principal Projection ──────────────────────────────────────
+
+class GPP(nn.Module):
+    def __init__(self, hidden: int, code_dim: int):
+        super().__init__()
+        self.down = nn.Linear(hidden, code_dim, bias=False)
+        self.up = nn.Linear(code_dim, hidden, bias=False)
+
+    def forward(self, x):
+        return self.up(F.silu(self.down(x)))
+
+
+# ─── VCR: Variance-Controlled Residual ─────────────────────────────────────
+
+class VCR(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.alpha = nn.Parameter(torch.tensor(1.0))
+        self.beta = nn.Parameter(torch.tensor(2.0))
+
+    def forward(self, x, delta):
+        v_exp = delta.float().norm() / x.float().norm().clamp(min=1e-8)
+        scale = torch.sigmoid(self.alpha * v_exp + self.beta)
+        return x + scale * delta
+
+
+# ─── RMSNorm ───────────────────────────────────────────────────────────────
+
 class RMSNorm(nn.Module):
     def __init__(self, dim: int, eps: float = 1e-6):
         super().__init__()
@@ -73,41 +100,36 @@ class RMSNorm(nn.Module):
         return (x.float() * norm).type_as(x) * self.weight
 
 
-class MLP(nn.Module):
-    def __init__(self, hidden: int, intermediate: int):
-        super().__init__()
-        self.gate = nn.Linear(hidden, intermediate, bias=False)
-        self.up = nn.Linear(hidden, intermediate, bias=False)
-        self.down = nn.Linear(intermediate, hidden, bias=False)
-
-    def forward(self, x):
-        return self.down(F.silu(self.gate(x)) * self.up(x))
-
+# ─── Transformer Block ────────────────────────────────────────────────────
 
 class TransformerBlock(nn.Module):
-    def __init__(self, hidden: int, intermediate: int, num_heads: int, num_kv_heads: int):
+    def __init__(self, hidden: int, code_dim: int, num_heads: int, num_kv_heads: int):
         super().__init__()
         self.ln1 = RMSNorm(hidden)
-        self.attn = ALiBiAttention(hidden, num_heads, num_kv_heads)
+        self.attn = GQAAttention(hidden, num_heads, num_kv_heads)
+        self.vcr_attn = VCR()
         self.ln2 = RMSNorm(hidden)
-        self.mlp = MLP(hidden, intermediate)
+        self.mlp = GPP(hidden, code_dim)
+        self.vcr_mlp = VCR()
 
     def forward(self, x):
-        x = x + self.attn(self.ln1(x))
-        x = x + self.mlp(self.ln2(x))
+        x = self.vcr_attn(x, self.attn(self.ln1(x)))
+        x = self.vcr_mlp(x, self.mlp(self.ln2(x)))
         return x
 
 
+# ─── TinyModel V3 ──────────────────────────────────────────────────────────
+
 class TinyModel(nn.Module):
-    def __init__(self, vocab_size=1757, hidden=128, intermediate=640,
-                 num_layers=3, num_heads=8, num_kv_heads=4, max_seq_len=2048,
+    def __init__(self, vocab_size=4096, hidden=128, code_dim=96,
+                 num_layers=6, num_heads=8, num_kv_heads=4, max_seq_len=2048,
                  tie_weights=True):
         super().__init__()
         self.hidden = hidden
         self.max_seq_len = max_seq_len
         self.token_embed = nn.Embedding(vocab_size, hidden)
         self.blocks = nn.ModuleList([
-            TransformerBlock(hidden, intermediate, num_heads, num_kv_heads)
+            TransformerBlock(hidden, code_dim, num_heads, num_kv_heads)
             for _ in range(num_layers)
         ])
         self.ln_f = RMSNorm(hidden)
@@ -115,17 +137,18 @@ class TinyModel(nn.Module):
         if tie_weights:
             self.lm_head.weight = self.token_embed.weight
 
-    def reset_weights(self, std=0.02):
+    def reset_weights(self):
         for m in self.modules():
-            if isinstance(m, nn.Linear):
-                nn.init.normal_(m.weight, mean=0.0, std=std)
-                if m.bias is not None:
-                    nn.init.zeros_(m.bias)
-            elif isinstance(m, nn.Embedding):
-                nn.init.normal_(m.weight, mean=0.0, std=std)
-            elif isinstance(m, nn.LayerNorm):
-                nn.init.ones_(m.weight)
+            if isinstance(m, (nn.Linear, nn.Embedding)) and m.weight.dim() >= 2:
+                fan_in = m.weight.shape[1] if m.weight.dim() >= 2 else 1
+                nn.init.normal_(m.weight, std=1.0 / math.sqrt(fan_in))
+            elif isinstance(m, nn.Linear) and m.bias is not None:
                 nn.init.zeros_(m.bias)
+            elif isinstance(m, RMSNorm):
+                nn.init.ones_(m.weight)
+        for m in self.modules():
+            if isinstance(m, RPW):
+                nn.init.normal_(m.W_phi, std=0.02)
 
     def forward(self, input_ids, labels=None):
         x = self.token_embed(input_ids)
@@ -145,7 +168,7 @@ class TinyModel(nn.Module):
             )
         return logits, loss
 
-    def generate(self, input_ids, max_new_tokens=128, temperature=0.7, top_p=0.9):
+    def generate(self, input_ids, max_new_tokens=128, temperature=0.7, top_p=0.9, stream_callback=None):
         self.eval()
         for _ in range(max_new_tokens):
             seq_len = input_ids.size(1)
@@ -167,65 +190,66 @@ class TinyModel(nn.Module):
             next_token = torch.multinomial(probs, num_samples=1)
             input_ids = torch.cat([input_ids, next_token], dim=1)
 
+            if stream_callback:
+                stream_callback(next_token.item())
+
             if next_token.item() == 2:
                 break
         return input_ids
 
 
-class LoRALinear(nn.Module):
-    def __init__(self, original: nn.Linear, r: int = 8, alpha: float = 16, dropout: float = 0.0):
-        super().__init__()
-        self.linear = original
-        self.r = r
-        self.alpha = alpha
-        self.scaling = alpha / r
-        self.dropout = nn.Dropout(dropout) if dropout > 0 else nn.Identity()
-        in_dim, out_dim = original.in_features, original.out_features
-        self.lora_A = nn.Parameter(torch.zeros(r, in_dim))
-        self.lora_B = nn.Parameter(torch.zeros(out_dim, r))
-        nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
-        self.linear.weight.requires_grad = False
-        if self.linear.bias is not None:
-            self.linear.bias.requires_grad = False
-
-    def forward(self, x):
-        return self.linear(x) + self.dropout(x) @ self.lora_A.T @ self.lora_B.T * self.scaling
-
-
-def apply_lora(model, target_modules=None, r=8, alpha=16, dropout=0.0):
-    if target_modules is None:
-        target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate", "up", "down"]
-    lora_params = 0
-    for name, module in model.named_modules():
-        if not isinstance(module, nn.Linear):
-            continue
-        key = name.split(".")[-1]
-        if key not in target_modules:
-            continue
-        parent = model
-        parts = name.split(".")
-        for p in parts[:-1]:
-            parent = getattr(parent, p)
-        lora = LoRALinear(module, r=r, alpha=alpha, dropout=dropout)
-        setattr(parent, parts[-1], lora)
-        lora_params += 2 * r * module.in_features + module.out_features * r
-    n = sum(p.numel() for p in model.parameters() if p.requires_grad)
-    print(f"LoRA applied: {lora_params:,} trainable params (total trainable: {n:,})")
+# ─── Restore from V2 checkpoint ────────────────────────────────────────────
+
+def restore_from_v2(v2_ckpt_path: str, strict: bool = False) -> TinyModel:
+    raw = torch.load(v2_ckpt_path, map_location="cpu", weights_only=True)
+    v2_sd = raw["model"]
+    v2_embed = v2_sd["token_embed.weight"]
+
+    model = TinyModel(vocab_size=4096, hidden=128, code_dim=96,
+                      num_layers=6, num_heads=8, num_kv_heads=4,
+                      max_seq_len=2048, tie_weights=True)
+    model.reset_weights()
+
+    restore_map = {}
+    restore_map["token_embed.weight"] = v2_embed
+    restore_map["ln_f.weight"] = v2_sd["ln_f.weight"]
+    for i in range(3):
+        restore_map[f"blocks.{i}.ln1.weight"] = v2_sd[f"blocks.{i}.ln1.weight"]
+        restore_map[f"blocks.{i}.ln2.weight"] = v2_sd[f"blocks.{i}.ln2.weight"]
+
+    restored = 0
+    for k, v in restore_map.items():
+        if k in model.state_dict():
+            target = model.state_dict()[k]
+            if target.shape == v.shape:
+                target.copy_(v)
+                restored += 1
+            elif target.dim() == 2 and v.dim() == 2 and target.shape[1] == v.shape[1]:
+                d = min(target.shape[0], v.shape[0])
+                target[:d].copy_(v[:d])
+                restored += 1
+
+    model.lm_head.weight = model.token_embed.weight
+
+    total_v2 = sum(p.numel() for p in v2_sd.values())
+    restored_params = sum(v.numel() for v in restore_map.values())
+    v3_params = sum(p.numel() for p in model.parameters())
+    print(f"V2 checkpoint: {len(v2_sd)} keys, {total_v2:,} params")
+    print(f"V3 model: {v3_params:,} params")
+    print(f"Restored: {restored} tensors ({restored_params:,} params, {100*restored_params/v3_params:.1f}%)")
     return model
 
 
+# ─── Helpers ───────────────────────────────────────────────────────────────
+
 def count_params(model):
     return sum(p.numel() for p in model.parameters())
 
 
 def create_model():
-    model = TinyModel(
-        vocab_size=1757, hidden=128, intermediate=640,
-        num_layers=3, num_heads=8, num_kv_heads=4,
-        max_seq_len=2048, tie_weights=True,
-    )
+    model = restore_from_v2("checkpoint.pt")
     n = count_params(model)
-    print(f"TinyModel: {n:,} params ({n/1e6:.2f}M)")
+    print(f"TinyModel V3: {n:,} params ({n/1e6:.2f}M)")
     return model
 
 
@@ -236,8 +260,3 @@ if __name__ == "__main__":
     print(f"Forward OK: logits {logits.shape}, loss {loss.item():.4f}")
     gen = m.generate(x, max_new_tokens=10)
     print(f"Generate OK: {gen.shape}")
-    m2 = create_model()
-    apply_lora(m2)
-    x2 = torch.randint(0, 100, (1, 16))
-    logits2, _ = m2(x2)
-    print(f"LoRA forward OK: {logits2.shape}")

prepare_tiny_data.py

@@ -0,0 +1,210 @@
+"""
+Prepare data for Lumia-Tiny training.
+
+Usage:
+  python3 scripts/prepare_tiny_data.py                    # use existing data/data.jsonl
+  python3 scripts/prepare_tiny_data.py --generate 50      # generate N synthetic examples
+  python3 scripts/prepare_tiny_data.py --hf-sample 100    # sample from HF dataset
+  python3 scripts/prepare_tiny_data.py --to-jsonl         # convert to simple format
+"""
+
+import os, sys, json, argparse, random
+from pathlib import Path
+
+
+SYSTEM_PROMPTS = [
+    "You are a helpful AI assistant who solves problems step by step.",
+    "You are a precise programming assistant who writes clean, correct code.",
+    "You are a math tutor who explains concepts clearly.",
+    "You are a reasoning assistant who thinks through problems carefully.",
+]
+
+INSTRUCTIONS = [
+    "What is 2+2?",
+    "Explain how a binary search works.",
+    "Write a Python function to reverse a linked list.",
+    "What is the capital of France?",
+    "Explain the concept of recursion.",
+    "Write a function to check if a string is a palindrome.",
+    "What is the difference between TCP and UDP?",
+    "Explain how gradient descent works.",
+    "Write a quicksort implementation.",
+    "What is the time complexity of binary search?",
+    "Explain the concept of overfitting in machine learning.",
+    "Write a function to find the nth Fibonacci number.",
+    "What is the difference between a list and a tuple in Python?",
+    "Explain the CAP theorem.",
+    "Write a function to merge two sorted arrays.",
+    "What is the Pythagorean theorem?",
+    "Explain how HTTP works.",
+    "Write a function to calculate the factorial of a number.",
+    "What is the difference between SQL and NoSQL?",
+    "Explain the concept of polymorphism in OOP.",
+    "What is 15 * 7?",
+    "Write a function to find the maximum subarray sum.",
+    "Explain how DNS resolution works.",
+    "What is the difference between stack and heap memory?",
+    "Write a function to detect cycles in a linked list.",
+    "What is the derivative of x^2?",
+    "Explain the concept of normalization in databases.",
+    "Write a function to perform binary search in a sorted array.",
+    "What is the difference between symmetric and asymmetric encryption?",
+    "Explain how a hash table works.",
+    "What is 144 / 12?",
+    "Write a function to find all prime numbers up to n.",
+    "Explain the difference between REST and GraphQL.",
+    "What is the chain rule in calculus?",
+    "Write a function to serialize a binary tree.",
+    "What is the difference between HTTP and HTTPS?",
+    "Explain the concept of attention in transformers.",
+    "Write a function to compute the edit distance between two strings.",
+    "What is the integral of 2x?",
+    "Explain how a blockchain works.",
+]
+
+
+RESPONSES = [
+    "4",
+    "Binary search works by repeatedly dividing a sorted array in half, comparing the target value to the middle element, and eliminating the half that cannot contain the target.",
+    "```python\ndef reverse_linked_list(head):\n    prev = None\n    curr = head\n    while curr:\n        next_node = curr.next\n        curr.next = prev\n        prev = curr\n        curr = next_node\n    return prev\n```",
+    "Paris",
+    "Recursion is a programming technique where a function calls itself to solve a problem by breaking it down into smaller subproblems. Each recursive call works on a smaller instance until reaching a base case.",
+    "```python\ndef is_palindrome(s):\n    s = ''.join(c.lower() for c in s if c.isalnum())\n    return s == s[::-1]\n```",
+    "TCP is connection-oriented and guarantees delivery with error checking. UDP is connectionless and faster but does not guarantee delivery.",
+    "Gradient descent is an optimization algorithm that iteratively adjusts parameters to minimize a loss function by moving in the direction of the negative gradient.",
+    "```python\ndef quicksort(arr):\n    if len(arr) <= 1:\n        return arr\n    pivot = arr[len(arr)//2]\n    left = [x for x in arr if x < pivot]\n    middle = [x for x in arr if x == pivot]\n    right = [x for x in arr if x > pivot]\n    return quicksort(left) + middle + quicksort(right)\n```",
+    "O(log n)",
+    "Overfitting occurs when a model learns training data too well, including noise, and performs poorly on new data. It can be addressed by regularization, dropout, or early stopping.",
+    "```python\ndef fibonacci(n):\n    if n <= 1:\n        return n\n    a, b = 0, 1\n    for _ in range(2, n+1):\n        a, b = b, a + b\n    return b\n```",
+    "Lists are mutable, tuples are immutable. Lists use square brackets [], tuples use parentheses ().",
+    "The CAP theorem states that a distributed system cannot simultaneously guarantee Consistency, Availability, and Partition tolerance.",
+    "```python\ndef merge_sorted(a, b):\n    result = []\n    i = j = 0\n    while i < len(a) and j < len(b):\n        if a[i] < b[j]:\n            result.append(a[i]); i += 1\n        else:\n            result.append(b[j]); j += 1\n    result.extend(a[i:]); result.extend(b[j:])\n    return result\n```",
+    "a² + b² = c², where a and b are legs and c is the hypotenuse.",
+    "HTTP is a request-response protocol where a client sends a request to a server, which replies with a status code and body.",
+    "```python\ndef factorial(n):\n    if n <= 1:\n        return 1\n    return n * factorial(n-1)\n```",
+    "SQL databases are relational with structured schemas and ACID properties. NoSQL databases are non-relational, schema-flexible, and scale horizontally.",
+    "Polymorphism allows objects of different types to respond to the same interface. In Python, duck typing lets any object with the required methods be used.",
+    "105",
+    "```python\ndef max_subarray_sum(arr):\n    max_ending = max_sofar = arr[0]\n    for x in arr[1:]:\n        max_ending = max(x, max_ending + x)\n        max_sofar = max(max_sofar, max_ending)\n    return max_sofar\n```",
+    "DNS converts domain names to IP addresses. The resolver queries root -> TLD -> authoritative servers to find the IP.",
+    "Stack is LIFO (last-in-first-out) for local variables and function calls. Heap is for dynamically allocated memory with longer lifespan.",
+    "```python\ndef has_cycle(head):\n    slow = fast = head\n    while fast and fast.next:\n        slow = slow.next\n        fast = fast.next.next\n        if slow == fast:\n            return True\n    return False\n```",
+    "2x",
+    "Normalization organizes data to reduce redundancy. Forms: 1NF (atomic columns), 2NF (no partial dependency), 3NF (no transitive dependency).",
+    "```python\ndef binary_search(arr, target):\n    lo, hi = 0, len(arr) - 1\n    while lo <= hi:\n        mid = (lo + hi) // 2\n        if arr[mid] == target:\n            return mid\n        elif arr[mid] < target:\n            lo = mid + 1\n        else:\n            hi = mid - 1\n    return -1\n```",
+    "Symmetric encryption uses the same key for both encryption and decryption. Asymmetric uses a public key to encrypt and a private key to decrypt.",
+    "A hash table uses a hash function to map keys to array indices, providing O(1) average lookup. Collisions are handled by chaining or open addressing.",
+    "12",
+    "```python\ndef sieve(n):\n    primes = [True] * (n+1)\n    primes[0] = primes[1] = False\n    for i in range(2, int(n**0.5)+1):\n        if primes[i]:\n            for j in range(i*i, n+1, i):\n                primes[j] = False\n    return [i for i, p in enumerate(primes) if p]\n```",
+    "REST uses standard HTTP methods with resource-based URLs. GraphQL uses a single endpoint with a query language for flexible data fetching.",
+    "d/dx f(g(x)) = f'(g(x)) * g'(x)",
+    "```python\ndef serialize_tree(root):\n    def encode(node):\n        if not node:\n            return 'null'\n        return f\"{node.val},{encode(node.left)},{encode(node.right)}\"\n    return encode(root)\n```",
+    "HTTPS adds TLS encryption on top of HTTP, providing confidentiality and integrity.",
+    "Attention computes weighted combinations of values based on query-key similarity. The Transformer uses multi-head attention to capture different relationship types.",
+    "```python\ndef edit_distance(a, b):\n    m, n = len(a), len(b)\n    dp = [[0]*(n+1) for _ in range(m+1)]\n    for i in range(m+1): dp[i][0] = i\n    for j in range(n+1): dp[0][j] = j\n    for i in range(1, m+1):\n        for j in range(1, n+1):\n            if a[i-1] == b[j-1]:\n                dp[i][j] = dp[i-1][j-1]\n            else:\n                dp[i][j] = 1 + min(dp[i-1][j], dp[i][j-1], dp[i-1][j-1])\n    return dp[m][n]\n```",
+    "x² + C",
+    "A blockchain is a distributed ledger where data is stored in blocks linked by cryptographic hashes. Each block contains a hash of the previous block, forming a chain.",
+]
+
+
+def generate_synthetic(n=50):
+    random.seed(42)
+    samples = []
+    for i in range(n):
+        sys_idx = i % len(SYSTEM_PROMPTS)
+        inst_idx = i % len(INSTRUCTIONS)
+        resp_idx = i % len(RESPONSES)
+        samples.append({
+            "system": SYSTEM_PROMPTS[sys_idx],
+            "instruction": INSTRUCTIONS[inst_idx],
+            "input": "",
+            "output": RESPONSES[resp_idx],
+        })
+    return samples
+
+
+def load_and_split(data_path, test_ratio=0.1):
+    with open(data_path) as f:
+        data = [json.loads(line) for line in f]
+    random.seed(42)
+    random.shuffle(data)
+    split = int(len(data) * (1 - test_ratio))
+    return data[:split], data[split:]
+
+
+def convert_to_messages(data):
+    out = []
+    for item in data:
+        user_msg = item["instruction"]
+        if item.get("input", ""):
+            user_msg += "\n" + item["input"]
+        out.append({
+            "messages": [
+                {"role": "system", "content": item.get("system", "")},
+                {"role": "user", "content": user_msg},
+                {"role": "assistant", "content": item["output"]},
+            ]
+        })
+    return out
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Prepare data for Lumia-Tiny")
+    parser.add_argument("--generate", type=int, default=None,
+                        help="Generate N synthetic examples")
+    parser.add_argument("--to-jsonl", action="store_true",
+                        help="Convert data/data.jsonl to messages format")
+    parser.add_argument("--output", default="data/tiny_data.jsonl",
+                        help="Output path")
+    args = parser.parse_args()
+
+    output_path = args.output
+    os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
+
+    if args.generate:
+        samples = generate_synthetic(args.generate)
+        with open(output_path, "w") as f:
+            for s in samples:
+                f.write(json.dumps(s) + "\n")
+        print(f" Generated {len(samples)} synthetic examples → {output_path}")
+        return
+
+    if args.to_jsonl:
+        data_path = "data/data.jsonl"
+        if not os.path.exists(data_path):
+            print(f" Error: {data_path} not found")
+            sys.exit(1)
+        with open(data_path) as f:
+            data = [json.loads(line) for line in f]
+        messages = convert_to_messages(data)
+        base = os.path.splitext(output_path)[0]
+        with open(f"{base}_messages.jsonl", "w") as f:
+            for m in messages:
+                f.write(json.dumps(m) + "\n")
+        print(f" Converted {len(messages)} examples → {base}_messages.jsonl")
+        return
+
+    # Default: just report stats
+    data_path = "data/data.jsonl"
+    if os.path.exists(data_path):
+        with open(data_path) as f:
+            lines = f.readlines()
+        print(f" Data file: {data_path}")
+        print(f" Samples:   {len(lines)}")
+        print(f" Size:      {os.path.getsize(data_path):,} bytes")
+        sample = json.loads(lines[0])
+        print(f" Fields:    {list(sample.keys())}")
+        print(f" Keys:      system, instruction, input, output")
+        print(f"")
+        print(f" To generate synthetic data:")
+        print(f"   python3 scripts/prepare_tiny_data.py --generate 200")
+        print(f"")
+        print(f" To convert to messages format:")
+        print(f"   python3 scripts/prepare_tiny_data.py --to-jsonl")
+    else:
+        print(f" No data found. Generate with:")
+        print(f"   python3 scripts/prepare_tiny_data.py --generate 200")
+
+
+if __name__ == "__main__":
+    main()

quantize_gguf.py

@@ -0,0 +1,119 @@
+"""
+Export TinyModel weights → GGUF INT4 (Q4_K_M).
+
+Usage:
+  python3 scripts/quantize_gguf.py                          # from outputs/tiny-sft/final/model.pt
+  python3 scripts/quantize_gguf.py --checkpoint path/to/model.pt
+  python3 scripts/quantize_gguf.py --checkpoint path/to/model.pt --output model.gguf
+"""
+
+import os, sys, argparse, json
+import torch
+import gguf
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+from scripts.model_tiny import TinyModel
+
+
+def export_gguf(checkpoint_path, output_path):
+    print(f" Loading checkpoint: {checkpoint_path}")
+
+    model = TinyModel(
+        vocab_size=1757, hidden=128, intermediate=640,
+        num_layers=3, num_heads=8, num_kv_heads=4,
+        max_seq_len=2048, tie_weights=True,
+    )
+    state = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
+    model.load_state_dict(state)
+    model.eval()
+
+    n = sum(p.numel() for p in model.parameters())
+    print(f" Params: {n:,}")
+
+    print(f" Writing GGUF: {output_path}")
+    writer = gguf.GGUFWriter(output_path, "tiny")
+
+    # Metadata
+    writer.add_context_length(2048)
+    writer.add_embedding_length(model.hidden)
+    writer.add_block_count(len(model.blocks))
+    writer.add_head_count(model.blocks[0].attn.num_heads)
+    writer.add_head_count_kv(model.blocks[0].attn.num_kv_heads)
+    writer.add_feed_forward_length(model.blocks[0].mlp.up.weight.shape[0])
+    writer.add_layer_norm_rms_eps(1e-6)
+
+    # Tensor names for llama-like GGUF format
+    name_map = {
+        "token_embed.weight": "token_embd.weight",
+        "ln_f.weight": "output_norm.weight",
+        "lm_head.weight": "output.weight",
+    }
+
+    def tensor_name(key):
+        parts = key.split(".")
+        if parts[0] == "blocks":
+            blk = int(parts[1])
+            sub = parts[2]
+            if sub == "ln1":
+                return f"blk.{blk}.attn_norm.{parts[3]}"
+            elif sub == "ln2":
+                return f"blk.{blk}.ffn_norm.{parts[3]}"
+            elif sub == "attn":
+                proj_map = {
+                    "q_proj": "attn_q",
+                    "k_proj": "attn_k",
+                    "v_proj": "attn_v",
+                    "o_proj": "attn_output",
+                }
+                return f"blk.{blk}.{proj_map[parts[3]]}.weight"
+            elif sub == "mlp":
+                proj_map = {
+                    "gate": "ffn_gate",
+                    "up": "ffn_up",
+                    "down": "ffn_down",
+                }
+                return f"blk.{blk}.{proj_map[parts[3]]}.weight"
+        return name_map.get(key, key)
+
+    # Write all tensors as fp32 first, GGUF will quantize
+    for key, param in model.state_dict().items():
+        tname = tensor_name(key)
+        data = param.contiguous().float().numpy()
+        writer.add_tensor(tname, data)
+
+    writer.write_header_to_file()
+    writer.write_kv_data_to_file()
+    writer.write_tensors_to_file()
+    writer.close()
+
+    print(f" Done → {output_path}")
+    print(f" Run GGUF quantization: the gguf library handles Q4_K_M inline")
+
+    import struct, os
+    fsize = os.path.getsize(output_path)
+    print(f" Raw size: {fsize/1024**2:.1f}MB")
+
+    return True
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--checkpoint", default=None)
+    parser.add_argument("--output", default="outputs/tiny-sft/tiny.gguf")
+    parser.add_argument("--quantize", default="q4_k_m",
+                        choices=["q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "q4_k_m", "q5_k_m", "q6_k", "q8_k_m"])
+    args = parser.parse_args()
+
+    if args.checkpoint is None:
+        args.checkpoint = "outputs/tiny-sft/final/model.pt"
+        if not os.path.exists(args.checkpoint):
+            print(f"No checkpoint found at {args.checkpoint}")
+            print("Train first: bash scripts/train_tiny.sh")
+            sys.exit(1)
+
+    export_gguf(args.checkpoint, args.output)
+    print(f"GGUF file: {args.output}")
+
+
+if __name__ == "__main__":
+    main()

tokenizer.json

@@ -1 +1 @@
-{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"content":"<unk>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":1,"content":"<s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":2,"content":"</s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":3,"content":"<pad>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":4,"content":"<|system|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":5,"content":"<|user|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":6,"content":"<|assistant|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":7,"content":"<think>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":8,"content":"</think>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":9,"content":"[INST]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":10,"content":"[/INST]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":11,"content":"<|begin_of_thought|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":12,"content":"<|end_of_thought|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":13,"content":"<|reflect|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":14,"content":"<|revise|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":15,"content":"<|verify|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":16,"content":"<|code|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":17,"content":"<|text|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":18,"content":"<|math|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":19,"content":"<|think|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":20,"content":"<|answer|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":21,"content":"<|step|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":22,"content":"<|reason|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":23,"content":"<|check|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":24,"content":"<|output|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":25,"content":"<|plan|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":26,"content":"<|solve|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":27,"content":"<|analyze|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":28,"content":"<|conclude|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":29,"content":"<|approach|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":30,"content":"<|alternative|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":31,"content":"<|summary|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":32,"content":"<|question|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":33,"content":"<|hint|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":34,"content":"<|example|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":35,"content":"<|correct|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":36,"content":"<|incorrect|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":37,"content":"<|feedback|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":38,"content":"<|start|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":39,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":40,"content":"<|sep|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":41,"content":"<|cls|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":42,"content":"<|tool|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":43,"content":"<|function|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":44,"content":"<|result|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":45,"content":"<|input|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":46,"content":"<|detect|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":47,"content":"<|context|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":48,"content":"<|proof|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":49,"content":"<|lemma|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":50,"content":"<|theorem|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}],"normalizer":null,"pre_tokenizer":{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":true},"post_processor":null,"decoder":{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":true},"model":{"type":"BPE","dropout":null,"unk_token":"<unk>","byte_fallback":true,"vocab":{"<unk>":0,"<s>":1,"</s>":2,"<pad>":3,"<|system|>":4,"<|user|>":5,"<|assistant|>":6,"<think>":7,"</think>":8,"[INST]":9,"[/INST]":10,"<|begin_of_thought|>":11,"<|end_of_thought|>":12,"<|reflect|>":13,"<|revise|>":14,"<|verify|>":15,"<|code|>":16,"<|text|>":17,"<|math|>":18,"<|think|>":19,"<|answer|>":20,"<|step|>":21,"<|reason|>":22,"<|check|>":23,"<|output|>":24,"<|plan|>":25,"<|solve|>":26,"<|analyze|>":27,"<|conclude|>":28,"<|approach|>":29,"<|alternative|>":30,"<|summary|>":31,"<|question|>":32,"<|hint|>":33,"<|example|>":34,"<|correct|>":35,"<|incorrect|>":36,"<|feedback|>":37,"<|start|>":38,"<|end|>":39,"<|sep|>":40,"<|cls|>":41,"<|tool|>":42,"<|function|>":43,"<|result|>":44,"<|input|>":45,"<|detect|>":46,"<|context|>":47,"<|proof|>":48,"<|lemma|>":49,"<|theorem|>":50,"Ā":51,"ā":52,"Ă":53,"ă":54,"Ą":55,"ą":56,"Ć":57,"ć":58,"Ĉ":59,"ĉ":60,"Ċ":61,"ċ":62,"Č":63,"č":64,"Ď":65,"ď":66,"Đ":67,"đ":68,"Ē":69,"ē":70,"Ĕ":71,"ĕ":72,"Ė":73,"ė":74,"Ę":75,"ę":76,"Ě":77,"ě":78,"Ĝ":79,"ĝ":80,"Ğ":81,"ğ":82,"Ġ":83,"!":84,"\"":85,"#":86,"$":87,"%":88,"&":89,"'":90,"(":91,")":92,"*":93,"+":94,",":95,"-":96,".":97,"/":98,"0":99,"1":100,"2":101,"3":102,"4":103,"5":104,"6":105,"7":106,"8":107,"9":108,":":109,";":110,"<":111,"=":112,">":113,"?":114,"@":115,"A":116,"B":117,"C":118,"D":119,"E":120,"F":121,"G":122,"H":123,"I":124,"J":125,"K":126,"L":127,"M":128,"N":129,"O":130,"P":131,"Q":132,"R":133,"S":134,"T":135,"U":136,"V":137,"W":138,"X":139,"Y":140,"Z":141,"[":142,"\\":143,"]":144,"^":145,"_":146,"`":147,"a":148,"b":149,"c":150,"d":151,"e":152,"f":153,"g":154,"h":155,"i":156,"j":157,"k":158,"l":159,"m":160,"n":161,"o":162,"p":163,"q":164,"r":165,"s":166,"t":167,"u":168,"v":169,"w":170,"x":171,"y":172,"z":173,"{":174,"|":175,"}":176,"~":177,"ġ":178,"Ģ":179,"ģ":180,"Ĥ":181,"ĥ":182,"Ħ":183,"ħ":184,"Ĩ":185,"ĩ":186,"Ī":187,"ī":188,"Ĭ":189,"ĭ":190,"Į":191,"į":192,"İ":193,"ı":194,"IJ":195,"ij":196,"Ĵ":197,"ĵ":198,"Ķ":199,"ķ":200,"ĸ":201,"Ĺ":202,"ĺ":203,"Ļ":204,"ļ":205,"Ľ":206,"ľ":207,"Ŀ":208,"ŀ":209,"Ł":210,"ł":211,"¡":212,"¢":213,"£":214,"¤":215,"¥":216,"¦":217,"§":218,"¨":219,"©":220,"ª":221,"«":222,"¬":223,"Ń":224,"®":225,"¯":226,"°":227,"±":228,"²":229,"³":230,"´":231,"µ":232,"¶":233,"·":234,"¸":235,"¹":236,"º":237,"»":238,"¼":239,"½":240,"¾":241,"¿":242,"À":243,"Á":244,"Â":245,"Ã":246,"Ä":247,"Å":248,"Æ":249,"Ç":250,"È":251,"É":252,"Ê":253,"Ë":254,"Ì":255,"Í":256,"Î":257,"Ï":258,"Ð":259,"Ñ":260,"Ò":261,"Ó":262,"Ô":263,"Õ":264,"Ö":265,"×":266,"Ø":267,"Ù":268,"Ú":269,"Û":270,"Ü":271,"Ý":272,"Þ":273,"ß":274,"à":275,"á":276,"â":277,"ã":278,"ä":279,"å":280,"æ":281,"ç":282,"è":283,"é":284,"ê":285,"ë":286,"ì":287,"í":288,"î":289,"ï":290,"ð":291,"ñ":292,"ò":293,"ó":294,"ô":295,"õ":296,"ö":297,"÷":298,"ø":299,"ù":300,"ú":301,"û":302,"ü":303,"ý":304,"þ":305,"ÿ":306,"the":307,"be":308,"to":309,"of":310,"and":311,"in":312,"that":313,"have":314,"it":315,"for":316,"not":317,"on":318,"with":319,"he":320,"as":321,"you":322,"do":323,"at":324,"this":325,"but":326,"his":327,"by":328,"from":329,"they":330,"we":331,"say":332,"her":333,"she":334,"or":335,"an":336,"will":337,"my":338,"one":339,"all":340,"would":341,"there":342,"their":343,"what":344,"so":345,"up":346,"out":347,"if":348,"about":349,"who":350,"get":351,"which":352,"go":353,"me":354,"when":355,"make":356,"can":357,"like":358,"time":359,"no":360,"just":361,"him":362,"know":363,"take":364,"people":365,"into":366,"year":367,"your":368,"good":369,"some":370,"could":371,"them":372,"see":373,"other":374,"than":375,"then":376,"now":377,"look":378,"only":379,"come":380,"its":381,"over":382,"think":383,"also":384,"back":385,"after":386,"use":387,"two":388,"how":389,"our":390,"work":391,"first":392,"well":393,"way":394,"even":395,"new":396,"want":397,"because":398,"any":399,"these":400,"give":401,"day":402,"most":403,"us":404,"is":405,"was":406,"are":407,"were":408,"been":409,"has":410,"had":411,"did":412,"does":413,"am":414,"being":415,"having":416,"doing":417,"saying":418,"going":419,"getting":420,"making":421,"knowing":422,"taking":423,"thinking":424,"coming":425,"came":426,"goes":427,"gone":428,"went":429,"saw":430,"seen":431,"seeing":432,"said":433,"says":434,"got":435,"gotten":436,"made":437,"makes":438,"knew":439,"known":440,"knows":441,"thought":442,"thinks":443,"took":444,"taken":445,"takes":446,"gave":447,"given":448,"gives":449,"giving":450,"find":451,"found":452,"finds":453,"finding":454,"tell":455,"told":456,"tells":457,"telling":458,"ask":459,"asked":460,"asks":461,"asking":462,"show":463,"showed":464,"shown":465,"shows":466,"showing":467,"try":468,"tried":469,"tries":470,"trying":471,"leave":472,"left":473,"leaves":474,"leaving":475,"call":476,"called":477,"calls":478,"calling":479,"keep":480,"kept":481,"keeps":482,"keeping":483,"let":484,"lets":485,"letting":486,"begin":487,"began":488,"begun":489,"begins":490,"beginning":491,"seem":492,"seemed":493,"seems":494,"seeming":495,"help":496,"helped":497,"helps":498,"helping":499,"turn":500,"turned":501,"turns":502,"turning":503,"start":504,"started":505,"starts":506,"starting":507,"bring":508,"brought":509,"brings":510,"bringing":511,"happen":512,"happened":513,"happens":514,"happening":515,"write":516,"wrote":517,"written":518,"writes":519,"writing":520,"provide":521,"provided":522,"provides":523,"providing":524,"consider":525,"considered":526,"considers":527,"considering":528,"appear":529,"appeared":530,"appears":531,"appearing":532,"follow":533,"followed":534,"follows":535,"following":536,"change":537,"changed":538,"changes":539,"changing":540,"form":541,"formed":542,"forms":543,"forming":544,"need":545,"needed":546,"needs":547,"needing":548,"set":549,"sets":550,"setting":551,"put":552,"puts":553,"putting":554,"run":555,"runs":556,"running":557,"move":558,"moved":559,"moves":560,"moving":561,"stand":562,"stood":563,"stands":564,"standing":565,"win":566,"won":567,"wins":568,"winning":569,"play":570,"played":571,"plays":572,"playing":573,"point":574,"points":575,"pointed":576,"pointing":577,"large":578,"small":579,"big":580,"little":581,"long":582,"short":583,"high":584,"low":585,"old":586,"young":587,"great":588,"important":589,"different":590,"same":591,"many":592,"much":593,"more":594,"few":595,"own":596,"very":597,"such":598,"still":599,"too":600,"here":601,"where":602,"why":603,"while":604,"though":605,"although":606,"until":607,"since":608,"before":609,"during":610,"without":611,"within":612,"between":613,"through":614,"across":615,"around":616,"above":617,"below":618,"under":619,"again":620,"ever":621,"never":622,"always":623,"often":624,"sometimes":625,"together":626,"alone":627,"already":628,"yet":629,"almost":630,"quite":631,"rather":632,"bad":633,"better":634,"worse":635,"best":636,"worst":637,"less":638,"every":639,"each":640,"both":641,"either":642,"neither":643,"none":644,"thing":645,"things":646,"ways":647,"times":648,"years":649,"days":650,"week":651,"weeks":652,"month":653,"months":654,"part":655,"parts":656,"place":657,"places":658,"case":659,"cases":660,"world":661,"worlds":662,"number":663,"numbers":664,"group":665,"groups":666,"system":667,"systems":668,"program":669,"programs":670,"data":671,"information":672,"problem":673,"problems":674,"solution":675,"solutions":676,"method":677,"methods":678,"result":679,"results":680,"process":681,"processes":682,"function":683,"functions":684,"value":685,"values":686,"type":687,"types":688,"state":689,"states":690,"model":691,"models":692,"level":693,"levels":694,"line":695,"lines":696,"file":697,"files":698,"code":699,"codes":700,"list":701,"lists":702,"array":703,"arrays":704,"object":705,"objects":706,"class":707,"classes":708,"property":709,"properties":710,"input":711,"inputs":712,"output":713,"outputs":714,"return":715,"returns":716,"define":717,"defined":718,"defines":719,"defining":720,"declare":721,"declared":722,"import":723,"imports":724,"export":725,"exports":726,"include":727,"includes":728,"public":729,"private":730,"protected":731,"static":732,"final":733,"const":734,"void":735,"int":736,"float":737,"double":738,"char":739,"bool":740,"string":741,"true":742,"false":743,"null":744,"None":745,"nil":746,"undefined":747,"else":748,"elif":749,"switch":750,"default":751,"break":752,"catch":753,"finally":754,"throw":755,"raise":756,"except":757,"yield":758,"await":759,"async":760,"defer":761,"lambda":762,"map":763,"filter":764,"reduce":765,"sort":766,"delete":767,"free":768,"alloc":769,"realloc":770,"print":771,"printf":772,"println":773,"log":774,"debug":775,"error":776,"len":777,"size":778,"length":779,"count":780,"sum":781,"max":782,"min":783,"abs":784,"pow":785,"sqrt":786,"floor":787,"ceil":788,"round":789,"sin":790,"cos":791,"tan":792,"atan":793,"exp":794,"zero":795,"three":796,"four":797,"five":798,"six":799,"seven":800,"eight":801,"nine":802,"ten":803,"second":804,"third":805,"last":806,"next":807,"previous":808,"current":809,"initial":810,"primary":811,"secondary":812,"main":813,"basic":814,"advanced":815,"simple":816,"complex":817,"single":818,"multiple":819,"add":820,"sub":821,"mul":822,"div":823,"mod":824,"inc":825,"dec":826,"push":827,"pop":828,"shift":829,"unshift":830,"insert":831,"remove":832,"append":833,"prepend":834,"concat":835,"join":836,"split":837,"slice":838,"open":839,"close":840,"read":841,"load":842,"save":843,"create":844,"update":845,"select":846,"merge":847,"end":848,"stop":849,"pause":850,"resume":851,"enable":852,"disable":853,"allow":854,"deny":855,"grant":856,"revoke":857,"user":858,"users":859,"name":860,"names":861,"id":862,"ids":863,"key":864,"keys":865,"field":866,"fields":867,"table":868,"tables":869,"row":870,"rows":871,"column":872,"columns":873,"index":874,"indexes":875,"indices":876,"query":877,"queries":878,"avg":879,"total":880,"page":881,"pages":882,"home":883,"login":884,"logout":885,"signup":886,"errors":887,"warning":888,"warnings":889,"info":890,"success":891,"failure":892,"status":893,"message":894,"messages":895,"request":896,"requests":897,"response":898,"responses":899,"client":900,"server":901,"api":902,"endpoint":903,"route":904,"http":905,"https":906,"url":907,"uri":908,"port":909,"host":910,"config":911,"configuration":912,"settings":913,"option":914,"options":915,"param":916,"params":917,"parameter":918,"parameters":919,"arg":920,"args":921,"argument":922,"arguments":923,"kwargs":924,"path":925,"dir":926,"directory":927,"dirs":928,"folder":929,"folders":930,"item":931,"items":932,"element":933,"elements":934,"entry":935,"entries":936,"note":937,"notes":938,"text":939,"texts":940,"content":941,"contents":942,"source":943,"sources":944,"target":945,"targets":946,"ref":947,"refs":948,"struct":949,"structs":950,"union":951,"unions":952,"enum":953,"enums":954,"impl":955,"implement":956,"implementation":957,"interface":958,"abstract":959,"virtual":960,"override":961,"overload":962,"base":963,"derived":964,"parent":965,"child":966,"children":967,"root":968,"leaf":969,"node":970,"nodes":971,"edge":972,"edges":973,"tree":974,"graph":975,"queue":976,"stack":977,"heap":978,"dict":979,"dictionary":980,"hash":981,"hashmap":982,"link":983,"links":984,"linked":985,"pointer":986,"pointers":987,"thread":988,"threads":989,"task":990,"tasks":991,"sync":992,"lock":993,"mutex":994,"semaphore":995,"buffer":996,"buffers":997,"cache":998,"cached":999,"pool":1000,"pools":1001,"memory":1002,"disk":1003,"network":1004,"socket":1005,"sockets":1006,"stream":1007,"streams":1008,"packet":1009,"packets":1010,"frame":1011,"meta":1012,"metadata":1013,"header":1014,"headers":1015,"body":1016,"payload":1017,"token":1018,"tokens":1019,"session":1020,"cookie":1021,"cookies":1022,"auth":1023,"register":1024,"password":1025,"salt":1026,"encrypt":1027,"decrypt":1028,"encode":1029,"decode":1030,"cert":1031,"certificate":1032,"train":1033,"training":1034,"trained":1035,"test":1036,"testing":1037,"tested":1038,"valid":1039,"validate":1040,"validation":1041,"eval":1042,"evaluate":1043,"layer":1044,"layers":1045,"weight":1046,"weights":1047,"bias":1048,"biases":1049,"loss":1050,"losses":1051,"grad":1052,"grads":1053,"lr":1054,"learning_rate":1055,"optimizer":1056,"adam":1057,"sgd":1058,"batch":1059,"batches":1060,"epoch":1061,"epochs":1062,"step":1063,"steps":1064,"dataset":1065,"dataloader":1066,"tensor":1067,"tensors":1068,"gpu":1069,"cpu":1070,"tpu":1071,"device":1072,"devices":1073,"math":1074,"physics":1075,"chemistry":1076,"biology":1077,"science":1078,"compute":1079,"calculate":1080,"computation":1081,"calculation":1082,"equation":1083,"formula":1084,"expression":1085,"theorem":1086,"proof":1087,"prove":1088,"lemma":1089,"axiom":1090,"corollary":1091,"derivative":1092,"integral":1093,"limit":1094,"sequence":1095,"series":1096,"matrix":1097,"vector":1098,"subset":1099,"intersection":1100,"complement":1101,"space":1102,"ring":1103,"module":1104,"algebra":1105,"analysis":1106,"topology":1107,"geometry":1108,"statistics":1109,"probability":1110,"distribution":1111,"random":1112,"sample":1113,"mean":1114,"median":1115,"mode":1116,"variance":1117,"std":1118,"deviation":1119,"linear":1120,"nonlinear":1121,"convex":1122,"concave":1123,"smooth":1124,"algorithm":1125,"complexity":1126,"runtime":1127,"search":1128,"traverse":1129,"recursive":1130,"iterative":1131,"dynamic":1132,"greedy":1133,"optimization":1134,"constraint":1135,"feasible":1136,"optimal":1137,"example":1138,"question":1139,"answer":1140,"hint":1141,"reason":1142,"analyze":1143,"approach":1144,"solve":1145,"verify":1146,"check":1147,"conclude":1148,"summarize":1149,"explain":1150,"describe":1151,"correct":1152,"incorrect":1153,"right":1154,"wrong":1155,"positive":1156,"negative":1157,"yes":1158,"maybe":1159,"==":1160,"!=":1161,"<=":1162,">=":1163,"&&":1164,"||":1165,"++":1166,"--":1167,"+=":1168,"-=":1169,"*=":1170,"/=":1171,"->":1172,"=>":1173,"::":1174,"..":1175,"...":1176,"/*":1177,"*/":1178,"//":1179,"<!--":1180,"-->":1181,"10":1182,"11":1183,"12":1184,"13":1185,"14":1186,"15":1187,"16":1188,"17":1189,"18":1190,"19":1191,"20":1192,"30":1193,"40":1194,"50":1195,"60":1196,"70":1197,"80":1198,"90":1199,"100":1200,"-1":1201,"-2":1202,"0x":1203,"0b":1204,"0o":1205,"Ġthe":1206,"Ġto":1207,"Ġof":1208,"Ġand":1209,"Ġa":1210,"Ġin":1211,"Ġthat":1212,"Ġis":1213,"Ġwas":1214,"Ġfor":1215,"Ġon":1216,"Ġwith":1217,"Ġas":1218,"Ġby":1219,"Ġat":1220,"Ġfrom":1221,"Ġor":1222,"Ġan":1223,"Ġwill":1224,"Ġwould":1225,"Ġnot":1226,"Ġbut":1227,"Ġare":1228,"Ġwere":1229,"Ġbeen":1230,"Ġhave":1231,"Ġhas":1232,"Ġhad":1233,"Ġdo":1234,"Ġdoes":1235,"Ġdid":1236,"Ġcan":1237,"Ġcould":1238,"Ġshould":1239,"Ġmay":1240,"Ġmight":1241,"Ġshall":1242,"Ġmust":1243,"Ġif":1244,"Ġelse":1245,"Ġwhen":1246,"Ġwhile":1247,"Ġbecause":1248,"Ġso":1249,"Ġthen":1250,"Ġthan":1251,"Ġalso":1252,"Ġeven":1253,"Ġonly":1254,"Ġjust":1255,"Ġvery":1256,"Ġtoo":1257,"Ġstill":1258,"Ġalready":1259,"Ġhere":1260,"Ġthere":1261,"Ġwhere":1262,"Ġwhy":1263,"Ġhow":1264,"Ġwhat":1265,"Ġwhich":1266,"Ġthis":1267,"Ġthese":1268,"Ġthose":1269,"Ġit":1270,"Ġits":1271,"Ġthey":1272,"Ġthem":1273,"Ġwe":1274,"Ġus":1275,"Ġour":1276,"Ġyou":1277,"Ġyour":1278,"Ġhe":1279,"Ġhim":1280,"Ġhis":1281,"Ġshe":1282,"Ġher":1283,"Ġhers":1284,"Ġone":1285,"Ġno":1286,"Ġall":1287,"Ġany":1288,"Ġsome":1289,"Ġeach":1290,"Ġevery":1291,"Ġboth":1292,"Ġneither":1293,"Ġeither":1294,"Ġmore":1295,"Ġmost":1296,"Ġfew":1297,"Ġother":1298,"Ġanother":1299,"Ġsuch":1300,"Ġsame":1301,"Ġdifferent":1302,"Ġown":1303,"Ġlike":1304,"Ġwell":1305,"Ġgood":1306,"Ġbad":1307,"Ġbetter":1308,"Ġbest":1309,"Ġnew":1310,"Ġold":1311,"Ġbig":1312,"Ġsmall":1313,"Ġlong":1314,"Ġshort":1315,"Ġhigh":1316,"Ġlow":1317,"Ġlarge":1318,"Ġlittle":1319,"Ġgreat":1320,"Ġimportant":1321,"Ġup":1322,"Ġdown":1323,"Ġout":1324,"Ġoff":1325,"Ġover":1326,"Ġunder":1327,"Ġagain":1328,"Ġback":1329,"Ġabout":1330,"Ġaround":1331,"Ġbetween":1332,"Ġthrough":1333,"Ġbefore":1334,"Ġafter":1335,"Ġduring":1336,"Ġuntil":1337,"Ġsince":1338,"Ġfirst":1339,"Ġlast":1340,"Ġnext":1341,"Ġprevious":1342,"Ġfinal":1343,"Ġget":1344,"Ġgot":1345,"Ġmake":1346,"Ġmade":1347,"Ġtake":1348,"Ġtook":1349,"Ġgive":1350,"Ġgave":1351,"Ġuse":1352,"Ġused":1353,"Ġusing":1354,"Ġneed":1355,"Ġneeds":1356,"Ġneeded":1357,"Ġwant":1358,"Ġwants":1359,"Ġwanted":1360,"Ġlet":1361,"Ġlets":1362,"Ġwork":1363,"Ġworks":1364,"Ġworked":1365,"Ġworking":1366,"Ġhelp":1367,"Ġhelps":1368,"Ġhelped":1369,"Ġcall":1370,"Ġcalls":1371,"Ġcalled":1372,"Ġcalling":1373,"Ġset":1374,"Ġsets":1375,"Ġsetting":1376,"Ġput":1377,"Ġputs":1378,"Ġputting":1379,"Ġrun":1380,"Ġruns":1381,"Ġran":1382,"Ġrunning":1383,"Ġkeep":1384,"Ġkeeps":1385,"Ġkept":1386,"Ġfind":1387,"Ġfinds":1388,"Ġfound":1389,"Ġshow":1390,"Ġshows":1391,"Ġtry":1392,"Ġtries":1393,"Ġtried":1394,"Ġtrying":1395,"Ġstart":1396,"Ġstarts":1397,"Ġstarted":1398,"Ġchange":1399,"Ġchanges":1400,"Ġchanged":1401,"Ġchanging":1402,"Ġfollow":1403,"Ġfollows":1404,"Ġknow":1405,"Ġknows":1406,"Ġknown":1407,"Ġthink":1408,"Ġthinks":1409,"Ġthought":1410,"Ġsay":1411,"Ġsays":1412,"Ġsaid":1413,"Ġsee":1414,"Ġsees":1415,"Ġsaw":1416,"Ġseen":1417,"Ġcome":1418,"Ġcomes":1419,"Ġcame":1420,"Ġgo":1421,"Ġgoes":1422,"Ġwent":1423,"Ġgone":1424,"Ġbring":1425,"Ġbrings":1426,"Ġbrought":1427,"Ġtell":1428,"Ġtells":1429,"Ġtold":1430,"Ġleave":1431,"Ġleaves":1432,"Ġleft":1433,"Ġhappen":1434,"Ġhappens":1435,"Ġprovide":1436,"Ġprovides":1437,"Ġprovided":1438,"Ġconsider":1439,"Ġconsiders":1440,"Ġappear":1441,"Ġappears":1442,"Ġappeared":1443,"Ġform":1444,"Ġforms":1445,"Ġformed":1446,"Ġseem":1447,"Ġseems":1448,"Ġseemed":1449,"Ġpoint":1450,"Ġpoints":1451,"Ġpointed":1452,"Ġturn":1453,"Ġturns":1454,"Ġturned":1455,"Ġturning":1456,"Ġplay":1457,"Ġplays":1458,"Ġthings":1459,"Ġthing":1460,"Ġtime":1461,"Ġtimes":1462,"Ġyear":1463,"Ġyears":1464,"Ġpeople":1465,"Ġplace":1466,"Ġplaces":1467,"Ġpart":1468,"Ġparts":1469,"Ġworld":1470,"Ġnumber":1471,"Ġnumbers":1472,"Ġsystem":1473,"Ġsystems":1474,"Ġgroup":1475,"Ġgroups":1476,"Ġline":1477,"Ġlines":1478,"Ġfile":1479,"Ġfiles":1480,"Ġcode":1481,"Ġdata":1482,"Ġfunction":1483,"Ġfunctions":1484,"Ġvalue":1485,"Ġvalues":1486,"Ġtype":1487,"Ġtypes":1488,"Ġclass":1489,"Ġclasses":1490,"Ġobject":1491,"Ġobjects":1492,"Ġmethod":1493,"Ġmethods":1494,"Ġresult":1495,"Ġresults":1496,"Ġprocess":1497,"Ġprocesses":1498,"Ġstate":1499,"Ġstates":1500,"Ġmodel":1501,"Ġmodels":1502,"Ġlevel":1503,"Ġlevels":1504,"Ġcase":1505,"Ġcases":1506,"Ġexample":1507,"Ġexamples":1508,"Ġinput":1509,"Ġinputs":1510,"Ġoutput":1511,"Ġoutputs":1512,"Ġerror":1513,"Ġerrors":1514,"Ġstatus":1515,"Ġmessage":1516,"Ġmessages":1517,"Ġrequest":1518,"Ġrequests":1519,"Ġresponse":1520,"Ġresponses":1521,"Ġreturn":1522,"Ġreturns":1523,"Ġimport":1524,"Ġimports":1525,"Ġexport":1526,"Ġexports":1527,"Ġdefine":1528,"Ġdefines":1529,"Ġdefined":1530,"Ġdeclare":1531,"Ġdeclares":1532,"Ġinclude":1533,"Ġincludes":1534,"Ġconfig":1535,"Ġsettings":1536,"Ġoption":1537,"Ġoptions":1538,"Ġparam":1539,"Ġparams":1540,"Ġargs":1541,"Ġtrue":1542,"Ġfalse":1543,"Ġnull":1544,"ĠNone":1545,"Ġundefined":1546,"Ġlen":1547,"Ġsize":1548,"Ġcount":1549,"Ġsum":1550,"Ġmax":1551,"Ġmin":1552,"Ġprint":1553,"Ġlog":1554,"Ġdebug":1555,"Ġinfo":1556,"Ġwarn":1557,"Ġtrain":1558,"Ġtest":1559,"Ġeval":1560,"Ġvalid":1561,"Ġval":1562,"Ġpage":1563,"Ġhome":1564,"Ġname":1565,"Ġkey":1566,"Ġkeys":1567,"Ġpath":1568,"Ġdir":1569,"Ġroot":1570,"Ġitem":1571,"Ġitems":1572,"Ġsource":1573,"Ġtarget":1574,"Ġbase":1575,"Ġmain":1576,"Ġprimary":1577,"Ġadd":1578,"Ġremove":1579,"Ġcreate":1580,"Ġdelete":1581,"Ġupdate":1582,"Ġopen":1583,"Ġclose":1584,"Ġread":1585,"Ġwrite":1586,"Ġload":1587,"Ġsave":1588,"Ġpush":1589,"Ġpop":1590,"Ġinsert":1591,"Ġappend":1592,"Ġsplit":1593,"Ġbegin":1594,"Ġend":1595,"Ġstop":1596,"Ġenable":1597,"Ġdisable":1598,"Ġuser":1599,"Ġusers":1600,"Ġadmin":1601,"Ġmanager":1602,"Ġstring":1603,"Ġint":1604,"Ġfloat":1605,"Ġdouble":1606,"Ġbool":1607,"Ġvoid":1608,"Ġlist":1609,"Ġdict":1610,"Ġtuple":1611,"Ġarray":1612,"Ġapi":1613,"Ġurl":1614,"Ġuri":1615,"Ġendpoint":1616,"Ġroute":1617,"Ġhttp":1618,"Ġhttps":1619,"Ġclient":1620,"Ġserver":1621,"Ġsocket":1622,"Ġmath":1623,"Ġscience":1624,"Ġanalysis":1625,"Ġtheory":1626,"Ġproof":1627,"Ġtheorem":1628,"Ġlemma":1629,"Ġequation":1630,"Ġformula":1631,"ing":1632,"ed":1633,"ly":1634,"tion":1635,"sion":1636,"ment":1637,"ness":1638,"ity":1639,"able":1640,"ible":1641,"al":1642,"ial":1643,"ical":1644,"ous":1645,"eous":1646,"ious":1647,"ive":1648,"ative":1649,"ful":1650,"wise":1651,"ward":1652,"un":1653,"re":1654,"im":1655,"ir":1656,"il":1657,"dis":1658,"mis":1659,"non":1660,"pre":1661,"pro":1662,"per":1663,"trans":1664,"inter":1665,"intra":1666,"extra":1667,"super":1668,"sur":1669,"semi":1670,"multi":1671,"mono":1672,"bi":1673,"tri":1674,"anti":1675,"counter":1676,"down":1677,"co":1678,"con":1679,"com":1680,"col":1681,"cor":1682,"de":1683,"di":1684,"dif":1685,"ex":1686,"fore":1687,"macro":1688,"micro":1689,"mid":1690,"post":1691,"tele":1692,"ultra":1693,"-ing":1694,"-ed":1695,"-ly":1696,"-tion":1697,"-sion":1698,"-ment":1699,"-ness":1700,"-ity":1701,"-able":1702,"-ible":1703,"-al":1704,"-ous":1705,"-ive":1706,"-ful":1707,"-less":1708,"un-":1709,"re-":1710,"pre-":1711,"non-":1712,"anti-":1713,"counter-":1714,"self-":1715,"all-":1716,"well-":1717,"so-":1718,"to-":1719,"in-":1720,"'t":1721,"'s":1722,"'m":1723,"'re":1724,"'ve":1725,"'ll":1726,"'d":1727,"n't":1728,"don't":1729,"can't":1730,"won't":1731,"isn't":1732,"aren't":1733,"wasn't":1734,"weren't":1735,"hasn't":1736,"haven't":1737,"hadn't":1738,"doesn't":1739,"didn't":1740,"couldn't":1741,"shouldn't":1742,"wouldn't":1743,"mustn't":1744,"needn't":1745,"mightn't":1746,"<-":1747,"\n":1748,"\t":1749,"  ":1750,"   ":1751,"    ":1752,"     ":1753,"ĠĠ":1754,"ĠĠĠ":1755,"ĠĠĠĠ":1756},"merges":[]}}
+{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"content":"<unk>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":1,"content":"<s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":2,"content":"</s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":3,"content":"<pad>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":4,"content":"<|system|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":5,"content":"<|user|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":6,"content":"<|assistant|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":7,"content":"<think>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":8,"content":"</think>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":9,"content":"[INST]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":10,"content":"[/INST]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":11,"content":"<|begin_of_thought|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":12,"content":"<|end_of_thought|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":13,"content":"<|reflect|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":14,"content":"<|revise|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":15,"content":"<|verify|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":16,"content":"<|code|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":17,"content":"<|text|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":18,"content":"<|math|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":19,"content":"<|think|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":20,"content":"<|answer|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":21,"content":"<|step|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":22,"content":"<|reason|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":23,"content":"<|check|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":24,"content":"<|output|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":25,"content":"<|plan|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":26,"content":"<|solve|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":27,"content":"<|analyze|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":28,"content":"<|conclude|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":29,"content":"<|approach|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":30,"content":"<|alternative|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":31,"content":"<|summary|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":32,"content":"<|question|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":33,"content":"<|hint|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":34,"content":"<|example|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":35,"content":"<|correct|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":36,"content":"<|incorrect|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":37,"content":"<|feedback|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":38,"content":"<|start|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":39,"content":"<|end|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":40,"content":"<|sep|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":41,"content":"<|cls|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":42,"content":"<|tool|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":43,"content":"<|function|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":44,"content":"<|result|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":45,"content":"<|input|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":46,"content":"<|detect|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":47,"content":"<|context|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":48,"content":"<|proof|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":49,"content":"<|lemma|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true},{"id":50,"content":"<|theorem|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":true}],"normalizer":null,"pre_tokenizer":{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":true},"post_processor":null,"decoder":{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":true},"model":{"type":"BPE","dropout":null,"unk_token":"<unk>","byte_fallback":true,"vocab":{"<unk>":0,"<s>":1,"</s>":2,"<pad>":3,"<|system|>":4,"<|user|>":5,"<|assistant|>":6,"<think>":7,"</think>":8,"[INST]":9,"[/INST]":10,"<|begin_of_thought|>":11,"<|end_of_thought|>":12,"<|reflect|>":13,"<|revise|>":14,"<|verify|>":15,"<|code|>":16,"<|text|>":17,"<|math|>":18,"<|think|>":19,"<|answer|>":20,"<|step|>":21,"<|reason|>":22,"<|check|>":23,"<|output|>":24,"<|plan|>":25,"<|solve|>":26,"<|analyze|>":27,"<|conclude|>":28,"<|approach|>":29,"<|alternative|>":30,"<|summary|>":31,"<|question|>":32,"<|hint|>":33,"<|example|>":34,"<|correct|>":35,"<|incorrect|>":36,"<|feedback|>":37,"<|start|>":38,"<|end|>":39,"<|sep|>":40,"<|cls|>":41,"<|tool|>":42,"<|function|>":43,"<|result|>":44,"<|input|>":45,"<|detect|>":46,"<|context|>":47,"<|proof|>":48,"<|lemma|>":49,"<|theorem|>":50,"Ā":51,"ā":52,"Ă":53,"ă":54,"Ą":55,"ą":56,"Ć":57,"ć":58,"Ĉ":59,"ĉ":60,"Ċ":61,"ċ":62,"Č":63,"č":64,"Ď":65,"ď":66,"Đ":67,"đ":68,"Ē":69,"ē":70,"Ĕ":71,"ĕ":72,"Ė":73,"ė":74,"Ę":75,"ę":76,"Ě":77,"ě":78,"Ĝ":79,"ĝ":80,"Ğ":81,"ğ":82,"Ġ":83,"!":84,"\"":85,"#":86,"$":87,"%":88,"&":89,"'":90,"(":91,")":92,"*":93,"+":94,",":95,"-":96,".":97,"/":98,"0":99,"1":100,"2":101,"3":102,"4":103,"5":104,"6":105,"7":106,"8":107,"9":108,":":109,";":110,"<":111,"=":112,">":113,"?":114,"@":115,"A":116,"B":117,"C":118,"D":119,"E":120,"F":121,"G":122,"H":123,"I":124,"J":125,"K":126,"L":127,"M":128,"N":129,"O":130,"P":131,"Q":132,"R":133,"S":134,"T":135,"U":136,"V":137,"W":138,"X":139,"Y":140,"Z":141,"[":142,"\\":143,"]":144,"^":145,"_":146,"`":147,"a":148,"b":149,"c":150,"d":151,"e":152,"f":153,"g":154,"h":155,"i":156,"j":157,"k":158,"l":159,"m":160,"n":161,"o":162,"p":163,"q":164,"r":165,"s":166,"t":167,"u":168,"v":169,"w":170,"x":171,"y":172,"z":173,"{":174,"|":175,"}":176,"~":177,"ġ":178,"Ģ":179,"ģ":180,"Ĥ":181,"ĥ":182,"Ħ":183,"ħ":184,"Ĩ":185,"ĩ":186,"Ī":187,"ī":188,"Ĭ":189,"ĭ":190,"Į":191,"į":192,"İ":193,"ı":194,"IJ":195,"ij":196,"Ĵ":197,"ĵ":198,"Ķ":199,"ķ":200,"ĸ":201,"Ĺ":202,"ĺ":203,"Ļ":204,"ļ":205,"Ľ":206,"ľ":207,"Ŀ":208,"ŀ":209,"Ł":210,"ł":211,"¡":212,"¢":213,"£":214,"¤":215,"¥":216,"¦":217,"§":218,"¨":219,"©":220,"ª":221,"«":222,"¬":223,"Ń":224,"®":225,"¯":226,"°":227,"±":228,"²":229,"³":230,"´":231,"µ":232,"¶":233,"·":234,"¸":235,"¹":236,"º":237,"»":238,"¼":239,"½":240,"¾":241,"¿":242,"À":243,"Á":244,"Â":245,"Ã":246,"Ä":247,"Å":248,"Æ":249,"Ç":250,"È":251,"É":252,"Ê":253,"Ë":254,"Ì":255,"Í":256,"Î":257,"Ï":258,"Ð":259,"Ñ":260,"Ò":261,"Ó":262,"Ô":263,"Õ":264,"Ö":265,"×":266,"Ø":267,"Ù":268,"Ú":269,"Û":270,"Ü":271,"Ý":272,"Þ":273,"ß":274,"à":275,"á":276,"â":277,"ã":278,"ä":279,"å":280,"æ":281,"ç":282,"è":283,"é":284,"ê":285,"ë":286,"ì":287,"í":288,"î":289,"ï":290,"ð":291,"ñ":292,"ò":293,"ó":294,"ô":295,"õ":296,"ö":297,"÷":298,"ø":299,"ù":300,"ú":301,"û":302,"ü":303,"ý":304,"þ":305,"ÿ":306,"the":307,"be":308,"to":309,"of":310,"and":311,"in":312,"that":313,"have":314,"it":315,"for":316,"not":317,"on":318,"with":319,"he":320,"as":321,"you":322,"do":323,"at":324,"this":325,"but":326,"his":327,"by":328,"from":329,"they":330,"we":331,"say":332,"her":333,"she":334,"or":335,"an":336,"will":337,"my":338,"one":339,"all":340,"would":341,"there":342,"their":343,"what":344,"so":345,"up":346,"out":347,"if":348,"about":349,"who":350,"get":351,"which":352,"go":353,"me":354,"when":355,"make":356,"can":357,"like":358,"time":359,"no":360,"just":361,"him":362,"know":363,"take":364,"people":365,"into":366,"year":367,"your":368,"good":369,"some":370,"could":371,"them":372,"see":373,"other":374,"than":375,"then":376,"now":377,"look":378,"only":379,"come":380,"its":381,"over":382,"think":383,"also":384,"back":385,"after":386,"use":387,"two":388,"how":389,"our":390,"work":391,"first":392,"well":393,"way":394,"even":395,"new":396,"want":397,"because":398,"any":399,"these":400,"give":401,"day":402,"most":403,"us":404,"is":405,"was":406,"are":407,"were":408,"been":409,"has":410,"had":411,"did":412,"does":413,"am":414,"being":415,"having":416,"doing":417,"saying":418,"going":419,"getting":420,"making":421,"knowing":422,"taking":423,"thinking":424,"coming":425,"came":426,"goes":427,"gone":428,"went":429,"saw":430,"seen":431,"seeing":432,"said":433,"says":434,"got":435,"gotten":436,"made":437,"makes":438,"knew":439,"known":440,"knows":441,"thought":442,"thinks":443,"took":444,"taken":445,"takes":446,"gave":447,"given":448,"gives":449,"giving":450,"find":451,"found":452,"finds":453,"finding":454,"tell":455,"told":456,"tells":457,"telling":458,"ask":459,"asked":460,"asks":461,"asking":462,"show":463,"showed":464,"shown":465,"shows":466,"showing":467,"try":468,"tried":469,"tries":470,"trying":471,"leave":472,"left":473,"leaves":474,"leaving":475,"call":476,"called":477,"calls":478,"calling":479,"keep":480,"kept":481,"keeps":482,"keeping":483,"let":484,"lets":485,"letting":486,"begin":487,"began":488,"begun":489,"begins":490,"beginning":491,"seem":492,"seemed":493,"seems":494,"seeming":495,"help":496,"helped":497,"helps":498,"helping":499,"turn":500,"turned":501,"turns":502,"turning":503,"start":504,"started":505,"starts":506,"starting":507,"bring":508,"brought":509,"brings":510,"bringing":511,"happen":512,"happened":513,"happens":514,"happening":515,"write":516,"wrote":517,"written":518,"writes":519,"writing":520,"provide":521,"provided":522,"provides":523,"providing":524,"consider":525,"considered":526,"considers":527,"considering":528,"appear":529,"appeared":530,"appears":531,"appearing":532,"follow":533,"followed":534,"follows":535,"following":536,"change":537,"changed":538,"changes":539,"changing":540,"form":541,"formed":542,"forms":543,"forming":544,"need":545,"needed":546,"needs":547,"needing":548,"set":549,"sets":550,"setting":551,"put":552,"puts":553,"putting":554,"run":555,"runs":556,"running":557,"move":558,"moved":559,"moves":560,"moving":561,"stand":562,"stood":563,"stands":564,"standing":565,"win":566,"won":567,"wins":568,"winning":569,"play":570,"played":571,"plays":572,"playing":573,"point":574,"points":575,"pointed":576,"pointing":577,"large":578,"small":579,"big":580,"little":581,"long":582,"short":583,"high":584,"low":585,"old":586,"young":587,"great":588,"important":589,"different":590,"same":591,"many":592,"much":593,"more":594,"few":595,"own":596,"very":597,"such":598,"still":599,"too":600,"here":601,"where":602,"why":603,"while":604,"though":605,"although":606,"until":607,"since":608,"before":609,"during":610,"without":611,"within":612,"between":613,"through":614,"across":615,"around":616,"above":617,"below":618,"under":619,"again":620,"ever":621,"never":622,"always":623,"often":624,"sometimes":625,"together":626,"alone":627,"already":628,"yet":629,"almost":630,"quite":631,"rather":632,"bad":633,"better":634,"worse":635,"best":636,"worst":637,"less":638,"every":639,"each":640,"both":641,"either":642,"neither":643,"none":644,"thing":645,"things":646,"ways":647,"times":648,"years":649,"days":650,"week":651,"weeks":652,"month":653,"months":654,"part":655,"parts":656,"place":657,"places":658,"case":659,"cases":660,"world":661,"worlds":662,"number":663,"numbers":664,"group":665,"groups":666,"system":667,"systems":668,"program":669,"programs":670,"data":671,"information":672,"problem":673,"problems":674,"solution":675,"solutions":676,"method":677,"methods":678,"result":679,"results":680,"process":681,"processes":682,"function":683,"functions":684,"value":685,"values":686,"type":687,"types":688,"state":689,"states":690,"model":691,"models":692,"level":693,"levels":694,"line":695,"lines":696,"file":697,"files":698,"code":699,"codes":700,"list":701,"lists":702,"array":703,"arrays":704,"object":705,"objects":706,"class":707,"classes":708,"property":709,"properties":710,"input":711,"inputs":712,"output":713,"outputs":714,"return":715,"returns":716,"define":717,"defined":718,"defines":719,"defining":720,"declare":721,"declared":722,"import":723,"imports":724,"export":725,"exports":726,"include":727,"includes":728,"public":729,"private":730,"protected":731,"static":732,"final":733,"const":734,"void":735,"int":736,"float":737,"double":738,"char":739,"bool":740,"string":741,"true":742,"false":743,"null":744,"None":745,"nil":746,"undefined":747,"else":748,"elif":749,"switch":750,"default":751,"break":752,"catch":753,"finally":754,"throw":755,"raise":756,"except":757,"yield":758,"await":759,"async":760,"defer":761,"lambda":762,"map":763,"filter":764,"reduce":765,"sort":766,"delete":767,"free":768,"alloc":769,"realloc":770,"print":771,"printf":772,"println":773,"log":774,"debug":775,"error":776,"len":777,"size":778,"length":779,"count":780,"sum":781,"max":782,"min":783,"abs":784,"pow":785,"sqrt":786,"floor":787,"ceil":788,"round":789,"sin":790,"cos":791,"tan":792,"atan":793,"exp":794,"zero":795,"three":796,"four":797,"five":798,"six":799,"seven":800,"eight":801,"nine":802,"ten":803,"second":804,"third":805,"last":806,"next":807,"previous":808,"current":809,"initial":810,"primary":811,"secondary":812,"main":813,"basic":814,"advanced":815,"simple":816,"complex":817,"single":818,"multiple":819,"add":820,"sub":821,"mul":822,"div":823,"mod":824,"inc":825,"dec":826,"push":827,"pop":828,"shift":829,"unshift":830,"insert":831,"remove":832,"append":833,"prepend":834,"concat":835,"join":836,"split":837,"slice":838,"open":839,"close":840,"read":841,"load":842,"save":843,"create":844,"update":845,"select":846,"merge":847,"end":848,"stop":849,"pause":850,"resume":851,"enable":852,"disable":853,"allow":854,"deny":855,"grant":856,"revoke":857,"user":858,"users":859,"name":860,"names":861,"id":862,"ids":863,"key":864,"keys":865,"field":866,"fields":867,"table":868,"tables":869,"row":870,"rows":871,"column":872,"columns":873,"index":874,"indexes":875,"indices":876,"query":877,"queries":878,"avg":879,"total":880,"page":881,"pages":882,"home":883,"login":884,"logout":885,"signup":886,"errors":887,"warning":888,"warnings":889,"info":890,"success":891,"failure":892,"status":893,"message":894,"messages":895,"request":896,"requests":897,"response":898,"responses":899,"client":900,"server":901,"api":902,"endpoint":903,"route":904,"http":905,"https":906,"url":907,"uri":908,"port":909,"host":910,"config":911,"configuration":912,"settings":913,"option":914,"options":915,"param":916,"params":917,"parameter":918,"parameters":919,"arg":920,"args":921,"argument":922,"arguments":923,"kwargs":924,"path":925,"dir":926,"directory":927,"dirs":928,"folder":929,"folders":930,"item":931,"items":932,"element":933,"elements":934,"entry":935,"entries":936,"note":937,"notes":938,"text":939,"texts":940,"content":941,"contents":942,"source":943,"sources":944,"target":945,"targets":946,"ref":947,"refs":948,"struct":949,"structs":950,"union":951,"unions":952,"enum":953,"enums":954,"impl":955,"implement":956,"implementation":957,"interface":958,"abstract":959,"virtual":960,"override":961,"overload":962,"base":963,"derived":964,"parent":965,"child":966,"children":967,"root":968,"leaf":969,"node":970,"nodes":971,"edge":972,"edges":973,"tree":974,"graph":975,"queue":976,"stack":977,"heap":978,"dict":979,"dictionary":980,"hash":981,"hashmap":982,"link":983,"links":984,"linked":985,"pointer":986,"pointers":987,"thread":988,"threads":989,"task":990,"tasks":991,"sync":992,"lock":993,"mutex":994,"semaphore":995,"buffer":996,"buffers":997,"cache":998,"cached":999,"pool":1000,"pools":1001,"memory":1002,"disk":1003,"network":1004,"socket":1005,"sockets":1006,"stream":1007,"streams":1008,"packet":1009,"packets":1010,"frame":1011,"meta":1012,"metadata":1013,"header":1014,"headers":1015,"body":1016,"payload":1017,"token":1018,"tokens":1019,"session":1020,"cookie":1021,"cookies":1022,"auth":1023,"register":1024,"password":1025,"salt":1026,"encrypt":1027,"decrypt":1028,"encode":1029,"decode":1030,"cert":1031,"certificate":1032,"train":1033,"training":1034,"trained":1035,"test":1036,"testing":1037,"tested":1038,"valid":1039,"validate":1040,"validation":1041,"eval":1042,"evaluate":1043,"layer":1044,"layers":1045,"weight":1046,"weights":1047,"bias":1048,"biases":1049,"loss":1050,"losses":1051,"grad":1052,"grads":1053,"lr":1054,"learning_rate":1055,"optimizer":1056,"adam":1057,"sgd":1058,"batch":1059,"batches":1060,"epoch":1061,"epochs":1062,"step":1063,"steps":1064,"dataset":1065,"dataloader":1066,"tensor":1067,"tensors":1068,"gpu":1069,"cpu":1070,"tpu":1071,"device":1072,"devices":1073,"math":1074,"physics":1075,"chemistry":1076,"biology":1077,"science":1078,"compute":1079,"calculate":1080,"computation":1081,"calculation":1082,"equation":1083,"formula":1084,"expression":1085,"theorem":1086,"proof":1087,"prove":1088,"lemma":1089,"axiom":1090,"corollary":1091,"derivative":1092,"integral":1093,"limit":1094,"sequence":1095,"series":1096,"matrix":1097,"vector":1098,"subset":1099,"intersection":1100,"complement":1101,"space":1102,"ring":1103,"module":1104,"algebra":1105,"analysis":1106,"topology":1107,"geometry":1108,"statistics":1109,"probability":1110,"distribution":1111,"random":1112,"sample":1113,"mean":1114,"median":1115,"mode":1116,"variance":1117,"std":1118,"deviation":1119,"linear":1120,"nonlinear":1121,"convex":1122,"concave":1123,"smooth":1124,"algorithm":1125,"complexity":1126,"runtime":1127,"search":1128,"traverse":1129,"recursive":1130,"iterative":1131,"dynamic":1132,"greedy":1133,"optimization":1134,"constraint":1135,"feasible":1136,"optimal":1137,"example":1138,"question":1139,"answer":1140,"hint":1141,"reason":1142,"analyze":1143,"approach":1144,"solve":1145,"verify":1146,"check":1147,"conclude":1148,"summarize":1149,"explain":1150,"describe":1151,"correct":1152,"incorrect":1153,"right":1154,"wrong":1155,"positive":1156,"negative":1157,"yes":1158,"maybe":1159,"==":1160,"!=":1161,"<=":1162,">=":1163,"&&":1164,"||":1165,"++":1166,"--":1167,"+=":1168,"-=":1169,"*=":1170,"/=":1171,"->":1172,"=>":1173,"::":1174,"..":1175,"...":1176,"/*":1177,"*/":1178,"//":1179,"<!--":1180,"-->":1181,"10":1182,"11":1183,"12":1184,"13":1185,"14":1186,"15":1187,"16":1188,"17":1189,"18":1190,"19":1191,"20":1192,"30":1193,"40":1194,"50":1195,"60":1196,"70":1197,"80":1198,"90":1199,"100":1200,"-1":1201,"-2":1202,"0x":1203,"0b":1204,"0o":1205,"Ġthe":1206,"Ġto":1207,"Ġof":1208,"Ġand":1209,"Ġa":1210,"Ġin":1211,"Ġthat":1212,"Ġis":1213,"Ġwas":1214,"Ġfor":1215,"Ġon":1216,"Ġwith":1217,"Ġas":1218,"Ġby":1219,"Ġat":1220,"Ġfrom":1221,"Ġor":1222,"Ġan":1223,"Ġwill":1224,"Ġwould":1225,"Ġnot":1226,"Ġbut":1227,"Ġare":1228,"Ġwere":1229,"Ġbeen":1230,"Ġhave":1231,"Ġhas":1232,"Ġhad":1233,"Ġdo":1234,"Ġdoes":1235,"Ġdid":1236,"Ġcan":1237,"Ġcould":1238,"Ġshould":1239,"Ġmay":1240,"Ġmight":1241,"Ġshall":1242,"Ġmust":1243,"Ġif":1244,"Ġelse":1245,"Ġwhen":1246,"Ġwhile":1247,"Ġbecause":1248,"Ġso":1249,"Ġthen":1250,"Ġthan":1251,"Ġalso":1252,"Ġeven":1253,"Ġonly":1254,"Ġjust":1255,"Ġvery":1256,"Ġtoo":1257,"Ġstill":1258,"Ġalready":1259,"Ġhere":1260,"Ġthere":1261,"Ġwhere":1262,"Ġwhy":1263,"Ġhow":1264,"Ġwhat":1265,"Ġwhich":1266,"Ġthis":1267,"Ġthese":1268,"Ġthose":1269,"Ġit":1270,"Ġits":1271,"Ġthey":1272,"Ġthem":1273,"Ġwe":1274,"Ġus":1275,"Ġour":1276,"Ġyou":1277,"Ġyour":1278,"Ġhe":1279,"Ġhim":1280,"Ġhis":1281,"Ġshe":1282,"Ġher":1283,"Ġhers":1284,"Ġone":1285,"Ġno":1286,"Ġall":1287,"Ġany":1288,"Ġsome":1289,"Ġeach":1290,"Ġevery":1291,"Ġboth":1292,"Ġneither":1293,"Ġeither":1294,"Ġmore":1295,"Ġmost":1296,"Ġfew":1297,"Ġother":1298,"Ġanother":1299,"Ġsuch":1300,"Ġsame":1301,"Ġdifferent":1302,"Ġown":1303,"Ġlike":1304,"Ġwell":1305,"Ġgood":1306,"Ġbad":1307,"Ġbetter":1308,"Ġbest":1309,"Ġnew":1310,"Ġold":1311,"Ġbig":1312,"Ġsmall":1313,"Ġlong":1314,"Ġshort":1315,"Ġhigh":1316,"Ġlow":1317,"Ġlarge":1318,"Ġlittle":1319,"Ġgreat":1320,"Ġimportant":1321,"Ġup":1322,"Ġdown":1323,"Ġout":1324,"Ġoff":1325,"Ġover":1326,"Ġunder":1327,"Ġagain":1328,"Ġback":1329,"Ġabout":1330,"Ġaround":1331,"Ġbetween":1332,"Ġthrough":1333,"Ġbefore":1334,"Ġafter":1335,"Ġduring":1336,"Ġuntil":1337,"Ġsince":1338,"Ġfirst":1339,"Ġlast":1340,"Ġnext":1341,"Ġprevious":1342,"Ġfinal":1343,"Ġget":1344,"Ġgot":1345,"Ġmake":1346,"Ġmade":1347,"Ġtake":1348,"Ġtook":1349,"Ġgive":1350,"Ġgave":1351,"Ġuse":1352,"Ġused":1353,"Ġusing":1354,"Ġneed":1355,"Ġneeds":1356,"Ġneeded":1357,"Ġwant":1358,"Ġwants":1359,"Ġwanted":1360,"Ġlet":1361,"Ġlets":1362,"Ġwork":1363,"Ġworks":1364,"Ġworked":1365,"Ġworking":1366,"Ġhelp":1367,"Ġhelps":1368,"Ġhelped":1369,"Ġcall":1370,"Ġcalls":1371,"Ġcalled":1372,"Ġcalling":1373,"Ġset":1374,"Ġsets":1375,"Ġsetting":1376,"Ġput":1377,"Ġputs":1378,"Ġputting":1379,"Ġrun":1380,"Ġruns":1381,"Ġran":1382,"Ġrunning":1383,"Ġkeep":1384,"Ġkeeps":1385,"Ġkept":1386,"Ġfind":1387,"Ġfinds":1388,"Ġfound":1389,"Ġshow":1390,"Ġshows":1391,"Ġtry":1392,"Ġtries":1393,"Ġtried":1394,"Ġtrying":1395,"Ġstart":1396,"Ġstarts":1397,"Ġstarted":1398,"Ġchange":1399,"Ġchanges":1400,"Ġchanged":1401,"Ġchanging":1402,"Ġfollow":1403,"Ġfollows":1404,"Ġknow":1405,"Ġknows":1406,"Ġknown":1407,"Ġthink":1408,"Ġthinks":1409,"Ġthought":1410,"Ġsay":1411,"Ġsays":1412,"Ġsaid":1413,"Ġsee":1414,"Ġsees":1415,"Ġsaw":1416,"Ġseen":1417,"Ġcome":1418,"Ġcomes":1419,"Ġcame":1420,"Ġgo":1421,"Ġgoes":1422,"Ġwent":1423,"Ġgone":1424,"Ġbring":1425,"Ġbrings":1426,"Ġbrought":1427,"Ġtell":1428,"Ġtells":1429,"Ġtold":1430,"Ġleave":1431,"Ġleaves":1432,"Ġleft":1433,"Ġhappen":1434,"Ġhappens":1435,"Ġprovide":1436,"Ġprovides":1437,"Ġprovided":1438,"Ġconsider":1439,"Ġconsiders":1440,"Ġappear":1441,"Ġappears":1442,"Ġappeared":1443,"Ġform":1444,"Ġforms":1445,"Ġformed":1446,"Ġseem":1447,"Ġseems":1448,"Ġseemed":1449,"Ġpoint":1450,"Ġpoints":1451,"Ġpointed":1452,"Ġturn":1453,"Ġturns":1454,"Ġturned":1455,"Ġturning":1456,"Ġplay":1457,"Ġplays":1458,"Ġthings":1459,"Ġthing":1460,"Ġtime":1461,"Ġtimes":1462,"Ġyear":1463,"Ġyears":1464,"Ġpeople":1465,"Ġplace":1466,"Ġplaces":1467,"Ġpart":1468,"Ġparts":1469,"Ġworld":1470,"Ġnumber":1471,"Ġnumbers":1472,"Ġsystem":1473,"Ġsystems":1474,"Ġgroup":1475,"Ġgroups":1476,"Ġline":1477,"Ġlines":1478,"Ġfile":1479,"Ġfiles":1480,"Ġcode":1481,"Ġdata":1482,"Ġfunction":1483,"Ġfunctions":1484,"Ġvalue":1485,"Ġvalues":1486,"Ġtype":1487,"Ġtypes":1488,"Ġclass":1489,"Ġclasses":1490,"Ġobject":1491,"Ġobjects":1492,"Ġmethod":1493,"Ġmethods":1494,"Ġresult":1495,"Ġresults":1496,"Ġprocess":1497,"Ġprocesses":1498,"Ġstate":1499,"Ġstates":1500,"Ġmodel":1501,"Ġmodels":1502,"Ġlevel":1503,"Ġlevels":1504,"Ġcase":1505,"Ġcases":1506,"Ġexample":1507,"Ġexamples":1508,"Ġinput":1509,"Ġinputs":1510,"Ġoutput":1511,"Ġoutputs":1512,"Ġerror":1513,"Ġerrors":1514,"Ġstatus":1515,"Ġmessage":1516,"Ġmessages":1517,"Ġrequest":1518,"Ġrequests":1519,"Ġresponse":1520,"Ġresponses":1521,"Ġreturn":1522,"Ġreturns":1523,"Ġimport":1524,"Ġimports":1525,"Ġexport":1526,"Ġexports":1527,"Ġdefine":1528,"Ġdefines":1529,"Ġdefined":1530,"Ġdeclare":1531,"Ġdeclares":1532,"Ġinclude":1533,"Ġincludes":1534,"Ġconfig":1535,"Ġsettings":1536,"Ġoption":1537,"Ġoptions":1538,"Ġparam":1539,"Ġparams":1540,"Ġargs":1541,"Ġtrue":1542,"Ġfalse":1543,"Ġnull":1544,"ĠNone":1545,"Ġundefined":1546,"Ġlen":1547,"Ġsize":1548,"Ġcount":1549,"Ġsum":1550,"Ġmax":1551,"Ġmin":1552,"Ġprint":1553,"Ġlog":1554,"Ġdebug":1555,"Ġinfo":1556,"Ġwarn":1557,"Ġtrain":1558,"Ġtest":1559,"Ġeval":1560,"Ġvalid":1561,"Ġval":1562,"Ġpage":1563,"Ġhome":1564,"Ġname":1565,"Ġkey":1566,"Ġkeys":1567,"Ġpath":1568,"Ġdir":1569,"Ġroot":1570,"Ġitem":1571,"Ġitems":1572,"Ġsource":1573,"Ġtarget":1574,"Ġbase":1575,"Ġmain":1576,"Ġprimary":1577,"Ġadd":1578,"Ġremove":1579,"Ġcreate":1580,"Ġdelete":1581,"Ġupdate":1582,"Ġopen":1583,"Ġclose":1584,"Ġread":1585,"Ġwrite":1586,"Ġload":1587,"Ġsave":1588,"Ġpush":1589,"Ġpop":1590,"Ġinsert":1591,"Ġappend":1592,"Ġsplit":1593,"Ġbegin":1594,"Ġend":1595,"Ġstop":1596,"Ġenable":1597,"Ġdisable":1598,"Ġuser":1599,"Ġusers":1600,"Ġadmin":1601,"Ġmanager":1602,"Ġstring":1603,"Ġint":1604,"Ġfloat":1605,"Ġdouble":1606,"Ġbool":1607,"Ġvoid":1608,"Ġlist":1609,"Ġdict":1610,"Ġtuple":1611,"Ġarray":1612,"Ġapi":1613,"Ġurl":1614,"Ġuri":1615,"Ġendpoint":1616,"Ġroute":1617,"Ġhttp":1618,"Ġhttps":1619,"Ġclient":1620,"Ġserver":1621,"Ġsocket":1622,"Ġmath":1623,"Ġscience":1624,"Ġanalysis":1625,"Ġtheory":1626,"Ġproof":1627,"Ġtheorem":1628,"Ġlemma":1629,"Ġequation":1630,"Ġformula":1631,"ing":1632,"ed":1633,"ly":1634,"tion":1635,"sion":1636,"ment":1637,"ness":1638,"ity":1639,"able":1640,"ible":1641,"al":1642,"ial":1643,"ical":1644,"ous":1645,"eous":1646,"ious":1647,"ive":1648,"ative":1649,"ful":1650,"wise":1651,"ward":1652,"un":1653,"re":1654,"im":1655,"ir":1656,"il":1657,"dis":1658,"mis":1659,"non":1660,"pre":1661,"pro":1662,"per":1663,"trans":1664,"inter":1665,"intra":1666,"extra":1667,"super":1668,"sur":1669,"semi":1670,"multi":1671,"mono":1672,"bi":1673,"tri":1674,"anti":1675,"counter":1676,"down":1677,"co":1678,"con":1679,"com":1680,"col":1681,"cor":1682,"de":1683,"di":1684,"dif":1685,"ex":1686,"fore":1687,"macro":1688,"micro":1689,"mid":1690,"post":1691,"tele":1692,"ultra":1693,"-ing":1694,"-ed":1695,"-ly":1696,"-tion":1697,"-sion":1698,"-ment":1699,"-ness":1700,"-ity":1701,"-able":1702,"-ible":1703,"-al":1704,"-ous":1705,"-ive":1706,"-ful":1707,"-less":1708,"un-":1709,"re-":1710,"pre-":1711,"non-":1712,"anti-":1713,"counter-":1714,"self-":1715,"all-":1716,"well-":1717,"so-":1718,"to-":1719,"in-":1720,"'t":1721,"'s":1722,"'m":1723,"'re":1724,"'ve":1725,"'ll":1726,"'d":1727,"n't":1728,"don't":1729,"can't":1730,"won't":1731,"isn't":1732,"aren't":1733,"wasn't":1734,"weren't":1735,"hasn't":1736,"haven't":1737,"hadn't":1738,"doesn't":1739,"didn't":1740,"couldn't":1741,"shouldn't":1742,"wouldn't":1743,"mustn't":1744,"needn't":1745,"mightn't":1746,"<-":1747,"\n":1748,"\t":1749,"  ":1750,"   ":1751,"    ":1752,"     ":1753,"ĠĠ":1754,"ĠĠĠ":1755,"ĠĠĠĠ":1756,"action":1757,"actually":1758,"address":1759,"agree":1760,"along":1761,"American":1762,"among":1763,"amount":1764,"animal":1765,"another":1766,"anything":1767,"area":1768,"argue":1769,"arm":1770,"article":1771,"artist":1772,"author":1773,"available":1774,"avoid":1775,"away":1776,"ball":1777,"bank":1778,"bar":1779,"battle":1780,"beauty":1781,"become":1782,"became":1783,"becoming":1784,"bed":1785,"behavior":1786,"behind":1787,"believe":1788,"benefit":1789,"beyond":1790,"bit":1791,"black":1792,"blood":1793,"board":1794,"book":1795,"born":1796,"boss":1797,"bother":1798,"bottle":1799,"bottom":1800,"box":1801,"boy":1802,"brain":1803,"bridge":1804,"brief":1805,"bright":1806,"broad":1807,"brother":1808,"budget":1809,"build":1810,"building":1811,"burn":1812,"business":1813,"buy":1814,"campaign":1815,"capital":1816,"car":1817,"care":1818,"career":1819,"carry":1820,"category":1821,"cause":1822,"central":1823,"century":1824,"certain":1825,"chair":1826,"chairman":1827,"challenge":1828,"chance":1829,"character":1830,"charge":1831,"choice":1832,"choose":1833,"chosen":1834,"church":1835,"citizen":1836,"city":1837,"civil":1838,"claim":1839,"clear":1840,"clearly":1841,"club":1842,"coach":1843,"cold":1844,"collection":1845,"college":1846,"color":1847,"comfortable":1848,"comment":1849,"committee":1850,"common":1851,"community":1852,"company":1853,"compare":1854,"competition":1855,"complete":1856,"completely":1857,"condition":1858,"conference":1859,"Congress":1860,"connect":1861,"conscious":1862,"contain":1863,"continue":1864,"contract":1865,"control":1866,"conversation":1867,"cost":1868,"country":1869,"couple":1870,"course":1871,"court":1872,"cover":1873,"crime":1874,"cultural":1875,"culture":1876,"cup":1877,"customer":1878,"cut":1879,"dark":1880,"daughter":1881,"deal":1882,"death":1883,"debate":1884,"decade":1885,"decide":1886,"decision":1887,"deep":1888,"defense":1889,"degree":1890,"democrat":1891,"democratic":1892,"design":1893,"despite":1894,"detail":1895,"determine":1896,"develop":1897,"development":1898,"die":1899,"difference":1900,"difficult":1901,"dinner":1902,"direction":1903,"director":1904,"discover":1905,"discuss":1906,"discussion":1907,"disease":1908,"dog":1909,"door":1910,"doubt":1911,"draw":1912,"dream":1913,"drive":1914,"driver":1915,"drop":1916,"drug":1917,"early":1918,"east":1919,"eat":1920,"economic":1921,"economy":1922,"edition":1923,"editor":1924,"education":1925,"effect":1926,"effort":1927,"election":1928,"employee":1929,"encourage":1930,"enemy":1931,"energy":1932,"enjoy":1933,"enough":1934,"enter":1935,"entire":1936,"environment":1937,"environmental":1938,"especially":1939,"establish":1940,"evening":1941,"event":1942,"everybody":1943,"everyone":1944,"everything":1945,"evidence":1946,"exactly":1947,"examine":1948,"executive":1949,"exist":1950,"expect":1951,"experience":1952,"explanation":1953,"extremely":1954,"eye":1955,"face":1956,"fact":1957,"factor":1958,"fail":1959,"fall":1960,"family":1961,"far":1962,"fast":1963,"father":1964,"fear":1965,"feature":1966,"federal":1967,"feel":1968,"feeling":1969,"fight":1970,"figure":1971,"fill":1972,"film":1973,"financial":1974,"fine":1975,"finish":1976,"firm":1977,"fish":1978,"fly":1979,"focus":1980,"food":1981,"foot":1982,"force":1983,"foreign":1984,"forget":1985,"former":1986,"forward":1987,"freedom":1988,"friendly":1989,"front":1990,"full":1991,"fund":1992,"future":1993,"game":1994,"garden":1995,"gas":1996,"general":1997,"generation":1998,"gentleman":1999,"girl":2000,"glad":2001,"glass":2002,"goal":2003,"god":2004,"gold":2005,"government":2006,"governor":2007,"green":2008,"ground":2009,"grow":2010,"growth":2011,"guess":2012,"gun":2013,"guy":2014,"half":2015,"hand":2016,"handle":2017,"hang":2018,"happy":2019,"hard":2020,"head":2021,"health":2022,"hear":2023,"heart":2024,"heat":2025,"heavy":2026,"hell":2027,"herself":2028,"hide":2029,"history":2030,"hit":2031,"hold":2032,"honest":2033,"hope":2034,"hospital":2035,"hotel":2036,"house":2037,"huge":2038,"human":2039,"hundred":2040,"husband":2041,"idea":2042,"identify":2043,"image":2044,"imagine":2045,"impact":2046,"imply":2047,"improve":2048,"including":2049,"increase":2050,"indeed":2051,"indicate":2052,"individual":2053,"industry":2054,"influence":2055,"inform":2056,"inside":2057,"instead":2058,"institution":2059,"interest":2060,"international":2061,"interview":2062,"introduce":2063,"investment":2064,"involve":2065,"issue":2066,"itself":2067,"job":2068,"journal":2069,"journey":2070,"judge":2071,"jump":2072,"justice":2073,"kill":2074,"kind":2075,"kitchen":2076,"knowledge":2077,"land":2078,"language":2079,"late":2080,"later":2081,"latter":2082,"laugh":2083,"launch":2084,"law":2085,"lawyer":2086,"lay":2087,"lead":2088,"leader":2089,"leading":2090,"learn":2091,"least":2092,"legal":2093,"letter":2094,"lie":2095,"life":2096,"lift":2097,"light":2098,"likely":2099,"listen":2100,"live":2101,"local":2102,"lord":2103,"lose":2104,"lost":2105,"lot":2106,"love":2107,"luck":2108,"lunch":2109,"machine":2110,"maintain":2111,"major":2112,"majority":2113,"manage":2114,"management":2115,"manager":2116,"manner":2117,"manufacturer":2118,"mark":2119,"market":2120,"marriage":2121,"master":2122,"material":2123,"matter":2124,"may":2125,"meaning":2126,"measure":2127,"media":2128,"medical":2129,"meet":2130,"meeting":2131,"member":2132,"mention":2133,"middle":2134,"might":2135,"military":2136,"million":2137,"mind":2138,"minute":2139,"miss":2140,"mission":2141,"mistake":2142,"mix":2143,"modern":2144,"mom":2145,"moment":2146,"money":2147,"moral":2148,"morning":2149,"mother":2150,"motion":2151,"movement":2152,"movie":2153,"music":2154,"narrative":2155,"nation":2156,"national":2157,"native":2158,"natural":2159,"nature":2160,"near":2161,"nearly":2162,"necessarily":2163,"necessary":2164,"neck":2165,"neighbor":2166,"nevertheless":2167,"night":2168,"nor":2169,"normal":2170,"north":2171,"nothing":2172,"notice":2173,"notion":2174,"nowhere":2175,"nuclear":2176,"occur":2177,"ocean":2178,"offer":2179,"office":2180,"officer":2181,"official":2182,"oil":2183,"OK":2184,"once":2185,"online":2186,"operate":2187,"operation":2188,"opinion":2189,"opportunity":2190,"opposition":2191,"order":2192,"organization":2193,"original":2194,"otherwise":2195,"ought":2196,"outside":2197,"overcome":2198,"owner":2199,"pain":2200,"paint":2201,"pair":2202,"paper":2203,"park":2204,"parliament":2205,"participant":2206,"particular":2207,"particularly":2208,"partner":2209,"party":2210,"pass":2211,"passage":2212,"past":2213,"patient":2214,"pattern":2215,"pay":2216,"peace":2217,"pension":2218,"percent":2219,"perfect":2220,"perform":2221,"performance":2222,"perhaps":2223,"period":2224,"permit":2225,"person":2226,"personal":2227,"perspective":2228,"phone":2229,"physical":2230,"pick":2231,"picture":2232,"piece":2233,"plan":2234,"plant":2235,"player":2236,"please":2237,"pleasure":2238,"plus":2239,"pocket":2240,"police":2241,"policy":2242,"political":2243,"politician":2244,"politics":2245,"poor":2246,"popular":2247,"population":2248,"position":2249,"possibility":2250,"possible":2251,"potentially":2252,"power":2253,"practice":2254,"prepare":2255,"presence":2256,"present":2257,"president":2258,"pressure":2259,"pretty":2260,"prevent":2261,"price":2262,"principle":2263,"prison":2264,"privilege":2265,"probably":2266,"procedure":2267,"produce":2268,"product":2269,"production":2270,"professional":2271,"professor":2272,"profile":2273,"profit":2274,"project":2275,"promise":2276,"promote":2277,"proper":2278,"proposal":2279,"propose":2280,"protect":2281,"protection":2282,"publication":2283,"publish":2284,"pull":2285,"purpose":2286,"pursue":2287,"quality":2288,"quarter":2289,"quick":2290,"quickly":2291,"quiet":2292,"race":2293,"radio":2294,"range":2295,"rate":2296,"reach":2297,"react":2298,"reaction":2299,"reader":2300,"reading":2301,"ready":2302,"real":2303,"reality":2304,"realize":2305,"really":2306,"reasonable":2307,"receive":2308,"recent":2309,"recently":2310,"recognize":2311,"recommend":2312,"record":2313,"recover":2314,"red":2315,"reflect":2316,"reform":2317,"region":2318,"relate":2319,"relationship":2320,"relative":2321,"relatively":2322,"release":2323,"relevant":2324,"relief":2325,"religion":2326,"religious":2327,"rely":2328,"remain":2329,"remember":2330,"remind":2331,"repeat":2332,"replace":2333,"report":2334,"reporter":2335,"represent":2336,"representation":2337,"republican":2338,"reputation":2339,"require":2340,"research":2341,"resource":2342,"respond":2343,"responsibility":2344,"responsible":2345,"rest":2346,"restaurant":2347,"retain":2348,"retire":2349,"reveal":2350,"review":2351,"revolution":2352,"rich":2353,"ride":2354,"rise":2355,"risk":2356,"river":2357,"road":2358,"rock":2359,"role":2360,"roll":2361,"room":2362,"rule":2363,"safe":2364,"safety":2365,"sale":2366,"scale":2367,"scene":2368,"schedule":2369,"school":2370,"scientist":2371,"score":2372,"screen":2373,"sea":2374,"season":2375,"seat":2376,"secret":2377,"section":2378,"security":2379,"seed":2380,"seek":2381,"self":2382,"sell":2383,"senate":2384,"senator":2385,"send":2386,"sense":2387,"serious":2388,"serve":2389,"service":2390,"settle":2391,"sexual":2392,"shadow":2393,"shape":2394,"share":2395,"sharp":2396,"sheet":2397,"ship":2398,"shock":2399,"shoe":2400,"shoot":2401,"shop":2402,"shot":2403,"shoulder":2404,"shut":2405,"sick":2406,"side":2407,"sight":2408,"sign":2409,"signal":2410,"significance":2411,"significant":2412,"silence":2413,"similar":2414,"simply":2415,"sing":2416,"sister":2417,"sit":2418,"site":2419,"situation":2420,"skill":2421,"skin":2422,"smile":2423,"society":2424,"soft":2425,"soldier":2426,"solid":2427,"somebody":2428,"somehow":2429,"someone":2430,"something":2431,"somewhat":2432,"son":2433,"song":2434,"soon":2435,"sound":2436,"south":2437,"speak":2438,"speaker":2439,"special":2440,"specific":2441,"speech":2442,"speed":2443,"spend":2444,"spin":2445,"spirit":2446,"spiritual":2447,"spokesman":2448,"sport":2449,"spot":2450,"spread":2451,"spring":2452,"staff":2453,"stage":2454,"standard":2455,"star":2456,"statement":2457,"station":2458,"stay":2459,"stick":2460,"stock":2461,"store":2462,"story":2463,"straight":2464,"strange":2465,"strategic":2466,"strategy":2467,"street":2468,"strength":2469,"stress":2470,"stretch":2471,"strike":2472,"strong":2473,"structure":2474,"struggle":2475,"student":2476,"study":2477,"subject":2478,"succeed":2479,"successful":2480,"suddenly":2481,"suffer":2482,"sufficient":2483,"suggest":2484,"suggestion":2485,"summer":2486,"supply":2487,"support":2488,"suppose":2489,"sure":2490,"surface":2491,"surgery":2492,"surprise":2493,"survey":2494,"survive":2495,"suspect":2496,"sustain":2497,"symbol":2498,"talent":2499,"talk":2500,"tape":2501,"taste":2502,"tax":2503,"teach":2504,"teacher":2505,"teaching":2506,"team":2507,"tear":2508,"technical":2509,"technique":2510,"technology":2511,"telephone":2512,"television":2513,"temperature":2514,"tend":2515,"term":2516,"testify":2517,"thank":2518,"themselves":2519,"therefore":2520,"thick":2521,"thin":2522,"thirty":2523,"threat":2524,"threaten":2525,"thus":2526,"ticket":2527,"tight":2528,"till":2529,"tiny":2530,"tip":2531,"title":2532,"today":2533,"tomorrow":2534,"tone":2535,"tonight":2536,"tool":2537,"top":2538,"totally":2539,"touch":2540,"tough":2541,"tour":2542,"toward":2543,"town":2544,"track":2545,"trade":2546,"tradition":2547,"traditional":2548,"traffic":2549,"transfer":2550,"transform":2551,"travel":2552,"treat":2553,"treatment":2554,"trial":2555,"trip":2556,"troop":2557,"trouble":2558,"truck":2559,"truly":2560,"trust":2561,"truth":2562,"tube":2563,"twice":2564,"typical":2565,"uncle":2566,"understand":2567,"understanding":2568,"unfortunately":2569,"unique":2570,"unit":2571,"United":2572,"universe":2573,"university":2574,"unless":2575,"unlike":2576,"unlikely":2577,"unusual":2578,"upper":2579,"urban":2580,"urge":2581,"used":2582,"useful":2583,"usual":2584,"usually":2585,"variety":2586,"various":2587,"vehicle":2588,"version":2589,"veteran":2590,"victim":2591,"victory":2592,"video":2593,"view":2594,"village":2595,"violence":2596,"visit":2597,"voice":2598,"volume":2599,"vote":2600,"voter":2601,"wage":2602,"wait":2603,"walk":2604,"wall":2605,"war":2606,"warm":2607,"warn":2608,"wash":2609,"watch":2610,"water":2611,"wave":2612,"weak":2613,"weapon":2614,"wear":2615,"weather":2616,"web":2617,"wedding":2618,"weekend":2619,"welcome":2620,"welfare":2621,"west":2622,"western":2623,"whatever":2624,"wheel":2625,"whenever":2626,"whereas":2627,"whether":2628,"white":2629,"whole":2630,"whom":2631,"whose":2632,"wide":2633,"widely":2634,"wife":2635,"wild":2636,"wind":2637,"window":2638,"wine":2639,"wing":2640,"winner":2641,"winter":2642,"wire":2643,"wish":2644,"woman":2645,"wonder":2646,"wonderful":2647,"wood":2648,"word":2649,"worker":2650,"working":2651,"works":2652,"worry":2653,"worth":2654,"writer":2655,"yard":2656,"yeah":2657,"youth":2658,"zone":2659,"ability":2660,"abroad":2661,"absent":2662,"absolute":2663,"absorb":2664,"abuse":2665,"academic":2666,"accept":2667,"access":2668,"accident":2669,"accompany":2670,"accomplish":2671,"account":2672,"accurate":2673,"accuse":2674,"achieve":2675,"acknowledge":2676,"acquire":2677,"adapt":2678,"addition":2679,"adjust":2680,"administration":2681,"admit":2682,"adopt":2683,"advance":2684,"advantage":2685,"adventure":2686,"advertise":2687,"advice":2688,"advise":2689,"advocate":2690,"affair":2691,"affect":2692,"afford":2693,"agency":2694,"agenda":2695,"agent":2696,"aggression":2697,"aggressive":2698,"aid":2699,"aim":2700,"air":2701,"airport":2702,"alarm":2703,"alcohol":2704,"alert":2705,"alive":2706,"alliance":2707,"allocate":2708,"ally":2709,"alter":2710,"alternative":2711,"amaze":2712,"ambition":2713,"amendment":2714,"amid":2715,"amongst":2716,"analyst":2717,"angle":2718,"angry":2719,"anniversary":2720,"announce":2721,"annual":2722,"anticipate":2723,"anxiety":2724,"anxious":2725,"apart":2726,"apartment":2727,"apologize":2728,"apparent":2729,"appeal":2730,"appearance":2731,"appetite":2732,"apple":2733,"applicant":2734,"application":2735,"apply":2736,"appoint":2737,"appreciate":2738,"appropriate":2739,"approval":2740,"approve":2741,"architecture":2742,"archive":2743,"arrange":2744,"arrangement":2745,"arrest":2746,"arrival":2747,"arrive":2748,"arrow":2749,"articulate":2750,"artificial":2751,"aside":2752,"aspect":2753,"assault":2754,"assemble":2755,"assembly":2756,"assert":2757,"assess":2758,"assessment":2759,"asset":2760,"assign":2761,"assist":2762,"assistance":2763,"associate":2764,"association":2765,"assume":2766,"assumption":2767,"atmosphere":2768,"attach":2769,"attack":2770,"attempt":2771,"attend":2772,"attention":2773,"attitude":2774,"attorney":2775,"attract":2776,"attraction":2777,"attractive":2778,"attribute":2779,"audience":2780,"auto":2781,"automatic":2782,"automatically":2783,"autonomy":2784,"avenue":2785,"average":2786,"award":2787,"aware":2788,"awareness":2789,"awful":2790,"background":2791,"bacteria":2792,"balance":2793,"bare":2794,"barely":2795,"barrier":2796,"basis":2797,"basket":2798,"bath":2799,"battery":2800,"bay":2801,"beach":2802,"bean":2803,"bear":2804,"beat":2805,"beautiful":2806,"bedroom":2807,"beer":2808,"behalf":2809,"behave":2810,"belief":2811,"believable":2812,"bell":2813,"belong":2814,"bench":2815,"bend":2816,"beneath":2817,"beneficial":2818,"beside":2819,"bet":2820,"betray":2821,"bible":2822,"bicycle":2823,"bid":2824,"bike":2825,"bill":2826,"bind":2827,"biological":2828,"birth":2829,"biscuit":2830,"bishop":2831,"bite":2832,"bitter":2833,"blade":2834,"blame":2835,"blank":2836,"blast":2837,"bleed":2838,"blend":2839,"bless":2840,"blind":2841,"block":2842,"blow":2843,"blue":2844,"blur":2845,"boast":2846,"boat":2847,"bomb":2848,"bond":2849,"bone":2850,"bonus":2851,"boom":2852,"boost":2853,"border":2854,"bore":2855,"borrow":2856,"bound":2857,"boundary":2858,"bowl":2859,"branch":2860,"brand":2861,"brave":2862,"bread":2863,"breadth":2864,"breast":2865,"breath":2866,"breathe":2867,"breathing":2868,"breed":2869,"brick":2870,"bride":2871,"briefly":2872,"brilliant":2873,"broadcast":2874,"broken":2875,"bronze":2876,"brow":2877,"brown":2878,"brush":2879,"bubble":2880,"bucket":2881,"buddy":2882,"buffalo":2883,"bunch":2884,"burden":2885,"burglar":2886,"burst":2887,"bury":2888,"bus":2889,"butter":2890,"button":2891,"cabin":2892,"cabinet":2893,"cable":2894,"cake":2895,"calendar":2896,"calm":2897,"camera":2898,"camp":2899,"campus":2900,"canal":2901,"cancel":2902,"candidate":2903,"candle":2904,"cap":2905,"capable":2906,"capacity":2907,"captain":2908,"capture":2909,"carbon":2910,"card":2911,"careful":2912,"carefully":2913,"carrier":2914,"cart":2915,"carve":2916,"cast":2917,"castle":2918,"casualty":2919,"catalog":2920,"catalogue":2921,"cattle":2922,"celebrate":2923,"celebration":2924,"cell":2925,"cellular":2926,"census":2927,"centimeter":2928,"ceremony":2929,"certainly":2930,"chain":2931,"chamber":2932,"champion":2933,"championship":2934,"channel":2935,"chapter":2936,"characteristic":2937,"charity":2938,"chart":2939,"chase":2940,"cheap":2941,"cheat":2942,"cheek":2943,"cheese":2944,"chemical":2945,"chest":2946,"chicken":2947,"chief":2948,"childhood":2949,"chip":2950,"chocolate":2951,"chorus":2952,"christian":2953,"Christmas":2954,"chronic":2955,"chunk":2956,"circle":2957,"circuit":2958,"circumstance":2959,"cite":2960,"civilian":2961,"clarify":2962,"clarity":2963,"clash":2964,"classic":2965,"classical":2966,"classification":2967,"classroom":2968,"clause":2969,"clean":2970,"clever":2971,"click":2972,"cliff":2973,"climate":2974,"climb":2975,"clinic":2976,"clinical":2977,"clock":2978,"clone":2979,"closed":2980,"closely":2981,"closer":2982,"closet":2983,"closing":2984,"cloth":2985,"clothe":2986,"clothes":2987,"clothing":2988,"cloud":2989,"cluster":2990,"coal":2991,"coalition":2992,"coast":2993,"coat":2994,"coffee":2995,"cognitive":2996,"coin":2997,"collapse":2998,"collar":2999,"colleague":3000,"collect":3001,"collective":3002,"colonial":3003,"colony":3004,"combat":3005,"combine":3006,"combined":3007,"comedy":3008,"comfort":3009,"command":3010,"commander":3011,"commerce":3012,"commercial":3013,"commission":3014,"commit":3015,"commitment":3016,"commodity":3017,"communicate":3018,"communication":3019,"communist":3020,"compact":3021,"companion":3022,"comparison":3023,"compel":3024,"compensate":3025,"compensation":3026,"compete":3027,"competitive":3028,"competitor":3029,"complain":3030,"complaint":3031,"complicate":3032,"complicated":3033,"comply":3034,"component":3035,"compose":3036,"composition":3037,"compound":3038,"comprehensive":3039,"comprise":3040,"compromise":3041,"compulsory":3042,"computer":3043,"conceal":3044,"concede":3045,"conceive":3046,"concentrate":3047,"concentration":3048,"concept":3049,"conception":3050,"concern":3051,"concerning":3052,"concert":3053,"conclusion":3054,"concrete":3055,"condemn":3056,"conduct":3057,"confess":3058,"confession":3059,"confidence":3060,"confident":3061,"confidential":3062,"confine":3063,"confirm":3064,"conflict":3065,"confront":3066,"confusion":3067,"congratulate":3068,"congress":3069,"connection":3070,"consciousness":3071,"consecutive":3072,"consensus":3073,"consent":3074,"consequence":3075,"consequently":3076,"conservation":3077,"conservative":3078,"considerable":3079,"considerably":3080,"consist":3081,"consistent":3082,"consistently":3083,"constant":3084,"constantly":3085,"constitute":3086,"constitution":3087,"constitutional":3088,"construct":3089,"construction":3090,"consult":3091,"consultant":3092,"consume":3093,"consumer":3094,"consumption":3095,"contact":3096,"contemporary":3097,"contend":3098,"contest":3099,"context":3100,"continent":3101,"continually":3102,"continuity":3103,"continuous":3104,"continuously":3105,"contradiction":3106,"contrary":3107,"contribute":3108,"contribution":3109,"contributor":3110,"controversial":3111,"controversy":3112,"convenience":3113,"convenient":3114,"convention":3115,"conventional":3116,"conversion":3117,"convert":3118,"convey":3119,"convict":3120,"conviction":3121,"convince":3122,"cook":3123,"cool":3124,"cooperate":3125,"cooperation":3126,"coordinate":3127,"coordination":3128,"cope":3129,"copper":3130,"copy":3131,"copyright":3132,"core":3133,"corn":3134,"corner":3135,"corporate":3136,"corporation":3137,"correction":3138,"correctly":3139,"correlate":3140,"correlation":3141,"correspond":3142,"correspondent":3143,"corridor":3144,"corruption":3145,"costly":3146,"cotton":3147,"council":3148,"counsel":3149,"counselor":3150,"counterpart":3151,"county":3152,"coup":3153,"courage":3154,"cousin":3155,"coverage":3156,"crack":3157,"craft":3158,"crash":3159,"creative":3160,"creativity":3161,"creator":3162,"creature":3163,"credibility":3164,"credit":3165,"creep":3166,"crew":3167,"criminal":3168,"crisis":3169,"criterion":3170,"critic":3171,"critical":3172,"criticism":3173,"criticize":3174,"crop":3175,"cross":3176,"crowd":3177,"crown":3178,"crucial":3179,"crude":3180,"cruel":3181,"cruise":3182,"crush":3183,"cry":3184,"crystal":3185,"cube":3186,"cuisine":3187,"cultivate":3188,"curious":3189,"currency":3190,"curriculum":3191,"curtain":3192,"curve":3193,"custody":3194,"custom":3195,"customary":3196,"cutting":3197,"cycle":3198,"dad":3199,"damage":3200,"damn":3201,"dance":3202,"danger":3203,"dangerous":3204,"dare":3205,"database":3206,"dawn":3207,"dead":3208,"deadline":3209,"deadly":3210,"deaf":3211,"dealer":3212,"dear":3213,"debt":3214,"decay":3215,"deceive":3216,"decent":3217,"decisive":3218,"deck":3219,"declaration":3220,"decline":3221,"decorate":3222,"decrease":3223,"decree":3224,"dedicate":3225,"deem":3226,"defeat":3227,"defend":3228,"defendant":3229,"defender":3230,"defensive":3231,"deficit":3232,"definite":3233,"definitely":3234,"definition":3235,"defy":3236,"delay":3237,"delegate":3238,"delegation":3239,"deliberate":3240,"deliberately":3241,"delicate":3242,"delicious":3243,"delight":3244,"deliver":3245,"delivery":3246,"demand":3247,"democracy":3248,"demographic":3249,"demonstrate":3250,"demonstration":3251,"denial":3252,"denote":3253,"depart":3254,"department":3255,"departure":3256,"depend":3257,"dependence":3258,"dependent":3259,"depict":3260,"deposit":3261,"depress":3262,"depression":3263,"deprive":3264,"depth":3265,"deputy":3266,"derive":3267,"descend":3268,"description":3269,"desert":3270,"deserve":3271,"designate":3272,"designer":3273,"desirable":3274,"desire":3275,"desk":3276,"desperate":3277,"desperately":3278,"destination":3279,"destroy":3280,"destruction":3281,"detailed":3282,"detain":3283,"detect":3284,"detection":3285,"detective":3286,"detention":3287,"deteriorate":3288,"determination":3289,"determined":3290,"developmental":3291,"devote":3292,"diabetes":3293,"diagnose":3294,"diagnosis":3295,"dialogue":3296,"diameter":3297,"diamond":3298,"diary":3299,"dictate":3300,"diet":3301,"differ":3302,"differentiate":3303,"differently":3304,"difficulty":3305,"dig":3306,"digest":3307,"digital":3308,"dignity":3309,"dilemma":3310,"dimension":3311,"diminish":3312,"dioxide":3313,"dip":3314,"diplomat":3315,"diplomatic":3316,"direct":3317,"directly":3318,"dirty":3319,"disability":3320,"disadvantage":3321,"disagree":3322,"disappear":3323,"disappoint":3324,"disappointment":3325,"disaster":3326,"disastrous":3327,"disc":3328,"discharge":3329,"discipline":3330,"disclose":3331,"discount":3332,"discourse":3333,"discovery":3334,"discrepancy":3335,"discretion":3336,"discrimination":3337,"dismiss":3338,"disorder":3339,"dispatch":3340,"display":3341,"disposal":3342,"dispose":3343,"dispute":3344,"disrupt":3345,"dissolve":3346,"distance":3347,"distant":3348,"distinct":3349,"distinction":3350,"distinctive":3351,"distinguish":3352,"distort":3353,"distract":3354,"distress":3355,"distribute":3356,"distributor":3357,"district":3358,"disturb":3359,"dive":3360,"diverse":3361,"diversity":3362,"divide":3363,"division":3364,"divorce":3365,"dock":3366,"doctor":3367,"doctrine":3368,"document":3369,"documentary":3370,"dollar":3371,"domain":3372,"dome":3373,"domestic":3374,"dominant":3375,"dominate":3376,"donation":3377,"donor":3378,"dose":3379,"dot":3380,"doubtful":3381,"downtown":3382,"draft":3383,"drag":3384,"drain":3385,"drama":3386,"dramatic":3387,"dramatically":3388,"drastic":3389,"drawing":3390,"drink":3391,"drought":3392,"drown":3393,"drum":3394,"drunk":3395,"dry":3396,"dual":3397,"dubious":3398,"duck":3399,"due":3400,"dull":3401,"dump":3402,"durable":3403,"duration":3404,"dust":3405,"duty":3406,"dynamics":3407,"eager":3408,"eagle":3409,"ear":3410,"earning":3411,"earth":3412,"ease":3413,"easily":3414,"eastern":3415,"echo":3416,"eclipse":3417,"ecological":3418,"ecology":3419,"economics":3420,"economist":3421,"ecosystem":3422,"edit":3423,"editorial":3424,"educate":3425,"educational":3426,"educator":3427,"effective":3428,"effectively":3429,"effectiveness":3430,"efficiency":3431,"efficient":3432,"efficiently":3433,"elaborate":3434,"elbow":3435,"elderly":3436,"elect":3437,"electoral":3438,"electric":3439,"electrical":3440,"electricity":3441,"electronic":3442,"electronics":3443,"elegant":3444,"elementary":3445,"eliminate":3446,"elimination":3447,"elite":3448,"elsewhere":3449,"email":3450,"embargo":3451,"embark":3452,"embarrass":3453,"embassy":3454,"embed":3455,"embody":3456,"embrace":3457,"emerge":3458,"emergence":3459,"emergency":3460,"emission":3461,"emotion":3462,"emotional":3463,"emphasis":3464,"emphasize":3465,"empire":3466,"empirical":3467,"employ":3468,"employer":3469,"employment":3470,"empower":3471,"enact":3472,"encompass":3473,"encounter":3474,"encouragement":3475,"endanger":3476,"endeavor":3477,"endorse":3478,"endorsement":3479,"endure":3480,"enforce":3481,"enforcement":3482,"engage":3483,"engagement":3484,"engine":3485,"engineering":3486,"enhance":3487,"enjoyment":3488,"enlarge":3489,"enormous":3490,"enrich":3491,"enroll":3492,"ensemble":3493,"ensure":3494,"enterprise":3495,"entertain":3496,"entertainment":3497,"enthusiasm":3498,"enthusiast":3499,"enthusiastic":3500,"entirely":3501,"entitle":3502,"entity":3503,"entrepreneur":3504,"envelope":3505,"epidemic":3506,"episode":3507,"equal":3508,"equality":3509,"equip":3510,"equipment":3511,"equivalent":3512,"era":3513,"erect":3514,"erupt":3515,"escalate":3516,"escape":3517,"essay":3518,"essence":3519,"essential":3520,"essentially":3521,"establishment":3522,"estate":3523,"estimate":3524,"estimation":3525,"eternal":3526,"ethical":3527,"ethics":3528,"ethnic":3529,"evacuate":3530,"evaluation":3531,"evenly":3532,"eventually":3533,"everyday":3534,"evident":3535,"evil":3536,"evoke":3537,"evolution":3538,"evolutionary":3539,"evolve":3540,"exact":3541,"exaggerate":3542,"examination":3543,"exceed":3544,"excellence":3545,"excellent":3546,"exception":3547,"exceptional":3548,"excess":3549,"excessive":3550,"exchange":3551,"excite":3552,"excitement":3553,"exciting":3554,"exclude":3555,"exclusion":3556,"exclusive":3557,"exclusively":3558,"excuse":3559,"execute":3560,"execution":3561,"exemplify":3562,"exercise":3563,"exert":3564,"exhaust":3565,"exhibit":3566,"exhibition":3567,"exile":3568,"existence":3569,"exit":3570,"expand":3571,"expansion":3572,"expectation":3573,"expedition":3574,"expel":3575,"expenditure":3576,"expense":3577,"expensive":3578,"expert":3579,"expertise":3580,"explicit":3581,"explicitly":3582,"explode":3583,"exploit":3584,"exploitation":3585,"exploration":3586,"explore":3587,"explosion":3588,"explosive":3589,"expose":3590,"exposure":3591,"express":3592,"extend":3593,"extension":3594,"extensive":3595,"extensively":3596,"extent":3597,"external":3598,"extinct":3599,"extinction":3600,"extract":3601,"extraordinary":3602,"extreme":3603,"fabric":3604,"fabulous":3605,"facade":3606,"facial":3607,"facilitate":3608,"facility":3609,"faction":3610,"faculty":3611,"fade":3612,"fair":3613,"fairly":3614,"fairness":3615,"faith":3616,"faithful":3617,"fake":3618,"fame":3619,"familiar":3620,"famine":3621,"fan":3622,"fancy":3623,"fantasy":3624,"fare":3625,"fascinate":3626,"fascinating":3627,"fashion":3628,"fat":3629,"fate":3630,"fatigue":3631,"fault":3632,"favor":3633,"favorable":3634,"favorite":3635,"fax":3636,"feast":3637,"feather":3638,"federation":3639,"fee":3640,"feed":3641,"feedback":3642,"fellow":3643,"fellowship":3644,"female":3645,"fence":3646,"fertile":3647,"fertilizer":3648,"festival":3649,"fetch":3650,"fever":3651,"fiber":3652,"fiction":3653,"fierce":3654,"fifteen":3655,"fifty":3656,"fig":3657,"fighter":3658,"finance":3659,"financially":3660,"financing":3661,"finger":3662,"finished":3663,"fire":3664,"firmly":3665,"fiscal":3666,"fisherman":3667,"fishing":3668,"fitness":3669,"fix":3670,"fixture":3671,"flag":3672,"flame":3673,"flash":3674,"flat":3675,"flavor":3676,"flee":3677,"fleet":3678,"flesh":3679,"flexibility":3680,"flexible":3681,"flight":3682,"flock":3683,"flood":3684,"flour":3685,"flow":3686,"flower":3687,"fluid":3688,"flush":3689,"folk":3690,"football":3691,"footnote":3692,"footstep":3693,"forbid":3694,"forbidden":3695,"forecast":3696,"forehead":3697,"foreigner":3698,"forest":3699,"forever":3700,"forge":3701,"forgive":3702,"fork":3703,"formal":3704,"format":3705,"formation":3706,"formulate":3707,"fort":3708,"forth":3709,"fortunate":3710,"fortune":3711,"forum":3712,"fossil":3713,"foster":3714,"foundation":3715,"founder":3716,"fountain":3717,"fraction":3718,"fracture":3719,"fragile":3720,"fragment":3721,"framework":3722,"franchise":3723,"frank":3724,"frankly":3725,"fraud":3726,"freely":3727,"freeze":3728,"freight":3729,"frequency":3730,"frequent":3731,"frequently":3732,"fresh":3733,"freshman":3734,"friction":3735,"friendship":3736,"frighten":3737,"frog":3738,"frontier":3739,"frost":3740,"frown":3741,"frozen":3742,"fruit":3743,"frustrate":3744,"frustration":3745,"fuel":3746,"fulfill":3747,"fun":3748,"functional":3749,"fundamental":3750,"funding":3751,"funeral":3752,"funny":3753,"fur":3754,"furious":3755,"furniture":3756,"further":3757,"furthermore":3758,"fury":3759,"fusion":3760,"gain":3761,"galaxy":3762,"gallery":3763,"gallon":3764,"gambling":3765,"gap":3766,"garage":3767,"garbage":3768,"garlic":3769,"garment":3770,"gasoline":3771,"gate":3772,"gather":3773,"gathering":3774,"gauge":3775,"gaze":3776,"gear":3777,"gender":3778,"gene":3779,"generally":3780,"generate":3781,"generator":3782,"generous":3783,"genetic":3784,"genetics":3785,"genius":3786,"genocide":3787,"genre":3788,"gentle":3789,"gently":3790,"genuine":3791,"genuinely":3792,"gesture":3793,"giant":3794,"gift":3795,"gigantic":3796,"glimpse":3797,"global":3798,"globalization":3799,"globe":3800,"glory":3801,"glove":3802,"glow":3803,"glucose":3804,"goddess":3805,"golden":3806,"golf":3807,"goodness":3808,"goods":3809,"gorgeous":3810,"gospel":3811,"gossip":3812,"govern":3813,"governance":3814,"grab":3815,"grace":3816,"grade":3817,"gradually":3818,"graduate":3819,"graduation":3820,"grain":3821,"gram":3822,"grammar":3823,"grand":3824,"grandfather":3825,"grandmother":3826,"graphic":3827,"grasp":3828,"grass":3829,"grateful":3830,"grave":3831,"gravity":3832,"gray":3833,"greatly":3834,"greenhouse":3835,"greet":3836,"grief":3837,"grin":3838,"grind":3839,"grip":3840,"grocery":3841,"gross":3842,"groundwater":3843,"guarantee":3844,"guard":3845,"guardian":3846,"guest":3847,"guidance":3848,"guide":3849,"guideline":3850,"guilty":3851,"guitar":3852,"gulf":3853,"gut":3854,"gym":3855,"habit":3856,"habitat":3857,"hair":3858,"hall":3859,"halt":3860,"hammer":3861,"handful":3862,"handling":3863,"handwriting":3864,"handy":3865,"happiness":3866,"harassment":3867,"harbor":3868,"hardly":3869,"hardware":3870,"harm":3871,"harmful":3872,"harmony":3873,"harvest":3874,"hat":3875,"hate":3876,"haul":3877,"hay":3878,"hazard":3879,"headache":3880,"headline":3881,"headquarters":3882,"heal":3883,"healthcare":3884,"healthy":3885,"hearing":3886,"heating":3887,"heaven":3888,"heavily":3889,"hedge":3890,"heel":3891,"height":3892,"helicopter":3893,"helmet":3894,"helpful":3895,"herb":3896,"heritage":3897,"hero":3898,"heroin":3899,"hesitate":3900,"hidden":3901,"hierarchy":3902,"highlight":3903,"highly":3904,"highway":3905,"hike":3906,"hill":3907,"himself":3908,"hip":3909,"hire":3910,"historian":3911,"historic":3912,"historical":3913,"hobby":3914,"holder":3915,"holding":3916,"hole":3917,"holiday":3918,"hollow":3919,"holy":3920,"homeland":3921,"homeless":3922,"homework":3923,"honesty":3924,"honey":3925,"honor":3926,"hook":3927,"hopeful":3928,"hopefully":3929,"horizon":3930,"horizontal":3931,"hormone":3932,"horn":3933,"horrible":3934,"horror":3935,"horse":3936,"hospitality":3937,"hostage":3938,"hostile":3939,"hot":3940,"hotline":3941,"hour":3942,"housing":3943,"hover":3944,"humane":3945,"humanitarian":3946,"humanity":3947,"humor":3948,"hunger":3949,"hungry":3950,"hunt":3951,"hunter":3952,"hunting":3953,"hurt":3954,"hut":3955,"hydrogen":3956,"hygiene":3957,"hypothesis":3958,"ice":3959,"icon":3960,"ideal":3961,"identical":3962,"identification":3963,"identity":3964,"ideology":3965,"ignorance":3966,"ignore":3967,"ill":3968,"illegal":3969,"illness":3970,"illusion":3971,"illustrate":3972,"illustration":3973,"imaginary":3974,"imagination":3975,"imitate":3976,"immediate":3977,"immediately":3978,"immense":3979,"immigrant":3980,"immigration":3981,"immune":3982,"immunity":3983,"implication":3984,"implicit":3985,"importance":3986,"impose":3987,"impossible":3988,"impress":3989,"impression":3990,"impressive":3991,"imprison":3992,"improbable":3993,"improvement":3994,"impulse":3995,"inability":3996,"inappropriate":3997,"incentive":3998,"incidence":3999,"incident":4000,"inclination":4001,"incline":4002,"inclusion":4003,"inclusive":4004,"income":4005,"incorporate":4006,"increasingly":4007,"incredible":4008,"incur":4009,"independence":4010,"independent":4011,"independently":4012,"indication":4013,"indicator":4014,"indictment":4015,"indigenous":4016,"indirect":4017,"indispensable":4018,"individuality":4019,"indoor":4020,"induce":4021,"indulge":4022,"industrial":4023,"industrialize":4024,"inequality":4025,"inevitable":4026,"inevitably":4027,"infant":4028,"infect":4029,"infection":4030,"infer":4031,"inference":4032,"inferior":4033,"infinite":4034,"infinity":4035,"inflation":4036,"inflict":4037,"influential":4038,"informal":4039,"infrastructure":4040,"ingredient":4041,"inhabit":4042,"inhabitant":4043,"inherent":4044,"inherit":4045,"inhibit":4046,"initially":4047,"initiate":4048,"initiative":4049,"inject":4050,"injection":4051,"injure":4052,"injury":4053,"inmate":4054,"inner":4055,"innocent":4056,"innovation":4057,"innovative":4058,"inquiry":4059,"insect":4060,"insertion":4061,"insight":4062,"insist":4063,"inspect":4064,"inspection":4065,"inspector":4066,"inspiration":4067,"inspire":4068,"install":4069,"installation":4070,"installment":4071,"instance":4072,"instant":4073,"instantly":4074,"instinct":4075,"institute":4076,"institutional":4077,"instruct":4078,"instruction":4079,"instructor":4080,"instrument":4081,"instrumental":4082,"insufficient":4083,"insult":4084,"insurance":4085,"intact":4086,"integrate":4087,"integration":4088,"integrity":4089,"intellectual":4090,"intelligence":4091,"intelligent":4092,"intend":4093,"intense":4094,"intensity":4095},"merges":[]}}

train_tiny.py

@@ -6,10 +6,12 @@ from torch.optim import AdamW
 from torch.optim.lr_scheduler import LambdaLR
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
-from scripts.model_tiny import TinyModel
+from scripts.model_tiny import restore_from_v2
 
 
 def _format_item(item, tokenizer):
+    if "text" in item:
+        return item["text"]
     if "messages" in item:
         return tokenizer.apply_chat_template(item["messages"], tokenize=False)
     system = item.get("system", "")
@@ -150,14 +152,8 @@ def train():
     vocab_size = tok.vocab_size
     max_seq = d_cfg.get("max_seq_length", 2048)
 
-    model = TinyModel(
-        vocab_size=vocab_size, hidden=128, intermediate=640,
-        num_layers=3, num_heads=8, num_kv_heads=4,
-        max_seq_len=max_seq, tie_weights=True,
-    ).to(device)
-    model.reset_weights()
-    n_params = sum(p.numel() for p in model.parameters())
-    print(f"Params: {n_params:,}")
+    # ── Create model with V2 restoration ────────────────────────────────────
+    model = restore_from_v2("checkpoint.pt").to(device)
 
     hf_repo = d_cfg.get("hf_repo")
     is_streaming = bool(hf_repo)
@@ -177,11 +173,11 @@ def train():
         eval_ds = ListDataset([_encode(_format_item(x, tok), tok, max_seq) for x in eval_raw])
         print(f"Train: {len(train_ds.samples)}  Eval: {len(eval_ds.samples)}")
 
-    bs = t_cfg.get("per_device_train_batch_size", 2)
-    eval_bs = t_cfg.get("per_device_eval_batch_size", 2)
-    ga_steps = t_cfg.get("gradient_accumulation_steps", 1)
-    lr = t_cfg.get("learning_rate", 5e-4)
-    epochs = t_cfg.get("num_train_epochs", 5)
+    bs = t_cfg.get("per_device_train_batch_size", 8)
+    eval_bs = t_cfg.get("per_device_eval_batch_size", 8)
+    ga_steps = t_cfg.get("gradient_accumulation_steps", 4)
+    lr = t_cfg.get("learning_rate", 3e-4)
+    epochs = t_cfg.get("num_train_epochs", 1)
     max_grad_norm = t_cfg.get("max_grad_norm", 1.0)
     log_steps = t_cfg.get("logging_steps", 5)
     save_steps = t_cfg.get("save_steps", 200)
@@ -194,8 +190,8 @@ def train():
     hf_repo_id = t_cfg.get("hf_repo_id", "")
     hf_token = os.environ.get("HF_TOKEN", "")
 
-    # Resume
-    ckpt_path = os.path.join(output_dir, "checkpoint.pt")
+    n_params = sum(p.numel() for p in model.parameters())
+
     step = 0
     opt_step = 0
     start_epoch = 1
@@ -203,15 +199,20 @@ def train():
     best_acc = 0.0
     cumul_tokens = 0
     global_step_offset = 0
+    ckpt_path = os.path.join(output_dir, "checkpoint.pt")
 
-    # Separate norms from weight decay
+    # ── Optimiser ──────────────────────────────────────────────────────────
     decay_params = []
     no_decay_params = []
     for name, param in model.named_parameters():
+        if not param.requires_grad:
+            continue
         if "norm" in name or "ln_" in name:
             no_decay_params.append(param)
         else:
             decay_params.append(param)
+    if not decay_params and not no_decay_params:
+        print("[WARNING] No trainable parameters found!")
     optimizer = AdamW([
         {"params": decay_params, "weight_decay": t_cfg.get("weight_decay", 0.1)},
         {"params": no_decay_params, "weight_decay": 0.0},
@@ -257,9 +258,9 @@ def train():
     eff_bs = bs * ga_steps
     print()
     print("=" * 72)
-    print(f"  Lumia-Tiny Training")
+    print(f"  PCT-V3 Training")
     print(f"  Model: {n_params:,} params | Device: {device.upper()}")
-    print(f"  Vocab: {vocab_size} | Depth: 3 | Hidden: 128 | FF: 640 | Heads: 8/4")
+    print(f"  Vocab: {vocab_size} | Depth: 6 | Hidden: 128 | Code: 96 | Heads: 8/4 | RPW+GPP+VCR")
     print(f"  Epochs: {epochs} | Batch: {bs} | GA: {ga_steps} | Eff BS: {eff_bs} | LR: {lr}")
     print(f"  Warmup: {warmup_opt_steps} opt steps | Total: {total_opt_steps} opt steps")
     print("=" * 72)

train_tiny.yaml

@@ -1,39 +1,27 @@
-# Tiny Training Config — Custom PyTorch (ALiBi+SiLU+RMSNorm, V2)
+# PCT-V3 Training Config — RPW+GPP+VCR
 # Script: scripts/model_tiny.py
 
-model:
-  name_or_path: "tiny-custom-pytorch-v2"
-
 data:
-  # Local JSONL (used when hf_repo is not set)
-  train_file: "data/tiny_data.jsonl"
-  eval_split_ratio: 0.1
-
-  # HF streaming (overrides train_file when set)
-  hf_repo: "trl-lib/Capybara"
-  hf_repo_name: ~
+  hf_repo: "HuggingFaceFW/fineweb-edu"
+  hf_repo_name: "sample-100BT"
   hf_split: "train"
   hf_num_eval: 50
-
   max_seq_length: 2048
 
 training:
   output_dir: "outputs/tiny-sft"
-  run_name: "lumia-tiny-v2"
-  per_device_train_batch_size: 16
-  per_device_eval_batch_size: 16
-  gradient_accumulation_steps: 1
+  run_name: "pct-v3-fwe"
+  per_device_train_batch_size: 8
+  per_device_eval_batch_size: 8
+  gradient_accumulation_steps: 4
   max_grad_norm: 1.0
-  num_train_epochs: 5
+  num_train_epochs: 1
   max_steps: 50000
-  learning_rate: 5.0e-4
+  learning_rate: 3.0e-4
   lr_scheduler_type: "cosine"
-  warmup_ratio: 0.1
+  warmup_ratio: 0.01
   weight_decay: 0.1
   use_cpu: false
   logging_steps: 5
-  save_steps: 200
-  save_total_limit: 3
-  eval_steps: 200
+  save_steps: 500
   seed: 42
-  hf_repo_id: "samcheng0/lumia-tiny"