Nexus-Erebus-3M (avg 32.50)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+nexus_erebus.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,82 @@
+---
+license: apache-2.0
+library_name: transformers
+pipeline_tag: text-generation
+language:
+- en
+tags:
+- small-language-model
+- slm
+- from-scratch
+- tiny
+- nexus-erebus
+- arithmetic
+---
+
+![Nexus-Erebus](./nexus_erebus.png)
+
+# Nexus-Erebus-3M
+
+A ~3M-parameter language model trained from scratch by **Ideoa Labs**.
+
+It uses a digit-atomic tokenizer that emits digit spans **least-significant-digit first**, which
+aligns carry propagation with the direction the model reads. Text goes in and comes out in normal
+order; the reversal happens inside the tokenizer.
+
+## Model details
+
+| | |
+|---|---|
+| Parameters | ~3.0M |
+| Architecture | Llama-style decoder |
+| Hidden size | 192 |
+| Layers | 5 |
+| Attention heads | 4 |
+| Vocab size | 4096 |
+| Context length | 512 |
+| Precision | bfloat16 |
+
+## Results
+
+0-shot, `acc_norm` for multiple choice, accuracy for ArithMark-2. Full test sets, no subsampling.
+
+| Task | Score |
+|---|---:|
+| ARC-easy | 26.98 |
+| ARC-challenge | 21.67 |
+| HellaSwag | 26.74 |
+| PIQA | 50.49 |
+| ArithMark-2 | 36.60 |
+| **Average** | **32.50** |
+
+## Usage
+
+The tokenizer ships with the model and requires `trust_remote_code=True`.
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+tok = AutoTokenizer.from_pretrained("MaliosDark/Nexus-Erebus-3M", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained("MaliosDark/Nexus-Erebus-3M")
+
+prompt = "16 + 4 * 3 ="
+print(tok.decode(model.generate(**tok(prompt, return_tensors="pt"), max_new_tokens=6)[0]))
+```
+
+## Reproducing the ArithMark-2 score
+
+```bash
+python benchmark_nexus_arithmark.py MaliosDark/Nexus-Erebus-3M
+```
+
+## Training
+
+Pretrained from scratch on TinyStories plus synthetic integer arithmetic covering addition,
+subtraction, multiplication, exact division, and mixed and parenthesised multi-operator
+expressions. Then fine-tuned on the official train splits of the public benchmarks plus more
+synthetic arithmetic. No evaluation items were used at any stage.
+
+## License
+
+Apache-2.0.
+
+Built by Ideoa Labs.

benchmark_nexus_arithmark.py

@@ -0,0 +1,47 @@
+"""Self-contained ArithMark-2 verification for Nexus-Erebus models.
+
+Reproduces the reported ArithMark-2 score. No local files needed beyond this repo.
+The model ships a digit-atomic, least-significant-digit-first tokenizer, so it must
+be loaded with trust_remote_code=True. Text goes in and comes out in normal order;
+the digit reversal happens inside the tokenizer.
+
+    pip install torch transformers datasets
+    python benchmark_nexus_arithmark.py            # uses this repo
+    python benchmark_nexus_arithmark.py <model_id>
+"""
+import sys, ast, torch
+from datasets import load_dataset
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+MODEL = sys.argv[1] if len(sys.argv) > 1 else "."
+dev = "cuda" if torch.cuda.is_available() else "cpu"
+
+tok = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16).to(dev).eval()
+ds = load_dataset("AxiomicLabs/ArithMark-2.0", split="train")
+
+
+@torch.no_grad()
+def avg_logprob(ctx: str, ending: str) -> float:
+    """Mean log-prob of `ending` conditioned on `ctx` (the leaderboard's scoring)."""
+    ctx_ids = tok(ctx, return_tensors="pt").input_ids.to(dev)
+    full_ids = tok(ctx + ending, return_tensors="pt").input_ids.to(dev)
+    if full_ids.shape[1] <= ctx_ids.shape[1]:
+        return -1e9
+    logits = model(full_ids).logits[:, :-1, :]
+    logp = torch.log_softmax(logits, dim=-1)
+    tgt = full_ids[:, 1:]
+    sel = logp.gather(2, tgt.unsqueeze(-1)).squeeze(-1)[:, ctx_ids.shape[1] - 1:]
+    return sel.mean().item()
+
+
+correct = 0
+for i, e in enumerate(ds):
+    endings = e["endings"] if isinstance(e["endings"], list) else ast.literal_eval(e["endings"])
+    scores = [avg_logprob(e["ctx"], end) for end in endings]
+    if max(range(len(scores)), key=lambda j: scores[j]) == int(e["label"]):
+        correct += 1
+    if (i + 1) % 500 == 0:
+        print(f"  {i+1}/{len(ds)}  running acc: {correct/(i+1):.4f}", flush=True)
+
+print(f"\nArithMark-2 accuracy for {MODEL}: {correct/len(ds)*100:.2f}%  ({correct}/{len(ds)})")

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": 0,
+  "head_dim": 48,
+  "hidden_act": "silu",
+  "hidden_size": 192,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "model_name": "scratch/nexus-3m-base",
+  "model_type": "llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 5,
+  "num_key_value_heads": 4,
+  "pad_token_id": 1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.6",
+  "unsloth_version": "2026.4.6",
+  "use_cache": true,
+  "vocab_size": 4096
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": [
+    0
+  ],
+  "max_length": 512,
+  "pad_token_id": 1,
+  "transformers_version": "4.57.6"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2edd88e498dd0d13eeca297ff421f6e47e9bb0561f8db89ade3111754a82612b
+size 6005832

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenization_nexus.py

@@ -0,0 +1,47 @@
+"""Nexus-Erebus digit-atomic, least-significant-digit-first tokenizer.
+
+Digits are never merged by BPE, and every maximal run of digits is reversed at
+encode time so the model reads and writes numbers least-significant-digit first.
+This aligns carry propagation with the left-to-right direction the model reads,
+which is what lets a tiny model do integer arithmetic.
+
+The transform is an involution, so decoding simply applies it again to restore
+ordinary left-to-right numbers. Callers see normal text in and normal text out.
+
+Load with:
+    AutoTokenizer.from_pretrained(repo, trust_remote_code=True)
+"""
+import re
+from transformers import PreTrainedTokenizerFast
+
+_DIGIT_RUN = re.compile(r"\d+")
+
+
+def rev_digits(text: str) -> str:
+    """Reverse each maximal run of digits. Involution: rev(rev(x)) == x."""
+    return _DIGIT_RUN.sub(lambda m: m.group(0)[::-1], text)
+
+
+class NexusLSDTokenizer(PreTrainedTokenizerFast):
+    """PreTrainedTokenizerFast that applies the LSD-first digit transform."""
+
+    def _t(self, x):
+        if x is None:
+            return None
+        if isinstance(x, str):
+            return rev_digits(x)
+        if isinstance(x, (list, tuple)):
+            return type(x)(self._t(i) for i in x)
+        return x
+
+    def _batch_encode_plus(self, batch_text_or_text_pairs, *args, **kwargs):
+        return super()._batch_encode_plus(self._t(batch_text_or_text_pairs), *args, **kwargs)
+
+    def _encode_plus(self, text, text_pair=None, *args, **kwargs):
+        return super()._encode_plus(self._t(text), self._t(text_pair), *args, **kwargs)
+
+    # NOTE: do not override tokenize(); it routes through _encode_plus, so the
+    # transform would be applied twice and cancel out.
+
+    def _decode(self, *args, **kwargs):
+        return rev_digits(super()._decode(*args, **kwargs))

tokenizer_config.json

@@ -0,0 +1,35 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|pad|>",
+  "padding_side": "left",
+  "tokenizer_class": "NexusLSDTokenizer",
+  "unk_token": "<|endoftext|>",
+  "auto_map": {
+    "AutoTokenizer": [
+      null,
+      "tokenization_nexus.NexusLSDTokenizer"
+    ]
+  }
+}