Upload 7 files

README.md

@@ -1,3 +1,78 @@
----
-license: apache-2.0
----
+---
+language:
+  - en
+tags:
+  - mixtral
+  - MixtralForCausalLM
+  - mixture-of-experts
+  - moe
+  - digit-recognition
+  - pattern-recognition
+  - toy-model
+  - educational
+license: apache-2.0
+library_name: transformers
+pipeline_tag: text-generation
+---
+
+# AtoMixtral-58K-5x5-DigitMesh
+
+A minimal 58K parameter Mixture-of-Experts (MoE) model for 5×5 digit mesh recognition, built on the MixtralForCausalLM architecture.
+
+## Model Description
+
+**AtoMixtral-58K-5x5-DigitMesh** is an ultra-lightweight MoE causal language model for efficient digit recognition from 5×5 binary mesh patterns. With only 58K parameters and 2 experts, this "atom-sized" MoE model demonstrates effective pattern recognition with sparse expert activation.
+
+### Key Specifications
+
+- **Architecture**: MixtralForCausalLM (Mixture-of-Experts)
+- **Parameters**: ~58K
+- **Experts**: 2 local experts, 1 active per token
+- **Input**: 5×5 binary mesh (25 tokens)
+- **Output**: Digit tokens (D0-D9)
+- **Vocabulary Size**: 14 tokens
+- **Context Length**: 32 tokens
+- **Hidden Size**: 32, Layers: 2, Attention Heads: 4
+
+## Quick Start
+
+### Serving with vLLM
+
+```bash
+python -m vllm.entrypoints.openai.api_server \
+  --model models/atoMixtral-58K-5x5-DigitMesh \
+  --max-model-len 32
+```
+
+### Test Example
+
+```bash
+curl http://localhost:8000/v1/completions \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "models/atoMixtral-58K-5x5-DigitMesh",
+    "prompt": "1 1 1 1 1 0 0 0 0 1 0 0 0 1 0 0 0 1 0 0 0 1 0 0 0 <SEP>",
+    "max_tokens": 1,
+    "temperature": 0
+  }'
+```
+Expected output: `D7`
+
+## Input Format
+
+25 space-separated binary values (0 or 1) representing a 5×5 grid, followed by `<SEP>`:
+
+```
+[5 values] [5 values] [5 values] [5 values] [5 values] <SEP>
+```
+
+## Use Cases
+
+- MoE architecture research at minimal scale
+- Educational demonstrations of sparse expert models
+- Resource-constrained digit recognition
+- Pattern recognition proof-of-concepts
+
+## License
+
+Apache-2.0

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 2,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "head_dim": null,
+  "hidden_act": "silu",
+  "hidden_size": 32,
+  "initializer_range": 0.02,
+  "intermediate_size": 128,
+  "max_position_embeddings": 32,
+  "model_type": "mixtral",
+  "num_attention_heads": 4,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "num_local_experts": 2,
+  "output_router_logits": false,
+  "pad_token_id": 13,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000.0,
+  "router_aux_loss_coef": 0.01,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.3",
+  "use_cache": true,
+  "vocab_size": 14
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 2,
+  "pad_token_id": 13,
+  "transformers_version": "4.57.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:758d40bfd4c5c98429c8a744a023a087c5c89b8d53a7827ddd387deaee575913
+size 237264

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<SEP>",
+  "eos_token": "<SEP>",
+  "pad_token": "<PAD>"
+}

tokenizer.json

@@ -0,0 +1,51 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 2,
+      "content": "<SEP>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 13,
+      "content": "<PAD>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "WhitespaceSplit"
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "WordLevel",
+    "vocab": {
+      "0": 0,
+      "1": 1,
+      "<SEP>": 2,
+      "D0": 3,
+      "D1": 4,
+      "D2": 5,
+      "D3": 6,
+      "D4": 7,
+      "D5": 8,
+      "D6": 9,
+      "D7": 10,
+      "D8": 11,
+      "D9": 12,
+      "<PAD>": 13
+    },
+    "unk_token": "<unk>"
+  }
+}

tokenizer_config.json

@@ -0,0 +1,27 @@
+{
+  "added_tokens_decoder": {
+    "2": {
+      "content": "<SEP>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<PAD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<SEP>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<SEP>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<PAD>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}