Upload 7 files

README.md

@@ -0,0 +1,75 @@
+---
+language:
+  - en
+license: apache-2.0
+tags:
+  - gpt
+  - text-generation
+  - causal-lm
+  - pytorch
+  - safetensors
+  - custom-trained
+---
+
+# gpt-model-2-decoder-100000-tiny-stories-fp16
+
+A custom GPT-style language model trained from scratch using PyTorch.
+
+## Model Details
+
+| Parameter | Value |
+|-----------|-------|
+| Architecture | GPT (Decoder-only Transformer) |
+| Hidden size (`d_model`) | 768 |
+| Attention heads | 8 |
+| Transformer blocks | 1 |
+| Max sequence length | 1024 |
+| Vocabulary size | 32000 |
+| Dropout | 0.2 |
+
+## Tokenizer
+
+Custom BPE tokenizer trained with the HuggingFace `tokenizers` library.
+
+**Special tokens:** `<|endoftext|>` · `<|pad|>` · `<|unk|>`
+
+## Quick Start
+
+You can easily load this model and tokenizer using the `transformers` library. Because the model uses a custom architecture, you must pass `trust_remote_code=True`.
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+# Load tokenizer and model
+tokenizer = AutoTokenizer.from_pretrained("sdkjfgndjfg/gpt-model-2-decoder-100000-tiny-stories-fp16", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained("sdkjfgndjfg/gpt-model-2-decoder-100000-tiny-stories-fp16", trust_remote_code=True)
+
+# Set up device
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model.to(device)
+
+# Generate text
+prompt = "The transformer is based on"
+inputs = tokenizer(prompt, return_tensors="pt").to(device)
+output_ids = model.generate(
+    **inputs, 
+    max_new_tokens=50, 
+    do_sample=True, 
+    temperature=0.8,
+    pad_token_id=tokenizer.eos_token_id
+)
+
+print(tokenizer.decode(output_ids[0], skip_special_tokens=True))
+```
+
+## Training Details
+
+- **Optimizer**: AdamW (lr=3e-4, betas=(0.9, 0.95), weight_decay=0.1)
+- **Scheduler**: CosineAnnealingLR (eta_min=1e-5)
+- **Loss**: CrossEntropyLoss (next-token prediction)
+- **Gradient clipping**: max_norm=1.0
+
+## License
+
+Apache 2.0

config.json

@@ -0,0 +1,19 @@
+{
+  "architectures": [
+    "GPTCustomForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "modeling_gpt_custom.GPTCustomConfig",
+    "AutoModelForCausalLM": "modeling_gpt_custom.GPTCustomForCausalLM"
+  },
+  "d_model": 768,
+  "dropout": 0.2,
+  "dtype": "float32",
+  "is_decoder": true,
+  "max_seq_len": 1024,
+  "model_type": "gpt-custom",
+  "num_heads": 8,
+  "number_of_transformer_block": 1,
+  "transformers_version": "5.13.1",
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.13.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad530812d6fe637ece6944fa122f6f1725f66c3630c6a2dd6f0fdbd18117a8e7
+size 236502600

modeling_gpt_custom.py

@@ -0,0 +1,134 @@
+"""
+Required for AutoModelForCausalLM(trust_remote_code=True) to know how to build your custom PyTorch architecture.
+(Note: Big companies like Meta/Mistral don't upload files like this because they merge their architecture code directly into the official `transformers` GitHub repository.)
+"""
+
+from __future__ import annotations
+
+import torch
+import torch.nn as nn
+from transformers import PretrainedConfig, PreTrainedModel
+from transformers.generation import GenerationMixin
+from transformers.modeling_outputs import CausalLMOutput
+
+class GPTCustomConfig(PretrainedConfig):
+    model_type = "gpt-custom"
+    attribute_map = {
+        "num_hidden_layers":  "number_of_transformer_block",
+        "hidden_size":        "d_model",
+        "num_attention_heads": "num_heads",
+    }
+    
+    def __init__(
+        self,
+        vocab_size: int = 32000,
+        d_model: int = 768,
+        num_heads: int = 8,
+        number_of_transformer_block: int = 6,
+        max_seq_len: int = 1024,
+        dropout: float = 0.2,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.vocab_size                  = vocab_size
+        self.d_model                     = d_model
+        self.num_heads                   = num_heads
+        self.number_of_transformer_block = number_of_transformer_block
+        self.max_seq_len                 = max_seq_len
+        self.dropout                     = dropout
+
+class _GPTBlock(nn.Module):
+    def __init__(self, config: GPTCustomConfig) -> None:
+        super().__init__()
+        self.layer_norm_1 = nn.LayerNorm(config.d_model)
+        self.layer_norm_2 = nn.LayerNorm(config.d_model)
+        self.multihead_attention = nn.MultiheadAttention(
+            embed_dim=config.d_model,
+            num_heads=config.num_heads,
+            batch_first=True,
+        )
+        self.gelu      = nn.GELU()
+        self.ffn_1     = nn.Linear(config.d_model, config.d_model * 4)
+        self.ffn_2     = nn.Linear(config.d_model * 4, config.d_model)
+        self.mha_drop  = nn.Dropout(config.dropout)
+        self.ffn_drop  = nn.Dropout(config.dropout)
+        self.register_buffer(
+            "causal_mask",
+            torch.triu(
+                torch.full((config.max_seq_len, config.max_seq_len), float("-inf")),
+                diagonal=1,
+            ),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        _, seq_len, _ = x.size()
+        ln1 = self.layer_norm_1(x)
+        attn_out, _ = self.multihead_attention(
+            ln1, ln1, ln1,
+            attn_mask=self.causal_mask[:seq_len, :seq_len],
+        )
+        x = x + self.mha_drop(attn_out)
+        ln2    = self.layer_norm_2(x)
+        ff_out = self.ffn_2(self.gelu(self.ffn_1(ln2)))
+        return x + self.ffn_drop(ff_out)
+
+class GPTCustomForCausalLM(PreTrainedModel, GenerationMixin):
+    config_class = GPTCustomConfig
+
+    def __init__(self, config: GPTCustomConfig) -> None:
+        super().__init__(config)
+        self.token_embedding      = nn.Embedding(config.vocab_size, config.d_model)
+        self.positional_encoding  = nn.Embedding(config.max_seq_len, config.d_model)
+        self.emb_dropout          = nn.Dropout(config.dropout)
+        self.transformer_blocks = nn.ModuleList(
+            [_GPTBlock(config) for _ in range(config.number_of_transformer_block)]
+        )
+        self.layer_norm_final   = nn.LayerNorm(config.d_model)
+        self.final_linear_layer = nn.Linear(config.d_model, config.vocab_size, bias=False)
+        self.final_linear_layer.weight = self.token_embedding.weight
+        self.config.is_decoder = True
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: torch.Tensor | None = None,
+        labels: torch.Tensor | None = None,
+        **kwargs,
+    ) -> CausalLMOutput:
+        batch_size, seq_len = input_ids.shape
+        position_ids = (
+            torch.arange(seq_len, device=input_ids.device)
+            .unsqueeze(0)
+            .expand(batch_size, -1)
+        )
+        x = self.token_embedding(input_ids) + self.positional_encoding(position_ids)
+        x = self.emb_dropout(x)
+        for block in self.transformer_blocks:
+            x = block(x)
+        logits = self.final_linear_layer(self.layer_norm_final(x))
+        loss = None
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss = nn.functional.cross_entropy(
+                shift_logits.view(-1, self.config.vocab_size),
+                shift_labels.view(-1),
+            )
+        return CausalLMOutput(loss=loss, logits=logits)
+
+    def get_input_embeddings(self) -> nn.Embedding:
+        return self.token_embedding
+
+    def set_input_embeddings(self, value: nn.Embedding) -> None:
+        self.token_embedding = value
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids: torch.Tensor,
+        **kwargs,
+    ) -> dict:
+        return {"input_ids": input_ids}
+
+    def tie_weights(self, **kwargs) -> None:
+        self.final_linear_layer.weight = self.token_embedding.weight

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}