Add main & ema weights for ace

README.md

@@ -0,0 +1,140 @@
+# haznitrama/babybabellm-gpt_bert-ace-causal
+
+GPT-BERT style BabyBabyLLM monolingual model for language **ace**.
+
+This repository mirrors the layout of the multi-all reference models: it may contain both *main* and *EMA* variants.
+
+**Default variant exposed to generic loaders:** `ema`
+
+## Variants Available
+ema, main
+
+## Files
+- model.safetensors (alias of default variant)
+- model_ema.safetensors
+- pytorch_model.bin (legacy PyTorch format)
+
+## Configuration
+```json
+{
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
+  "intermediate_size": 1280,
+  "max_position_embeddings": 512,
+  "position_bucket_size": 32,
+  "num_attention_heads": 6,
+  "num_hidden_layers": 12,
+  "vocab_size": 8192,
+  "layer_norm_eps": 1e-05,
+  "auto_map": {
+    "AutoConfig": "configuration_gpt_bert.GPTBertConfig",
+    "AutoModel": "modeling_gpt_bert.GPTBertForMaskedLM",
+    "AutoModelForCausalLM": "modeling_gpt_bert.GPTBertForMaskedLM",
+    "AutoModelForMaskedLM": "modeling_gpt_bert.GPTBertForMaskedLM"
+  },
+  "return_dict": true,
+  "output_hidden_states": false,
+  "torchscript": false,
+  "dtype": "float32",
+  "pruned_heads": {},
+  "tie_word_embeddings": true,
+  "chunk_size_feed_forward": 0,
+  "is_encoder_decoder": false,
+  "is_decoder": false,
+  "cross_attention_hidden_size": null,
+  "add_cross_attention": false,
+  "tie_encoder_decoder": false,
+  "architectures": [
+    "GPTBertForMaskedLM"
+  ],
+  "finetuning_task": null,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "task_specific_params": null,
+  "problem_type": null,
+  "tokenizer_class": null,
+  "prefix": null,
+  "bos_token_id": null,
+  "pad_token_id": null,
+  "eos_token_id": null,
+  "sep_token_id": null,
+  "decoder_start_token_id": null,
+  "max_length": 20,
+  "min_length": 0,
+  "do_sample": false,
+  "early_stopping": false,
+  "num_beams": 1,
+  "num_beam_groups": 1,
+  "diversity_penalty": 0.0,
+  "temperature": 1.0,
+  "top_k": 50,
+  "top_p": 1.0,
+  "typical_p": 1.0,
+  "repetition_penalty": 1.0,
+  "length_penalty": 1.0,
+  "no_repeat_ngram_size": 0,
+  "encoder_no_repeat_ngram_size": 0,
+  "bad_words_ids": null,
+  "num_return_sequences": 1,
+  "output_scores": false,
+  "return_dict_in_generate": false,
+  "forced_bos_token_id": null,
+  "forced_eos_token_id": null,
+  "remove_invalid_values": false,
+  "exponential_decay_length_penalty": null,
+  "suppress_tokens": null,
+  "begin_suppress_tokens": null,
+  "_name_or_path": "",
+  "transformers_version": "4.56.1",
+  "tf_legacy_loss": false,
+  "use_bfloat16": false,
+  "model_type": "gpt_bert",
+  "output_attentions": false
+}
+```
+Tokenizer file: `tokenizer_ace_vs8192.json`
+
+## Quick Usage
+```python
+from transformers import AutoTokenizer, AutoModelForMaskedLM
+model_id = 'haznitrama/babybabellm-gpt_bert-ace-causal'
+tok = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForMaskedLM.from_pretrained(model_id, trust_remote_code=True)
+out = model(**tok('Hello world', return_tensors='pt'))
+```
+Select a specific variant explicitly (when both present):
+```python
+# Load EMA weights explicitly if both are present
+from safetensors.torch import load_file
+import torch
+from transformers import AutoConfig, AutoModelForMaskedLM
+model_id = 'haznitrama/babybabellm-gpt_bert-ace-causal'
+config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
+model = AutoModelForMaskedLM.from_config(config, trust_remote_code=True)
+state_dict = torch.load('pytorch_model.bin')  # or load_file('model_ema.safetensors')
+model.load_state_dict(state_dict, strict=False)
+```
+
+### Causal LM Wrapper
+This repo includes a lightweight GPTBertForCausalLM wrapper.
+Generation example:
+```python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+mid='haznitrama/babybabellm-gpt_bert-ace-causal'
+tok=AutoTokenizer.from_pretrained(mid)
+model=AutoModelForCausalLM.from_pretrained(mid, trust_remote_code=True)
+print(tok.decode(model.generate(**tok('Hello', return_tensors='pt'), max_new_tokens=20)[0], skip_special_tokens=True))
+```
+
+## Notes
+- Converted on 2025-09-16T06:15:08.548402Z
+- Safe serialization (safetensors) used; `pytorch_model.bin` added for legacy tools.
+- Requires `trust_remote_code=True` due to custom architecture.
+- EMA (Exponential Moving Average) weights can yield slightly better evaluation metrics; choose according to your needs.

config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "GPTBertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoConfig": "configuration_gpt_bert.GPTBertConfig",
+    "AutoModel": "modeling_gpt_bert.GPTBertForMaskedLM",
+    "AutoModelForCausalLM": "modeling_gpt_bert.GPTBertForMaskedLM",
+    "AutoModelForMaskedLM": "modeling_gpt_bert.GPTBertForMaskedLM"
+  },
+  "bos_token_id": 1,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
+  "intermediate_size": 1280,
+  "layer_norm_eps": 1e-05,
+  "mask_token_id": 4,
+  "max_position_embeddings": 512,
+  "model_type": "gpt_bert",
+  "num_attention_heads": 6,
+  "num_hidden_layers": 12,
+  "pad_token_id": 3,
+  "position_bucket_size": 32,
+  "transformers_version": "4.56.1",
+  "vocab_size": 8192
+}

configuration_gpt_bert.py

@@ -0,0 +1,22 @@
+from transformers import PretrainedConfig
+
+class GPTBertConfig(PretrainedConfig):
+    model_type = 'gpt_bert'
+    def __init__(self, **kwargs):
+        self.attention_probs_dropout_prob = kwargs.pop('attention_probs_dropout_prob', 0.1)
+        self.hidden_dropout_prob = kwargs.pop('hidden_dropout_prob', 0.1)
+        self.hidden_size = kwargs.pop('hidden_size', 768)
+        self.intermediate_size = kwargs.pop('intermediate_size', 2560)
+        self.max_position_embeddings = kwargs.pop('max_position_embeddings', 512)
+        self.position_bucket_size = kwargs.pop('position_bucket_size', 32)
+        self.num_attention_heads = kwargs.pop('num_attention_heads', 12)
+        self.num_hidden_layers = kwargs.pop('num_hidden_layers', 12)
+        self.vocab_size = kwargs.pop('vocab_size', 16384)
+        self.layer_norm_eps = kwargs.pop('layer_norm_eps', 1e-5)
+        self.auto_map = {
+            'AutoConfig': 'configuration_gpt_bert.GPTBertConfig',
+            'AutoModel': 'modeling_gpt_bert.GPTBertForCausalLM',
+            'AutoModelForCausalLM': 'modeling_gpt_bert.GPTBertForCausalLM',
+            'AutoModelForMaskedLM': 'modeling_gpt_bert.GPTBertForMaskedLM',
+        }
+        super().__init__(**kwargs)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8f00b242a43531ca7223948132041c432f71d0b46ece868224b242f9f26092c
+size 144750928

model_ema.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8f00b242a43531ca7223948132041c432f71d0b46ece868224b242f9f26092c
+size 144750928

modeling_gpt_bert.py

@@ -0,0 +1,130 @@
+import math, torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import MaskedLMOutput, CausalLMOutputWithCrossAttentions
+from .configuration_gpt_bert import GPTBertConfig
+
+class GeGLU(nn.Module):
+    def forward(self, x):
+        x, gate = x.chunk(2, dim=-1)
+        return x * F.gelu(gate, approximate='tanh')
+
+class FeedForward(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.mlp = nn.Sequential(
+            nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False)
+            ,nn.Linear(config.hidden_size, 2*config.intermediate_size, bias=False)
+            ,GeGLU()
+            ,nn.LayerNorm(config.intermediate_size, eps=config.layer_norm_eps, elementwise_affine=False)
+            ,nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
+            ,nn.Dropout(config.hidden_dropout_prob)
+        )
+        self._init(config.hidden_size)
+    def _init(self, h):
+        std = math.sqrt(2.0 / (5.0 * h))
+        nn.init.trunc_normal_(self.mlp[1].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+        nn.init.trunc_normal_(self.mlp[-2].weight, mean=0.0, std=std, a=-2*std, b=2*std)
+    def forward(self, x): return self.mlp(x)
+
+class Attention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        if config.hidden_size % config.num_attention_heads != 0:
+            raise ValueError('hidden not divisible by heads')
+        self.nh = config.num_attention_heads
+        self.dh = config.hidden_size // config.num_attention_heads
+        self.qkv = nn.Linear(config.hidden_size, 3*config.hidden_size, bias=False)
+        self.o = nn.Linear(config.hidden_size, config.hidden_size, bias=False)
+        self.drop = nn.Dropout(config.hidden_dropout_prob)
+    def forward(self, x, attn_mask, rel):
+        B,S,H = x.shape
+        qkv = self.qkv(x).view(B,S,3,self.nh,self.dh).permute(2,0,3,1,4)
+        q,k,v = qkv[0],qkv[1],qkv[2]
+        attn = (q @ k.transpose(-1,-2)) / math.sqrt(self.dh)
+        if attn_mask is not None: attn = attn.masked_fill(attn_mask[:,None,:, :]==0, float('-inf'))
+        attn = torch.softmax(attn, dim=-1)
+        attn = self.drop(attn)
+        y = attn @ v
+        y = y.transpose(1,2).contiguous().view(B,S,H)
+        return self.o(y)
+
+class Block(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.attn = Attention(config)
+        self.ff = FeedForward(config)
+    def forward(self, x, attn_mask, rel):
+        x = x + self.attn(x, attn_mask, rel)
+        x = x + self.ff(x)
+        return x
+
+class Encoder(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.layers = nn.ModuleList([Block(config) for _ in range(config.num_hidden_layers)])
+    def forward(self, x, attn_mask, rel):
+        for layer in self.layers:
+            x = layer(x, attn_mask, rel)
+        return x
+
+class Embedding(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.word_embedding = nn.Embedding(config.vocab_size, config.hidden_size)
+        self.pos_embedding = nn.Embedding(config.max_position_embeddings, config.hidden_size)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+    def forward(self, input_ids):
+        B,S = input_ids.shape
+        pos = torch.arange(0,S, device=input_ids.device).unsqueeze(0).expand(B,S)
+        x = self.word_embedding(input_ids) + self.pos_embedding(pos)
+        return self.dropout(x), None
+
+class CoreModel(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.embedding = Embedding(config)
+        self.transformer = Encoder(config)
+        self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps, elementwise_affine=False)
+        self.head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        self.head.weight = self.embedding.word_embedding.weight
+    def forward(self, input_ids, attention_mask=None):
+        x,_ = self.embedding(input_ids)
+        if attention_mask is not None: attn = attention_mask.unsqueeze(1)
+        else: attn = None
+        x = self.transformer(x, attn, None)
+        x = self.layer_norm(x)
+        return self.head(x)
+
+class GPTBertForMaskedLM(PreTrainedModel):
+    config_class = GPTBertConfig
+    base_model_prefix = 'gpt_bert'
+    def __init__(self, config: GPTBertConfig):
+        super().__init__(config)
+        self.model = CoreModel(config)
+    def forward(self, input_ids, attention_mask=None, labels=None):
+        logits = self.model(input_ids, attention_mask)
+        loss=None
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss(ignore_index=-100)
+            loss = loss_fct(logits.view(-1, logits.size(-1)), labels.view(-1))
+        return MaskedLMOutput(loss=loss, logits=logits)
+
+class GPTBertForCausalLM(PreTrainedModel):
+    config_class = GPTBertConfig
+    base_model_prefix = 'gpt_bert'
+    def __init__(self, config: GPTBertConfig):
+        super().__init__(config)
+        self.model = CoreModel(config)
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        return {'input_ids': input_ids}
+    def forward(self, input_ids, attention_mask=None, labels=None):
+        logits = self.model(input_ids, attention_mask)
+        loss=None
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = nn.CrossEntropyLoss(ignore_index=-100)
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+        return CausalLMOutputWithCrossAttentions(loss=loss, logits=logits)

original_project_config.json

@@ -0,0 +1,12 @@
+{
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
+  "intermediate_size": 1280,
+  "max_position_embeddings": 512,
+  "position_bucket_size": 32,
+  "num_attention_heads": 6,
+  "num_hidden_layers": 12,
+  "vocab_size": 8192,
+  "layer_norm_eps": 1e-05
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60831fe97fd7fa50ab77364bbc6ccd2d9aad85e48ca260b7104e5d1936870d41
+size 144791119

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,141 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<special_0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<special_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<special_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<special_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<special_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<special_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<special_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<special_7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<special_8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<special_9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<special_10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}