Upload continued MLM checkpoint

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,76 @@
 ---
+language:
+- fr
+- de
+- en
+- lb
+base_model: jhu-clsp/mmBERT-base
+library_name: transformers
 license: mit
+tags:
+- modernbert
+- mmbert
+- masked-language-modeling
+- impresso
 ---
+
+# multilingualmodernimpressoBERT
+
+Source card for the future Hugging Face model repository:
+
+```text
+impresso-project/mmbert-multilingual-impresso-continued-mlm
+```
+
+This model is a continued-MLM adaptation of `jhu-clsp/mmBERT-base` on multilingual Impresso newspaper text.
+
+The checkpoint is intended as a domain-adapted base model for downstream media-agency token classification.
+
+## License
+
+This continued-pretraining checkpoint is released under the MIT license, matching the license of the base model `jhu-clsp/mmBERT-base`.
+
+The model is derived from `jhu-clsp/mmBERT-base`; users should cite and comply with the base model terms. The continued-MLM training corpus consists of Impresso newspaper text samples used for domain adaptation. The model weights are published separately from the source text; users remain responsible for checking whether their downstream use of Impresso-derived models and outputs is compatible with their institutional, corpus, and application-specific requirements.
+
+## Continued MLM Run
+
+The first completed workbench run used:
+
+- source compiled Impresso files: `fr`, `de`, `en`, `lb`
+- source filtering: OCR quality at least `0.90`, minimum text length `200` characters
+- source sampling policy: up to 300,000 texts per language, with smaller languages exhausted
+- sampled corpus: 872,889 train rows and 8,818 validation rows after split
+- training subset: 100,000 train rows
+- validation subset: 2,000 validation rows
+- base model: `jhu-clsp/mmBERT-base`
+- objective: masked language modeling
+- MLM probability: `0.15`
+- max sequence length: `256`
+- padding: fixed max-length padding
+- epochs: `1`
+- per-device train batch size: `1`
+- gradient accumulation steps: `8`
+- effective train batch size per device: `8`
+- gradient checkpointing: enabled
+- learning rate: `2e-5`
+- weight decay: `0.01`
+- warmup: 750 steps, computed as 6 percent of the capped optimizer steps
+- intermediate checkpoint saving: disabled; final model only
+- random seed: `42`
+
+Training was run locally on Apple Silicon using the PyTorch MPS backend. The completed run took about 8 hours and 6 minutes. Observed memory use stayed below roughly 30 GB.
+
+## Metrics
+
+Final run metrics:
+
+| metric | value |
+| --- | ---: |
+| train loss | 1.7775 |
+| eval loss | 1.7172 |
+| train runtime | 29,134 seconds |
+| train samples / second | 3.432 |
+| train steps / second | 0.429 |
+| eval samples / second | 14.885 |
+
+The validation loss stayed close to the train loss and the run completed without divergence, so this checkpoint is suitable for downstream comparison against the original `jhu-clsp/mmBERT-base`.

config.json

@@ -0,0 +1,79 @@
+{
+  "architectures": [
+    "ModernBertForMaskedLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "classifier_activation": "gelu",
+  "classifier_bias": false,
+  "classifier_dropout": 0.0,
+  "classifier_pooling": "mean",
+  "cls_token_id": 1,
+  "decoder_bias": true,
+  "deterministic_flash_attn": false,
+  "dtype": "float32",
+  "embedding_dropout": 0.0,
+  "eos_token_id": 1,
+  "global_attn_every_n_layers": 3,
+  "gradient_checkpointing": false,
+  "hidden_activation": "gelu",
+  "hidden_size": 768,
+  "initializer_cutoff_factor": 2.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 1152,
+  "layer_norm_eps": 1e-05,
+  "layer_types": [
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "local_attention": 128,
+  "mask_token_id": 4,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mlp_dropout": 0.0,
+  "model_type": "modernbert",
+  "norm_bias": false,
+  "norm_eps": 1e-05,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 22,
+  "pad_token_id": 0,
+  "position_embedding_type": "sans_pos",
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 160000,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 160000,
+      "rope_type": "default"
+    }
+  },
+  "sep_token_id": 1,
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.9.0",
+  "use_cache": false,
+  "vocab_size": 256000
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1edd72e430ece6e02d016da16e97e518e32e3e2a69b539bdb080f34ad10db153
+size 1231159208

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:609d8f4c067cd3950f88594c5a802616cea245823836ef5848ee4fc40aab5b6f
+size 34363188

tokenizer_config.json

@@ -0,0 +1,25 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<bos>",
+  "eos_token": "<eos>",
+  "extra_special_tokens": [
+    "<start_of_turn>",
+    "<end_of_turn>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "sep_token": "<eos>",
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<unk>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:078fd0af81dc073195349062b2b9c2417551d6d2001517362ce19bfb912d251e
+size 5265