Upload PIBot Joint BERT model package

README.md

@@ -0,0 +1,154 @@
+---
+language: es
+license: mit
+tags:
+  - pytorch
+  - bert
+  - joint-bert
+  - intent-classification
+  - slot-filling
+  - named-entity-recognition
+  - multi-head-classification
+  - spanish
+  - macroeconomics
+pipeline_tag: token-classification
+library_name: transformers
+---
+
+# PIBot Joint BERT
+
+Modelo **JointBERT multi-cabeza** para consultas macroeconómicas en español.  
+Predice simultáneamente **5 cabezas de intención** + **slot filling BIO** por token.
+
+## Configuración del checkpoint
+
+| Parámetro | Valor |
+|---|---|
+| **Task** | `pibimacecv5` |
+| **Base model** | `dccuchile/bert-base-spanish-wwm-cased` (BETO) |
+| **Model type** | `bert` |
+| **Custom code** | Sí (`trust_remote_code=True`) |
+
+## Cabezas de intención
+
+| Cabeza | Archivo de labels |
+|---|---|
+| `calc_mode` | `labels/calc_mode_label.txt` |
+| `activity` | `labels/activity_label.txt` |
+| `region` | `labels/region_label.txt` |
+| `investment` | `labels/investment_label.txt` |
+| `req_form` | `labels/req_form_label.txt` |
+
+## Slot filling
+
+- Etiquetado BIO por token (`labels/slot_label.txt`)
+
+## Estructura del repositorio
+
+```text
+├── config.json                 # Configuración del modelo
+├── model.safetensors           # Pesos del modelo
+├── training_args.bin           # Argumentos de entrenamiento
+├── tokenizer.json              # Tokenizer
+├── tokenizer_config.json
+├── vocab.txt
+├── special_tokens_map.json
+├── modeling_jointbert.py       # Código custom (AutoModel)
+├── module.py                   # Clasificadores por cabeza
+├── __init__.py
+├── labels/
+│   ├── calc_mode_label.txt
+│   ├── activity_label.txt
+│   ├── region_label.txt
+│   ├── investment_label.txt
+│   ├── req_form_label.txt
+│   └── slot_label.txt
+└── README.md
+```
+
+## Uso rápido
+
+### Descargar y cargar el modelo
+
+```python
+import torch
+from pathlib import Path
+from transformers import AutoConfig, AutoTokenizer, AutoModel
+
+repo_id = "tu-usuario/pibot-jointbert"  # Cambiar por tu repo
+
+# 1. Cargar tokenizer y config
+tokenizer = AutoTokenizer.from_pretrained(repo_id, trust_remote_code=True)
+config = AutoConfig.from_pretrained(repo_id, trust_remote_code=True)
+
+# 2. Cargar training args y labels
+from huggingface_hub import hf_hub_download
+
+args_path = hf_hub_download(repo_id, "training_args.bin")
+train_args = torch.load(args_path, weights_only=False)
+
+def read_labels(repo_id, filename):
+    path = hf_hub_download(repo_id, f"labels/{filename}")
+    return [line.strip() for line in open(path, encoding="utf-8") if line.strip()]
+
+calc_mode_labels = read_labels(repo_id, "calc_mode_label.txt")
+activity_labels = read_labels(repo_id, "activity_label.txt")
+region_labels = read_labels(repo_id, "region_label.txt")
+investment_labels = read_labels(repo_id, "investment_label.txt")
+req_form_labels = read_labels(repo_id, "req_form_label.txt")
+slot_labels = read_labels(repo_id, "slot_label.txt")
+
+# 3. Instanciar modelo
+model = AutoModel.from_pretrained(
+    repo_id,
+    config=config,
+    trust_remote_code=True,
+    args=train_args,
+    calc_mode_label_lst=calc_mode_labels,
+    activity_label_lst=activity_labels,
+    region_label_lst=region_labels,
+    investment_label_lst=investment_labels,
+    req_form_label_lst=req_form_labels,
+    slot_label_lst=slot_labels,
+)
+model.eval()
+```
+
+### Inferencia
+
+```python
+text = "cual fue el ultimo imacec"
+inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=64)
+
+with torch.no_grad():
+    outputs = model(**inputs)
+
+# outputs contiene logits de cada cabeza de intención y slot filling
+calc_mode_logits = outputs[1]   # (batch, num_calc_mode_labels)
+activity_logits = outputs[2]    # (batch, num_activity_labels)
+region_logits = outputs[3]      # (batch, num_region_labels)
+investment_logits = outputs[4]  # (batch, num_investment_labels)
+req_form_logits = outputs[5]    # (batch, num_req_form_labels)
+slot_logits = outputs[6]        # (batch, seq_len, num_slot_labels)
+```
+
+### Uso en endpoint (snapshot local)
+
+```python
+from huggingface_hub import snapshot_download
+
+# Descargar checkpoint completo
+model_dir = snapshot_download("tu-usuario/pibot-jointbert")
+
+# Cargar labels desde carpeta local
+from pathlib import Path
+labels_dir = Path(model_dir) / "labels"
+labels = {}
+for f in labels_dir.glob("*_label.txt"):
+    labels[f.stem] = [l.strip() for l in f.read_text(encoding="utf-8").splitlines() if l.strip()]
+```
+
+## Referencia
+
+- Paper base: [BERT for Joint Intent Classification and Slot Filling](https://arxiv.org/abs/1810.04805v2)
+- Implementación original: [monologg/JointBERT](https://github.com/monologg/JointBERT)

__init__.py

@@ -0,0 +1 @@
+from .modeling_jointbert import JointBERT

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "dccuchile/bert-base-spanish-wwm-cased",
+  "architectures": [
+    "JointBERT"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "finetuning_task": "pibimacecv5",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.46.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 31002
+}

labels/activity_label.txt

@@ -0,0 +1,3 @@
+none
+specific
+general

labels/calc_mode_label.txt

@@ -0,0 +1,4 @@
+original
+prev_period
+yoy
+contribution

labels/investment_label.txt

@@ -0,0 +1,3 @@
+none
+specific
+general

labels/region_label.txt

@@ -0,0 +1,3 @@
+none
+specific
+general

labels/req_form_label.txt

@@ -0,0 +1,3 @@
+latest
+point
+range

labels/slot_label.txt

@@ -0,0 +1,15 @@
+O
+B-ACTIVITY
+B-FREQUENCY
+B-INDICATOR
+B-INVESTMENT
+B-PERIOD
+B-REGION
+B-SEASONALITY
+I-ACTIVITY
+I-FREQUENCY
+I-INDICATOR
+I-INVESTMENT
+I-PERIOD
+I-REGION
+I-SEASONALITY

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d6084a789c8845d2d09b6ff593aff3639c66db677c9c7bd7399fca3eed7b902
+size 439525264

modeling_jointbert.py

@@ -0,0 +1,125 @@
+import torch.nn as nn
+from transformers import PreTrainedModel, AutoModel
+from .module import CalcModeClassifier, ActivityClassifier, RegionClassifier, InvestmentClassifier, ReqFormClassifier, SlotClassifier
+
+try:
+    from torchcrf import CRF
+except ImportError:
+    CRF = None
+
+class JointBERT(PreTrainedModel):
+    def __init__(self, config, args, calc_mode_label_lst, activity_label_lst, region_label_lst, investment_label_lst, req_form_label_lst, slot_label_lst):
+        super(JointBERT, self).__init__(config)
+        self.args = args
+        
+        self.num_calc_mode_labels = len(calc_mode_label_lst)
+        self.num_activity_labels = len(activity_label_lst)
+        self.num_region_labels = len(region_label_lst)
+        self.num_investment_labels = len(investment_label_lst)
+        self.num_req_form_labels = len(req_form_label_lst)
+        self.num_slot_labels = len(slot_label_lst)
+        
+        # Usar AutoModel para soportar cualquier encoder transformer
+        self.encoder = AutoModel.from_pretrained(args.model_name_or_path, config=config)
+
+        self.calc_mode_classifier = CalcModeClassifier(config.hidden_size, self.num_calc_mode_labels, args.dropout_rate)
+        self.activity_classifier = ActivityClassifier(config.hidden_size, self.num_activity_labels, args.dropout_rate)
+        self.region_classifier = RegionClassifier(config.hidden_size, self.num_region_labels, args.dropout_rate)
+        self.investment_classifier = InvestmentClassifier(config.hidden_size, self.num_investment_labels, args.dropout_rate)
+        self.req_form_classifier = ReqFormClassifier(config.hidden_size, self.num_req_form_labels, args.dropout_rate)
+        self.slot_classifier = SlotClassifier(config.hidden_size, self.num_slot_labels, args.dropout_rate)
+
+        if args.use_crf:
+            if CRF is None:
+                raise ImportError("torchcrf no está instalado. Instala con: pip install pytorch-crf o ejecuta sin --use_crf")
+            crf_init_errors = []
+            for init_fn in (
+                lambda: CRF(self.num_slot_labels, pad_idx=None, use_gpu=False),
+                lambda: CRF(self.num_slot_labels, batch_first=True),
+                lambda: CRF(num_tags=self.num_slot_labels, batch_first=True),
+                lambda: CRF(self.num_slot_labels),
+                lambda: CRF(num_tags=self.num_slot_labels),
+            ):
+                try:
+                    self.crf = init_fn()
+                    break
+                except TypeError as e:
+                    crf_init_errors.append(str(e))
+            else:
+                raise TypeError("No se pudo inicializar CRF con las firmas conocidas: " + " | ".join(crf_init_errors))
+
+    def forward(self, input_ids, attention_mask, token_type_ids=None,
+        calc_mode_label_ids=None, activity_label_ids=None, region_label_ids=None, investment_label_ids=None, req_form_label_ids=None, slot_labels_ids=None):
+        outputs = self.encoder(input_ids, attention_mask=attention_mask,
+                              token_type_ids=token_type_ids)  # sequence_output, pooled_output, (hidden_states), (attentions)
+        sequence_output = outputs[0]
+        pooled_output = outputs[1]  # [CLS]
+
+        calc_mode_logits = self.calc_mode_classifier(pooled_output)
+        activity_logits = self.activity_classifier(pooled_output)
+        region_logits = self.region_classifier(pooled_output)
+        investment_logits = self.investment_classifier(pooled_output)
+        req_form_logits = self.req_form_classifier(pooled_output)
+        slot_logits = self.slot_classifier(sequence_output)
+
+        total_loss = 0   
+        # 1. Calc Mode CrossEntropy
+        if calc_mode_label_ids is not None:
+            calc_mode_loss_fct = nn.CrossEntropyLoss()
+            calc_mode_loss = calc_mode_loss_fct(calc_mode_logits.view(-1, self.num_calc_mode_labels), calc_mode_label_ids.view(-1))
+            total_loss += calc_mode_loss
+
+        # 2. Activity CrossEntropy
+        if activity_label_ids is not None:
+            activity_loss_fct = nn.CrossEntropyLoss()
+            activity_loss = activity_loss_fct(activity_logits.view(-1, self.num_activity_labels), activity_label_ids.view(-1))
+            total_loss += activity_loss
+            
+        # 3. Region CrossEntropy
+        if region_label_ids is not None:
+            region_loss_fct = nn.CrossEntropyLoss()
+            region_loss = region_loss_fct(region_logits.view(-1, self.num_region_labels), region_label_ids.view(-1))
+            total_loss += region_loss
+
+        # 4. Investment CrossEntropy
+        if investment_label_ids is not None:
+            investment_loss_fct = nn.CrossEntropyLoss()
+            investment_loss = investment_loss_fct(investment_logits.view(-1, self.num_investment_labels), investment_label_ids.view(-1))
+            total_loss += investment_loss
+
+        # 5. Req Form CrossEntropy
+        if req_form_label_ids is not None:
+            req_form_loss_fct = nn.CrossEntropyLoss()
+            req_form_loss = req_form_loss_fct(req_form_logits.view(-1, self.num_req_form_labels), req_form_label_ids.view(-1))
+            total_loss += req_form_loss
+
+        # 6. Slot Softmax
+        if slot_labels_ids is not None and self.args.slot_loss_coef != 0:
+            if self.args.use_crf:
+                # CRF doesn't handle ignore_index (-100), so we replace it with PAD (0)
+                slot_labels_ids_crf = slot_labels_ids.clone()
+                slot_labels_ids_crf[slot_labels_ids_crf == self.args.ignore_index] = 0
+                if hasattr(self.crf, 'viterbi_decode'):
+                    # TorchCRF API: forward returns log-likelihood per batch item
+                    slot_loss = -self.crf(slot_logits, slot_labels_ids_crf, attention_mask.bool()).mean()
+                else:
+                    # pytorch-crf API
+                    slot_loss = self.crf(slot_logits, slot_labels_ids_crf, mask=attention_mask.bool(), reduction='mean')
+                    slot_loss = -1 * slot_loss  # negative log-likelihood
+            else:
+                slot_loss_fct = nn.CrossEntropyLoss(ignore_index=self.args.ignore_index)
+                # Only keep active parts of the loss
+                if attention_mask is not None:
+                    active_loss = attention_mask.view(-1) == 1
+                    active_logits = slot_logits.view(-1, self.num_slot_labels)[active_loss]
+                    active_labels = slot_labels_ids.view(-1)[active_loss]
+                    slot_loss = slot_loss_fct(active_logits, active_labels)
+                else:
+                    slot_loss = slot_loss_fct(slot_logits.view(-1, self.num_slot_labels), slot_labels_ids.view(-1))
+            total_loss += self.args.slot_loss_coef * slot_loss
+
+        outputs = ((calc_mode_logits, activity_logits, region_logits, investment_logits, req_form_logits, slot_logits),) + outputs[2:]  # add hidden states and attention if they are here 
+
+        outputs = (total_loss,) + outputs
+
+        return outputs  # (loss), logits, (hidden_states), (attentions) # Logits is a tuple of all classifier logits

module.py

@@ -0,0 +1,62 @@
+import torch.nn as nn
+
+class CalcModeClassifier(nn.Module):
+    def __init__(self, input_dim, num_calc_mode_labels, dropout_rate=0.):
+        super(CalcModeClassifier, self).__init__()
+        self.dropout = nn.Dropout(dropout_rate)
+        self.linear = nn.Linear(input_dim, num_calc_mode_labels)
+
+    def forward(self, x):
+        x = self.dropout(x)
+        return self.linear(x)
+    
+class ActivityClassifier(nn.Module):
+    def __init__(self, input_dim, num_activity_labels, dropout_rate=0.):
+        super(ActivityClassifier, self).__init__()
+        self.dropout = nn.Dropout(dropout_rate)
+        self.linear = nn.Linear(input_dim, num_activity_labels)
+
+    def forward(self, x):
+        x = self.dropout(x)
+        return self.linear(x)
+    
+class RegionClassifier(nn.Module):
+    def __init__(self, input_dim, num_region_labels, dropout_rate=0.):
+        super(RegionClassifier, self).__init__()
+        self.dropout = nn.Dropout(dropout_rate)
+        self.linear = nn.Linear(input_dim, num_region_labels)
+
+    def forward(self, x):
+        x = self.dropout(x)
+        return self.linear(x)
+
+class InvestmentClassifier(nn.Module):
+    def __init__(self, input_dim, num_investment_labels, dropout_rate=0.):
+        super(InvestmentClassifier, self).__init__()
+        self.dropout = nn.Dropout(dropout_rate)
+        self.linear = nn.Linear(input_dim, num_investment_labels)
+
+    def forward(self, x):
+        x = self.dropout(x)
+        return self.linear(x)
+    
+class ReqFormClassifier(nn.Module):
+    def __init__(self, input_dim, num_req_form_labels, dropout_rate=0.):
+        super(ReqFormClassifier, self).__init__()
+        self.dropout = nn.Dropout(dropout_rate)
+        self.linear = nn.Linear(input_dim, num_req_form_labels)
+
+    def forward(self, x):
+        x = self.dropout(x)
+        return self.linear(x)
+
+class SlotClassifier(nn.Module):
+    def __init__(self, input_dim, num_slot_labels, dropout_rate=0.):
+        super(SlotClassifier, self).__init__()
+        self.dropout = nn.Dropout(dropout_rate)
+        self.linear = nn.Linear(input_dim, num_slot_labels)
+
+    def forward(self, x):
+        x = self.dropout(x)
+        return self.linear(x)
+    

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,57 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": false,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91f6bf5e8606b9c4a901c47a0cde8195ea2546e401ff0cef645286e98ffab140
+size 2040