Duplicate from FFZG-cleopatra/Croatian-News-Sentiment-Classifier-V1

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Croatian Sentiment News Classifier
+emoji: 🦀
+colorFrom: pink
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.29.0
+app_file: app.py
+pinned: false
+duplicated_from: FFZG-cleopatra/Croatian-News-Sentiment-Classifier-V1
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,78 @@
+import datasets
+import numpy as np
+import torch
+import transformers
+from config import epochs, batch_size, learning_rate, id2label
+from model import tokenizer, multitask_model
+from mtm import MultitaskTrainer, NLPDataCollator, DataLoaderWithTaskname
+import pandas as pd
+from datasets import Dataset, DatasetDict
+from data_predict import convert_to_stsb_features,convert_to_features
+import gradio as gr
+from huggingface_hub import hf_hub_download,snapshot_download
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Version 1 -  Croatian Document + Slovenian Document.
+model_link = hf_hub_download(repo_id="FFZG-cleopatra/Croatian-Document-News-Sentiment-Classifier",filename = "pytorch_model.bin")
+
+multitask_model.load_state_dict(torch.load(model_link, map_location=device))
+multitask_model.to(device)
+
+def predict_sentiment(sentence = "Volim ti"):
+    # gather everyone if you want to have a single DatasetDict
+    document = DatasetDict({
+        # "train": Dataset.from_pandas(df_document_sl_hr_train),
+        # "valid": Dataset.from_pandas(df_document_sl_hr_valid),
+        "test": Dataset.from_dict({"content":[sentence]})
+    })
+    
+    dataset_dict = {
+        "document": document,
+    }
+    
+    for task_name, dataset in dataset_dict.items():
+        print(task_name)
+        print(dataset_dict[task_name]["test"][0])
+        print()
+    
+    
+    convert_func_dict = {
+        "document": convert_to_stsb_features,
+        # "paragraph": convert_to_stsb_features,
+        # "sentence": convert_to_stsb_features,
+    }
+    
+    features_dict = convert_to_features(dataset_dict, convert_func_dict)
+   
+    predictions = []
+    
+    for _, batch in enumerate(features_dict["document"]['test']):
+        for key, value in batch.items():
+            batch[key] = batch[key].to(device)
+        
+        task_model = multitask_model.get_model("document")
+        classifier_output = task_model.forward(
+                torch.unsqueeze(batch["input_ids"], 0),
+                torch.unsqueeze(batch["attention_mask"], 0),)
+        
+        print(tokenizer.decode(batch["input_ids"],skip_special_tokens=True))
+        print("logits:",classifier_output.logits)
+        prediction =torch.max(classifier_output.logits, axis=1)
+        predictions.append(prediction.indices.item())
+    
+    print("predictions:", predictions[0] , id2label[predictions[0]] )
+    return id2label[predictions[0]]
+
+
+interface = gr.Interface(
+    fn=predict_sentiment,
+    inputs='text',
+    outputs=['label'],
+    title='Croatian News Sentiment Analysis 1.0',
+    description='Get the positive/neutral/negative sentiment for the given input.'
+)
+
+
+interface.launch(inline = False)
+

config.py

@@ -0,0 +1,19 @@
+import os
+
+print(os.listdir())
+model_name = "EMBEDDIA/crosloengual-bert" #"FFZG-cleopatra/dummy-model" #"FFZG-cleopatra/Croatian-News-Classifier"
+
+print("model-name:",model_name)
+
+output_path = ""
+drop_out = 0.3
+max_length = 512
+epochs = 5
+label2id = {'neutral': 0, 'negative': 1, 'positive': 2}
+id2label = {0: 'neutral', 1: 'negative', 2: 'positive'}
+output_dir=""
+batch_size=16 #32
+learning_rate=2e-5
+
+from pip import _internal
+print(_internal.main(['list']))

data_predict.py

@@ -0,0 +1,48 @@
+from datasets import Dataset, DatasetDict
+import pandas as pd
+from config import max_length, label2id
+from model import tokenizer
+import os
+import torch
+
+
+def convert_to_stsb_features(example_batch):
+    inputs = example_batch['content']
+    features = tokenizer.batch_encode_plus(
+        inputs, truncation=True, max_length=max_length, padding='max_length')
+
+    # features["labels"] = [label2id[i] for i in example_batch["sentiment"]]
+    features["labels"] = [0]*len(example_batch["content"]) #[i for i in range(len(example_batch["content"]))]
+    # features["nid"] = [int(i) for i in example_batch["nid"]]
+    return features
+
+
+
+
+def convert_to_features(dataset_dict, convert_func_dict):
+    columns_dict = {
+        "document": ['input_ids', 'attention_mask', 'labels'],
+        # "paragraph": ['input_ids', 'attention_mask', 'labels'],
+        # "sentence": ['input_ids', 'attention_mask', 'labels'],
+    }
+    features_dict = {}
+
+    for task_name, dataset in dataset_dict.items():
+        features_dict[task_name] = {}
+        print(task_name)
+        for phase, phase_dataset in dataset.items():
+            features_dict[task_name][phase] = phase_dataset.map(
+                convert_func_dict[task_name],
+                batched=True,
+                load_from_cache_file=False,
+            )
+            print(task_name, phase, len(phase_dataset),
+                  len(features_dict[task_name][phase]))
+            features_dict[task_name][phase].set_format(
+                type="torch",
+                columns=columns_dict[task_name],
+            )
+            print("=>",task_name, phase, len(phase_dataset),
+                  len(features_dict[task_name][phase]))
+    return features_dict
+

model.py

@@ -0,0 +1,19 @@
+import transformers
+from mtm import MultitaskModel
+from config import model_name, drop_out
+
+multitask_model = MultitaskModel.create(
+    model_name=model_name,
+    model_type_dict={
+        "document": transformers.AutoModelForSequenceClassification,
+        "paragraph": transformers.AutoModelForSequenceClassification,
+        "sentence": transformers.AutoModelForSequenceClassification,
+    },
+    model_config_dict={
+        "document": transformers.AutoConfig.from_pretrained(model_name, num_labels=3, hidden_dropout_prob=drop_out, attention_probs_dropout_prob=drop_out),
+        "paragraph": transformers.AutoConfig.from_pretrained(model_name, num_labels=3, hidden_dropout_prob=drop_out, attention_probs_dropout_prob=drop_out),
+        "sentence": transformers.AutoConfig.from_pretrained(model_name, num_labels=3, hidden_dropout_prob=drop_out, attention_probs_dropout_prob=drop_out),
+    },
+)
+
+tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)

mtm.py

@@ -0,0 +1,214 @@
+import transformers
+import torch
+import torch.nn as nn
+from torch.utils.data.sampler import RandomSampler
+from torch.utils.data.distributed import DistributedSampler
+from torch.utils.data.dataloader import DataLoader
+from transformers.data.data_collator import DataCollator
+from transformers.data.data_collator import DataCollatorWithPadding, InputDataClass
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
+from transformers import is_torch_tpu_available
+import numpy as np
+
+class MultitaskModel(transformers.PreTrainedModel):
+    def __init__(self, encoder, taskmodels_dict):
+        """
+        Setting MultitaskModel up as a PretrainedModel allows us
+        to take better advantage of Trainer features
+        """
+        super().__init__(transformers.PretrainedConfig())
+
+        self.encoder = encoder
+        self.taskmodels_dict = nn.ModuleDict(taskmodels_dict)
+
+    @classmethod
+    def create(cls, model_name, model_type_dict, model_config_dict):
+        """
+        This creates a MultitaskModel using the model class and config objects
+        from single-task models. 
+
+        We do this by creating each single-task model, and having them share
+        the same encoder transformer.
+        """
+        shared_encoder = None
+        taskmodels_dict = {}
+        do = nn.Dropout(p=0.2)
+        for task_name, model_type in model_type_dict.items():
+            model = model_type.from_pretrained(
+                model_name,
+                config=model_config_dict[task_name],
+            )
+            if shared_encoder is None:
+                shared_encoder = getattr(
+                    model, cls.get_encoder_attr_name(model))
+            else:
+                setattr(model, cls.get_encoder_attr_name(
+                    model), shared_encoder)
+            taskmodels_dict[task_name] = model
+        return cls(encoder=shared_encoder, taskmodels_dict=taskmodels_dict)
+
+    @classmethod
+    def get_encoder_attr_name(cls, model):
+        """
+        The encoder transformer is named differently in each model "architecture".
+        This method lets us get the name of the encoder attribute
+        """
+        model_class_name = model.__class__.__name__
+        if model_class_name.startswith("Bert"):
+            return "bert"
+        elif model_class_name.startswith("Roberta"):
+            return "roberta"
+        elif model_class_name.startswith("Albert"):
+            return "albert"
+        else:
+            raise KeyError(f"Add support for new model {model_class_name}")
+
+    def forward(self, task_name, **kwargs):
+        return self.taskmodels_dict[task_name](**kwargs)
+
+    def get_model(self, task_name):
+        return self.taskmodels_dict[task_name]
+
+class NLPDataCollator(DataCollatorWithPadding):  # DataCollatorWithPadding
+    """
+    Extending the existing DataCollator to work with NLP dataset batches
+    """
+
+    def collate_batch(self, features: List[Union[InputDataClass, Dict]]) -> Dict[str, torch.Tensor]:
+        first = features[0]
+        batch = None
+        if isinstance(first, dict):
+            # NLP data sets current works presents features as lists of dictionary
+            # (one per example), so we  will adapt the collate_batch logic for that
+            if "labels" in first and first["labels"] is not None:
+                if first["labels"].dtype == torch.int64:
+                    labels = torch.tensor([f["labels"]
+                                           for f in features], dtype=torch.long)
+                else:
+                    labels = torch.tensor([f["labels"]
+                                           for f in features], dtype=torch.float)
+                batch = {"labels": labels}
+            for k, v in first.items():
+                if k != "labels" and v is not None and not isinstance(v, str):
+                    batch[k] = torch.stack([f[k] for f in features])
+            return batch
+        else:
+            # otherwise, revert to using the default collate_batch
+            return DataCollatorWithPadding().collate_batch(features)
+
+
+class StrIgnoreDevice(str):
+    """
+    This is a hack. The Trainer is going call .to(device) on every input
+    value, but we need to pass in an additional `task_name` string.
+    This prevents it from throwing an error
+    """
+
+    def to(self, device):
+        return self
+
+
+class DataLoaderWithTaskname:
+    """
+    Wrapper around a DataLoader to also yield a task name
+    """
+
+    def __init__(self, task_name, data_loader):
+        self.task_name = task_name
+        self.data_loader = data_loader
+
+        self.batch_size = data_loader.batch_size
+        self.dataset = data_loader.dataset
+
+    def __len__(self):
+        return len(self.data_loader)
+
+    def __iter__(self):
+        for batch in self.data_loader:
+            batch["task_name"] = StrIgnoreDevice(self.task_name)
+            yield batch
+
+
+class MultitaskDataloader:
+    """
+    Data loader that combines and samples from multiple single-task
+    data loaders.
+    """
+
+    def __init__(self, dataloader_dict):
+        self.dataloader_dict = dataloader_dict
+        self.num_batches_dict = {
+            task_name: len(dataloader)
+            for task_name, dataloader in self.dataloader_dict.items()
+        }
+        self.task_name_list = list(self.dataloader_dict)
+        self.dataset = [None] * sum(
+            len(dataloader.dataset)
+            for dataloader in self.dataloader_dict.values()
+        )
+
+    def __len__(self):
+        return sum(self.num_batches_dict.values())
+
+    def __iter__(self):
+        """
+        For each batch, sample a task, and yield a batch from the respective
+        task Dataloader.
+
+        We use size-proportional sampling, but you could easily modify this
+        to sample from some-other distribution.
+        """
+        task_choice_list = []
+        for i, task_name in enumerate(self.task_name_list):
+            task_choice_list += [i] * self.num_batches_dict[task_name]
+        task_choice_list = np.array(task_choice_list)
+        np.random.shuffle(task_choice_list)
+        dataloader_iter_dict = {
+            task_name: iter(dataloader)
+            for task_name, dataloader in self.dataloader_dict.items()
+        }
+        for task_choice in task_choice_list:
+            task_name = self.task_name_list[task_choice]
+            yield next(dataloader_iter_dict[task_name])
+
+
+class MultitaskTrainer(transformers.Trainer):
+
+    def get_single_train_dataloader(self, task_name, train_dataset):
+        """
+        Create a single-task data loader that also yields task names
+        """
+        if self.train_dataset is None:
+            raise ValueError("Trainer: training requires a train_dataset.")
+        if False and is_torch_tpu_available():
+            train_sampler = get_tpu_sampler(train_dataset)
+        else:
+            train_sampler = (
+                RandomSampler(train_dataset)
+                if self.args.local_rank == -1
+                else DistributedSampler(train_dataset)
+            )
+
+        data_loader = DataLoaderWithTaskname(
+            task_name=task_name,
+            data_loader=DataLoader(
+                train_dataset,
+                batch_size=self.args.train_batch_size,
+                sampler=train_sampler,
+                collate_fn=self.data_collator.collate_batch,
+            ),
+        )
+        return data_loader
+
+    def get_train_dataloader(self):
+        """
+        Returns a MultitaskDataloader, which is not actually a Dataloader
+        but an iterable that returns a generator that samples from each 
+        task Dataloader
+        """
+        return MultitaskDataloader({
+            task_name: self.get_single_train_dataloader(
+                task_name, task_dataset)
+            for task_name, task_dataset in self.train_dataset.items()
+        })
+    

requirements.txt

@@ -0,0 +1,8 @@
+nltk
+datasets==1.6.2
+torch==1.8.1
+transformers==4.8.2
+pytorch-lightning==1.4.9
+tokenizers==0.10.3
+numpy==1.21.2
+scikit-learn==0.24.1