Upload model checkpoint and constants (http)

README.md

@@ -0,0 +1,121 @@
+---
+language: ['ar']
+tags:
+  - diacritization
+  - nlp
+  - arabic
+metrics:
+  - DER
+  - WER
+  - SER
+model_type: to_push
+datasets:
+  - transformer-text-only
+---
+
+# Model Card
+
+This is a To_Push model for Arabic text diacritization.
+
+## Model Details
+
+**Source checkpoint:** `/home/rufael/Projects/forced_alignment/Diac/results/to_push/transformer-text-only/tashkeela/tensorboard/version_0/checkpoints/best_model.ckpt`
+
+**Included config files:**
+- `/home/rufael/Projects/forced_alignment/Diac/results/to_push/transformer-text-only/tashkeela/tensorboard/version_0/hparams.yaml`
+
+## Evaluation Results
+
+### Evaluation on clartts
+
+### DER (Diacritic Error Rate)
+
+| Configuration | With case ending | Without case ending |
+|---|---|---|
+| **Including no diacritic** | 10.33% | 8.45% |
+| **Excluding no diacritic** | 12.72% | 10.33% |
+
+
+### WER (Word Error Rate)
+
+| Configuration | With case ending | Without case ending |
+|---|---|---|
+| **Including no diacritic** | 30.16% | 19.71% |
+| **Excluding no diacritic** | 29.91% | 19.60% |
+
+
+### SER (Sentence Error Rate)
+
+| Configuration | With case ending | Without case ending |
+|---|---|---|
+| **Including no diacritic** | 91.62% | 79.34% |
+| **Excluding no diacritic** | 91.62% | 79.34% |
+
+
+## How to Use
+
+### Installation
+
+```bash
+pip install torch
+```
+
+### Loading the Model
+
+```python
+from diac.models import DiacritizationModule
+
+model = DiacritizationModule.from_pretrained(
+    "rufaelfekadu/diac-transformer-text-only-tashkeela",
+    tokenizer_constants_path="constants/"  # Path to constants directory
+)
+```
+
+### Running Inference
+
+```python
+# Predict diacritization for a text file
+model.predict_file(
+    input_file="path/to/input.txt",
+    output_file="path/to/output.txt"
+)
+
+# Or predict for a single text string
+diacritized_text = model.predict_text("مرحبا بك")
+```
+
+### Running Evaluation
+
+To evaluate the model on your own test set:
+
+1. **Run inference** to generate predictions:
+
+```bash
+python inference.py \
+    --config configs/<model>.yml \
+    --opts \
+    DATA.TEST_PATH path/to/test.txt \
+    INFERENCE.MODEL_PATH <path_to_checkpoint> \
+    INFERENCE.OUTPUT_PATH path/to/predictions.txt
+```
+
+2. **Prepare reference file** (if needed):
+
+```bash
+python src/diac/utils/prep_ref.py \
+    --input_file path/to/test.txt \
+    -o path/to/output_dir
+```
+
+3. **Calculate metrics** (DER, WER, SER):
+
+```bash
+python src/diac/utils/eval.py \
+    -ofp path/to/predictions.txt \
+    -tfp path/to/reference.txt \
+    --style Fadel
+```
+
+The evaluation script will output DER, WER, and SER metrics with different configurations:
+- With/without case ending
+- Including/excluding no diacritic

best_model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5364a8e32a8847fd7152689508aed769c2675bc72120f751aa38e4096c2245a
+size 8538913

hparams.yaml

@@ -0,0 +1,307 @@
+config: !!python/object/new:yacs.config.CfgNode
+  dictitems:
+    CONSTANTS_PATH: constants/
+    DATA: !!python/object/new:yacs.config.CfgNode
+      dictitems:
+        MAX_LENGTH: 270
+        TEST_PATH: data/clartts/ClarTTS_test_Ref.txt
+        TRAIN_PATH: data/tashkeela/train.txt
+        VAL_PATH: data/tashkeela/val.txt
+      state:
+        __deprecated_keys__: !!set {}
+        __immutable__: true
+        __new_allowed__: false
+        __renamed_keys__: {}
+    INFERENCE: !!python/object/new:yacs.config.CfgNode
+      dictitems:
+        ASR_MODEL_NAME: sashat/whisper-medium-ClassicalAr
+        BATCH_SIZE: 16
+        BUFFER_SIZE: 25
+        DEVICE: cuda
+        FORCED_IDS: null
+        MAX_LENGTH: 300
+        MODEL_PATH: null
+        OUTPUT_PATH: results/transformer-text-only_clartts.txt
+        USE_ASR: false
+        WINDOW_SIZE: 50
+      state:
+        __deprecated_keys__: !!set {}
+        __immutable__: true
+        __new_allowed__: false
+        __renamed_keys__: {}
+    MODEL: !!python/object/new:yacs.config.CfgNode
+      dictitems:
+        ASR_VOCAB_SIZE: 91
+        DFF: 128
+        DROPOUT_RATE: 0.2
+        D_MODEL: 256
+        LOAD_TEXT_BRANCH_ONLY: false
+        MAXLEN: 272
+        NUM_BLOCKS: 2
+        NUM_HEADS: 4
+        OUTPUT_SIZE: 19
+        PRETRAINED_PATH: null
+        TYPE: Transformer
+        USE_ASR: false
+        VOCAB_SIZE: 77
+        WITH_CONN: false
+      state:
+        __deprecated_keys__: !!set {}
+        __immutable__: true
+        __new_allowed__: false
+        __renamed_keys__: {}
+    TRAIN: !!python/object/new:yacs.config.CfgNode
+      dictitems:
+        ACCUMULATE_GRAD_BATCHES: 1
+        BATCH_SIZE: 128
+        DEVICE: cuda
+        EARLY_STOPPING_PATIENCE: 10
+        EVAL_FREQ: 1
+        LEARNING_RATE: 0.001
+        NUM_EPOCHS: 100
+        SAVE_DIR: results/transformer-text-only/tashkeela
+        SAVE_FREQ: 30
+        VAL_CHECK_INTERVAL: 1.0
+      state:
+        __deprecated_keys__: !!set {}
+        __immutable__: true
+        __new_allowed__: false
+        __renamed_keys__: {}
+  state:
+    __deprecated_keys__: !!set {}
+    __immutable__: true
+    __new_allowed__: false
+    __renamed_keys__: {}
+tokenizer: !!python/object:tokenizer.ArabicDiacritizationTokenizer
+  constants: !!python/object:tokenizer.TokenizerConstants
+    arabic_letters_list: "\u0621\u0622\u0623\u0624\u0625\u0626\u0627\u0628\u0629\u062A\
+      \u062B\u062C\u062D\u062E\u062F\u0630\u0631\u0632\u0633\u0634\u0635\u0636\u0637\
+      \u0638\u0639\u063A\u0641\u0642\u0643\u0644\u0645\u0646\u0647\u0648\u0649\u064A"
+    characters_mapping:
+      ? '
+
+        '
+      : 4
+      ' ': 5
+      '!': 6
+      '"': 7
+      '''': 8
+      (: 9
+      ): 10
+      '*': 11
+      ',': 12
+      '-': 13
+      .: 14
+      /: 15
+      '0': 16
+      '1': 17
+      '2': 18
+      '3': 19
+      '4': 20
+      '5': 21
+      '6': 22
+      '7': 23
+      '8': 24
+      '9': 25
+      ':': 26
+      ;: 27
+      <EOS>: 2
+      <PAD>: 0
+      <SOS>: 1
+      <UNK>: 3
+      '[': 28
+      ']': 29
+      '`': 30
+      '{': 31
+      '}': 32
+      '~': 33
+      "\xAB": 34
+      "\xBB": 35
+      "\u060C": 36
+      "\u061B": 37
+      "\u061F": 38
+      "\u0621": 39
+      "\u0622": 40
+      "\u0623": 41
+      "\u0624": 42
+      "\u0625": 43
+      "\u0626": 44
+      "\u0627": 45
+      "\u0628": 46
+      "\u0629": 47
+      "\u062A": 48
+      "\u062B": 49
+      "\u062C": 50
+      "\u062D": 51
+      "\u062E": 52
+      "\u062F": 53
+      "\u0630": 54
+      "\u0631": 55
+      "\u0632": 56
+      "\u0633": 57
+      "\u0634": 58
+      "\u0635": 59
+      "\u0636": 60
+      "\u0637": 61
+      "\u0638": 62
+      "\u0639": 63
+      "\u063A": 64
+      "\u0641": 65
+      "\u0642": 66
+      "\u0643": 67
+      "\u0644": 68
+      "\u0645": 69
+      "\u0646": 70
+      "\u0647": 71
+      "\u0648": 72
+      "\u0649": 73
+      "\u064A": 74
+      "\u200F": 75
+      "\u2013": 76
+    classes_mapping:
+      ? ''
+      : 0
+      <EOS>: 17
+      <N/A>: 18
+      <PAD>: 15
+      <SOS>: 16
+      "\u064B": 2
+      "\u064C": 4
+      "\u064D": 6
+      "\u064E": 1
+      "\u064F": 3
+      "\u0650": 5
+      "\u0651": 8
+      "\u0651\u064B": 10
+      "\u0651\u064C": 12
+      "\u0651\u064D": 14
+      "\u0651\u064E": 9
+      "\u0651\u064F": 11
+      "\u0651\u0650": 13
+      "\u0652": 7
+    diacritics_list:
+    - "\u064E"
+    - "\u064B"
+    - "\u0650"
+    - "\u064D"
+    - "\u064F"
+    - "\u064C"
+    - "\u0652"
+    - "\u0651"
+    expanded_vocabulary:
+      ? '
+
+        '
+      : 4
+      ' ': 5
+      '!': 6
+      '"': 7
+      '''': 8
+      (: 9
+      ): 10
+      '*': 11
+      ',': 12
+      '-': 13
+      .: 14
+      /: 15
+      '0': 16
+      '1': 17
+      '2': 18
+      '3': 19
+      '4': 20
+      '5': 21
+      '6': 22
+      '7': 23
+      '8': 24
+      '9': 25
+      ':': 26
+      ;: 27
+      <EOS>: 2
+      <PAD>: 0
+      <SOS>: 1
+      <UNK>: 3
+      '[': 28
+      ']': 29
+      '`': 30
+      '{': 31
+      '}': 32
+      '~': 33
+      "\xAB": 34
+      "\xBB": 35
+      "\u060C": 36
+      "\u061B": 37
+      "\u061F": 38
+      "\u0621": 39
+      "\u0622": 40
+      "\u0623": 41
+      "\u0624": 42
+      "\u0625": 43
+      "\u0626": 44
+      "\u0627": 45
+      "\u0628": 46
+      "\u0629": 47
+      "\u062A": 48
+      "\u062B": 49
+      "\u062C": 50
+      "\u062D": 51
+      "\u062E": 52
+      "\u062F": 53
+      "\u0630": 54
+      "\u0631": 55
+      "\u0632": 56
+      "\u0633": 57
+      "\u0634": 58
+      "\u0635": 59
+      "\u0636": 60
+      "\u0637": 61
+      "\u0638": 62
+      "\u0639": 63
+      "\u063A": 64
+      "\u0641": 65
+      "\u0642": 66
+      "\u0643": 67
+      "\u0644": 68
+      "\u0645": 69
+      "\u0646": 70
+      "\u0647": 71
+      "\u0648": 72
+      "\u0649": 73
+      "\u064A": 74
+      "\u064B": 78
+      "\u064C": 80
+      "\u064D": 82
+      "\u064E": 77
+      "\u064F": 79
+      "\u0650": 81
+      "\u0651": 84
+      "\u0651\u064B": 86
+      "\u0651\u064C": 88
+      "\u0651\u064D": 90
+      "\u0651\u064E": 85
+      "\u0651\u064F": 87
+      "\u0651\u0650": 89
+      "\u0652": 83
+      "\u200F": 75
+      "\u2013": 76
+    rev_classes_mapping:
+      0: ''
+      1: "\u064E"
+      2: "\u064B"
+      3: "\u064F"
+      4: "\u064C"
+      5: "\u0650"
+      6: "\u064D"
+      7: "\u0652"
+      8: "\u0651"
+      9: "\u0651\u064E"
+      10: "\u0651\u064B"
+      11: "\u0651\u064F"
+      12: "\u0651\u064C"
+      13: "\u0651\u0650"
+      14: "\u0651\u064D"
+      15: <PAD>
+      16: <SOS>
+      17: <EOS>
+      18: <N/A>
+  constants_path: constants/
+  with_extra_train: false