Instructions to use OpenFormosa/PangolinTokenizer with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use OpenFormosa/PangolinTokenizer with Transformers:
# Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("OpenFormosa/PangolinTokenizer", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 1,628 Bytes
82e506e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | {
"ok": true,
"failures": [],
"sections": {
"Timestamp / rich transcription checks": {
"rich_transcription_token_ids": {
"<|transcript_start|>": 114670,
"<|transcript_end|>": 114671,
"<|segment_start|>": 114672,
"<|segment_end|>": 114673,
"<|speaker|>": 114674,
"<|start_time|>": 114675,
"<|end_time|>": 114676,
"<|duration|>": 114677,
"<|content|>": 114678,
"<|non_speech_event|>": 114679
},
"timestamp_precision_digits": 2,
"json_roundtrip_ok": true,
"dense_timestamp_tokens_found": [],
"missing_rich_transcription_tokens": [],
"timestamp_strings_present": {
"0.00": true,
"3.42": true,
"10.25": true,
"3575.50": true
},
"non_speech_labels_present": {
"[Silence]": true,
"[Noise]": true,
"[Music]": true,
"[Unintelligible Speech]": true
},
"text_roundtrip_ok": {
"traditional_chinese": true,
"bopomofo_mixed_romanization": true,
"json_syntax": true
},
"required_fields_ok": true,
"parse_error": null
}
},
"rich_transcription_token_ids": {
"<|transcript_start|>": 114670,
"<|transcript_end|>": 114671,
"<|segment_start|>": 114672,
"<|segment_end|>": 114673,
"<|speaker|>": 114674,
"<|start_time|>": 114675,
"<|end_time|>": 114676,
"<|duration|>": 114677,
"<|content|>": 114678,
"<|non_speech_event|>": 114679
},
"timestamp_precision_digits": 2,
"json_roundtrip_ok": true,
"dense_timestamp_tokens_found": []
}
|