Add 🤗 Transformers support (#20)

- Upload processor (2af778cf06cfcc4bb943ed309c66a12008d98475)
- Upload Nemotron3_5AsrForRNNT (6f4cda233ca61c8aab428544a3344d96904f9983)
- Add 🤗 Transformers usage to model card (65267afc897a237850eda75e6ce45eb278fa73a4)
- Rebase model card on main; add 🤗 Transformers usage only (153610ad16586dfc19b57ad9060b9bb6e367b04d)
- Drop revision pin from Transformers snippets (not needed post-merge) (be133a9c2559b9f39bad8c7a789832bdc21a3fde)
- Add Pipeline usage snippet to Transformers section (92bc43f5297e5b145773660d07935052f9419a9e)
- config: drop unused joint_hidden_size, add default_prompt_id=101 (auto) (3afa791cd14f1c185541cb7b8c015096db946b66)
- config: drop redundant decoder_start_token_id (carried by generation_config) (299cc89044b022111fef860bee4244296b0eaf95)


Co-authored-by: Eustache Le Bihan <eustlb@users.noreply.huggingface.co>

README.md

@@ -49,6 +49,7 @@ datasets:
 - europarl
 thumbnail: null
 tags:
+- transformers
 - speech-recognition
 - cache-aware ASR
 - automatic-speech-recognition
@@ -368,6 +369,8 @@ pip install git+https://github.com/NVIDIA/NeMo.git@main#egg=nemo_toolkit[asr]
 
 The model is available for use in the NeMo Framework, and can be used as a pre-trained checkpoint for inference or for fine-tuning on another dataset.
 
+You can also run it with [🤗 Transformers](https://github.com/huggingface/transformers) (more below).
+
 ### Loading the Model
 
 ```python
@@ -409,6 +412,137 @@ Latency is defined by the `att_context_size` param, where att_context_size = `{n
 
 Here, chunk size = current frame + right context; each chunk is processed in non-overlapping fashion.
 
+### 🤗 Transformers usage
+
+This checkpoint also runs with [🤗 Transformers](https://github.com/huggingface/transformers). The target language is passed through the processor's `language` argument: a locale such as `en-US`/`de-DE`, a bare code such as `de`, or `auto` for automatic language detection. In `auto` mode the model appends an `<xx-XX>` language tag after the transcript's terminal punctuation; it is a special token, so decoding with `skip_special_tokens=True` strips it (clean transcript) and `skip_special_tokens=False` keeps it for language labeling.
+
+Until Nemotron3_5Asr is part of an official Transformers release, install Transformers from source:
+
+```bash
+pip install git+https://github.com/huggingface/transformers
+```
+
+<details>
+  <summary>➡️ Pipeline</summary>
+
+```python
+from transformers import pipeline
+
+pipe = pipeline("automatic-speech-recognition", model="nvidia/nemotron-3.5-asr-streaming-0.6b")
+out = pipe("https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/bcn_weather.mp3")
+print(out)
+```
+
+The pipeline uses the default language prompt (index 0, `en-US`). For explicit language conditioning or automatic detection, pass the processor's `language` argument (see the AutoModel example below).
+</details>
+
+<details>
+  <summary>➡️ Offline transcription</summary>
+
+```python
+from transformers import AutoModelForRNNT, AutoProcessor
+from transformers.audio_utils import load_audio
+
+model_id = "nvidia/nemotron-3.5-asr-streaming-0.6b"
+processor = AutoProcessor.from_pretrained(model_id)
+model = AutoModelForRNNT.from_pretrained(model_id, device_map="auto")
+
+audio = load_audio(
+    "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/bcn_weather.mp3",
+    sampling_rate=processor.feature_extractor.sampling_rate,
+)
+
+# Condition on a known language ...
+inputs = processor(audio, sampling_rate=processor.feature_extractor.sampling_rate, language="en-US")
+inputs.to(model.device, dtype=model.dtype)
+output = model.generate(**inputs, return_dict_in_generate=True)
+print(processor.decode(output.sequences, skip_special_tokens=True))
+
+# ... or let the model detect it and keep the emitted <xx-XX> language tag.
+inputs = processor(audio, sampling_rate=processor.feature_extractor.sampling_rate, language="auto")
+inputs.to(model.device, dtype=model.dtype)
+output = model.generate(**inputs, return_dict_in_generate=True)
+print(processor.decode(output.sequences, skip_special_tokens=False))
+```
+</details>
+
+<details>
+  <summary>➡️ Streaming transcription</summary>
+
+```python
+from threading import Thread
+from transformers import AutoModelForRNNT, AutoProcessor, TextIteratorStreamer
+from transformers.audio_utils import load_audio
+
+model_id = "nvidia/nemotron-3.5-asr-streaming-0.6b"
+processor = AutoProcessor.from_pretrained(model_id)
+model = AutoModelForRNNT.from_pretrained(model_id, device_map="auto")
+
+processor.set_num_lookahead_tokens(6)
+print(f"Streaming latency: {processor.streaming_latency_ms} ms")
+
+# The language prompt rides along on every chunk; use a locale (e.g. "de-DE") or "auto".
+language = "en-US"
+
+sampling_rate = processor.feature_extractor.sampling_rate
+audio = load_audio(
+    "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/obama.mp3",
+    sampling_rate=sampling_rate,
+)
+
+first_chunk_inputs = processor(
+    audio[: processor.num_samples_first_audio_chunk],
+    sampling_rate=sampling_rate,
+    is_streaming=True,
+    is_first_audio_chunk=True,
+    language=language,
+    return_tensors="pt",
+)
+first_chunk_inputs = first_chunk_inputs.to(model.device, dtype=model.dtype)
+
+
+def input_features_generator():
+    yield first_chunk_inputs.input_features[:, : processor.num_mel_frames_first_audio_chunk, :]
+
+    mel_frame_idx = processor.num_mel_frames_first_audio_chunk
+    hop_length = processor.feature_extractor.hop_length
+    n_fft = processor.feature_extractor.n_fft
+
+    start_idx = mel_frame_idx * hop_length - n_fft // 2
+    while (end_idx := start_idx + processor.num_samples_per_audio_chunk) < audio.shape[0]:
+        inputs = processor(
+            audio[start_idx:end_idx],
+            sampling_rate=sampling_rate,
+            is_streaming=True,
+            is_first_audio_chunk=False,
+            language=language,
+            return_tensors="pt",
+        )
+        inputs = inputs.to(model.device, dtype=model.dtype)
+        yield inputs.input_features
+
+        mel_frame_idx += processor.num_mel_frames_per_audio_chunk
+        start_idx = mel_frame_idx * hop_length - n_fft // 2
+
+
+streamer = TextIteratorStreamer(processor.tokenizer, skip_special_tokens=True)
+generate_kwargs = {
+    **first_chunk_inputs,
+    "input_features": input_features_generator(),
+    "streamer": streamer,
+}
+thread = Thread(target=model.generate, kwargs=generate_kwargs)
+thread.start()
+
+print("Model output (streaming):", end=" ", flush=True)
+for text_chunk in streamer:
+    print(text_chunk, end="", flush=True)
+thread.join()
+```
+</details>
+
+For more details about usage, please refer to the [Transformers documentation](https://huggingface.co/docs/transformers/en/model_doc/nemotron3_5_asr).
+
 ### Input(s): <br>
 
 **Input Type(s):** Audio, Lang ID <br>

config.json

@@ -0,0 +1,54 @@
+{
+  "architectures": [
+    "Nemotron3_5AsrForRNNT"
+  ],
+  "blank_token_id": 13087,
+  "decoder_hidden_size": 640,
+  "dtype": "float32",
+  "durations": [],
+  "encoder_config": {
+    "activation_dropout": 0.1,
+    "attention_bias": false,
+    "attention_dropout": 0.1,
+    "conv_kernel_size": 9,
+    "convolution_bias": false,
+    "default_num_lookahead_tokens": 3,
+    "dropout": 0.1,
+    "dropout_positions": 0.0,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "layerdrop": 0.1,
+    "max_position_embeddings": 5000,
+    "model_type": "nemotron_asr_streaming_encoder",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 8,
+    "num_mel_bins": 128,
+    "scale_input": false,
+    "sliding_window": 57,
+    "subsampling_conv_channels": 256,
+    "subsampling_conv_kernel_size": 3,
+    "subsampling_conv_stride": 2,
+    "subsampling_factor": 8,
+    "supported_num_lookahead_tokens": [
+      3,
+      0,
+      6,
+      13
+    ]
+  },
+  "hidden_act": "relu",
+  "initializer_range": 0.02,
+  "is_encoder_decoder": true,
+  "max_symbols_per_step": 10,
+  "model_type": "nemotron3_5_asr",
+  "num_decoder_layers": 2,
+  "num_prompts": 128,
+  "pad_token_id": 0,
+  "prompt_intermediate_size": 2048,
+  "transformers_version": "5.13.0.dev0",
+  "vocab_size": 13088,
+  "default_prompt_id": 101
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 13087,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 0,
+  "transformers_version": "5.13.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eebdd6590289cb3030f310858f3df93256600a800a3e8200c5993d5f967e174
+size 2552062944

processor_config.json

@@ -0,0 +1,147 @@
+{
+  "blank_token": "<blank>",
+  "default_num_lookahead_tokens": 3,
+  "feature_extractor": {
+    "feature_extractor_type": "NemotronAsrStreamingFeatureExtractor",
+    "feature_size": 128,
+    "hop_length": 160,
+    "n_fft": 512,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "preemphasis": 0.97,
+    "return_attention_mask": true,
+    "sampling_rate": 16000,
+    "win_length": 400
+  },
+  "num_prompts": 128,
+  "processor_class": "Nemotron3_5AsrProcessor",
+  "prompt_dictionary": {
+    "af-ZA": 54,
+    "am-ET": 49,
+    "ar": 7,
+    "ar-AR": 7,
+    "auto": 101,
+    "ay-BO": 81,
+    "az-AZ": 66,
+    "bg": 30,
+    "bg-BG": 30,
+    "bn-IN": 36,
+    "cs": 22,
+    "cs-CZ": 22,
+    "da": 25,
+    "da-DK": 25,
+    "de": 9,
+    "de-DE": 9,
+    "el": 21,
+    "el-GR": 21,
+    "en": 0,
+    "en-GB": 1,
+    "en-US": 0,
+    "enGB": 1,
+    "es": 3,
+    "es-ES": 2,
+    "es-US": 3,
+    "esES": 2,
+    "et": 60,
+    "et-EE": 60,
+    "fa-IR": 38,
+    "fi": 26,
+    "fi-FI": 26,
+    "fr": 8,
+    "fr-CA": 100,
+    "fr-FR": 8,
+    "gn-PY": 82,
+    "gu-IN": 42,
+    "ha-NG": 50,
+    "haw-US": 97,
+    "he-IL": 64,
+    "hi": 6,
+    "hi-HI": 6,
+    "hi-IN": 6,
+    "hr": 29,
+    "hr-HR": 29,
+    "hu": 23,
+    "hu-HU": 23,
+    "hy-AM": 68,
+    "id-ID": 34,
+    "ig-NG": 53,
+    "it": 15,
+    "it-IT": 15,
+    "ja-JA": 10,
+    "ja-JP": 10,
+    "ka-GE": 67,
+    "km-KH": 47,
+    "kn-IN": 43,
+    "ko": 14,
+    "ko-KO": 14,
+    "ko-KR": 14,
+    "ku-TR": 65,
+    "ky-KG": 71,
+    "ln-CD": 58,
+    "lt": 31,
+    "lt-LT": 31,
+    "lv": 61,
+    "lv-LV": 61,
+    "mi-NZ": 96,
+    "ml-IN": 44,
+    "mr-IN": 41,
+    "ms-MY": 35,
+    "mt-MT": 102,
+    "nah-MX": 83,
+    "nb": 103,
+    "nb-NO": 103,
+    "ne-NP": 46,
+    "nl": 16,
+    "nl-NL": 16,
+    "nn": 104,
+    "nn-NO": 104,
+    "no": 27,
+    "no-NO": 27,
+    "ny-MW": 57,
+    "or-KE": 59,
+    "pl": 17,
+    "pl-PL": 17,
+    "pt": 13,
+    "pt-BR": 12,
+    "pt-PT": 13,
+    "qu-PE": 80,
+    "ro": 20,
+    "ro-RO": 20,
+    "ru": 11,
+    "ru-RU": 11,
+    "rw-RW": 55,
+    "si-LK": 45,
+    "sk": 28,
+    "sk-SK": 28,
+    "sl": 62,
+    "sl-SI": 62,
+    "sm-WS": 98,
+    "so-SO": 56,
+    "sv": 24,
+    "sv-SE": 24,
+    "sw-KE": 48,
+    "ta-IN": 39,
+    "te-IN": 40,
+    "tg-TJ": 70,
+    "th-TH": 32,
+    "to-TO": 99,
+    "tr": 18,
+    "tr-TR": 18,
+    "uk": 19,
+    "uk-UA": 19,
+    "ur-PK": 37,
+    "uz-UZ": 69,
+    "vi-VN": 33,
+    "yo-NG": 52,
+    "zh-CN": 4,
+    "zh-TW": 5,
+    "zh-ZH": 4,
+    "zu-ZA": 51
+  },
+  "supported_num_lookahead_tokens": [
+    3,
+    0,
+    6,
+    13
+  ]
+}

tokenizer_config.json

@@ -0,0 +1,50 @@
+{
+  "backend": "tokenizers",
+  "clean_up_tokenization_spaces": false,
+  "extra_special_tokens": [
+    "<ar-AR>",
+    "<bg-BG>",
+    "<cs-CZ>",
+    "<da-DK>",
+    "<de-DE>",
+    "<el-GR>",
+    "<en-GB>",
+    "<en-US>",
+    "<es-ES>",
+    "<es-US>",
+    "<et-EE>",
+    "<fi-FI>",
+    "<fr-CA>",
+    "<fr-FR>",
+    "<he-IL>",
+    "<hi-IN>",
+    "<hr-HR>",
+    "<hu-HU>",
+    "<it-IT>",
+    "<ja-JP>",
+    "<ko-KR>",
+    "<lt-LT>",
+    "<lv-LV>",
+    "<nb-NO>",
+    "<nl-NL>",
+    "<nn-NO>",
+    "<pl-PL>",
+    "<pt-BR>",
+    "<pt-PT>",
+    "<ro-RO>",
+    "<ru-RU>",
+    "<sk-SK>",
+    "<sl-SL>",
+    "<sv-SE>",
+    "<th-TH>",
+    "<tr-TR>",
+    "<uk-UA>",
+    "<vi-VN>",
+    "<zh-CN>"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "processor_class": "Nemotron3_5AsrProcessor",
+  "tokenizer_class": "ParakeetTokenizer",
+  "unk_token": "<unk>"
+}