Extract alignment information from transcript with transformers
Following #14, is it possible using transformers to extract timestamp information and how is this information accessed?
So far, I can only extract the full transcribed text from the audio.
You can get the token timestamps like this:
import sys
import torch
from transformers import AutoModelForRNNT, AutoProcessor
from transformers.audio_utils import load_audio
model_id = "nvidia/nemotron-3.5-asr-streaming-0.6b"
processor = AutoProcessor.from_pretrained(model_id)
model = AutoModelForRNNT.from_pretrained(
model_id,
dtype=torch.float16,
device_map="cuda:0",
).eval()
audio = load_audio(
sys.argv[1],
sampling_rate=processor.feature_extractor.sampling_rate,
)
inputs = processor(
audio,
sampling_rate=processor.feature_extractor.sampling_rate,
language="en-US",
return_tensors="pt",
).to(model.device, dtype=model.dtype)
with torch.inference_mode():
output = model.generate(**inputs, return_dict_in_generate=True)
text, timestamps = processor.decode(
output.sequences,
durations=output.durations,
skip_special_tokens=True,
)
print(text[0])
for token in timestamps[0]:
print(token)