import numpy as np import torch from typing import Dict, List, Tuple, Optional def chunk_audio_with_stride( audio: np.ndarray, sampling_rate: int, chunk_length_s: float = 30.0, stride_length_s: Tuple[float, float] = (5.0, 5.0) ) -> List[Dict]: """ Split audio into overlapping chunks with stride. Args: audio: Audio array (mono, 1D) sampling_rate: Sampling rate of the audio chunk_length_s: Length of each chunk in seconds stride_length_s: Tuple of (left_stride, right_stride) in seconds Returns: List of dicts with keys: - 'audio': audio chunk array - 'stride': tuple of (left_stride_samples, right_stride_samples) - 'is_first': whether this is the first chunk - 'is_last': whether this is the last chunk """ chunk_length = int(chunk_length_s * sampling_rate) if stride_length_s is None: stride_length_s = chunk_length_s / 6 if isinstance(stride_length_s, (int, float)): stride_length_s = [stride_length_s, stride_length_s] stride_left = int(stride_length_s[0] * sampling_rate) stride_right = int(stride_length_s[1] * sampling_rate) audio_length = len(audio) # If audio is shorter than chunk, return as single chunk if audio_length <= chunk_length: return [{ 'audio': audio, 'stride': (0, 0), 'is_first': True, 'is_last': True, 'start_sample': 0, 'end_sample': audio_length }] chunks = [] start = 0 while start < audio_length: end = min(start + chunk_length, audio_length) actual_stride_left = stride_left if start > 0 else 0 actual_stride_right = stride_right if end < audio_length else 0 chunk_start = max(0, start - actual_stride_left) chunk_end = min(audio_length, end + actual_stride_right) chunk_audio = audio[chunk_start:chunk_end] chunks.append({ 'audio': chunk_audio, 'stride': (actual_stride_left, actual_stride_right), 'is_first': (start == 0), 'is_last': (end >= audio_length), 'start_sample': start, 'end_sample': end, 'chunk_start': chunk_start, 'chunk_end': chunk_end }) start = end return chunks def process_logits_with_stride( all_logits: List[torch.Tensor], chunks_info: List[Dict], model_ratio: float ) -> torch.Tensor: """ Process logits from all chunks, removing stride parts and concatenating. Args: all_logits: List of logit tensors from each chunk [batch, time, vocab] chunks_info: List of chunk info dicts from chunk_audio_with_stride model_ratio: The downsampling ratio of the model (e.g. 320 for wav2vec2) This is the ratio between input samples and output frames Returns: Concatenated logits tensor """ processed_logits = [] for i, (logits, chunk_info) in enumerate(zip(all_logits, chunks_info)): stride_left_frames = int(chunk_info['stride'][0] / model_ratio) stride_right_frames = int(chunk_info['stride'][1] / model_ratio) if len(logits.shape) == 3: logits = logits[0] # [time, vocab] start_frame = stride_left_frames end_frame = logits.shape[0] - stride_right_frames trimmed_logits = logits[start_frame:end_frame] processed_logits.append(trimmed_logits) concatenated_logits = torch.cat(processed_logits, dim=0) return concatenated_logits.unsqueeze(0) def infer_fairseq_with_chunking( audio: np.ndarray, sampling_rate: int, model: torch.nn.Module, dictionary: Dict, device: torch.device, normalize_audio: bool = True, chunk_length_s: float = 30.0, stride_length_s: Tuple[float, float] = (5.0, 5.0), model_downsample_ratio: float = 320.0, decode_fn: callable = None ) -> str: """ Run fairseq inference with chunking and stride. Args: audio: Audio array (mono, 1D) sampling_rate: Sampling rate model: Fairseq checkpoint dictionary: Token dictionary for decoding device: torch device chunk_length_s: Length of each chunk in seconds stride_length_s: Tuple of (left_stride, right_stride) in seconds model_downsample_ratio: Downsampling ratio of the model decode_fn: Optional custom decoding function. If None, will use simple argmax Returns: ASR transcript """ chunks = chunk_audio_with_stride( audio, sampling_rate, chunk_length_s, stride_length_s ) all_logits = [] model.eval() with torch.no_grad(): for chunk_info in chunks: chunk_audio = chunk_info['audio'] # Normalize if needed if normalize_audio: chunk_audio = (chunk_audio - np.mean(chunk_audio)) / (np.std(chunk_audio) + 1e-5) try: chunk_tensor = torch.from_numpy(chunk_audio).float().unsqueeze(0).to(device) except TypeError: chunk_tensor = torch.FloatTensor(chunk_audio).unsqueeze(0).to(device) output = model(source=chunk_tensor, padding_mask=None) if isinstance(output, dict): logits = output['encoder_out'] # [time, batch, vocab] for fairseq logits = logits.transpose(0, 1) # [batch, time, vocab] else: logits = output all_logits.append(logits.cpu()) concatenated_logits = process_logits_with_stride( all_logits, chunks, model_downsample_ratio ) if decode_fn is not None: transcription = decode_fn(concatenated_logits, dictionary) else: transcription = simple_ctc_decode(concatenated_logits, dictionary) return transcription def simple_ctc_decode(emissions: torch.Tensor, dictionary: Dict) -> str: """ Simple greedy CTC decoding. Args: emissions: Logits tensor [batch, time, vocab] dictionary: Token dictionary {id: token} Returns: Decoded string """ # Get the most likely token at each timestep predictions = emissions.argmax(dim=-1) vocab_size = emissions.shape[-1] blank_idx = vocab_size - 1 # Convert to tokens tokens = [] prev_token_id = None for pred in predictions[0]: token_id = pred.item() # Skip blank tokens if token_id == blank_idx: prev_token_id = None continue # Skip special tokens and repeated tokens if token_id == prev_token_id or token_id < 4: # Skip , , , if token_id != prev_token_id and token_id < 4: prev_token_id = None continue # Get token from dictionary token = dictionary.get(token_id, f'') tokens.append(token) prev_token_id = token_id # Join character tokens and replace | with space transcription = ''.join(tokens).replace('|', ' ').strip() return transcription def map_to_pred_fairseq_chunked( batch: Dict, model: torch.nn.Module, dictionary: Dict, device: torch.device, normalize_audio: bool = True, chunk_length_s: float = 30.0, stride_length_s: Tuple[float, float] = (5.0, 5.0), model_downsample_ratio: float = 320.0, decode_fn: callable = None ) -> Dict: """ Map function compatible with HuggingFace datasets that uses chunking. Args: batch: Batch dict with 'audio' and 'sampling_rate' keys model: Fairseq checkpoint dictionary: Token dictionary device: torch device chunk_length_s: Chunk length in seconds stride_length_s: Stride lengths (left, right) in seconds model_downsample_ratio: Model downsampling ratio decode_fn: Optional custom decoding function Returns: Dict with 'prediction' key added """ transcriptions = [] for i in range(len(batch['audio'])): audio = batch['audio'][i] sr = batch['sampling_rate'][i] transcription = infer_fairseq_with_chunking( audio=audio, sampling_rate=sr, model=model, dictionary=dictionary, device=device, normalize_audio=normalize_audio, chunk_length_s=chunk_length_s, stride_length_s=stride_length_s, model_downsample_ratio=model_downsample_ratio, decode_fn=decode_fn ) transcriptions.append(transcription) batch['prediction'] = transcriptions return batch