Initial commit: per-language Chatterbox TTS demo

app.py

@@ -12,14 +12,20 @@ DEFAULT_CONFIG = {
     "text": 'Last month, we reached a new milestone with two billion views on our YouTube channel.',
 }
 
+LANGUAGE_CHOICES = [('en', 'English'), ('ar', 'Arabic'), ('da', 'Danish'), ('de', 'German'), ('el', 'Greek'), ('es', 'Spanish'), ('fi', 'Finnish'), ('fr', 'French'), ('he', 'Hebrew'), ('hi', 'Hindi'), ('it', 'Italian'), ('ja', 'Japanese'), ('ko', 'Korean'), ('ms', 'Malay'), ('nl', 'Dutch'), ('no', 'Norwegian'), ('pl', 'Polish'), ('pt', 'Portuguese'), ('ru', 'Russian'), ('sv', 'Swedish'), ('sw', 'Swahili'), ('tr', 'Turkish'), ('zh', 'Chinese')]
+DEFAULT_LANGUAGE = 'en'
+FIXED_LANGUAGE_ID = None
+
 EXAMPLES = [
-        ['Last month, we reached a new milestone with two billion views on our YouTube channel.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/en_f1.flac', 0.5, 0.8, 0, 0.5],
-        ["Bonjour, comment allez-vous? Aujourd'hui, il fait un temps magnifique pour se promener.", 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/fr_f1.flac', 0.5, 0.8, 0, 0.5],
-        ['Guten Tag! Heute ist ein wunderschöner Tag für einen Spaziergang im Park.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/de_f1.flac', 0.5, 0.8, 0, 0.5],
-        ['Hola, ¿cómo estás? Me encanta visitar nuevos lugares y conocer culturas diferentes.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/es_f1.flac', 0.5, 0.8, 0, 0.5],
-        ['Ciao! Oggi è una giornata perfetta per una passeggiata a Roma.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/it_m1.flac', 0.5, 0.8, 0, 0.5],
-        ['Olá, tudo bem? O Brasil é um país incrível com paisagens deslumbrantes.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/pt_m1.flac', 0.5, 0.8, 0, 0.5],
-        ['नमस्ते, आज का दिन बहुत खूबसूरत है।', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/hi_f1.flac', 0.5, 0.8, 0, 0.5]
+        ['Last month, we reached a new milestone with two billion views on our YouTube channel.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/en_f1.flac', 'en', 0.5, 0.8, 0, 0.5],
+        ["Bonjour, comment allez-vous? Aujourd'hui, il fait un temps magnifique pour se promener.", 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/fr_f1.flac', 'fr', 0.5, 0.8, 0, 0.5],
+        ['Guten Tag! Heute ist ein wunderschöner Tag für einen Spaziergang im Park.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/de_f1.flac', 'de', 0.5, 0.8, 0, 0.5],
+        ['Hola, ¿cómo estás? Me encanta visitar nuevos lugares y conocer culturas diferentes.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/es_f1.flac', 'es', 0.5, 0.8, 0, 0.5],
+        ['Ciao! Oggi è una giornata perfetta per una passeggiata a Roma.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/it_m1.flac', 'it', 0.5, 0.8, 0, 0.5],
+        ['Olá, tudo bem? O Brasil é um país incrível com paisagens deslumbrantes.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/pt_m1.flac', 'pt', 0.5, 0.8, 0, 0.5],
+        ['नमस्ते, आज का दिन बहुत खूबसूरत है।', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/hi_f1.flac', 'hi', 0.5, 0.8, 0, 0.5],
+        ['你好,今天天气真不错,我们一起去公园散步吧。', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/zh_f1.flac', 'zh', 0.5, 0.8, 0, 0.5],
+        ['مرحبا، كيف حالك اليوم؟ الطقس جميل للنزهة في الحديقة.', 'https://storage.googleapis.com/chatterbox-demo-samples/mtl_prompts/ar_m1.flac', 'ar', 0.5, 0.8, 0, 0.5]
 ]
 
 
@@ -53,6 +59,7 @@ def set_seed(seed: int, device: str):
 def generate_tts_audio(
     text_input: str,
     audio_prompt_path_input: str = None,
+    language_id_input: str = None,
     exaggeration_input: float = 0.5,
     temperature_input: float = 0.8,
     seed_num_input: int = 0,
@@ -64,16 +71,17 @@ def generate_tts_audio(
     current_model.to(device)
     if seed_num_input != 0:
         set_seed(int(seed_num_input), device)
-    print(f"Generating on {device} for text: '{text_input[:50]}...'")
     chosen_prompt = audio_prompt_path_input or default_audio_for_ui()
+    lang = language_id_input or DEFAULT_LANGUAGE
+    print(f"Generating on {device} (lang={lang}) for text: '{text_input[:50]}...'")
     generate_kwargs = {
         "exaggeration": exaggeration_input,
         "temperature": temperature_input,
         "cfg_weight": cfgw_input,
+        "language_id": lang,
     }
     if chosen_prompt:
         generate_kwargs["audio_prompt_path"] = chosen_prompt
-        print(f"Using audio prompt: {chosen_prompt}")
     wav = current_model.generate(text_input[:300], **generate_kwargs)
     return (current_model.sr, wav.squeeze(0).cpu().numpy())
 
@@ -99,6 +107,11 @@ with gr.Blocks() as demo:
                 label="Reference Audio File (Optional)",
                 value=default_audio_for_ui(),
             )
+            language = gr.Dropdown(
+                choices=[(name, code) for code, name in LANGUAGE_CHOICES],
+                value=DEFAULT_LANGUAGE,
+                label="Language",
+            )
             exaggeration = gr.Slider(0.25, 2, step=.05, label="Exaggeration (Neutral = 0.5)", value=.5)
             cfg_weight = gr.Slider(0.2, 1, step=.05, label="CFG/Pace", value=0.5)
             with gr.Accordion("More options", open=False):
@@ -108,12 +121,12 @@ with gr.Blocks() as demo:
         with gr.Column():
             audio_output = gr.Audio(label="Output Audio")
 
-    inputs = [text, ref_wav, exaggeration, temp, seed_num, cfg_weight]
+    inputs = [text, ref_wav, language, exaggeration, temp, seed_num, cfg_weight]
     run_btn.click(fn=generate_tts_audio, inputs=inputs, outputs=[audio_output])
 
     gr.Examples(
         examples=EXAMPLES,
-        inputs=[text, ref_wav, exaggeration, temp, seed_num, cfg_weight],
+        inputs=inputs,
         label="Examples",
     )
 

chatterbox/src/chatterbox/tts.py

@@ -247,6 +247,7 @@ class ChatterboxTTS:
         exaggeration=0.5,
         cfg_weight=0.5,
         temperature=0.8,
+        language_id="en",
     ):
         """
         Generate speech from text using the language-agnostic model.
@@ -278,7 +279,8 @@ class ChatterboxTTS:
 
         # Norm and tokenize text
         text = punc_norm(text)
-        text_tokens = self.tokenizer.text_to_tokens(text, language_id="en").to(self.device)
+        lang = (language_id or "en").lower() if language_id else None
+        text_tokens = self.tokenizer.text_to_tokens(text, language_id=lang).to(self.device)
         text_tokens = torch.cat([text_tokens, text_tokens], dim=0)  # Need two seqs for CFG
 
         sot = self.t3.hp.start_text_token