import gradio as gr import os from InferenceInterfaces.InferenceFastSpeech2 import InferenceFastSpeech2 import argparse # If the HF Space variable or env var is set, download models from Hub MODEL_REPO_ID = os.getenv("FastSpeechHifiGANckpt", os.getenv("MODEL_REPO_ID", "")) MODEL_DIR = os.path.expanduser("~/.cache/austrian_tts_models") if MODEL_REPO_ID else "Models" HF_MODEL_FILES = { "best.pt": "FastSpeech2_Austrian_vox107_wav2vec_espeak/best.pt", "best_HiFiGAN.pt": "HiFiGAN_aridialect/best.pt", } def download_hf_models(): """Download model checkpoints from HF Hub if MODEL_REPO_ID is set.""" if not MODEL_REPO_ID: return from huggingface_hub import hf_hub_download os.makedirs(MODEL_DIR, exist_ok=True) for repo_file, local_rel in HF_MODEL_FILES.items(): target = os.path.join(MODEL_DIR, local_rel) if os.path.exists(target): print(f"Already cached: {target}") continue os.makedirs(os.path.dirname(target), exist_ok=True) print(f"Downloading {repo_file} from {MODEL_REPO_ID} ...") cached = hf_hub_download(repo_id=MODEL_REPO_ID, filename=repo_file, repo_type="model") os.symlink(cached, target) print(f" -> {target}") HEADER = ( '
' '' '' '
\n' ) TTS_TITLE = 'Text-to-Speech' TTS_DESCRIPTION = ( HEADER + 'Type in a text and read it with the chosen voice' ) TTS_TITLE = 'Text-to-Speech
For further details see: Paper' TTS_MODELS = ( 'Text' ) TTS_DESCRIPTION = ( HEADER + #make footnote in description for FastSpeech 2 "" "
" "A demo interface for Text-to-Speech synthesis. The implementation is based on FastSpeech 21, using an adaptation of IMS-Toucan2 with standard/dialect embeddings3." "
" "For further information see our publication4 : https://www.isca-archive.org/sigul_2023/gutscher23_sigul.html" "
" "Geben Sie einen deutschen Text ein, wählen Sie eine:n Sprecher:in sowie einen Standard/Dialekt aus und lassen Sie den Text vorlesen." "
" "Type in a German text, select a speaker, and select a standard/dialect embedding to read the text." "
" ) REFERENCES = ( "[1] Y. Ren, C. Hu, X. Tan, T. Qin, S. Zhao, Z. Zhao, and T.-Y. Liu, “Fastspeech 2: Fast and high-quality end-to-end text to speech,” in *ICLR 2021 - 9th International Conference on Learning Representations*, 2021.
" "[2] https://github.com/DigitalPhonetics/IMS-Toucan
" "[3] https://huggingface.co/TalTechNLP/voxlingua107-xls-r-300m-wav2vec
" "[4] L. Gutscher, M. Pucher, and V. García, “Neural Speech Synthesis for Austrian Dialects with Standard German Grapheme-to-Phoneme Conversion and Dialect Embeddings,” in *Proc. 2nd Annual Meeting of the ELRA/ISCA SIG on Under-resourced Languages (SIGUL 2023)*, 2023.
" " https://www.isca-archive.org/sigul_2023/gutscher23_sigul.html
" ) FOOTER = ( 'Security Notice: Text wird beim klicken auf "Senden" temporär gespeichert.
' 'Data is stored temporarly when clicking "Absenden".
\n' '
\n' 'Lorenz Gutscher and ' 'Michael Pucher
\n' ) ARTICLE = (REFERENCES + FOOTER) SPEAKERS = ["Austrian Standard (m)", "Viennese (f)", "Viennese (m)", "young Viennese (f)", "Goisern (f1)", "Goisern (f2)", "Goisern (m1)", "Goisern (m2)", "Innervillgraten (f1)", "Innervillgraten (f2)", "Innervillgraten (m1)", "Innervillgraten (m2)"] DIALECT_EMBEDDINGS = ["Standard", "Viennese", "Goisern", "Innervillgraten"] default_spk = "Viennese (m)" default_lang_emb = "Viennese" default_interpolation = "No interpolation" def synthesize(input_txt: str, speaker: str, lang_emb_avg: str, interpolation: str): model_id="Austrian_vox107_wav2vec_espeak" filename="audios/german_for_HPO_2Austrian_extended_phonemes_Empfehlung.wav" sentence=input_txt print(lang_emb_avg) print(speaker) read_texts(model_id=model_id, sentence=sentence, filename=filename, language="de", speaker=speaker, lang_emb_avg=lang_emb_avg, input_is_phones=False, interpolation=interpolation) return filename def read_texts(model_id, sentence, filename, speaker=None, device="cpu", language="", input_is_phones=None, lang_emb_avg=None, interpolation="No interpolation"): if lang_emb_avg == "Standard" and interpolation == "No interpolation": lang_emb = 'Preprocessing/wav2vec_embeddings/at_emb_vox107_wav2vec.pt' elif interpolation == "0.5 Standard, 0.5 Viennese": lang_emb = "Preprocessing/wav2vec_embeddings/at_to_vd_emb_vox107_0_50.pt" elif interpolation == "0.5 Standard, 0.5 Goisern": lang_emb = "Preprocessing/wav2vec_embeddings/at_to_goi_emb_vox107_0_50.pt" elif interpolation == "0.5 Standard, 0.5 Innervillgraten": lang_emb = "Preprocessing/wav2vec_embeddings/at_to_ivg_emb_vox107_0_50.pt" # Viennese elif lang_emb_avg == "Viennese" and interpolation == "No interpolation": lang_emb = "Preprocessing/wav2vec_embeddings/vd_emb_vox107_wav2vec.pt" elif interpolation == "0.5 Viennese, 0.5 Goisern": lang_emb = "Preprocessing/wav2vec_embeddings/vd_to_goi_emb_vox107_0_50.pt" elif interpolation == "0.5 Viennese, 0.5 Innervillgraten": lang_emb = "Preprocessing/wav2vec_embeddings/vd_to_ivg_emb_vox107_0_50.pt" # Goisern elif lang_emb_avg == "Goisern" and interpolation == "No interpolation": lang_emb = "Preprocessing/wav2vec_embeddings/goi_emb_vox107_wav2vec.pt" elif interpolation == "0.5 Goisern, 0.5 Innervillgraten": lang_emb = "Preprocessing/wav2vec_embeddings/goi_to_ivg_emb_vox107_0_50.pt" # Innervillgraten elif lang_emb_avg == "Innervillgraten" and interpolation == "No interpolation": lang_emb = "Preprocessing/wav2vec_embeddings/ivg_emb_vox107_wav2vec.pt" print("speaker in read texts is: " + str(speaker)) tts = InferenceFastSpeech2(device=device, model_name=model_id, language=language, Avocodo=False, model_dir=MODEL_DIR) tts.set_language(language) tts.set_phoneme_input(input_is_phones) print("model_id is: " + model_id) print("lang_emb is: " + str(lang_emb_avg)) use_avg = True if speaker == "Austrian Standard (m)" and lang_emb_avg == "Standard": print("utterance embedding is set to spo_at_berlin_001:") #tts.set_utterance_embedding(utt_emb) tts.set_utterance_embedding("Utility/example_wavs/spo_at_berlin_001.wav") tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker == "Austrian Standard (m)": print("utterance embedding is set to spo_vd_berlin_001:") #tts.set_utterance_embedding(utt_emb) tts.set_utterance_embedding("Utility/example_wavs/spo_vd_vdftw_001278.wav") tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker== "Viennese (f)": print("utterance embedding is set to hga_vd_berlin_003:") tts.set_utterance_embedding("Utility/example_wavs/hga_vd_berlin_003.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker == "Viennese (m)" and lang_emb_avg == "Standard": print("utterance embedding is set to hpo_at_wean_0002:") tts.set_utterance_embedding("Utility/example_wavs/hpo_at_nordwind_005.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker == "Viennese (m)": print("utterance embedding is set to hpo_vd_wean_0002:") tts.set_utterance_embedding("Utility/example_wavs/hpo_vd_wean_0002.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker == "young Viennese (f)": print("utterance embedding is set to joe_vd_fritz_048:") tts.set_utterance_embedding("Utility/example_wavs/joe_vd_fritz_048.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker == "Goisern (f1)" and lang_emb_avg == "Standard": print("utterance embedding is set to gun_at_berlin_001:") tts.set_utterance_embedding("Utility/example_wavs/gun_at_berlin_001.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker == "Goisern (f1)": print("utterance embedding is set to gun_goi_goi_001:") tts.set_utterance_embedding("Utility/example_wavs/gun_goi_goi_001.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker == "Goisern (m1)" and lang_emb_avg == "Standard": print("utterance embedding is set to hoi_at_berlin_001:") tts.set_utterance_embedding("Utility/example_wavs/hoi_at_berlin_001.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker == "Goisern (m1)": print("utterance embedding is set to hoi_goi_goi_051:") tts.set_utterance_embedding("Utility/example_wavs/hoi_goi_goi_001.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker == "Goisern (f2)": print("utterance embedding is set to tfe_goi_goi_051:") tts.set_utterance_embedding("Utility/example_wavs/tfe_goi_goi_001.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker == "Goisern (m2)": print("utterance embedding is set to wke_goi_goi_051:") tts.set_utterance_embedding("Utility/example_wavs/wke_goi_goi_001.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker =="Innervillgraten (f1)" and lang_emb_avg == "Standard": print("utterance embedding is set to bsc_at_berlin_001:") tts.set_utterance_embedding("Utility/example_wavs/bsc_at_berlin_001.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker =="Innervillgraten (f1)": print("utterance embedding is set to bsc_ivg_ivg_009:") tts.set_utterance_embedding("Utility/example_wavs/bsc_ivg_ivg_009.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker =="Innervillgraten (m1)" and lang_emb_avg == "Standard": print("utterance embedding is set to csc_at_berlin_001:") tts.set_utterance_embedding("Utility/example_wavs/csc_at_berlin_001.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker =="Innervillgraten (m1)": print("utterance embedding is set to csc_ivg_ivg_009:") tts.set_utterance_embedding("Utility/example_wavs/csc_ivg_ivg_009.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker =="Innervillgraten (m2)": print("utterance embedding is set to lsc_ivg_ivg_009:") tts.set_utterance_embedding("Utility/example_wavs/lsc_ivg_ivg_009.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) elif speaker =="Innervillgraten (f2)": print("utterance embedding is set to psc_ivg_ivg_009:") tts.set_utterance_embedding("Utility/example_wavs/psc_ivg_ivg_009.wav") #tts.set_utterance_embedding(utt_emb) tts.set_language_embedding(lang_emb,use_avg=use_avg) if type(sentence) == str: sentence = [sentence] tts.read_to_file(text_list=sentence, file_location=filename) del tts # dropdown options INTERPOLATION_OPTIONS = { "No interpolation": ["Standard", "Viennese", "Goisern", "Innervillgraten"], "0.5 Standard, 0.5 Viennese": ["Standard", "Viennese"], "0.5 Standard, 0.5 Goisern": ["Standard", "Goisern"], "0.5 Standard, 0.5 Innervillgraten": ["Standard", "Innervillgraten"], "0.5 Viennese, 0.5 Goisern": ["Viennese", "Goisern"], "0.5 Viennese, 0.5 Innervillgraten": ["Viennese", "Innervillgraten"], "0.5 Goisern, 0.5 Innervillgraten": ["Goisern", "Innervillgraten"], } def tts_demo_fn(text, speaker='Wiener Dialekt (m)', lang_emb_avg="Standard", interpolation="No interpolation"): if len(text) == 0: raise ValueError('Empty text.') tts_audio = synthesize(text, speaker, lang_emb_avg, interpolation) return tts_audio def load_models(): download_hf_models() print("load_models (model_dir=" + MODEL_DIR + ")") def tts_demo(): interface = gr.Interface( fn=tts_demo_fn, inputs=[ gr.Textbox(label='Text'), gr.Radio( SPEAKERS, value=default_spk, label='Speaker' ), gr.Radio( DIALECT_EMBEDDINGS, value=default_lang_emb, label='Standard/Dialect Embedding', ), gr.Dropdown( choices=INTERPOLATION_OPTIONS, value=default_interpolation, label='Interpolation (overrides Standard/Dialect Embedding!))', ), ], outputs=[ gr.Audio(label='Audio'), ], title=TTS_TITLE, description=TTS_DESCRIPTION, article=ARTICLE, ) return interface def demo(): interface = gr.TabbedInterface( interface_list=[tts_demo()], tab_names=['TTS'], ) return interface def main(): """Main entry point of the program.""" parser = argparse.ArgumentParser(description='Sprechen wir.') parser.add_argument( '--queue', action='store_true', help=( 'Schreibens was!' ) ) parser.add_argument( '--threads', type=int, default=40, help='Allow up to `threads` to be processed in parallel.' ) parser.add_argument( '--share', action='store_true', help='Whether to create a publicly shareable link for the interface.' ) parser.add_argument( '--host', help=( 'Host to listen connections into. ' 'Set to "0.0.0.0" to make it accessible in local network.' ) ) parser.add_argument( '--port', help='Start gradio app on this port (if available).' ) parser.add_argument( '--debug', action='store_true', help='If True, blocks the main thread from running.' ) parser.add_argument( '--nolock', action='store_true', help=( 'If True, the interface will block the main thread while the ' 'server is running.' ) ) args = parser.parse_args() load_models() interface = demo() if args.queue: interface = interface.queue() interface.launch( server_name="127.0.0.1", server_port=8080, debug=False, share=False ) if __name__ == '__main__': try: main() except KeyboardInterrupt: print("exit") gr.close_all() exit()