Spaces:
Sleeping
Sleeping
chandrashekar8 commited on
Commit ·
7b93687
1
Parent(s): 4bc32f0
Bundle WhatsApp female speaker embedding
Browse files- WHATSAPP_HF.md +1 -0
- reference_audio/female_speaker_embedding.npy +3 -0
- voice_agent_standalone.py +11 -0
WHATSAPP_HF.md
CHANGED
|
@@ -30,5 +30,6 @@ If you need 24/7 availability, use an always-on Hugging Face deployment or equiv
|
|
| 30 |
- The planner model is optional and is not auto-downloaded on startup; set `AARA_ENABLE_PLANNER_DOWNLOAD=1` if you want it.
|
| 31 |
- WhatsApp replies now include a generated audio attachment, so the reply plays as a voice message instead of plain text.
|
| 32 |
- WhatsApp voice replies use an Indian female SpeechT5 checkpoint from Hugging Face for a clearer local voice.
|
|
|
|
| 33 |
- If you want inbound WhatsApp voice notes to be transcribed, set `TWILIO_ACCOUNT_SID` and `TWILIO_AUTH_TOKEN` so the Space can download Twilio media.
|
| 34 |
- If you want the bot to greet first, send a first message like `hello` in WhatsApp after joining the sandbox.
|
|
|
|
| 30 |
- The planner model is optional and is not auto-downloaded on startup; set `AARA_ENABLE_PLANNER_DOWNLOAD=1` if you want it.
|
| 31 |
- WhatsApp replies now include a generated audio attachment, so the reply plays as a voice message instead of plain text.
|
| 32 |
- WhatsApp voice replies use an Indian female SpeechT5 checkpoint from Hugging Face for a clearer local voice.
|
| 33 |
+
- The female speaker embedding is bundled in `reference_audio/female_speaker_embedding.npy`, so the Space does not need to fetch the xvector archive at runtime.
|
| 34 |
- If you want inbound WhatsApp voice notes to be transcribed, set `TWILIO_ACCOUNT_SID` and `TWILIO_AUTH_TOKEN` so the Space can download Twilio media.
|
| 35 |
- If you want the bot to greet first, send a first message like `hello` in WhatsApp after joining the sandbox.
|
reference_audio/female_speaker_embedding.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5718a29583dfdc17d760f5fb807fdd8ce68c1cf48d6acbec208337474bed178a
|
| 3 |
+
size 2176
|
voice_agent_standalone.py
CHANGED
|
@@ -1922,6 +1922,7 @@ class TTSEngine:
|
|
| 1922 |
SPEECH_T5_TTS_MODEL = "aryamannningombam/speecht5_finetuned_indian_female_v3"
|
| 1923 |
SPEECH_T5_PROCESSOR_MODEL = "microsoft/speecht5_tts"
|
| 1924 |
SPEECH_T5_VOCODER_MODEL = "microsoft/speecht5_hifigan"
|
|
|
|
| 1925 |
SPEAKER_ARCHIVE_REPO = "Matthijs/cmu-arctic-xvectors"
|
| 1926 |
FEMALE_SPEAKER_FILE = "spkrec-xvect/cmu_us_slt_arctic-wav-arctic_a0402.npy"
|
| 1927 |
MALE_SPEAKER_FILE = "spkrec-xvect/cmu_us_bdl_arctic-wav-arctic_a0153.npy"
|
|
@@ -2051,6 +2052,16 @@ class TTSEngine:
|
|
| 2051 |
self._speaker_embedding_cache[cache_key] = tensor
|
| 2052 |
return tensor
|
| 2053 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2054 |
speaker_file = self.MALE_SPEAKER_FILE if preference == "male" else self.FEMALE_SPEAKER_FILE
|
| 2055 |
if speaker_file in self._speaker_embedding_cache:
|
| 2056 |
return self._speaker_embedding_cache[speaker_file]
|
|
|
|
| 1922 |
SPEECH_T5_TTS_MODEL = "aryamannningombam/speecht5_finetuned_indian_female_v3"
|
| 1923 |
SPEECH_T5_PROCESSOR_MODEL = "microsoft/speecht5_tts"
|
| 1924 |
SPEECH_T5_VOCODER_MODEL = "microsoft/speecht5_hifigan"
|
| 1925 |
+
FEMALE_SPEAKER_EMBEDDING_FILE = "female_speaker_embedding.npy"
|
| 1926 |
SPEAKER_ARCHIVE_REPO = "Matthijs/cmu-arctic-xvectors"
|
| 1927 |
FEMALE_SPEAKER_FILE = "spkrec-xvect/cmu_us_slt_arctic-wav-arctic_a0402.npy"
|
| 1928 |
MALE_SPEAKER_FILE = "spkrec-xvect/cmu_us_bdl_arctic-wav-arctic_a0153.npy"
|
|
|
|
| 2052 |
self._speaker_embedding_cache[cache_key] = tensor
|
| 2053 |
return tensor
|
| 2054 |
|
| 2055 |
+
local_embedding_path = self.cfg.reference_audio_dir / self.FEMALE_SPEAKER_EMBEDDING_FILE
|
| 2056 |
+
if preference != "male" and local_embedding_path.exists():
|
| 2057 |
+
cache_key = str(local_embedding_path.resolve())
|
| 2058 |
+
if cache_key in self._speaker_embedding_cache:
|
| 2059 |
+
return self._speaker_embedding_cache[cache_key]
|
| 2060 |
+
embedding = np.load(local_embedding_path, allow_pickle=False)
|
| 2061 |
+
tensor = torch.from_numpy(np.asarray(embedding, dtype=np.float32).reshape(1, -1))
|
| 2062 |
+
self._speaker_embedding_cache[cache_key] = tensor
|
| 2063 |
+
return tensor
|
| 2064 |
+
|
| 2065 |
speaker_file = self.MALE_SPEAKER_FILE if preference == "male" else self.FEMALE_SPEAKER_FILE
|
| 2066 |
if speaker_file in self._speaker_embedding_cache:
|
| 2067 |
return self._speaker_embedding_cache[speaker_file]
|