chandrashekar8 commited on
Commit
7b93687
·
1 Parent(s): 4bc32f0

Bundle WhatsApp female speaker embedding

Browse files
WHATSAPP_HF.md CHANGED
@@ -30,5 +30,6 @@ If you need 24/7 availability, use an always-on Hugging Face deployment or equiv
30
  - The planner model is optional and is not auto-downloaded on startup; set `AARA_ENABLE_PLANNER_DOWNLOAD=1` if you want it.
31
  - WhatsApp replies now include a generated audio attachment, so the reply plays as a voice message instead of plain text.
32
  - WhatsApp voice replies use an Indian female SpeechT5 checkpoint from Hugging Face for a clearer local voice.
 
33
  - If you want inbound WhatsApp voice notes to be transcribed, set `TWILIO_ACCOUNT_SID` and `TWILIO_AUTH_TOKEN` so the Space can download Twilio media.
34
  - If you want the bot to greet first, send a first message like `hello` in WhatsApp after joining the sandbox.
 
30
  - The planner model is optional and is not auto-downloaded on startup; set `AARA_ENABLE_PLANNER_DOWNLOAD=1` if you want it.
31
  - WhatsApp replies now include a generated audio attachment, so the reply plays as a voice message instead of plain text.
32
  - WhatsApp voice replies use an Indian female SpeechT5 checkpoint from Hugging Face for a clearer local voice.
33
+ - The female speaker embedding is bundled in `reference_audio/female_speaker_embedding.npy`, so the Space does not need to fetch the xvector archive at runtime.
34
  - If you want inbound WhatsApp voice notes to be transcribed, set `TWILIO_ACCOUNT_SID` and `TWILIO_AUTH_TOKEN` so the Space can download Twilio media.
35
  - If you want the bot to greet first, send a first message like `hello` in WhatsApp after joining the sandbox.
reference_audio/female_speaker_embedding.npy ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5718a29583dfdc17d760f5fb807fdd8ce68c1cf48d6acbec208337474bed178a
3
+ size 2176
voice_agent_standalone.py CHANGED
@@ -1922,6 +1922,7 @@ class TTSEngine:
1922
  SPEECH_T5_TTS_MODEL = "aryamannningombam/speecht5_finetuned_indian_female_v3"
1923
  SPEECH_T5_PROCESSOR_MODEL = "microsoft/speecht5_tts"
1924
  SPEECH_T5_VOCODER_MODEL = "microsoft/speecht5_hifigan"
 
1925
  SPEAKER_ARCHIVE_REPO = "Matthijs/cmu-arctic-xvectors"
1926
  FEMALE_SPEAKER_FILE = "spkrec-xvect/cmu_us_slt_arctic-wav-arctic_a0402.npy"
1927
  MALE_SPEAKER_FILE = "spkrec-xvect/cmu_us_bdl_arctic-wav-arctic_a0153.npy"
@@ -2051,6 +2052,16 @@ class TTSEngine:
2051
  self._speaker_embedding_cache[cache_key] = tensor
2052
  return tensor
2053
 
 
 
 
 
 
 
 
 
 
 
2054
  speaker_file = self.MALE_SPEAKER_FILE if preference == "male" else self.FEMALE_SPEAKER_FILE
2055
  if speaker_file in self._speaker_embedding_cache:
2056
  return self._speaker_embedding_cache[speaker_file]
 
1922
  SPEECH_T5_TTS_MODEL = "aryamannningombam/speecht5_finetuned_indian_female_v3"
1923
  SPEECH_T5_PROCESSOR_MODEL = "microsoft/speecht5_tts"
1924
  SPEECH_T5_VOCODER_MODEL = "microsoft/speecht5_hifigan"
1925
+ FEMALE_SPEAKER_EMBEDDING_FILE = "female_speaker_embedding.npy"
1926
  SPEAKER_ARCHIVE_REPO = "Matthijs/cmu-arctic-xvectors"
1927
  FEMALE_SPEAKER_FILE = "spkrec-xvect/cmu_us_slt_arctic-wav-arctic_a0402.npy"
1928
  MALE_SPEAKER_FILE = "spkrec-xvect/cmu_us_bdl_arctic-wav-arctic_a0153.npy"
 
2052
  self._speaker_embedding_cache[cache_key] = tensor
2053
  return tensor
2054
 
2055
+ local_embedding_path = self.cfg.reference_audio_dir / self.FEMALE_SPEAKER_EMBEDDING_FILE
2056
+ if preference != "male" and local_embedding_path.exists():
2057
+ cache_key = str(local_embedding_path.resolve())
2058
+ if cache_key in self._speaker_embedding_cache:
2059
+ return self._speaker_embedding_cache[cache_key]
2060
+ embedding = np.load(local_embedding_path, allow_pickle=False)
2061
+ tensor = torch.from_numpy(np.asarray(embedding, dtype=np.float32).reshape(1, -1))
2062
+ self._speaker_embedding_cache[cache_key] = tensor
2063
+ return tensor
2064
+
2065
  speaker_file = self.MALE_SPEAKER_FILE if preference == "male" else self.FEMALE_SPEAKER_FILE
2066
  if speaker_file in self._speaker_embedding_cache:
2067
  return self._speaker_embedding_cache[speaker_file]