Desmond-Dong commited on
Commit
5f199de
·
1 Parent(s): 055a928

Refactor app.py to match official conversation app pattern

Browse files
Files changed (1) hide show
  1. reachy_mini_ha_voice/app.py +239 -218
reachy_mini_ha_voice/app.py CHANGED
@@ -1,11 +1,12 @@
1
  """Reachy Mini Home Assistant Voice Assistant."""
2
 
3
- import time
 
4
  import threading
 
5
  from pathlib import Path
6
  from queue import Queue
7
  from typing import Dict, List, Optional, Set, Union
8
- from pydantic import BaseModel
9
 
10
  import numpy as np
11
  from pymicro_wakeword import MicroWakeWord, MicroWakeWordFeatures
@@ -24,7 +25,7 @@ from .satellite import VoiceSatelliteProtocol
24
  from .util import get_mac
25
  from .zeroconf import HomeAssistantZeroconf
26
 
27
- _LOGGER = __import__('logging').getLogger(__name__)
28
  _MODULE_DIR = Path(__file__).parent
29
  _REPO_DIR = _MODULE_DIR.parent
30
  _WAKEWORDS_DIR = _REPO_DIR / "wakewords"
@@ -32,243 +33,263 @@ _SOUNDS_DIR = _REPO_DIR / "sounds"
32
 
33
 
34
  class ReachyMiniHAVoiceApp(ReachyMiniApp):
35
- """Home Assistant Voice Assistant for Reachy Mini."""
36
-
37
- custom_app_url: str = "http://0.0.0.0:8042"
38
-
39
- def run(self, reachy_mini: ReachyMini, stop_event: threading.Event):
40
- """Run the voice assistant."""
41
- _LOGGER.info("Starting Reachy Mini Home Assistant Voice Assistant")
42
 
43
- # Initialize server state
44
- state = self._init_state(reachy_mini)
45
 
46
- # Start audio processing thread
47
- audio_thread = threading.Thread(
48
- target=self._process_audio,
49
- args=(state,),
50
- daemon=True,
 
 
 
 
 
51
  )
52
- audio_thread.start()
53
 
54
- # Start ESPHome server in background thread
55
- server_thread = threading.Thread(
56
- target=self._run_server,
57
- args=(state, stop_event),
58
- daemon=True,
59
- )
60
- server_thread.start()
61
 
62
- # Main loop - wait for stop event
63
- while not stop_event.is_set():
64
- time.sleep(0.1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
 
66
- _LOGGER.info("Shutting down voice assistant")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
 
68
- def _init_state(self, reachy_mini: ReachyMini) -> ServerState:
69
- """Initialize server state."""
70
- # Load wake words
71
- available_wake_words = self._load_wake_words()
72
 
73
- # Load active wake words
74
- active_wake_words = set()
75
- wake_models: Dict[str, Union[MicroWakeWord, OpenWakeWord]] = {}
76
 
77
- # Use default wake word
78
- default_wake_word = "okay_nabu"
79
- if default_wake_word in available_wake_words:
80
- try:
81
- wake_word = available_wake_words[default_wake_word]
82
- wake_models[default_wake_word] = wake_word.load()
83
- active_wake_words.add(default_wake_word)
84
- _LOGGER.info("Loaded wake word: %s", default_wake_word)
85
- except Exception as e:
86
- _LOGGER.error("Failed to load wake word %s: %s", default_wake_word, e)
87
-
88
- # Load stop model
89
- stop_model = self._load_stop_model()
90
-
91
- return ServerState(
92
- name="ReachyMini",
93
- mac_address=get_mac(),
94
- audio_queue=Queue(),
95
- entities=[],
96
- available_wake_words=available_wake_words,
97
- wake_words=wake_models,
98
- active_wake_words=active_wake_words,
99
- stop_word=stop_model,
100
- music_player=ReachyMiniAudioPlayer(reachy_mini),
101
- tts_player=ReachyMiniAudioPlayer(reachy_mini),
102
- wakeup_sound=str(_SOUNDS_DIR / "wake_word_triggered.flac"),
103
- timer_finished_sound=str(_SOUNDS_DIR / "timer_finished.flac"),
104
- preferences=Preferences(),
105
- preferences_path=_REPO_DIR / "preferences.json",
106
- refractory_seconds=2.0,
107
- download_dir=_REPO_DIR / "local",
108
- reachy_integration=None,
109
- media_player_entity=None,
110
- )
111
 
112
- def _load_wake_words(self) -> Dict[str, AvailableWakeWord]:
113
- """Load available wake words."""
114
- available_wake_words: Dict[str, AvailableWakeWord] = {}
 
 
115
 
116
- for wake_word_dir in [_WAKEWORDS_DIR]:
117
- if not wake_word_dir.exists():
118
- continue
119
 
120
- for model_config_path in wake_word_dir.glob("*.json"):
121
- model_id = model_config_path.stem
122
- if model_id == "stop":
123
- continue
124
-
125
- try:
126
- import json
127
-
128
- with open(model_config_path, "r", encoding="utf-8") as f:
129
- model_config = json.load(f)
130
- model_type = WakeWordType(model_config.get("type", "microWakeWord"))
131
- if model_type == WakeWordType.OPEN_WAKE_WORD:
132
- wake_word_path = model_config_path.parent / model_config["model"]
133
- else:
134
- wake_word_path = model_config_path
135
-
136
- available_wake_words[model_id] = AvailableWakeWord(
137
- id=model_id,
138
- type=model_type,
139
- wake_word=model_config["wake_word"],
140
- trained_languages=model_config.get("trained_languages", []),
141
- wake_word_path=wake_word_path,
142
- )
143
- except Exception as e:
144
- _LOGGER.error("Error loading wake word config %s: %s", model_config_path, e)
145
 
146
- return available_wake_words
 
 
147
 
148
- def _load_stop_model(self) -> Optional[MicroWakeWord]:
149
- """Load stop word model."""
150
- stop_config_path = _WAKEWORDS_DIR / "stop.json"
151
- if not stop_config_path.exists():
152
- return None
153
 
154
  try:
155
- return MicroWakeWord.from_config(stop_config_path)
156
- except Exception as e:
157
- _LOGGER.error("Failed to load stop model: %s", e)
158
- return None
 
 
 
 
 
159
 
160
- def _run_server(self, state: ServerState, stop_event: threading.Event):
161
- """Run ESPHome server in a separate thread."""
162
- import asyncio
163
 
164
- async def server_loop():
165
- loop = asyncio.new_event_loop()
166
- asyncio.set_event_loop(loop)
167
 
168
- server = await loop.create_server(
169
- lambda: VoiceSatelliteProtocol(state), host="0.0.0.0", port=6053
170
- )
 
 
 
171
 
172
- # Auto discovery (zeroconf, mDNS)
173
- discovery = HomeAssistantZeroconf(port=6053, name="ReachyMini")
174
- await discovery.register_server()
175
 
176
- try:
177
- async with server:
178
- _LOGGER.info("ESPHome server started on port 6053")
179
- _LOGGER.info("mDNS service registered for auto-discovery")
180
-
181
- while not stop_event.is_set():
182
- await asyncio.sleep(0.1)
183
- finally:
184
- await discovery.unregister_server()
185
- _LOGGER.info("ESPHome server stopped")
186
-
187
- asyncio.run(server_loop())
188
-
189
- def _process_audio(self, state: ServerState):
190
- """Process audio from microphone."""
191
- # Start media
192
- state.music_player._robot.media.start_recording()
193
- state.music_player._robot.media.start_playing()
194
- time.sleep(1)
195
-
196
- wake_words: List[Union[MicroWakeWord, OpenWakeWord]] = []
197
- micro_features: Optional[MicroWakeWordFeatures] = None
198
- micro_inputs: List[np.ndarray] = []
199
-
200
- oww_features: Optional[OpenWakeWordFeatures] = None
201
- oww_inputs: List[np.ndarray] = []
202
- has_oww = False
203
-
204
- last_active: Optional[float] = None
205
-
206
- _LOGGER.info("Audio processing started")
207
-
208
- while True:
209
- # Get audio sample from Reachy Mini
210
- audio_frame = state.music_player._robot.media.get_audio_sample()
211
-
212
- if audio_frame is not None:
213
- # Send to satellite if connected
214
- if state.satellite is not None:
215
- # Convert to bytes for satellite
216
- audio_bytes = (audio_frame * 32767.0).astype(np.int16).tobytes()
217
- state.satellite.handle_audio(audio_bytes)
218
-
219
- # Update wake word models
220
- if (not wake_words) or (state.wake_words_changed and state.wake_words):
221
- state.wake_words_changed = False
222
- wake_words = [
223
- ww
224
- for ww in state.wake_words.values()
225
- if ww.id in state.active_wake_words
226
- ]
227
-
228
- has_oww = False
229
- for wake_word in wake_words:
230
- if isinstance(wake_word, OpenWakeWord):
231
- has_oww = True
232
-
233
- if micro_features is None:
234
- micro_features = MicroWakeWordFeatures()
235
-
236
- if has_oww and (oww_features is None):
237
- oww_features = OpenWakeWordFeatures.from_builtin()
238
-
239
- # Process wake words
240
- if wake_words:
241
- assert micro_features is not None
242
- micro_inputs.clear()
243
- # Convert float32 audio to int16 for microWakeWord
244
- audio_int16 = (audio_frame * 32767.0).astype(np.int16)
245
- micro_inputs.extend(micro_features.process_streaming(audio_int16))
246
-
247
- if has_oww:
248
- assert oww_features is not None
249
- oww_inputs.clear()
250
- oww_inputs.extend(oww_features.process_streaming(audio_frame))
251
-
252
- for wake_word in wake_words:
253
- activated = False
254
- if isinstance(wake_word, MicroWakeWord):
255
- for micro_input in micro_inputs:
256
- if wake_word.process_streaming(micro_input):
257
  activated = True
258
- elif isinstance(wake_word, OpenWakeWord):
259
- for oww_input in oww_inputs:
260
- for prob in wake_word.process_streaming(oww_input):
261
- if prob > 0.5:
262
- activated = True
263
-
264
- if activated:
265
- now = time.monotonic()
266
- if (last_active is None) or (
267
- (now - last_active) > state.refractory_seconds
268
- ):
269
- if state.satellite:
270
- state.satellite.wakeup(wake_word)
271
- last_active = now
272
 
273
  # Process stop word
274
  if state.stop_word is not None:
 
1
  """Reachy Mini Home Assistant Voice Assistant."""
2
 
3
+ import asyncio
4
+ import logging
5
  import threading
6
+ import time
7
  from pathlib import Path
8
  from queue import Queue
9
  from typing import Dict, List, Optional, Set, Union
 
10
 
11
  import numpy as np
12
  from pymicro_wakeword import MicroWakeWord, MicroWakeWordFeatures
 
25
  from .util import get_mac
26
  from .zeroconf import HomeAssistantZeroconf
27
 
28
+ _LOGGER = logging.getLogger(__name__)
29
  _MODULE_DIR = Path(__file__).parent
30
  _REPO_DIR = _MODULE_DIR.parent
31
  _WAKEWORDS_DIR = _REPO_DIR / "wakewords"
 
33
 
34
 
35
  class ReachyMiniHAVoiceApp(ReachyMiniApp):
36
+ """Reachy Mini Apps entry point for the voice assistant app."""
 
 
 
 
 
 
37
 
38
+ custom_app_url = "http://0.0.0.0:7860/"
 
39
 
40
+ def run(self, reachy_mini: ReachyMini, stop_event: threading.Event) -> None:
41
+ """Run the Reachy Mini voice assistant app."""
42
+ loop = asyncio.new_event_loop()
43
+ asyncio.set_event_loop(loop)
44
+
45
+ _run(
46
+ robot=reachy_mini,
47
+ app_stop_event=stop_event,
48
+ settings_app=self.settings_app,
49
+ instance_path=Path.cwd(),
50
  )
 
51
 
 
 
 
 
 
 
 
52
 
53
+ def _run(
54
+ robot: ReachyMini,
55
+ app_stop_event: threading.Event,
56
+ settings_app=None,
57
+ instance_path: Optional[Path] = None,
58
+ ) -> None:
59
+ """Run the voice assistant."""
60
+ _LOGGER.info("Starting Reachy Mini Home Assistant Voice Assistant")
61
+
62
+ # Initialize server state
63
+ state = _init_state(robot)
64
+
65
+ # Start audio processing thread
66
+ audio_thread = threading.Thread(
67
+ target=_process_audio,
68
+ args=(state,),
69
+ daemon=True,
70
+ )
71
+ audio_thread.start()
72
+
73
+ # Start ESPHome server in background thread
74
+ server_thread = threading.Thread(
75
+ target=_run_server,
76
+ args=(state, app_stop_event),
77
+ daemon=True,
78
+ )
79
+ server_thread.start()
80
+
81
+ # Main loop - wait for stop event
82
+ while not app_stop_event.is_set():
83
+ time.sleep(0.1)
84
+
85
+ _LOGGER.info("Shutting down voice assistant")
86
+
87
+
88
+ def _init_state(robot: ReachyMini) -> ServerState:
89
+ """Initialize server state."""
90
+ # Load wake words
91
+ available_wake_words = _load_wake_words()
92
+
93
+ # Load active wake words
94
+ active_wake_words = set()
95
+ wake_models: Dict[str, Union[MicroWakeWord, OpenWakeWord]] = {}
96
+
97
+ # Use default wake word
98
+ default_wake_word = "okay_nabu"
99
+ if default_wake_word in available_wake_words:
100
+ try:
101
+ wake_word = available_wake_words[default_wake_word]
102
+ wake_models[default_wake_word] = wake_word.load()
103
+ active_wake_words.add(default_wake_word)
104
+ _LOGGER.info("Loaded wake word: %s", default_wake_word)
105
+ except Exception as e:
106
+ _LOGGER.error("Failed to load wake word %s: %s", default_wake_word, e)
107
+
108
+ # Load stop model
109
+ stop_model = _load_stop_model()
110
+
111
+ return ServerState(
112
+ name="ReachyMini",
113
+ mac_address=get_mac(),
114
+ audio_queue=Queue(),
115
+ entities=[],
116
+ available_wake_words=available_wake_words,
117
+ wake_words=wake_models,
118
+ active_wake_words=active_wake_words,
119
+ stop_word=stop_model,
120
+ music_player=ReachyMiniAudioPlayer(robot),
121
+ tts_player=ReachyMiniAudioPlayer(robot),
122
+ wakeup_sound=str(_SOUNDS_DIR / "wake_word_triggered.flac"),
123
+ timer_finished_sound=str(_SOUNDS_DIR / "timer_finished.flac"),
124
+ preferences=Preferences(),
125
+ preferences_path=_REPO_DIR / "preferences.json",
126
+ refractory_seconds=2.0,
127
+ download_dir=_REPO_DIR / "local",
128
+ reachy_integration=None,
129
+ media_player_entity=None,
130
+ )
131
+
132
+
133
+ def _load_wake_words() -> Dict[str, AvailableWakeWord]:
134
+ """Load available wake words."""
135
+ available_wake_words: Dict[str, AvailableWakeWord] = {}
136
+
137
+ for wake_word_dir in [_WAKEWORDS_DIR]:
138
+ if not wake_word_dir.exists():
139
+ continue
140
+
141
+ for model_config_path in wake_word_dir.glob("*.json"):
142
+ model_id = model_config_path.stem
143
+ if model_id == "stop":
144
+ continue
145
 
146
+ try:
147
+ import json
148
+
149
+ with open(model_config_path, "r", encoding="utf-8") as f:
150
+ model_config = json.load(f)
151
+ model_type = WakeWordType(model_config.get("type", "microWakeWord"))
152
+ if model_type == WakeWordType.OPEN_WAKE_WORD:
153
+ wake_word_path = model_config_path.parent / model_config["model"]
154
+ else:
155
+ wake_word_path = model_config_path
156
+
157
+ available_wake_words[model_id] = AvailableWakeWord(
158
+ id=model_id,
159
+ type=model_type,
160
+ wake_word=model_config["wake_word"],
161
+ trained_languages=model_config.get("trained_languages", []),
162
+ wake_word_path=wake_word_path,
163
+ )
164
+ except Exception as e:
165
+ _LOGGER.error("Error loading wake word config %s: %s", model_config_path, e)
166
 
167
+ return available_wake_words
 
 
 
168
 
 
 
 
169
 
170
+ def _load_stop_model() -> Optional[MicroWakeWord]:
171
+ """Load stop word model."""
172
+ stop_config_path = _WAKEWORDS_DIR / "stop.json"
173
+ if not stop_config_path.exists():
174
+ return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
 
176
+ try:
177
+ return MicroWakeWord.from_config(stop_config_path)
178
+ except Exception as e:
179
+ _LOGGER.error("Failed to load stop model: %s", e)
180
+ return None
181
 
 
 
 
182
 
183
+ def _run_server(state: ServerState, stop_event: threading.Event):
184
+ """Run ESPHome server in a separate thread."""
185
+ async def server_loop():
186
+ loop = asyncio.get_running_loop()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
187
 
188
+ server = await loop.create_server(
189
+ lambda: VoiceSatelliteProtocol(state), host="0.0.0.0", port=6053
190
+ )
191
 
192
+ # Auto discovery (zeroconf, mDNS)
193
+ discovery = HomeAssistantZeroconf(port=6053, name="ReachyMini")
194
+ await discovery.register_server()
 
 
195
 
196
  try:
197
+ async with server:
198
+ _LOGGER.info("ESPHome server started on port 6053")
199
+ _LOGGER.info("mDNS service registered for auto-discovery")
200
+
201
+ while not stop_event.is_set():
202
+ await asyncio.sleep(0.1)
203
+ finally:
204
+ await discovery.unregister_server()
205
+ _LOGGER.info("ESPHome server stopped")
206
 
207
+ asyncio.run(server_loop())
 
 
208
 
 
 
 
209
 
210
+ def _process_audio(state: ServerState):
211
+ """Process audio from microphone."""
212
+ # Start media
213
+ state.music_player._robot.media.start_recording()
214
+ state.music_player._robot.media.start_playing()
215
+ time.sleep(1)
216
 
217
+ wake_words: List[Union[MicroWakeWord, OpenWakeWord]] = []
218
+ micro_features: Optional[MicroWakeWordFeatures] = None
219
+ micro_inputs: List[np.ndarray] = []
220
 
221
+ oww_features: Optional[OpenWakeWordFeatures] = None
222
+ oww_inputs: List[np.ndarray] = []
223
+ has_oww = False
224
+
225
+ last_active: Optional[float] = None
226
+
227
+ _LOGGER.info("Audio processing started")
228
+
229
+ while True:
230
+ # Get audio sample from Reachy Mini
231
+ audio_frame = state.music_player._robot.media.get_audio_sample()
232
+
233
+ if audio_frame is not None:
234
+ # Send to satellite if connected
235
+ if state.satellite is not None:
236
+ # Convert to bytes for satellite
237
+ audio_bytes = (audio_frame * 32767.0).astype(np.int16).tobytes()
238
+ state.satellite.handle_audio(audio_bytes)
239
+
240
+ # Update wake word models
241
+ if (not wake_words) or (state.wake_words_changed and state.wake_words):
242
+ state.wake_words_changed = False
243
+ wake_words = [
244
+ ww
245
+ for ww in state.wake_words.values()
246
+ if ww.id in state.active_wake_words
247
+ ]
248
+
249
+ has_oww = False
250
+ for wake_word in wake_words:
251
+ if isinstance(wake_word, OpenWakeWord):
252
+ has_oww = True
253
+
254
+ if micro_features is None:
255
+ micro_features = MicroWakeWordFeatures()
256
+
257
+ if has_oww and (oww_features is None):
258
+ oww_features = OpenWakeWordFeatures.from_builtin()
259
+
260
+ # Process wake words
261
+ if wake_words:
262
+ assert micro_features is not None
263
+ micro_inputs.clear()
264
+ # Convert float32 audio to int16 for microWakeWord
265
+ audio_int16 = (audio_frame * 32767.0).astype(np.int16)
266
+ micro_inputs.extend(micro_features.process_streaming(audio_int16))
267
+
268
+ if has_oww:
269
+ assert oww_features is not None
270
+ oww_inputs.clear()
271
+ oww_inputs.extend(oww_features.process_streaming(audio_frame))
272
+
273
+ for wake_word in wake_words:
274
+ activated = False
275
+ if isinstance(wake_word, MicroWakeWord):
276
+ for micro_input in micro_inputs:
277
+ if wake_word.process_streaming(micro_input):
278
+ activated = True
279
+ elif isinstance(wake_word, OpenWakeWord):
280
+ for oww_input in oww_inputs:
281
+ for prob in wake_word.process_streaming(oww_input):
282
+ if prob > 0.5:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
283
  activated = True
284
+
285
+ if activated:
286
+ now = time.monotonic()
287
+ if (last_active is None) or (
288
+ (now - last_active) > state.refractory_seconds
289
+ ):
290
+ if state.satellite:
291
+ state.satellite.wakeup(wake_word)
292
+ last_active = now
 
 
 
 
 
293
 
294
  # Process stop word
295
  if state.stop_word is not None: