// Streaming int8 phoneme engine. See pww_engine.h for the contract. #include "pww_engine.h" #include #include #include #include "model_int8.h" // Debug: when >= 0, pww_engine_step returns layer N's snapped int8 // output column instead of the final logits. int pww_dump_layer = -1; // Ring buffer per layer holding the int8 input history a causal conv // needs: (k-1)*dilation + 1 columns of in_c values. typedef struct { int8_t *buf; // hist * in_c, column-major by time step int hist; // number of columns int pos; // next write slot int primed; // columns written so far (zeros before that) } ring_t; struct pww_engine { ring_t rings[PWW_NUM_LAYERS]; // weights copied out of memory-mapped flash at create() time: flash // cache misses were 20x slower than the arithmetic pww_layer_t layers[PWW_NUM_LAYERS]; // block input snapshot (int8 col + its scale) for residual adds int8_t block_in[512]; float block_in_scale; float scratch_f[512]; int8_t col_a[512], col_b[512]; }; // Weight allocation. On ESP32 the engine is memory-latency bound, not // MAC bound: internal SRAM is several times faster to walk than octal // PSRAM, so fill the internal budget first and spill the rest to PSRAM. #ifdef ESP_PLATFORM #include "esp_heap_caps.h" #ifndef PWW_INTERNAL_WEIGHT_BUDGET #define PWW_INTERNAL_WEIGHT_BUDGET (160 * 1024) #endif static size_t s_internal_used = 0; static void *weights_alloc(size_t n) { if (s_internal_used + n <= PWW_INTERNAL_WEIGHT_BUDGET) { void *p = heap_caps_malloc(n, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); if (p) { s_internal_used += n; return p; } } void *p = heap_caps_malloc(n, MALLOC_CAP_SPIRAM); if (!p) p = malloc(n); return p; } size_t pww_internal_weight_bytes(void) { return s_internal_used; } // engine state (ring buffers, scratch) is touched every frame - never // let it land in PSRAM static void *state_alloc(size_t n) { void *p = heap_caps_calloc(1, n, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); return p ? p : calloc(1, n); } #else static void *weights_alloc(size_t n) { return malloc(n); } static void *state_alloc(size_t n) { return calloc(1, n); } #endif static int8_t quant_clamp(float v, float inv_scale) { float q = roundf(v * inv_scale); if (q > 127.f) q = 127.f; if (q < -127.f) q = -127.f; return (int8_t)q; } pww_engine_t *pww_engine_create(void) { pww_engine_t *e = (pww_engine_t *)state_alloc(sizeof(pww_engine_t)); if (!e) return NULL; for (int li = 0; li < PWW_NUM_LAYERS; li++) { e->layers[li] = PWW_LAYERS[li]; pww_layer_t *M = &e->layers[li]; size_t wn = (size_t)(M->is_dw ? M->out_c : M->in_c * M->out_c) * M->k; int8_t *wcopy = (int8_t *)weights_alloc(wn); float *scopy = (float *)malloc(sizeof(float) * M->out_c); float *bcopy = (float *)malloc(sizeof(float) * M->out_c); if (!wcopy || !scopy || !bcopy) { pww_engine_destroy(e); return NULL; } memcpy(wcopy, M->w, wn); memcpy(scopy, M->s, sizeof(float) * M->out_c); memcpy(bcopy, M->b, sizeof(float) * M->out_c); M->w = wcopy; M->s = scopy; M->b = bcopy; const pww_layer_t *L = &PWW_LAYERS[li]; int hist = (L->k - 1) * L->dilation + 1; // stride-2 stem consumes 2 input columns per step if (L->stride == 2) hist += 1; ring_t *r = &e->rings[li]; r->hist = hist; r->buf = (int8_t *)state_alloc((size_t)hist * L->in_c); if (!r->buf) { pww_engine_destroy(e); return NULL; } } pww_engine_reset(e); return e; } void pww_engine_destroy(pww_engine_t *e) { if (!e) return; for (int li = 0; li < PWW_NUM_LAYERS; li++) { free(e->rings[li].buf); if (e->layers[li].w && e->layers[li].w != PWW_LAYERS[li].w) { free((void *)e->layers[li].w); free((void *)e->layers[li].s); free((void *)e->layers[li].b); } } free(e); } void pww_engine_reset(pww_engine_t *e) { for (int li = 0; li < PWW_NUM_LAYERS; li++) { ring_t *r = &e->rings[li]; memset(r->buf, 0, (size_t)r->hist * PWW_LAYERS[li].in_c); r->pos = 0; r->primed = 0; } } static void ring_push(ring_t *r, const int8_t *col, int in_c) { memcpy(r->buf + (size_t)r->pos * in_c, col, (size_t)in_c); r->pos = (r->pos + 1) % r->hist; if (r->primed < r->hist) r->primed++; } // column at "delay" steps in the past (0 = newest) static const int8_t *ring_at(const ring_t *r, int delay, int in_c) { int idx = r->pos - 1 - delay; while (idx < 0) idx += r->hist; return r->buf + (size_t)idx * in_c; } // Runs one layer on the newest ring content, writing float pre-snap // output to out_f (out_c values). For stride-2 layers the newest TWO // columns have been pushed before calling. static void layer_forward(const pww_layer_t *L, const ring_t *r, float *out_f) { int k = L->k, d = L->dilation, in_c = L->in_c, out_c = L->out_c; // stride-2 layers get two pushes per step but output frame t only // consumes up to input column 2t: the newest tap sits one column back int off = (L->stride == 2) ? 1 : 0; if (L->is_dw) { for (int c = 0; c < out_c; c++) out_f[c] = 0.f; for (int i = 0; i < k; i++) { // tap i is the newest at i == k-1 const int8_t *col = ring_at(r, (k - 1 - i) * d + off, in_c); const int8_t *w = L->w + i; // w layout: (ch, 1, k) for (int c = 0; c < out_c; c++) out_f[c] += (float)((int32_t)col[c] * (int32_t)w[c * k]); } for (int c = 0; c < out_c; c++) out_f[c] = out_f[c] * L->s[c] + L->b[c]; } else if (k == 1) { // 1x1 conv = matrix-vector; 85% of all MACs live here. // Contiguous rows, 4-way unroll, single ring lookup. const int8_t *restrict col = ring_at(r, off, in_c); for (int c = 0; c < out_c; c++) { const int8_t *restrict w = L->w + (size_t)c * in_c; int32_t a0 = 0, a1 = 0, a2 = 0, a3 = 0; int j = 0; for (; j + 4 <= in_c; j += 4) { a0 += (int32_t)col[j] * (int32_t)w[j]; a1 += (int32_t)col[j + 1] * (int32_t)w[j + 1]; a2 += (int32_t)col[j + 2] * (int32_t)w[j + 2]; a3 += (int32_t)col[j + 3] * (int32_t)w[j + 3]; } int32_t acc = a0 + a1 + a2 + a3; for (; j < in_c; j++) acc += (int32_t)col[j] * (int32_t)w[j]; out_f[c] = (float)acc * L->s[c] + L->b[c]; } } else { // general conv: hoist the k column pointers out of the c loop const int8_t *cols[8]; for (int i = 0; i < k; i++) cols[i] = ring_at(r, (k - 1 - i) * d + off, in_c); for (int c = 0; c < out_c; c++) { const int8_t *restrict w = L->w + (size_t)c * in_c * k; int32_t acc = 0; for (int i = 0; i < k; i++) { const int8_t *restrict col = cols[i]; const int8_t *restrict wk = w + i; for (int j = 0; j < in_c; j++) acc += (int32_t)col[j] * (int32_t)wk[(size_t)j * k]; } out_f[c] = (float)acc * L->s[c] + L->b[c]; } } } int pww_engine_step(pww_engine_t *e, const float *mel0, const float *mel1, float *logits_out) { // quantize the two input mel columns to the input scale float inv_in = 1.0f / PWW_INPUT_SCALE; for (int j = 0; j < PWW_MELS; j++) e->col_a[j] = quant_clamp(mel0[j], inv_in); ring_push(&e->rings[0], e->col_a, PWW_MELS); for (int j = 0; j < PWW_MELS; j++) e->col_a[j] = quant_clamp(mel1[j], inv_in); ring_push(&e->rings[0], e->col_a, PWW_MELS); int8_t *cur = e->col_a; // int8 column flowing between layers float cur_scale = PWW_INPUT_SCALE; (void)cur_scale; for (int li = 0; li < PWW_NUM_LAYERS; li++) { const pww_layer_t *L = &e->layers[li]; ring_t *r = &e->rings[li]; if (li > 0) ring_push(r, cur, L->in_c); if (L->is_dw) { // save the block input column + scale for the residual 2 // layers later (dw -> pw(residual)) memcpy(e->block_in, ring_at(r, 0, L->in_c), (size_t)L->in_c); e->block_in_scale = (li == 0) ? PWW_INPUT_SCALE : e->layers[li - 1].out_scale; } layer_forward(L, r, e->scratch_f); if (L->residual) { for (int c = 0; c < L->out_c; c++) e->scratch_f[c] += (float)e->block_in[c] * e->block_in_scale; } if (L->relu) { for (int c = 0; c < L->out_c; c++) if (e->scratch_f[c] < 0.f) e->scratch_f[c] = 0.f; } if (li == PWW_NUM_LAYERS - 1) { // final logits: snap to grid to mirror the simulator, then // return as float float s = L->out_scale; for (int c = 0; c < L->out_c; c++) { int8_t q = quant_clamp(e->scratch_f[c], 1.0f / s); logits_out[c] = (float)q * s; } return 0; } float inv = 1.0f / L->out_scale; int8_t *nxt = (cur == e->col_a) ? e->col_b : e->col_a; for (int c = 0; c < L->out_c; c++) nxt[c] = quant_clamp(e->scratch_f[c], inv); if (pww_dump_layer == li) { for (int c = 0; c < L->out_c && c < 256; c++) logits_out[c] = (float)nxt[c]; return 0; } cur = nxt; } return -1; // unreachable }