update model

app-tts.js

@@ -1,6 +1,11 @@
 const generateBtn = document.getElementById('generateBtn');
 const speakerIdLabel = document.getElementById('speakerIdLabel');
 const speakerIdInput = document.getElementById('speakerId');
+const speakerIdSection = document.getElementById('speakerIdSection');
+const referenceAudioSection = document.getElementById('referenceAudioSection');
+const referenceTextSection = document.getElementById('referenceTextSection');
+const referenceAudioInput = document.getElementById('referenceAudio');
+const referenceTextInput = document.getElementById('referenceText');
 const speedInput = document.getElementById('speed');
 const speedValue = document.getElementById('speedValue');
 const textArea = document.getElementById('text');
@@ -13,6 +18,7 @@ let index = 0;
 let audioCtx = null;
 const worker = new Worker("/sherpa-onnx-tts.worker.js");
 let ttsInstanceInfo = {
+  modelType: 0,
   numSpeakers: 0,
   isReady: false,
 };
@@ -21,10 +27,12 @@ worker.onmessage = (e) => {
     Module.setStatus(e.data.status);
   }
   if (e.data.type === "sherpa-onnx-tts-ready") {
+    ttsInstanceInfo.modelType = e.data.modelType ?? 0;
     ttsInstanceInfo.numSpeakers = e.data.numSpeakers;
     ttsInstanceInfo.isReady = true;
     generateBtn.disabled = false;
     speakerIdLabel.innerHTML = `Speaker ID (0 - ${e.data.numSpeakers - 1}):`;
+    updateUiForModelType();
     return;
   }
   if (e.data.type === "sherpa-onnx-tts-result") {
@@ -92,23 +100,68 @@ speedInput.oninput = function() {
   speedValue.innerHTML = this.value;
 };
 
-generateBtn.onclick = function() {
-  let speakerId = speakerIdInput.value;
-  if (speakerId.trim().length == 0) {
-    alert('Please input a speakerId');
-    return;
+function updateUiForModelType() {
+  const isZipVoice = ttsInstanceInfo.modelType === 4;
+  speakerIdSection.classList.toggle('hidden', isZipVoice);
+  referenceAudioSection.classList.toggle('hidden', !isZipVoice);
+  referenceTextSection.classList.toggle('hidden', !isZipVoice);
+}
+
+function getMonoSamples(audioBuffer) {
+  if (audioBuffer.numberOfChannels === 1) {
+    return new Float32Array(audioBuffer.getChannelData(0));
   }
 
-  if (!speakerId.match(/^\d+$/)) {
-    alert(`Input speakerID ${
-        speakerId} is not a number.\nPlease enter a number between 0 and ${
-        ttsInstanceInfo.numSpeakers - 1}`);
-    return;
+  const samples = new Float32Array(audioBuffer.length);
+  for (let c = 0; c < audioBuffer.numberOfChannels; ++c) {
+    const channel = audioBuffer.getChannelData(c);
+    for (let i = 0; i < channel.length; ++i) {
+      samples[i] += channel[i];
+    }
   }
-  speakerId = parseInt(speakerId, 10);
-  if (speakerId > ttsInstanceInfo.numSpeakers - 1) {
-    alert(`Pleaser enter a number between 0 and ${ttsInstanceInfo.numSpeakers - 1}`);
-    return;
+
+  for (let i = 0; i < samples.length; ++i) {
+    samples[i] /= audioBuffer.numberOfChannels;
+  }
+
+  return samples;
+}
+
+async function readReferenceAudio(file) {
+  const arrayBuffer = await file.arrayBuffer();
+  const ctx = new AudioContext();
+  try {
+    const audioBuffer = await ctx.decodeAudioData(arrayBuffer.slice(0));
+    return {
+      samples: getMonoSamples(audioBuffer),
+      sampleRate: audioBuffer.sampleRate,
+    };
+  } finally {
+    await ctx.close();
+  }
+}
+
+generateBtn.onclick = async function() {
+  const isZipVoice = ttsInstanceInfo.modelType === 4;
+
+  let speakerId = speakerIdInput.value;
+  if (!isZipVoice) {
+    if (speakerId.trim().length == 0) {
+      alert('Please input a speakerId');
+      return;
+    }
+
+    if (!speakerId.match(/^\d+$/)) {
+      alert(`Input speakerID ${
+          speakerId} is not a number.\nPlease enter a number between 0 and ${
+          ttsInstanceInfo.numSpeakers - 1}`);
+      return;
+    }
+    speakerId = parseInt(speakerId, 10);
+    if (speakerId > ttsInstanceInfo.numSpeakers - 1) {
+      alert(`Pleaser enter a number between 0 and ${ttsInstanceInfo.numSpeakers - 1}`);
+      return;
+    }
   }
 
   let text = textArea.value.trim();
@@ -120,10 +173,43 @@ generateBtn.onclick = function() {
   console.log('speakerId', speakerId);
   console.log('speed', speedInput.value);
   console.log('text', text);
+
+  if (isZipVoice) {
+    if (!referenceAudioInput.files || referenceAudioInput.files.length === 0) {
+      alert('Please select a reference audio file');
+      return;
+    }
+
+    const referenceText = referenceTextInput.value.trim();
+    if (referenceText.length === 0) {
+      alert('Please input the reference text');
+      return;
+    }
+
+    const referenceAudio = await readReferenceAudio(referenceAudioInput.files[0]);
+    const genConfig = {
+      speed: parseFloat(speedInput.value),
+      referenceAudio: referenceAudio.samples,
+      referenceSampleRate: referenceAudio.sampleRate,
+      referenceText: referenceText,
+      numSteps: 4,
+      extra: {
+        min_char_in_sentence: 30,
+      },
+    };
+
+    worker.postMessage({
+      text,
+      genConfig,
+      type: "generateWithConfig",
+    }, [genConfig.referenceAudio.buffer]);
+    return;
+  }
+
   worker.postMessage({
     text,
     sid: speakerId,
-    speed: speedInput.value,
+    speed: parseFloat(speedInput.value),
     type: "generate",
   });
 };

index.html

@@ -14,6 +14,9 @@
     .loading {
       display: none !important;
     }
+    .hidden {
+      display: none !important;
+    }
   </style>
 </head>
 
@@ -27,10 +30,25 @@
     <div id="status">Loading...</div>
 
     <div id="singleAudioContent" class="tab-content loading">
-      <label for="speakerId" id="speakerIdLabel">Speaker ID: </label>
-      <input type="text" id="speakerId" name="speakerId" value="0" />
-      <br/>
-      <br/>
+      <div id="speakerIdSection">
+        <label for="speakerId" id="speakerIdLabel">Speaker ID: </label>
+        <input type="text" id="speakerId" name="speakerId" value="0" />
+        <br/>
+        <br/>
+      </div>
+      <div id="referenceAudioSection" class="hidden">
+        <label for="referenceAudio">Reference audio: </label>
+        <input type="file" id="referenceAudio" name="referenceAudio" accept="audio/*" />
+        <br/>
+        <br/>
+      </div>
+      <div id="referenceTextSection" class="hidden">
+        <label for="referenceText">Reference text: </label>
+        <br/>
+        <textarea id="referenceText" rows="3" placeholder="Please enter the transcription of the reference audio"></textarea>
+        <br/>
+        <br/>
+      </div>
       <label for="speed" id="speedLabel">Speed: </label>
       <input type="range" id="speed" name="speed" min="0.4" max="3.5" step="0.1" value="1.0" />
       <span id="speedValue"></span>

sherpa-onnx-tts.js

@@ -916,6 +916,12 @@ class OfflineTts {
   }
 }
 
+let modelType = 0;
+
+function getDefaultOfflineTtsModelType() {
+  return modelType;
+}
+
 function createOfflineTts(Module, myConfig) {
   const vits = {
     model: '',
@@ -955,10 +961,22 @@ function createOfflineTts(Module, myConfig) {
     lengthScale: 1.0,
   };
 
+  const offlineTtsZipVoiceModelConfig = {
+    tokens: '',
+    encoder: '',
+    decoder: '',
+    vocoder: '',
+    dataDir: '',
+    lexicon: '',
+    featScale: 0.1,
+    tShift: 0.5,
+    targetRMS: 0.1,
+    guidanceScale: 1.0,
+  };
+
   let ruleFsts = '';
 
-  let type = 0;
-  switch (type) {
+  switch (modelType) {
     case 0:
       // vits
       vits.model = './model.onnx';
@@ -992,6 +1010,16 @@ function createOfflineTts(Module, myConfig) {
       matcha.tokens = './tokens.txt';
       matcha.dataDir = './espeak-ng-data';
       break;
+    case 4:
+      // zipvoice zh-en
+      // https://k2-fsa.github.io/sherpa/onnx/tts/zipvoice.html
+      offlineTtsZipVoiceModelConfig.tokens = './tokens.txt';
+      offlineTtsZipVoiceModelConfig.encoder = './encoder.int8.onnx';
+      offlineTtsZipVoiceModelConfig.decoder = './decoder.int8.onnx';
+      offlineTtsZipVoiceModelConfig.vocoder = './vocos_24khz.onnx';
+      offlineTtsZipVoiceModelConfig.dataDir = './espeak-ng-data';
+      offlineTtsZipVoiceModelConfig.lexicon = './lexicon.txt';
+      break;
   }
 
   const offlineTtsModelConfig = {
@@ -999,6 +1027,7 @@ function createOfflineTts(Module, myConfig) {
     offlineTtsMatchaModelConfig: matcha,
     offlineTtsKokoroModelConfig: offlineTtsKokoroModelConfig,
     offlineTtsKittenModelConfig: offlineTtsKittenModelConfig,
+    offlineTtsZipVoiceModelConfig: offlineTtsZipVoiceModelConfig,
     numThreads: 1,
     debug: 1,
     provider: 'cpu',
@@ -1022,5 +1051,6 @@ if (typeof process == 'object' && typeof process.versions == 'object' &&
     typeof process.versions.node == 'string') {
   module.exports = {
     createOfflineTts,
+    getDefaultOfflineTtsModelType,
   };
 }

sherpa-onnx-wasm-main-tts.data

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7ce2d43070f87274774b8426e5ca7440a4fed2b1ecd8fe6dccb792f42a2016e
-size 96523617
+oid sha256:619888141185f0c6dea5926ab9bb8a525383d1546b96f92d7346323297a73899
+size 96524422

sherpa-onnx-wasm-main-tts.wasm

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20ec17318118c73835b33cf44ed73e34e368c612104e6a584bbd3ea565eb0750
-size 11964249
+oid sha256:6edcd44a15b7c385405a142b473d9f83eff0fb5c1ca683e2a729addedb0bd21b
+size 11967286