Real time api with websockets working example

Hi,
We are trying to create a working example of a real-time API (with a split backend and frontend) using WebSockets. So far, we’ve managed to overcome many issues and have it somewhat working. However, the responses we’re getting don’t make sense.

Could someone please review our code? Thank you!

code:
const initVAD = async () => {
try {
vadInstance.value = await window.vad.MicVAD.new({
onSpeechStart: () => {
console.log(‘Speech started’);
startRecording();
},
onSpeechEnd: () => {
console.log(‘Speech ended’);
stopRecording();
},
onVADMisfire: () => {
console.log(‘VAD misfire’);
if (isRecording.value) {
stopRecording();
}
},
minSpeechFrames: 4,
maxSilenceFrames: 10,
vadMode: 3,
intervalSize: 30,
});
await vadInstance.value.start();
} catch (error) {
console.error(‘Error initializing VAD:’, error);
}
};
const processAudioForStreaming = (audioData) => {
if (client.value && isConnected.value && isRecording.value) {
const isSilence = audioData.every((sample) => Math.abs(sample) < 0.01);
if (!isSilence) {
const pcmData = convertToPCM16(audioData);
client.value.appendInputAudio(pcmData);
}
}
};
const convertToPCM16 = (audioData) => {
const pcmData = new Int16Array(audioData.length);
for (let i = 0; i < audioData.length; i++) {
const s = Math.max(-1, Math.min(1, audioData[i]));
pcmData[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
}
return pcmData;
};
const startRecording = async () => {
if (!isRecording.value) {
try {
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true,
sampleRate: 24000,
},
});
audioStream.value = stream;
audioContext.value = new AudioContext();
source = audioContext.value.createMediaStreamSource(stream);
processor = audioContext.value.createScriptProcessor(4096, 1, 1);
processor.onaudioprocess = (e) => {
const audioData = e.inputBuffer.getChannelData(0);
processAudioForStreaming(audioData);
};
source.connect(processor);
processor.connect(audioContext.value.destination);
isRecording.value = true;
handleVoiceInput();
} catch (error) {
console.error(‘Error starting recording:’, error);
}
}
};

Receiving the answer

const stopRecording = () => {
if (isRecording.value) {
if (audioStream.value) {
audioStream.value.getTracks().forEach((track) => track.stop());
audioStream.value = null;
}
if (processor && source) {
processor.disconnect();
source.disconnect();
}
if (audioContext.value) {
audioContext.value.close().then(() => {
audioContext.value = null;
source = null;
processor = null;
});
}
isRecording.value = false;
client.value.createResponse();
}
};