Having a hard time getting realtime api to recognize system instruction in NodeJS

API
1

I am attempting to create a Realtime audio chatbot using OpenAI’s Realtime API in NodeJS. The bot is working functionally and there are no errors, however the system instruction I am attempting to pass into the session is not applying and it only responds as the default ‘ChatGPT assistant’ I have tried quite a few different system instructions and none of them work making me think it is either an issue with the code or with the API itself. I tried to follow the documentation as closely as I could. Am I missing something?

const express = require("express");
const { RealtimeClient } = require("@openai/realtime-api-beta");
const record = require("node-record-lpcm16");
const Speaker = require("speaker");
require("dotenv").config();


let recordingProcess = null;
let isStreaming = false;

// OpenAI Realtime API client
const realtimeClient = new RealtimeClient({
  apiKey: process.env.OPENAI_API_KEY,
});

let isConnected = false;

// Initialize OpenAI Realtime API session
const initializeRealtimeClient = async () => {
  if (isConnected) {
    console.log("Realtime client already connected.");
    return;
  }

  realtimeClient.updateSession({
    instructions: "You are Sara, A friendly chatbot who loves to talk about herself and is very intrested in marinebio",
    input_audio_transcription: { model: "whisper-1" },
    turn_detection: { type: "server_vad" },
    voice: "nova",
  });
;
 realtimeClient.on("conversation.updated", ({ delta }) => {

    if (delta?.audio) {
      playAudioResponse(delta.audio); // Play the audio response
    }
  });

  try {
    await realtimeClient.connect();
    console.log("Connected to OpenAI Realtime API.");
    isConnected = true;
  } catch (error) {
    console.error("Error connecting to Realtime API:", error);
    isConnected = false;
  }
};



// Start streaming audio
const startStreaming = async () => {
  if (isStreaming) {
    console.log("Streaming is already active.");
    return;
  }

  await initializeRealtimeClient();

  console.log("Starting audio streaming...");
  isStreaming = true;

  recordingProcess = record.record({
    sampleRate: 24000,
    channels: 1,
    threshold: 0.5,
    device: "VB-Cable"
  });

  const audioStream = recordingProcess.stream();

  audioStream.on("data", (chunk) => {
    if (!isConnected) {
      console.warn("Realtime client is not connected. Skipping audio chunk.");
      return;
    }

    const int16Array = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.byteLength / 2);

    if (int16Array.every((value) => value === 0)) {
      console.warn("Skipping zero-filled chunk...");
      return;
    }

    try {
      realtimeClient.appendInputAudio(int16Array);
    } catch (error) {
      console.error("Error appending audio chunk:", error);
    }
  });

  console.log("Audio streaming started.");
};


// Routes
app.get("/start-streaming", async (req, res) => {
  try {
    await startStreaming();
    res.send("Audio streaming started.");
  } catch (error) {
    console.error("Error starting streaming:", error);
    res.status(500).send("Error starting streaming.");
  }
});
1 Like