This is the function where RTC is made. Without code change, open
event from the data channel doesn’t fire at all. The thing is now the data channel doesn’t receive any event at all. Does this happen from time to time?
async () => {
const API_KEY = process.env.NEXT_PUBLIC_GPT_API_KEY;
// Create a peer connection
if (!rtcConnectionRef.current) {
rtcConnectionRef.current = new RTCPeerConnection();
// Set up to play remote audio from the model
const audioEl = document.createElement("audio");
audioEl.autoplay = true;
rtcConnectionRef.current!.ontrack = (e) =>
(audioEl.srcObject = e.streams[0]);
// Add local audio track for microphone input in the browser
const ms = await navigator.mediaDevices.getUserMedia({
audio: true,
});
rtcConnectionRef.current!.addTrack(ms.getTracks()[0]);
// Set up data channel for sending and receiving events
const dc = rtcConnectionRef.current!.createDataChannel("oai-events");
dc.addEventListener("message", handleRTCMessage);
dc.addEventListener("open", () => {
setIsRTCInitialized(true);
});
dc.addEventListener("error", (e) => {
console.error("data channel error", e);
});
dc.addEventListener("close", () => {
setIsRTCInitialized(false);
});
// Start the session using the Session Description Protocol (SDP)
const offer = await rtcConnectionRef.current!.createOffer();
await rtcConnectionRef.current!.setLocalDescription(offer);
const baseUrl = "https://api.openai.com/v1/realtime";
const model = "gpt-4o-realtime-preview-2024-12-17";
const sdpResponse = await fetch(`${baseUrl}?model=${model}`, {
method: "POST",
body: offer.sdp,
headers: {
Authorization: `Bearer ${API_KEY}`,
"Content-Type": "application/sdp",
},
});
const answer = {
type: "answer",
sdp: await sdpResponse.text(),
};
await rtcConnectionRef.current!.setRemoteDescription(
answer as RTCSessionDescriptionInit
);
dataChannel.current = dc;
dc.addEventListener("open", () => {
try {
if (!isInstructionSentRTC) {
dc.send(
JSON.stringify({
type: "session.update",
session: {
modalities: ["text", "audio"],
voice: "shimmer",
instructions:"Some Instructions",
},
})
);
console.log("session.update");
}
setIsInstructionSentRTC(true);
} catch (e) {
console.error("session.update error", e);
}
});
}