It will say that it’s called the function, and it hasn’t, then I’ll ask it to try again, and it’ll do it that time. Here’s some of my code:
dc.addEventListener("message", async (ev) => {
const msg = JSON.parse(ev.data);
console.log("Realtime event:", msg.type, msg);
if (msg.type === "response.done") {
if (msg?.response?.output) {
for (const item of msg.response.output) {
if (item.type === "function_call") {
const fnName = item.name;
const rawArgs = item.arguments || "";
console.log("Function call from response.done:", fnName, rawArgs);
const parsedArgs = JSON.parse(rawArgs || "{}");
const fn = fns[fnName];
if (!fn) {
console.warn("No local function named:", fnName);
return;
}
const result = fn(parsedArgs);
// Return the function output
const functionOutputEvent = {
type: "conversation.item.create",
item: {
type: "function_call_output",
call_id: item.call_id,
output: JSON.stringify(result),
},
};
dc.send(JSON.stringify(functionOutputEvent));
// Then ask the model to keep responding
dc.send(JSON.stringify({ type: "response.create" }));
}
}
}
}
});
function configureData() {
const dc = dataChannelRef.current;
if (!dc) return;
// We let the AI know about these local “tools” in a session.update:
const event = {
type: "session.update",
session: {
modalities: ["text", "audio"],
tools: [
{
type: "function",
name: "highlightColumn",
description: "Highlight an entire column (1–18) in the periodic table.",
parameters: {
type: "object",
properties: {
column: { type: "number", description: "A column from 1 to 18" },
},
required: ["column"],
},
},
{
type: "function",
name: "highlightGroupBlock",
description: 'Highlight an entire group block (e.g. "alkali metal").',
parameters: {
type: "object",
properties: {
groupBlock: {
type: "string",
description:
'A group label, e.g. "alkali metal", "alkaline earth metal", "transition metal", etc.',
},
},
required: ["groupBlock"],
},
},
{
type: "function",
name: "toggleElementHighlight",
description: "Toggle highlight on a single element by its name.",
parameters: {
type: "object",
properties: {
elementName: {
type: "string",
description: "name of the element i.e. 'Hydrogen' first letter must be capitalized. You MUST ensure" +
"that you've received confirmation of an element being highlighted before saying you've done it.",
},
},
required: ["elementName"],
},
},
{
type: "function",
name: "showElementModal",
description: "Open the info modal for an element based on its name.",
parameters: {
type: "object",
properties: {
elementName: {
type: "string",
description: "The name of the element i.e. 'Hydrogen' first letter must be capitalized. ou MUST ensure" +
"that you've received confirmation of an element opened before saying you've done it.",
},
},
required: ["elementName"],
},
},
{
type: "function",
name: "closeElementDialog",
description: "Close the currently open element dialog (if any).",
},
{
type: "function",
name: "resetHighlights",
description: "Clear all highlight states in the periodic table (columns, groups, etc.).",
},
],
"tool_choice": "auto"
},
};
dc.send(JSON.stringify(event));
}
function createDataChannel(pc: RTCPeerConnection) {
const dc = pc.createDataChannel("response");
dataChannelRef.current = dc;
dc.addEventListener("open", () => {
console.log("Data channel opened");
configureData();
});