Streaming while using Agent/Tools/Function calling in NodeJS

API
,
1

Hello! I’ve been trying to get streaming to work while using Function Calling (Tools).

This is my current code of my node server based on this documentation:

import express from 'express';
import http from 'http';
import { Server } from 'socket.io';
import bodyParser from 'body-parser';
import fetch from 'node-fetch';
import OpenAI from 'openai';

const app = express();
const server = http.createServer(app);
const io = new Server(server, {
  cors: {
    origin: "*", // Adjust this to match your frontend's origin in production for security
    methods: ["GET", "POST"]
  }
});

const port = 8000;

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
  dangerouslyAllowBrowser: true,
});

async function getLocation() {
  const response = await fetch("https://ipapi.co/json/");
  const locationData = await response.json();
  return locationData;
}

async function getCurrentWeather(latitude, longitude) {
  const url = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&hourly=apparent_temperature`;
  const response = await fetch(url);
  const weatherData = await response.json();
  return weatherData;
}


const tools = [
  {
    type: "function",
    function: {
      name: "getCurrentWeather",
      description: "Get the current weather in a given location",
      parameters: {
        type: "object",
        properties: {
          latitude: {
            type: "string",
          },
          longitude: {
            type: "string",
          },
        },
        required: ["latitude", "longitude"],
      },
    }
  },
  {
    type: "function",
    function: {
      name: "getLocation",
      description: "Get the user's location based on their IP address",
      parameters: {},
    }
  },
];

const availableTools = {
  getCurrentWeather,
  getLocation,
};

let messages = [
  {
    role: "system",
    content: `You are a helpful assistant. Only use the functions you have been provided with.`,
  },
];

async function agent(userInput) {
  console.log(`User question: ${userInput}`); // Log user's question

  let wasFunctionCalled = "No";
  let functionName = "None";
  let functionResponse = "None";

  messages.push({
    role: "user",
    content: userInput,
  });

  for (let i = 0; i < 5; i++) {
    const response = await openai.chat.completions.create({
      model: "gpt-4",
      messages: messages,
      tools: tools,
      temperature: 0.1,
    });

    const { finish_reason, message } = response.choices[0];

    if (finish_reason === "tool_calls" && message.tool_calls) {
      wasFunctionCalled = "Yes";
      functionName = message.tool_calls[0].function.name;
      const functionToCall = availableTools[functionName];
      const functionArgs = JSON.parse(message.tool_calls[0].function.arguments);
      const functionArgsArr = Object.values(functionArgs);
      const result = await functionToCall.apply(null, functionArgsArr);
      functionResponse = JSON.stringify(result);

      messages.push({
        role: "function",
        name: functionName,
        content: `The result of the last function was this: ${functionResponse}`,
      });
    } else if (finish_reason === "stop") {
      messages.push(message);
      console.log(`Was any function called: ${wasFunctionCalled}`);
      console.log(`Function Name: ${functionName}`);
      console.log(`Function response: ${functionResponse}`);
      console.log(`Final Response: ${message.content}`); // Log final response
      return message.content;
    }
  }
  console.log(`Was any function called: ${wasFunctionCalled}`);
  console.log(`Function Name: ${functionName}`);
  console.log(`Function response: ${functionResponse}`);
  console.log("The maximum number of iterations has been met without a suitable answer. Please try again with a more specific input.");
  return "The maximum number of iterations has been met without a suitable answer. Please try again with a more specific input.";
}

app.use(bodyParser.json());

io.on('connection', (socket) => {
  console.log('a user connected');

  socket.on('disconnect', () => {
    console.log('user disconnected');
  });

  socket.on('sendMessage', async (text) => {
    // Process the message using your existing `agent` function
    const response = await agent(text);
    // Emit a message back to the client
    io.emit('receiveMessage', { text: response, isFinal: true });
  });
});

server.listen(port, () => {
  console.log(`Server running at http://localhost:${port}`);
});

Is there a way to stream the final response? Thinking about it, it shouldn’t be so different compared to streaming without function calls, since the function call is simply adding the response of the function to messages (as you would if you were to be adding search results from a RAG system). But, it’s been super tricky to get streaming to work. If you’ve figured out a way to do it, please let me know how, it’ll be much appreciated!

Thank you in advance!