Incomplete stream chunks for completions API

API
,
1

Code:

import OpenAI from "openai";

const openai = new OpenAI();

async function main() {
  const response = await openai.completions
    .create({
      prompt: "Hello",
      model: "text-davinci-003",
      stream: true,
    })
    .asResponse();

  for await (const chunk of response.body) {
    console.log(chunk.toString());
    console.log("-".repeat(10));
  }
}

main();

Output

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_complet
----------
ion","created":1695192584,"choices":[{"text":"\"","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_complet
----------
ion","created":1695192584,"choices":[{"text":"\n","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_complet
----------
ion","created":1695192584,"choices":[{"text":"\n","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completi
----------
on","created":1695192584,"choices":[{"text":"❱","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completion"
----------
,"created":1695192584,"choices":[{"text":" local","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completio
----------
n","created":1695192584,"choices":[{"text":"\n\n","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completion"
----------
,"created":1695192584,"choices":[{"text":" Local","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completi
----------
on","created":1695192584,"choices":[{"text":" is","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completi
----------
on","created":1695192584,"choices":[{"text":" an","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completion","cr
----------
eated":1695192584,"choices":[{"text":" adjective","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completion
----------
","created":1695192584,"choices":[{"text":" that","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completion"
----------
,"created":1695192584,"choices":[{"text":" means","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completion","cr
----------
eated":1695192584,"choices":[{"text":" something","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completion","cr
----------
eated":1695192584,"choices":[{"text":" belonging","index":0,"logprobs":null,"finish_reason":null}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completion",
----------
"created":1695192584,"choices":[{"text":" to","index":0,"logprobs":null,"finish_reason":"length"}],"model":"text-davinci-003"}

data: {"warning":"This model version is deprecated. Migrate before January 4, 2024 to avoid disruption of service. Learn more https://platform.openai.com/docs/deprecations","id":"cmpl-80lJo9XeeHSV3FLtp9zr9Utz2BR8Y","object":"text_completion","created":1695192584,"choices":[{"text":"","index":0,"logprobs":null,"finish_reason":"length"}],"model":"text-davinci-003"}

data: [DONE]


----------

Is this intended behavior?

2

Another completions example

import OpenAI from "openai";

const openai = new OpenAI();

async function main() {
  const response = await openai.completions
    .create({
      prompt: "Hello",
      model: "gpt-3.5-turbo-instruct",
      stream: true,
    })
    .asResponse();

  for await (const chunk of response.body) {
    console.log(chunk.toString());
    console.log("-".repeat(10));
  }
}

main();

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","created
----------
":1695192817,"choices":[{"text":" everyone","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion",
----------
"created":1695192817,"choices":[{"text":",","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","cre
----------
ated":1695192817,"choices":[{"text":" this","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","c
----------
reated":1695192817,"choices":[{"text":" is","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","c
----------
reated":1695192817,"choices":[{"text":" Mr","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion",
----------
"created":1695192817,"choices":[{"text":".","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","creat
----------
ed":1695192817,"choices":[{"text":" Butler","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion",
----------
"created":1695192817,"choices":[{"text":".","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","
----------
created":1695192817,"choices":[{"text":" I","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","c
----------
reated":1695192817,"choices":[{"text":" am","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","created":1695192817,"choices":[{"text":" a","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","created":1695192817,"choices":[{"text":" virtual","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","created"
----------
:1695192817,"choices":[{"text":" assistant","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","created
----------
":1695192817,"choices":[{"text":" designed","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","c
----------
reated":1695192817,"choices":[{"text":" to","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","created
----------
":1695192817,"choices":[{"text":" help","index":0,"logprobs":null,"finish_reason":"length"}],"model":"gpt-3.5-turbo-instruct"}

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","cr
----------
eated":1695192
----------
817,"choices":[{"text":"","index":0,"logprobs":null,"finish_reason":"length"}],"model":"gpt-3.5-turbo-instruct"}

data: [DONE]


----------
3

Chat completions example

import OpenAI from "openai";

const openai = new OpenAI();

async function main() {
  const response = await openai.chat.completions
    .create({
      model: "gpt-4",
      messages: [{ role: "user", content: "Say this is a test!" }],
      stream: true,
    })
    .asResponse();

  for await (const chunk of response.body) {
    console.log(chunk.toString());
    console.log("-".repeat(10));
  }
}

main();

data: {"id":"chatcmpl-80lQJgffqBMRunnFqiHxPlnrdeKV9","object":"chat.completion.chunk","created":1695192987,"model":"gpt-4-0613","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}

data: {"id":"chatcmpl-80lQJgffqBMRunnFqiHxPlnrdeKV9","object":"chat.completion.chunk","created":1695192987,"model":"gpt-4-0613","choices":[{"index":0,"delta":{"content":"This"},"finish_reason":null}]}


----------
data: {"id":"chatcmpl-80lQJgffqBMRunnFqiHxPlnrdeKV9","object":"chat.completion.chunk","created":1695192987,"model":"gpt-4-0613","choices":[{"index":0,"delta":{"content":" is"},"finish_reason":null}]}


----------
data: {"id":"chatcmpl-80lQJgffqBMRunnFqiHxPlnrdeKV9","object":"chat.completion.chunk","created":1695192987,"model":"gpt-4-0613","choices":[{"index":0,"delta":{"content":" a"},"finish_reason":null}]}


----------
data: {"id":"chatcmpl-80lQJgffqBMRunnFqiHxPlnrdeKV9","object":"chat.completion.chunk","created":1695192987,"model":"gpt-4-0613","choices":[{"index":0,"delta":{"content":" test"},"finish_reason":null}]}


----------
data: {"id":"chatcmpl-80lQJgffqBMRunnFqiHxPlnrdeKV9","object":"chat.completion.chunk","created":1695192987,"model":"gpt-4-0613","choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]}


----------
data: {"id":"chatcmpl-80lQJgffqBMRunnFqiHxPlnrdeKV9","object":"chat.completion.chunk","created":1695192987,"model":"gpt-4-0613","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}

data: [DONE]


----------

No incomplete chunks

4

Hi,

When streaming responses, there is no set amount of “delta” per message, it could be a single token, or it could be 20 or some other arbitrary value, the hope is that the number of tokens sent per message is small as this allows for a low latency of first response which is typically the desired outcome from streaming.

Obviously the message deltas form a working whole when concatenated.

5

What part of your code is inserting the hyphens?
The first two seem complete if these were not there.

They are not being used as chunk separators for display, otherwise we’d see them between the deltas, not within.

6

What part of your code is inserting the hyphens?

console.log("-".repeat(10)); is inserting the hyphens to make it easier to see what the variable chunk looks like.

The first two seem complete if these were not there.

Yes, but I expect something like this:

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","created":1695192817,"choices":[{"text":" everyone","index":0,"logprobs":null,"finish_reason":null}],"model":"gpt-3.5-turbo-instruct"}

not:

data: {"id":"cmpl-80lNZieVFOo5xjO5t3YcobgIsNlpV","object":"text_completion","created"
7

Why is that a more valid delta than just “!” ?

8

Expensive AI answer, where lossless chat history gpt-4 has lost the overall plan within three turns and 3000 context, (demonstrating why I don’t code with it any more)

I see, you want to keep the streaming behavior but just want to ensure that each JSON object is logged completely before moving on to the next one. The issue here is that the stream chunks do not necessarily align with the boundaries of the JSON objects. The issue you’re experiencing is likely due to the way Node.js handles streams.

One way to handle this is to buffer the chunks until a complete JSON object has been received, and then log that object. Here’s an example of how you could do this:

async function main() {
  let buffer = '';
  for await (const chunk of response.body) {
    buffer += chunk.toString();
    let boundary = buffer.indexOf('\n');
    while (boundary !== -1) {
      const jsonStr = buffer.substring(0, boundary);
      buffer = buffer.substring(boundary + 1);
      const jsonObj = JSON.parse(jsonStr);
      console.log(jsonObj);
      console.log("-".repeat(10));
      boundary = buffer.indexOf('\n');
    }
  }
}

main();

This code will keep adding chunks to a buffer until it finds a newline character (\n), which signifies the end of a JSON object. It will then parse and log that object, remove it from the buffer, and continue with the next object. This should ensure that each JSON object is logged completely before moving on to the next one.

9

Thanks for the code! I’m curious what determines the split points.