Streaming chunk format with multiple choices or tool calls?

API
1

Hi!

Are there ever cases when one streaming chunk would return multiple choices or tool calls within the chunk? For example, when running the following curl command and setting n>1, I receive the following response with one choice per chunk and the index corresponding to which choice.

This works great for my use case. I just want to confirm that this is expected and there are no edge cases I’m missing in which there are multiple outputs per chunk. Thanks!

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"obfuscation":"EqbeT"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"obfuscation":"ZP"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"obfuscation":"3u8fyh"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}],"obfuscation":"sBp"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"obfuscation":"fYdMc"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"obfuscation":"St"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"obfuscation":"w7fjx"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"obfuscation":"Tm"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}],"obfuscation":"yWo"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"obfuscation":"lbo0G1"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"obfuscation":"vjFlxq"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}],"obfuscation":"pYxJH"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}],"obfuscation":"Z0z"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}],"obfuscation":"g7g"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}],"obfuscation":""}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}],"obfuscation":"Lun"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}],"obfuscation":"q3d"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}],"obfuscation":"RO1"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}],"obfuscation":"f7dt3"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}],"obfuscation":"nWcvq"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}],"obfuscation":"3"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}],"obfuscation":""}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}],"obfuscation":""}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"obfuscation":"3wyx11"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}],"obfuscation":"Q6t"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}],"obfuscation":"9ln"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}],"obfuscation":"h"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}],"obfuscation":"C"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"obfuscation":"mHzwfK"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"obfuscation":"HryenW"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"obfuscation":"M"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":1,"delta":{},"logprobs":null,"finish_reason":"stop"}],"obfuscation":"3"}

data: {"id":"chatcmpl-C6I0P9G7T9NwAZH0Y9yOPkFUzbAve","object":"chat.completion.chunk","created":1755615217,"model":"gpt-4.1-mini-2025-04-14","service_tier":"default","system_fingerprint":"fp_3f58d112f7","choices":[{"index":2,"delta":{},"logprobs":null,"finish_reason":"stop"}],"obfuscation":"7"}

data: [DONE]
2

Yes! It’s been happening to me for a while, and I thought it was just me. Something in the models must have been modified. :smiling_face_with_tear:

3

The data chunks have to follow the schema, closely matching the API docs:

image
image518×614 12.5 KB

While that looks like a total response’s description, the deltas are increments.

They all need to be collected from. Tool calls are particularly annoying because each chunk of a string comes in its own container, so they can’t simply be appended like the text of content.

There’s nothing saying that there can’t be multiple keys per chunk, or extended lengths, except by practice, so there should not be a “case” in your code; they all should be gathered while producing the output that you want seen.

So: you can just treat n>1 as needing a call sorter in front of it by index (and probably not a live printer unless you have a UI make a bunch of chat boxes each to fill in.) to the same working stream parser.

4

Thank you for the response!

Curious, do you have an existing example of a call that might return multiple content/tool calls in the same chunk?

This is usually the format that I see (with one tool call returned per chunk), so it would be helpful to reproduce for testing

data: {"id": "chatcmpl-id...", "choices": [{"delta": {"content": null, "function_call": null, "refusal": null, "role": "assistant", "tool_calls": null}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1754677519, "model": "gpt-4o-2024-08-06", "object": "chat.completion.chunk", "service_tier": "default", "system_fingerprint": "fp_ff25b2783a", "usage": null, "obfuscation": "cgODK1lrDeV"}
data: {"id": "chatcmpl-id...", "choices": [{"delta": {"content": null, "function_call": null, "refusal": null, "role": null, "tool_calls": [{"index": 0, "id": "cal_...", "function": {"arguments": "", "name": "tool_call_name_1"}, "type": "function"}]}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1754677519, "model": "gpt-4o-2024-08-06", "object": "chat.completion.chunk", "service_tier": "default", "system_fingerprint": "fp_ff25b2783a", "usage": null, "obfuscation": "xw9iRWYoPvy"}
data: {"id": "chatcmpl-...", "choices": [{"delta": {"content": null, "function_call": null, "refusal": null, "role": null, "tool_calls": [{"index": 0, "id": null, "function": {"arguments": "arg1\"}", "name": null}, "type": null}]}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1754677519, "model": "gpt-4o-2024-08-06", "object": "chat.completion.chunk", "service_tier": "default", "system_fingerprint": "fp_ff25b2783a", "usage": null, "obfuscation": ""}
data: {"id": "chatcmpl-...", "choices": [{"delta": {"content": null, "function_call": null, "refusal": null, "role": null, "tool_calls": [{"index": 1, "id": "some_id", "function": {"arguments": "", "name": "tool_call_name2"}, "type": "function"}]}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1754677519, "model": "gpt-4o-2024-08-06", "object": "chat.completion.chunk", "service_tier": "default", "system_fingerprint": "", "usage": null, "obfuscation": ""}
5

I only say “there’s nothing saying” because OpenAI has altered streaming behavior before, such as splitting data between http stream chunks (the underlying transmission), messing up some programmers’ naive code. Another instance that one might not expect is the AI model writing “content” and “tool_call” in the same response.

So if allowed, anticipate. They don’t do that now; the types are discrete chunks. It doesn’t take more work to put all chunks through the same pattern of parsing, though.

A case for this may arise if OpenAI moved towards buffering more than a few tokens worth per SSE data chunk, as there’s a lot of bandwidth for “hi”.