Every 20 or so instances I notice the audio get randomly chopped off from the model response. It seems like it fails to send the last one or two of the audio.delta packets and sends over the response.done event with the full correct audio transcription (without the end being chopped off). The number of audio tokens that it claimed to have sent back also makes sense so it is factoring in the audio.delta packets that actually were never received on the client side. I tried using
- Javascript client that has been released @openai-realtime-console
- Simple Python websockets client
Example event log where it did not vocalize whiskers!
04:03.67
server
conversation.item.input_audio_transcription.completed
{
"type": "conversation.item.input_audio_transcription.completed",
"event_id": "event_AqTGye66WAYaTZBMZhPSd",
"item_id": "item_AqTGxG9l5os8SKEeWBBTh",
"content_index": 0,
"transcript": "Tell me a joke about tabby cats.\n"
}
04:03.81
server
rate_limits.updated
{
"type": "rate_limits.updated",
"event_id": "event_AqTGyTYKgp2hSDIJBb9um",
"rate_limits": [
{
"name": "requests",
"limit": 20000,
"remaining": 19999,
"reset_seconds": 0.003
},
{
"name": "tokens",
"limit": 10000000,
"remaining": 9991225,
"reset_seconds": 0.052
}
]
}
04:03.84
server
response.output_item.added
{
"type": "response.output_item.added",
"event_id": "event_AqTGyuWlvHUuWj7Bzjr62",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"output_index": 0,
"item": {
"id": "item_AqTGxahUUQk7u1MTySIez",
"object": "realtime.item",
"type": "message",
"status": "in_progress",
"role": "assistant",
"content": []
}
}
04:03.87
server
conversation.item.created
{
"type": "conversation.item.created",
"event_id": "event_AqTGy3eb7hYgvyn5ALrtF",
"previous_item_id": "item_AqTGxG9l5os8SKEeWBBTh",
"item": {
"id": "item_AqTGxahUUQk7u1MTySIez",
"object": "realtime.item",
"type": "message",
"status": "in_progress",
"role": "assistant",
"content": []
}
}
04:03.87
server
response.content_part.added
{
"type": "response.content_part.added",
"event_id": "event_AqTGyc6euOwLpPRZHUyKr",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"part": {
"type": "audio",
"transcript": "Why do tabby cats make great bakers? Because they have the best whiskers!"
}
}
04:03.87
server
response.audio_transcript.delta (8)
{
"type": "response.audio_transcript.delta",
"event_id": "event_AqTGyzcgutOcR5rvar0Jg",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"delta": "Why"
}
04:03.94
server
response.audio.delta (4)
{
"type": "response.audio.delta",
"event_id": "event_AqTGySprZUHmmiPWmsRqx",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"delta": "[trimmed: 6400 bytes]"
}
04:03.99
server
response.audio_transcript.delta (4)
{
"type": "response.audio_transcript.delta",
"event_id": "event_AqTGyZ8L2wH9v3rW99WdU",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"delta": " make"
}
04:04.04
server
response.audio.delta (2)
{
"type": "response.audio.delta",
"event_id": "event_AqTGymhzh3QbAiBb8JLJr",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"delta": "[trimmed: 16000 bytes]"
}
04:04.12
server
response.audio_transcript.delta (2)
{
"type": "response.audio_transcript.delta",
"event_id": "event_AqTGzW96OpmqkNTpSv4k0",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"delta": "ers"
}
04:04.18
server
response.audio.delta (2)
{
"type": "response.audio.delta",
"event_id": "event_AqTGzc0TYbhJy0lWKyZBS",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"delta": "[trimmed: 16000 bytes]"
}
04:04.28
server
response.audio_transcript.delta (14)
{
"type": "response.audio_transcript.delta",
"event_id": "event_AqTGztTxizcXwGRvo1m2g",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"delta": " Because"
}
04:04.44
server
response.audio.delta (4)
{
"type": "response.audio.delta",
"event_id": "event_AqTGzlmpmPGlpB8m9JYq7",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"delta": "[trimmed: 48000 bytes]"
}
04:04.71
server
response.audio.done
{
"type": "response.audio.done",
"event_id": "event_AqTGzeVtdOMxiq0iUZhus",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0
}
04:04.74
server
response.audio_transcript.done
{
"type": "response.audio_transcript.done",
"event_id": "event_AqTGzi93fNQ9YxyYKR2kf",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"transcript": "Why do tabby cats make great bakers? Because they have the best whiskers!"
}
04:04.74
server
response.content_part.done
{
"type": "response.content_part.done",
"event_id": "event_AqTGzfbOca4W2gxH2X9eq",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"item_id": "item_AqTGxahUUQk7u1MTySIez",
"output_index": 0,
"content_index": 0,
"part": {
"type": "audio",
"transcript": "Why do tabby cats make great bakers? Because they have the best whiskers!"
}
}
04:04.74
server
response.output_item.done
{
"type": "response.output_item.done",
"event_id": "event_AqTGzaynfoyySC02FQGjc",
"response_id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"output_index": 0,
"item": {
"id": "item_AqTGxahUUQk7u1MTySIez",
"object": "realtime.item",
"type": "message",
"status": "completed",
"role": "assistant",
"content": [
{
"type": "audio",
"transcript": "Why do tabby cats make great bakers? Because they have the best whiskers!"
}
]
}
}
04:04.74
server
response.done
{
"type": "response.done",
"event_id": "event_AqTGzp28UfoGGD1J4336J",
"response": {
"object": "realtime.response",
"id": "resp_AqTGxN6mw6GHOYYzyO9OV",
"status": "completed",
"status_details": null,
"output": [
{
"id": "item_AqTGxahUUQk7u1MTySIez",
"object": "realtime.item",
"type": "message",
"status": "completed",
"role": "assistant",
"content": [
{
"type": "audio",
"transcript": "Why do tabby cats make great bakers? Because they have the best whiskers!"
}
]
}
],
"conversation_id": "conv_AqTD3lcrIAv7ksRkanhHH",
"modalities": [
"audio",
"text"
],
"voice": "alloy",
"output_audio_format": "pcm16",
"temperature": 0.8,
"max_output_tokens": 4096,
"usage": {
"total_tokens": 5340,
"input_tokens": 5220,
"output_tokens": 120,
"input_token_details": {
"text_tokens": 1401,
"audio_tokens": 3819,
"cached_tokens": 5056,
"cached_tokens_details": {
"text_tokens": 1344,
"audio_tokens": 3712
}
},
"output_token_details": {
"text_tokens": 30,
"audio_tokens": 90
}
},
"metadata": null
}
}