Error processing requests with images attached to them in gpt-4o

OpenAI:DEBUG:request https://api.openai.com/v1/chat/completions {
  method: 'post',
  path: '/chat/completions',
  body: {
    model: 'gpt-4o',
    messages: [ [Object], [Object] ],
    max_tokens: 1000,
    response_format: { type: 'json_object' }
  },
  stream: false
} {
  'content-length': '829',
  accept: 'application/json',
  'content-type': 'application/json',
  'user-agent': 'OpenAI/JS 4.52.3',
  'x-stainless-lang': 'js',
  'x-stainless-package-version': '4.52.3',
  'x-stainless-os': 'MacOS',
  'x-stainless-arch': 'x64',
  'x-stainless-runtime': 'node',
  'x-stainless-runtime-version': 'v18.17.0',
  authorization: 'hahahaha',
  'openai-project': 'pretendyouseeit'
}
OpenAI:DEBUG:response (error; retrying, 2 attempts remaining) 500 https://api.openai.com/v1/chat/completions {
  'alt-svc': 'h3=":443"; ma=86400',
  'cf-cache-status': 'DYNAMIC',
  'cf-ray': '89e1c20e498bc34a-EWR',
  connection: 'keep-alive',
  'content-length': '369',
  'content-type': 'application/json',
  date: 'Thu, 04 Jul 2024 20:10:19 GMT',
  'openai-processing-ms': '110',
  server: 'cloudflare',
  'set-cookie': '__cf_bm=kOddgENJ6xdc5zx7hYbfnHPvl1kZ0atRgIoqjiMJfkY-1720123819-1.0.1.1-cSZgFe4nnb1aW31Y38nenE9xvu1VERY9jiQ4ZlOVXHDxZ0bv6v63thAFthqI6HQDr3wEDWz3aIFbQjild76XhQ; path=/; expires=Thu, 04-Jul-24 20:40:19 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=HCv.HaKmbVjid0NoIWatS4YNWfXg5dj46ve5UR6uKto-1720123819438-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None',
  'strict-transport-security': 'max-age=31536000; includeSubDomains',
  'x-ratelimit-limit-requests': '500',
  'x-ratelimit-limit-tokens': '30000',
  'x-ratelimit-remaining-requests': '499',
  'x-ratelimit-remaining-tokens': '28154',
  'x-ratelimit-reset-requests': '120ms',
  'x-ratelimit-reset-tokens': '3.692s',
  'x-request-id': 'req_4050d7c77d5164bca9d9526aa9e4dc1d'
}
OpenAI:DEBUG:request https://api.openai.com/v1/chat/completions {
  method: 'post',
  path: '/chat/completions',
  body: {
    model: 'gpt-4o',
    messages: [ [Object], [Object] ],
    max_tokens: 1000,
    response_format: { type: 'json_object' }
  },
  stream: false
} {
  'content-length': '829',
  accept: 'application/json',
  'content-type': 'application/json',
  'user-agent': 'OpenAI/JS 4.52.3',
  'x-stainless-lang': 'js',
  'x-stainless-package-version': '4.52.3',
  'x-stainless-os': 'MacOS',
  'x-stainless-arch': 'x64',
  'x-stainless-runtime': 'node',
  'x-stainless-runtime-version': 'v18.17.0',
  authorization: 'Bearer nopenopenope',
  'openai-project': 'notthiseither'
}
OpenAI:DEBUG:response (error; retrying, 1 attempts remaining) 500 https://api.openai.com/v1/chat/completions {
  'alt-svc': 'h3=":443"; ma=86400',
  'cf-cache-status': 'DYNAMIC',
  'cf-ray': '89e1c2127d49c34a-EWR',
  connection: 'keep-alive',
  'content-length': '369',
  'content-type': 'application/json',
  date: 'Thu, 04 Jul 2024 20:10:20 GMT',
  'openai-processing-ms': '52',
  server: 'cloudflare',
  'set-cookie': '__cf_bm=6myj1FGCKOu1y6X.IH7TOPScgUTk4rx2lra6WBy1etI-1720123820-1.0.1.1-Wj2mro2DlPwFaqtQOJfP9mp5O2jke6kJ3HL9TiX3opx8NiRw_aWB8EFYMOwAGWoqNKqdVbk2G4YauAxvvniLFA; path=/; expires=Thu, 04-Jul-24 20:40:20 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=eQfIBKGEkFVNr5k6_1JMJq7s7rCNWFoY9ECCw2RBI34-1720123820063-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None',
  'strict-transport-security': 'max-age=31536000; includeSubDomains',
  'x-ratelimit-limit-requests': '500',
  'x-ratelimit-limit-tokens': '30000',
  'x-ratelimit-remaining-requests': '499',
  'x-ratelimit-remaining-tokens': '26650',
  'x-ratelimit-reset-requests': '120ms',
  'x-ratelimit-reset-tokens': '6.699s',
  'x-request-id': 'req_c8bb4c34d328f8d18abf871f1d405cb0'
}
OpenAI:DEBUG:request https://api.openai.com/v1/chat/completions {
  method: 'post',
  path: '/chat/completions',
  body: {
    model: 'gpt-4o',
    messages: [ [Object], [Object] ],
    max_tokens: 1000,
    response_format: { type: 'json_object' }
  },
  stream: false
} {
  'content-length': '829',
  accept: 'application/json',
  'content-type': 'application/json',
  'user-agent': 'OpenAI/JS 4.52.3',
  'x-stainless-lang': 'js',
  'x-stainless-package-version': '4.52.3',
  'x-stainless-os': 'MacOS',
  'x-stainless-arch': 'x64',
  'x-stainless-runtime': 'node',
  'x-stainless-runtime-version': 'v18.17.0',
  authorization: 'Bearer nopenopenope',
  'openai-project': 'notthiseither'
}
OpenAI:DEBUG:response (error; (error; not retryable)) 500 https://api.openai.com/v1/chat/completions {
  'alt-svc': 'h3=":443"; ma=86400',
  'cf-cache-status': 'DYNAMIC',
  'cf-ray': '89e1c21899f5c34a-EWR',
  connection: 'keep-alive',
  'content-length': '369',
  'content-type': 'application/json',
  date: 'Thu, 04 Jul 2024 20:10:21 GMT',
  'openai-processing-ms': '68',
  server: 'cloudflare',
  'set-cookie': '__cf_bm=3NLStCMH6snVxExvaVd0jXomdMRPgYb305QJEmwb9PE-1720123821-1.0.1.1-Esq8UBV3XeCEN27yiC_Wvk8gFRn9_0orWJFc99nOqfdfqu4U16XcRr8_aeVKJ9IX7zYu_fmdaIjamJYRWD82xg; path=/; expires=Thu, 04-Jul-24 20:40:21 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=WVEtIN9KAIEmTkLEATKZDmIOwgV6uG_1bKyKPEPn_B0-1720123821052-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None',
  'strict-transport-security': 'max-age=31536000; includeSubDomains',
  'x-ratelimit-limit-requests': '500',
  'x-ratelimit-limit-tokens': '30000',
  'x-ratelimit-remaining-requests': '499',
  'x-ratelimit-remaining-tokens': '25282',
  'x-ratelimit-reset-requests': '120ms',
  'x-ratelimit-reset-tokens': '9.434s',
  'x-request-id': 'req_ecaea928be490446af89acb8d5827807'
}
const response = await this.client.chat.completions.create({
      model: 'gpt-4o',
      messages: [
        {
          role: 'user',
          content: [
            {
              image_url: {
                url: 'https://imagedelivery.net/K1P8n1w6zwth8QgZbg5Qeg/9a01a34a-6a8a-493f-3f06-6dbaf7afc600/public',
              },
              type: 'image_url',
            },
            {
              type: 'text',
              text: 'Two lamb chops on cauliflower puree with Greek yogurt sauce',
            },
          ],
        },
        {
          role: 'system',
          content: `Using the images and description provided, estimate the:
- number of calores
- grams of protein
- grams of fat
- grams of carbohydrates.

Output must be a JSON object with the following keys: calories, description, gramsOfProtein, gramsOfFat, gramsOfCarbs.`,
        },
      ],
      max_tokens: 1000,
      response_format: { type: 'json_object' },
    });

What goes on?

You have your messages backwards and the first system message cannot be images, which may impact the execution.

system: how the AI should behave in general.
user: what task to perform (image attachment)

You don’t need to tell the AI what’s in the image, BTW; it could answer the user question without a picture.

Also, status 500 errors have been noted if sending a “project_id” without “organization_id” (which are taken automatically from environment variables also), or if there are mismatches between those and the key. Only an API key is really necessary to make calls to your own default organization or via the project where API keys came from.

Turn off response format.

I can confirm that adding the ord ID to my environment variables fixed the 500 issue. THANKYOUTHANKYOUTHANKYOU!

1 Like