Tool Calling - function that returns an Image

API
1

Hi,

is there a way to have a function that returns an image and send it back to the LLM? Something like the following

{

 "type": "function_call_output",
 "call_id": tool_call_id,
 "name": tool_name,
 "output": f"data:image/jpeg;base64,{image_data_base64}",
}
2

Welcome to the community.

Sure, here is a minimal example on how to return an image from a function call. 
from openai import OpenAI
# replace the line below with your own settings
client = OpenAI()

img_path=r"your_sample_image.png"

import json, base64, mimetypes
from pathlib import Path

def image_to_base64(path: str | Path) -> str:
    path = Path(path)
    b64 = base64.b64encode(path.read_bytes()).decode("ascii")
    mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
    return f"data:{mime};base64,{b64}"


def get_function_result(text):
    # you can change the function to respond to the 'text' argument
    return [{
             "type": "input_image",
             "detail":"low",
             "image_url": image_to_base64(img_path),
           }]

# 1. Function definition
tools = [
    {
        "type": "function",
        "name": "get_image",
        "description": "Retrieves a sample image",
        "parameters": {
            "type": "object",
            "properties": {
                "text": {
                    "type": "string",
                    "description": "A description of the image",
                },
            },
            "required": ["text"],
        },
    },
]


input_list = [
    {"role": "user", "content": "Retrieve a sample image and get a description. "}
]

# 2. Prompt the model with tools defined
response = client.responses.create(
    model="gpt-5-mini",
    tools=tools,
    input=input_list,
)

# Save function call outputs for subsequent requests
input_list += response.output

for item in response.output:
    if item.type == "function_call":
        if item.name == "get_image":
            # 3. Execute the function logic 
            print('argument:',json.loads(item.arguments))
            function_result = {
                "type": "function_call_output",
                "call_id": item.call_id,
                "output": get_function_result(json.loads(item.arguments).get('text')),
            }
            
            # 4. Provide function call results to the model
            input_list.append(function_result)


response = client.responses.create(
    model="gpt-5-mini",
    instructions="Use the tool to retrieve an image, then describe it",
    tools=tools,
    input=input_list,
)

# 5. The model should be able to give a response!
print("Final output:")
print("\n" + response.output_text)

See also: Images and files as function call outputs

1 Like