Hello,
I have created a chat bot by Chat Completions API and ReactJS. At the moment, the chatbot asks for answsers in form of markdown from OpenAI, and renders them by react-markdown, rehype-raw, and remark-gfm, etc.
Now, I would like to enable users to draw charts as ChatGPT does:
I understand that the chatbot can ask for Plotly JSON configuration from OpenAI and render the chart by react-plotly.js. But I would like to reverse-engineer how exactly ChatGPT achieves this chart drawing.
For instance, I can set up the system such that
- In the
systemmessage, I can say, whenever the user wants the bot to draw a graph, please return a valid Plotly JSON configuration (e.g., enclosed by<PlotlyConfiguration>and</PlotlyConfiguration>) - When rendering the message of the assistant, whenever we see
<PlotlyConfiguration>and</PlotlyConfiguration>, we render it with react-plotly.js.
Do you think this is how ChatGPT achieves the chart plotting? Do you think asking Chat Completions API to enclose Plotly JSON configurations with <PlotlyConfiguration> and </PlotlyConfiguration> is a good idea? Do you think we need to use advanced features such as function calling or structured outputs, etc?
Edit 1:, I realize that ChatGPT can render several charts in one response. With function calling and structured outputs, I wrote the following code, do you think the idea is good and whether it can be optimized?
async function runRealChart() {
try {
const openai = new OpenAI({ apiKey: "sk-ThIdpClUNt..." });
const draw_chart =
{
type: "function",
function: {
name: 'get_plotly_configuration_json_to_render_a_chart',
description: "Get a Plotly configuration JSON generated by AI, based on a two-dimensional array of data provided by the user. The JSON will be used to draw a chart later in our UI.",
parameters: {
type: 'object',
properties: {
data: {
type: 'array',
items: {
type: 'array',
items: {
type: 'string'
}
},
description: 'A two-dimensional array of data.'
}
},
required: ["data"],
additionalProperties: false,
}
}
}
const response_format = {
type: "json_schema",
json_schema: {
name: "response",
strict: true,
schema: {
type: "object",
properties: {
plotly_configuration_json: {
type: "string",
description: "The Plotly configuration JSON for the chart."
}
},
required: ["plotly_configuration_json"],
additionalProperties: false
}
}
}
const m0 = { role: "system", content: "You are a spreadsheet expert." }
const m1 = { role: "user", content: "I have the following data:\n\nDate\tRevenue\n2022-01-01\t100\n2022-01-02\t200\n2022-01-03\t300\n\n, could you draw a chart?\n\nI have the following data:\n\nDate\tRevenue\n2023-01-01\t1000\n2023-01-02\t2000\n2023-01-03\t3000\n\n, could you draw another chart?" }
const messages = [m0, m1]
const response = await openai.chat.completions.create({
model: "gpt-4o-2024-08-06",
messages: messages,
tools: [draw_chart],
});
console.log("response.choices[0].message", response.choices[0].message)
console.log("response.choices[0].message.content", response.choices[0].message.content)
console.log("response.choices[0].message.tool_calls", response.choices[0].message.tool_calls)
const configurations = [];
for (const tool_call of response.choices[0].message.tool_calls) {
if (tool_call.function.name === "get_plotly_configuration_json_to_render_a_chart") {
const arguments = JSON.parse(tool_call.function.arguments).data;
const messages2 = [
{ role: "system", content: "You are a chart expert." },
{ role: "user", content: "return a Plotly configuration JSON for the data: " + JSON.stringify(arguments) }
]
const response2 = await openai.chat.completions.create({
model: "gpt-4o-2024-08-06",
messages: messages2,
response_format: response_format
});
console.log("response2.choices[0].message", response2.choices[0].message)
console.log("response2.choices[0].message.content", response2.choices[0].message.content)
configurations.push({
role: "tool",
content: JSON.stringify({ plotly_configuration_json: JSON.parse(response2.choices[0].message.content).plotly_configuration_json }),
tool_call_id: tool_call.id
})
};
}
console.log("configurations", configurations)
if (configurations.length > 0) {
const messages3 = [
{ role: "system", content: "You are a spreadsheet expert. Do NOT mention the term 'Plotly configuration JSON' in the answer because the JSON data will be rendered as charts in the UI. Enclose the Plotly configuration JSON data with <PlotConfiguration> and </PlotConfiguration>." },
m1,
response.choices[0].message,
...configurations
];
console.log("messages3", messages3)
const response3 = await openai.chat.completions.create({
model: "gpt-4o-2024-08-06",
messages: messages3,
});
console.log("response3.choices[0].message", response3.choices[0].message)
console.log("response3.choices[0].message", response3.choices[0].message.content)
}
} catch (error) {
console.error("An error occurred:", error);
}
}
