What I am doing: Sending an backend API code snippet to the GPT-4o via both API and chat window to get the format of input and output of the API.
What I am experiencing so far: the response from the chat window is stable and reliable, while sometimes (about 1/2 to 1/3 of the times) the API call would return the wrong answer.
I wonder if there’s any configuration I am doing wrong.
Testable case:
API:
https://api.openai.com/v1/chat/completions
Input:
{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Given data structure following: \nCREATE TABLE ApiDesignProjects (\n id SERIAL PRIMARY KEY,\n user_id INT References Users(id),\n name VARCHAR(100),\n active BOOLEAN DEFAULT FALSE,\n deleted BOOLEAN DEFAULT FALSE,\n created_at TIMESTAMP DEFAULT NOW(),\n created_by INT,\n updated_at TIMESTAMP DEFAULT NOW(),\n updated_by INT\n);\n\nShow me the format of input and output of this function. \n\n`router.get('/component/list/:projectId', (request, response) => { console.log('api: GET /project/component/list/:projectId'); console.log('Get the list of components by project ID'); auth.verifyToken(request, response, {}).then(b_tokenValidated => { if (b_tokenValidated && b_tokenValidated.statusCode !== 401) { if (!validateBodyWithKeys(request.params, ['projectId'])) { generalApiErrorHandler(response, { status: 400, message: 'Input invalid.' }); return; } const getApiDesignComponentsByProjectId = project.getApiDesignComponentsByProjectId(request.params.projectId, b_tokenValidated.userId); getApiDesignComponentsByProjectId.then(res => { if (typeof res !== 'number') { generalApiResponseSender(response, res); } else { generalApiErrorHandler(response, res); } }).catch(err => { console.error('Error status: ', err); generalApiErrorHandler(response, err); }); } else { generalApiErrorHandler(response, { status: 401, message: \"You haven't logged in or token has expired.\" }); } }).catch((err) => { console.log('verifyToken failed: ', err); let errCode = 401; let errMsg = \"You haven't logged in or token has expired.\"; if (typeof err === 'number') { errCode = err; errMsg = \"Please see error code for more details.\"; } generalApiErrorHandler(response, { status: errCode, message: errMsg }); }); });\n\nrouter.get('/component/list/:projectId', (request, response) => { console.log('api: GET /project/component/list/:projectId'); console.log('Get the list of components by project ID'); auth.verifyToken(request, response, {}).then(b_tokenValidated => { if (b_tokenValidated && b_tokenValidated.statusCode !== 401) { if (!validateBodyWithKeys(request.params, ['projectId'])) { generalApiErrorHandler(response, { status: 400, message: 'Input invalid.' }); return; } const getApiDesignComponentsByProjectId = project.getApiDesignComponentsByProjectId(request.params.projectId, b_tokenValidated.userId); getApiDesignComponentsByProjectId.then(res => { if (typeof res !== 'number') { generalApiResponseSender(response, res); } else { generalApiErrorHandler(response, res); } }).catch(err => { console.error('Error status: ', err); generalApiErrorHandler(response, err); }); } else { generalApiErrorHandler(response, { status: 401, message: \"You haven't logged in or token has expired.\" }); } }).catch((err) => { console.log('verifyToken failed: ', err); let errCode = 401; let errMsg = \"You haven't logged in or token has expired.\"; if (typeof err === 'number') { errCode = err; errMsg = \"Please see error code for more details.\"; } generalApiErrorHandler(response, { status: errCode, message: errMsg }); }); });\n\nexport const getApiDesignComponentsByProjectId = async (projectId, myId) => { console.log(\"getApiDesignComponentsByProjectId is triggered\"); console.log(\"projectId: \", projectId); console.log(\"myId: \", myId); try { await checkProjectOwnership(projectId, myId); const query = ` SELECT c.id, ac.id AS category_id, ac.name AS category, t.id AS type_id, t.name AS type, c.inputs, c.outputs, c.tables, c.fields, c.rules, c.edit_rules, c.error_handlers, c.url, c.location_x, c.location_y, c.next_component_id, c.data_conditions, c.table_relations, c.group_fields, c.db_secret_id, c.custom_fields, c.select_order_by, c.select_limit, c.select_offset, c.special_fields, c2.outputs AS prev_component_output, d.name AS database_name, ca.type_id AS auth_type_id, ca.key_id AS auth_key_id, k.name AS auth_secret_name, cc.input_is_array, cc.input_key FROM ApiDesignComponents c LEFT JOIN ( SELECT c2.outputs, c2.next_component_id FROM ApiDesignComponents c2, ApiDesignComponentTypes t WHERE c2.deleted = false AND c2.type_id = t.id AND t.category_id != 5 ) c2 ON c.id = c2.next_component_id LEFT JOIN DatabaseConnectDetails d ON c.db_secret_id = d.id AND d.deleted = false LEFT JOIN ApiDesignComponentApiAuthentications ca ON c.id = ca.component_id AND ca.deleted = false AND ca.active = true LEFT JOIN ApiDesignComponentApiKeys k ON ca.key_id = k.id AND k.deleted = false LEFT JOIN ApiDesignComponentConfigs cc ON c.id = cc.component_id AND cc.deleted = false, ApiDesignProjects p, ApiDesignComponentTypes t, ApiDesignComponentCategories ac WHERE c.project_id = $1 AND c.deleted = false AND c.type_id = t.id AND t.category_id = ac.id; `; const values = [projectId]; const result = await pool.query(query, values).then(result => { return result.rows; }).catch(err => { console.log(\"query failed, error: \", err); throw { status: 500, message: err.message }; }); console.log(\"Result.length: \", result.length); return result; } catch(err) { console.error(\"query failed, err: \", err); throw err; } };`\n\nOnly answer with the input and output format in the following style:\n\ninput: { body: {key1: type1}, params: {key2: type2}, query: {key3: type3} }\noutput: {key1: type1}"}],
"temperature": 0
}
Desirable Output:
This always shows in chat window and sometimes shows in API call.
input: {
body: {},
params: { projectId: number },
query: {}
}
output: [
{
id: number,
category_id: number,
category: string,
type_id: number,
type: string,
inputs: any,
outputs: any,
tables: any,
fields: any,
rules: any,
edit_rules: any,
error_handlers: any,
url: string,
location_x: number,
location_y: number,
next_component_id: number | null,
data_conditions: any,
table_relations: any,
group_fields: any,
db_secret_id: number | null,
custom_fields: any,
select_order_by: string | null,
select_limit: number | null,
select_offset: number | null,
special_fields: any,
prev_component_output: any | null,
database_name: string | null,
auth_type_id: number | null,
auth_key_id: number | null,
auth_secret_name: string | null,
input_is_array: boolean | null,
input_key: string | null
}
]
Undesirable Output:
This sometimes shows in API calls.
Input and output format for the function:\n\nInput: \n```json\n{\n \"body\": {},\n \"params\": {\n \"projectId\": \"string\"\n },\n \"query\": {}\n}\n```\n\nOutput:\n```json\n[\n {\n \"id\": \"number\",\n \"category_id\": \"number\",\n \"category\": \"string\",\n \"type_id\": \"number\",\n \"type\": \"string\",\n \"inputs\": \"string\",\n \"outputs\": \"string\",\n \"tables\": \"string\",\n \"fields\": \"string\",\n \"rules\": \"string\",\n \"edit_rules\": \"string\",\n \"error_handlers\": \"string\",\n \"url\": \"string\",\n \"location_x\": \"number\",\n \"location_y\": \"number\",\n \"next_component_id\": \"number\",\n \"data_conditions\": \"string\",\n \"table_relations\": \"string\",\n \"group_fields\": \"string\",\n \"db_secret_id\": \"number\",\n \"custom_fields\": \"string\",\n \"select_order_by\": \"string\",\n \"select_limit\": \"number\",\n \"select_offset\": \"number\",\n \"special_fields\": \"string\",\n \"prev_component_output\": \"string\",\n \"database_name\": \"string\",\n \"auth_type_id\": \"number\",\n \"auth_key_id\": \"number\",\n \"auth_secret_name\": \"string\",\n \"input_is_array\": \"boolean\",\n \"input_key\": \"string\"\n }\n]\n```
I wonder why API would provide a lower and less reliable quality response then chat window. I have put temperature to 0 to make the output as fixed as possible.