It takes too much time to evaluate the new fine-tuned model against usage policies. It took 17 minutes for my newest fine-tuned model. Is there a way to improve this?
1 Like
You can still use the model during this. You may need to manually find it in your models list. My understanding is that you just canât use top_p and temperature until checks complete.
1 Like
How do we use the model? I donât get a fine tuned job id. Is there a way to use it on openai api?
A fine tuning model is indeed used on the pay-per-use API, the whole point of the exercise in creating one.
If you created via API call, youâd have a fine-tuning job object as the return, an ID you can use to poll and ultimately get the model name. (read API Reference)
Easier is the Platform siteâs dashboard â fine-tuning UI
Even easier is a Python script I pounded out for fun from different pieces, also making it non-dependent on openai or external libraries.
"""Call the OpenAI models endpoint, print filtered sorted models (non-SDK)"""
STARTS_WITH_FILTER = ["gpt-", "o", "ft:gpt", "ft:o4", "co"] # all chat models as reference
BLACKLIST_STRING = ["instruct", "moderation", "realtime"] # non-chat model detection
# CUSTOMIZED: get only finetune models, and don't report incremental step models
STARTS_WITH_FILTER = ["ft:",] # model name must match this starting
BLACKLIST_STRING = ["step"]
recent_days = 90 # model date cutoff, using "created" field
def get_openai_models(api_key: str | None = None) -> list[dict]:
"""
Fetch the list of available OpenAI models. Uses minimum library dependency.
Also passes optional OPENAI_ORG_ID and OPENAI_PROJECT_ID if environment variables.
Args:
api_key: Optional override for the bearer token. If omitted, the
OPENAI_API_KEY environment variable is retrieved.
Returns:
A list of dictionaries, each describing a model (the serverâs ``data``
array).
"""
import json
import os
import ssl
from urllib.request import Request, urlopen
# --- authentication token -------------------------------------------------
token = api_key or os.environ.get("OPENAI_API_KEY")
if not token:
raise ValueError("An OpenAI API key is required (env OPENAI_API_KEY).")
headers: dict[str, str] = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
}
if org_id := os.environ.get("OPENAI_ORG_ID"):
headers["OpenAI-Organization"] = org_id
if proj_id := os.environ.get("OPENAI_PROJECT_ID"):
headers["OpenAI-Project"] = proj_id
ssl_ctx = ssl.create_default_context()
ssl_ctx.minimum_version = ssl.TLSVersion.TLSv1_2 # enforce â„ TLS 1.2 against proxy
req = Request("https://api.openai.com/v1/models", headers=headers, method="GET")
with urlopen(req, context=ssl_ctx, timeout=30) as resp:
if resp.status != 200:
raise RuntimeError(f"OpenAI API request failed (HTTP {resp.status})")
body = resp.read().decode("utf-8")
data = json.loads(body).get("data")
if not isinstance(data, list):
raise RuntimeError("Unexpected response format: missing 'data' list")
# keep only the fields we care about (drops useless 'object': 'model')
whitelist = {"id", "created", "owned_by"}
for model_dict in data:
for k in list(model_dict): # list() avoids runtime mutation error
if k not in whitelist:
model_dict.pop(k, None) # drop any nonâwhitelisted key
return data
def model_filtering(model_dict_list):
model_dict_list[:] = [
d for d in model_dict_list
if any(d['id'].startswith(prefix) for prefix in STARTS_WITH_FILTER)
and not any(bl_item in d['id'] for bl_item in BLACKLIST_STRING)
]
def sort_models_by_owner_group(models: list[dict]) -> None:
"""
Sort *models* endpoint return data in place. See your own sorted fine-tune ids at the end.
Order rules
-----------
1. All models whose `owned_by` is a standard OpenAI model
come first (they share the same precedence).
2. The remaining groups are `owned_by` your organizations and sorted.
3. Inside each group, models are ordered by `id`.
"""
priority_owners = ("openai", "system", "openai-internal")
models.sort(
key=lambda d: (
0 if d["owned_by"] in priority_owners else 1, # bucket index
"" if d["owned_by"] in priority_owners else d["owned_by"],
d["id"],
)
)
def sort_models_by_created(
models: list[dict],
oldest_first: bool = False,
recent_days=0, # Optional
) -> None:
"""
Good for finding when new models are made, also adds a human-readable key.
Modifies the passed mutable model list of dicts, sorting by date.
"""
import datetime
if isinstance(recent_days, int) and recent_days > 0:
cutoff_ts = (
datetime.datetime.now(datetime.UTC)
- datetime.timedelta(days=recent_days)
).timestamp()
models[:] = [m for m in models if m.get("created", 0) >= cutoff_ts]
models.sort(key=lambda d: d.get("created", 0), reverse=not oldest_first)
import datetime
for d in models:
ts = d.get("created")
if isinstance(ts, (int, float)):
d["created_at"] = (
datetime.datetime.fromtimestamp(ts, datetime.UTC)
.isoformat(timespec="seconds")
.replace("+00:00", "")
)
try:
model_dict_list = get_openai_models()
model_filtering(model_dict_list)
sort_models_by_created(model_dict_list, recent_days=recent_days)
for model_dict in model_dict_list:
print(f"{model_dict["created_at"]}: {model_dict["id"]}")
# You want just IDs? Build and display the surviving model IDs as string list
filtered_models = [d['id'] for d in model_dict_list]
#print("\n" + ", ".join(filtered_models))
filtered_models_id_sorted = sorted(d['id'] for d in model_dict_list) # sort by just name
except Exception as err:
from urllib.error import HTTPError, URLError
import json
if isinstance(err, HTTPError):
# HTTP layer failure (4xx / 5xx)
print(f"[HTTPError] {err.code} {err.reason} â {err.url}")
try:
body = err.read().decode("utf-8", "replace")
print("â Response body (truncated):", body[:400])
except Exception:
pass
elif isinstance(err, URLError):
# DNS / TLS / connectivity
print(f"[URLError] {err.reason}")
import ssl
if isinstance(err.reason, ssl.SSLError):
print("â TLS handshake failed (check Python/OpenSSL â„ TLS1.2)")
elif isinstance(err, json.JSONDecodeError):
# Bad / unexpected JSON payload
print("[JSONDecodeError] Could not parse API response:", err)
elif isinstance(err, ValueError):
# Typically missing API key or bad parameter
print("[ValueError]", err)
else:
# Anything not specifically handled
print("[Unhandled]", err)
import sys, traceback
traceback.print_exc(file=sys.stderr)
The globals at the start are used by filtering functions to deliver just the recent date-sorted fine-tuning models of your organization.
I know that, but the model isnât ready while openai is âevaluating model against their policyâ which takes forever (~20 mins). And we donât get a fine-tuned-model id when we reach that point in the queue. So it is in fact NOT available to use before policy evals. Right?
It seems the solution is to adjust your expectations of fine tuning, vs âforeverâ (~5 billion more years of life possible on Earth)
Are you a bot? Or am I talking to a real human?
Detected bots get banned. You instead, get to choke on a model checker UI app in Python to monitor when new filtered chat model names come down the pike, since chaining some API calls per documentation is a real head-scratcherâŠ
import sys
import asyncio
import threading
from datetime import datetime, timedelta, UTC
from PyQt5.QtCore import Qt, pyqtSignal, QObject, QTimer
from PyQt5.QtWidgets import (
QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QTextEdit
)
from PyQt5.QtGui import QFont, QColor, QPalette
from openai import AsyncOpenAI
class ModelPoller(QObject):
new_model_detected = pyqtSignal(str, str) # (model_id, detection_time)
last_checked = pyqtSignal(str) # (check_time)
error_occurred = pyqtSignal(str)
def __init__(self):
super().__init__()
self._stop_event = threading.Event()
self._client = AsyncOpenAI()
self._known_models = set()
self.starts_with = ["gpt", "ft:gpt", "o"]
self.blacklist = ["instruct", "omni", "realtime", "audio", "search", "research", "tts"]
def stop(self):
self._stop_event.set()
def starts_with_any(self, model_name, prefixes):
return any(model_name.startswith(prefix) for prefix in prefixes)
def contains_blacklisted(self, model_name, blacklist_items):
return any(blacklisted in model_name for blacklisted in blacklist_items)
async def fetch_filtered_models(self):
try:
model_obj = await self._client.models.list()
except Exception as err:
self.error_occurred.emit(f"API call failed: {err}")
return None
model_dict = model_obj.model_dump().get('data', [])
model_list = sorted([model['id'] for model in model_dict])
filtered_models = [
model for model in model_list
if self.starts_with_any(model, self.starts_with) and not self.contains_blacklisted(model, self.blacklist)
]
return filtered_models
async def preload(self):
"""Fetch initial model list and set as known, but do not log anything."""
models = await self.fetch_filtered_models()
if models is None:
await asyncio.sleep(10)
return await self.preload()
self._known_models = set(models)
async def poll_loop(self):
await self.preload()
while not self._stop_event.is_set():
now = datetime.now(UTC)
next_minute = (now + timedelta(minutes=1)).replace(second=0, microsecond=0)
wait_seconds = (next_minute - now).total_seconds()
await asyncio.sleep(wait_seconds)
if self._stop_event.is_set():
break
check_time = datetime.now(UTC).replace(second=0, microsecond=0)
check_time_str = check_time.strftime('%Y-%m-%d %H:%M:%S UTC')
self.last_checked.emit(check_time_str)
models = await self.fetch_filtered_models()
if self._stop_event.is_set():
break
if models is None:
await asyncio.sleep(10)
continue
new_models = sorted(set(models) - self._known_models)
for model in new_models:
self.new_model_detected.emit(model, check_time_str)
self._known_models.update(new_models)
def run(self):
asyncio.set_event_loop(asyncio.new_event_loop())
loop = asyncio.get_event_loop()
loop.run_until_complete(self.poll_loop())
class MainWindow(QMainWindow):
def __init__(self):
super().__init__()
self.setWindowTitle("OpenAI Model Monitor")
self.setMinimumSize(400, 200)
self.setStyleSheet("""
QMainWindow { background: #23272e; }
QLabel, QTextEdit { color: #f0f0f0; font-size: 15px; }
QTextEdit { background: #181c20; border: 1px solid #353b45; }
""")
central = QWidget()
self.setCentralWidget(central)
vlayout = QVBoxLayout(central)
vlayout.setSpacing(10)
# Top row: Last checked (left), stretch, UTC clock (right)
hlayout = QHBoxLayout()
self.timeLabel = QLabel("Last checked: â")
self.timeLabel.setFont(QFont("Segoe UI", 13))
hlayout.addWidget(self.timeLabel)
hlayout.addStretch()
self.clockLabel = QLabel("--:--:--")
self.clockLabel.setFont(QFont("Consolas", 15, QFont.Bold))
hlayout.addWidget(self.clockLabel)
vlayout.addLayout(hlayout)
self.logArea = QTextEdit()
self.logArea.setReadOnly(True)
self.logArea.setFont(QFont("Consolas", 12))
self.logArea.setPlaceholderText(" - What awaits?...")
vlayout.addWidget(self.logArea, stretch=1)
# Live UTC clock timer
self.clock_timer = QTimer(self)
self.clock_timer.timeout.connect(self.update_clock)
self.clock_timer.start(1000)
self.update_clock()
# Model poller thread
self.poller = ModelPoller()
self.poller_thread = threading.Thread(target=self.poller.run, daemon=True)
self.poller.new_model_detected.connect(self.on_new_model_detected)
self.poller.last_checked.connect(self.on_last_checked)
self.poller.error_occurred.connect(self.on_error)
self.poller_thread.start()
def closeEvent(self, event):
self.poller.stop()
self.poller_thread.join(timeout=5)
super().closeEvent(event)
def update_clock(self):
now = datetime.now(UTC)
self.clockLabel.setText(now.strftime("%H:%M:%S"))
def on_new_model_detected(self, model, detection_time):
self.logArea.append(f"{detection_time}: {model}")
def on_last_checked(self, check_time):
self.timeLabel.setText(f"Last checked: {check_time}")
def on_error(self, msg):
self.logArea.append(f"[ERROR] {msg}")
def main():
app = QApplication(sys.argv)
app.setStyle("Fusion")
dark_palette = QPalette()
dark_palette.setColor(QPalette.Window, QColor(35, 39, 46))
dark_palette.setColor(QPalette.WindowText, Qt.white)
dark_palette.setColor(QPalette.Base, QColor(24, 28, 32))
dark_palette.setColor(QPalette.AlternateBase, QColor(35, 39, 46))
dark_palette.setColor(QPalette.ToolTipBase, Qt.white)
dark_palette.setColor(QPalette.ToolTipText, Qt.white)
dark_palette.setColor(QPalette.Text, Qt.white)
dark_palette.setColor(QPalette.Button, QColor(45, 54, 64))
dark_palette.setColor(QPalette.ButtonText, Qt.white)
dark_palette.setColor(QPalette.BrightText, Qt.red)
dark_palette.setColor(QPalette.Highlight, QColor(91, 138, 255))
dark_palette.setColor(QPalette.HighlightedText, Qt.white)
app.setPalette(dark_palette)
window = MainWindow()
window.show()
sys.exit(app.exec_())
if __name__ == "__main__":
main()