pip install markdownify playwright beautifulsoup4
playwright install
import asyncio
from bs4 import BeautifulSoup
from markdownify import markdownify as md
from playwright.async_api import async_playwright
async def fetch_openai_pricing(url: str) -> str:
async with async_playwright() as p:
browser = await p.chromium.launch(headless=False, slow_mo=100)
context = await browser.new_context(
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
viewport={"width": 1280, "height": 720},
)
page = await context.new_page()
try:
await page.goto(url, wait_until="networkidle", timeout=60000)
await asyncio.sleep(5)
pre_raw_content = await page.content() #Cloudflare Page
if "cloudflare" in pre_raw_content.lower():
await page.wait_for_timeout(10000)
raw_content = await page.content()
soup = BeautifulSoup(raw_content, "html.parser")
sections = {}
current_heading = None
for element in soup.find_all(["a"]):
if element.name in ["a"]:
current_heading = element.get_text(strip=True)
sections[current_heading] = []
elif element.name == "table" and current_heading:
for tag in element.find_all(True):
tag.attrs = {}
sections[current_heading].append(str(element))
soup = BeautifulSoup(raw_content, "html.parser")
markdown_content = md(str(soup))
return markdown_content
except Exception as e:
print(f"Error fetching OpenAI pricing: {e}")
return ""
finally:
await browser.close()
md_pricing = await fetch_openai_pricing("https://platform.openai.com/docs/pricing")
The above code can get you model + pricing