I can write you code. It has gpt-4-turbo prices hard-coded.
Scroll down to the bottom, edit in your own system and user message in the script, run, see what it would cost to send (the code can’t send anything).
This uses tiktoken, which you can install in your environment with
pip install tiktoken
import re
import tiktoken
class Tokenizer:
""" required: import tiktoken; import re;
usage example:
cl100 = Tokenizer()
number_of_tokens = cl100.count("my string")
"""
def __init__(self, model="cl100k_base"):
self.tokenizer = tiktoken.get_encoding(model)
self.chat_strip_match = re.compile(r'<\|.*?\|>')
self.intype = None
self.inprice = round(0.01/1000, 6) ### hardcoded GPT-4-Turbo prices
self.outprice = round(0.03/1000, 6)
def ucount(self, text):
encoded_text = self.tokenizer.encode(text)
return len(encoded_text)
def count(self, text):
text = self.chat_strip_match.sub('', text)
encoded_text = self.tokenizer.encode(text)
return len(encoded_text)
def outputprice(self, text):
return self.ucount(text) * self.outprice
def inputprice(self, text):
return self.ucount(text) * self.inprice
def message(self, message):
"""
Extends the input message dictionary or list of dictionaries with a 'tokens' field,
which contains the token count of the 'role' and 'content' fields
(and optionally the 'name' field). The token count is calculated using the
'scount' method, which strips out any text enclosed within "<|" and "|>" before counting the tokens.
Args:
message (dict or list): A dictionary or a list of dictionaries. The ChatML format.
Each dictionary must have a 'role' field and a 'content' field, and may optionally
have a 'name' field. The 'role' and 'content' fields are strings, and the
'name' field, if present, is also a string.
Returns:
The input message dictionary or list of dictionaries, extended with a 'tokens' field
in each dictionary. The 'tokens' field contains the token count of the 'role' and
'content' fields (and optionally the 'name' field), calculated using the 'scount'
method. The total token count also includes a fixed overhead of 3 control tokens.
Raises:
KeyError: If a dictionary does not have a 'role' or 'content' field.
"""
if isinstance(message, str):
self.intype = string
message = dict(message)
if isinstance(message, dict):
self.intype = dict
message = [message]
elif isinstance(message, list):
self.intype = list
else:
raise ValueError("no supported format in message")
for msg in message:
role_string = msg['role']
if 'name' in msg:
role_string += ':' + msg['name']
role_tokens = self.count(role_string)
content_tokens = self.count(msg['content'])
msg['tokens'] = 3 + role_tokens + content_tokens
msg['price'] = round(msg['tokens'] * self.inprice, 6)
return message if len(message) > 1 else message[0]
####### Actually using those functions starts here
token = Tokenizer()
system = {"role":"system", "content": """
You are an AI. You need some more instructions of how to behave though.
""".strip()}
user = {"role":"user", "content": """
I am a human, and I want to know how much it costs to send you a question.
I just type whatever else I want to ask.
""".strip()}
# concatenate message lists together like you do to send to API
messages = [system, user]
print("-- Here's messages after I add token count and token price metadata to them")
print(token.message(messages))
print("-- Here's the sum of all message prices")
total_price = sum(message['price'] for message in messages)
print(f"Input messages price: {total_price}")
# Example of measuring tokens in a file with a function you'd call
def measure_tokens_in_a_file(filename):
with open(filename, 'r') as file:
content = file.read()
print(f"This message has {token.count(content)} tokens.")
print(f"If it was sent as input, it would cost ${token.inputprice(content):.5f}")
print(f"If it was received as output, it would cost ${token.outputprice(content):.5f}")
# other uses of my functions
#print(token.count(user[0]['content'])) # the count() method just measures tokens
#print(token.message(system)) # the message() method gets input dict counts