added new models to count_tokens

This commit is contained in:
AlexHTW
2023-11-06 23:55:21 +01:00
parent 274e784e8e
commit 8a87563282

View File

@@ -436,7 +436,7 @@ class OpenAIHelper:
if model in GPT_3_MODELS + GPT_3_16K_MODELS:
tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n
tokens_per_name = -1 # if there's a name, the role is omitted
elif model in GPT_4_MODELS + GPT_4_32K_MODELS:
elif model in GPT_4_MODELS + GPT_4_32K_MODELS + GPT_4_128K_MODELS:
tokens_per_message = 3
tokens_per_name = 1
else: