MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-12 19:53:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tell embedding providers which tokenizer their model uses

Browse Source
This commit is contained in:
Thorsten Sommer committed 2026-09-12 20:16:27 +02:00
1 parent 5326a0bf6c
commit 672237a225
10 files changed
+116 -64
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/Tests/Models/TokenizerRuleTests.cs
+3
View File
@@ -33,6 +33,9 @@ public sealed class TokenizerRuleTests
[TestCase(LLMProviders.OPEN_AI, "gpt-4", "cl100k_base", Description = "The older encoding, which is where the 4 line stayed.")]
[TestCase(LLMProviders.OPEN_AI, "gpt-4-turbo", "cl100k_base")]
[TestCase(LLMProviders.OPEN_AI, "gpt-3.5-turbo", "cl100k_base")]
[TestCase(LLMProviders.OPEN_AI, "text-embedding-3-small", "cl100k_base", Description = "The embedding models stayed on the older encoding as well, and their dialog asks the same question.")]
[TestCase(LLMProviders.OPEN_AI, "text-embedding-3-large", "cl100k_base")]
[TestCase(LLMProviders.OPEN_AI, "text-embedding-ada-002", "cl100k_base")]
public void OpenAINamesAnEncodingRatherThanAFile(LLMProviders provider, string modelId, string encoding)
{
var tokenizer = provider.GetModelProfile(new Model(modelId, null)).Tokenizer;