feat: local tokenizer (#5508)

This commit is contained in:
jif-oai authored and GitHub committed 2025-10-22 16:01:02 +01:00
1 parent 00b1e130b3
commit fd0673e457
4 files changed
+218 -3

No files matched your search

+15
View File
@@ -0,0 +1,15 @@
[package]
edition.workspace = true
name = "codex-utils-tokenizer"
version.workspace = true
[lints]
workspace = true
[dependencies]
anyhow = { workspace = true }
thiserror = { workspace = true }
tiktoken-rs = "0.7"
[dev-dependencies]
pretty_assertions = { workspace = true }