mirror of
https://github.com/openai/codex.git
synced 2026-11-19 23:05:23 +00:00
feat: local tokenizer (#5508)
This commit is contained in:
1 parent
00b1e130b3
commit
fd0673e457
4 files changed
+218
-3
No files matched your search
@@ -0,0 +1,15 @@
|
||||
[package]
|
||||
edition.workspace = true
|
||||
name = "codex-utils-tokenizer"
|
||||
version.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tiktoken-rs = "0.7"
|
||||
|
||||
[dev-dependencies]
|
||||
pretty_assertions = { workspace = true }
|
||||
Reference in new issue
Block a user