diff --git a/crates/bpe-openai/Cargo.toml b/crates/bpe-openai/Cargo.toml index cc385c8..c20ee2c 100644 --- a/crates/bpe-openai/Cargo.toml +++ b/crates/bpe-openai/Cargo.toml @@ -21,7 +21,7 @@ unicode-normalization = "0.1" [dev-dependencies] bpe = { version = "0.2", path = "../bpe", features = ["rand"] } -tiktoken-rs = "0.6" +tiktoken-rs = "0.7" [build-dependencies] base64 = "0.22" diff --git a/crates/bpe-openai/src/lib.rs b/crates/bpe-openai/src/lib.rs index b6d9add..51f514f 100644 --- a/crates/bpe-openai/src/lib.rs +++ b/crates/bpe-openai/src/lib.rs @@ -242,12 +242,12 @@ mod tests { #[test] fn test_cl100k() { - test_equivalence(cl100k_base(), &cl100k_base_singleton().lock()); + test_equivalence(cl100k_base(), cl100k_base_singleton()); } #[test] fn test_o200k() { - test_equivalence(o200k_base(), &o200k_base_singleton().lock()); + test_equivalence(o200k_base(), o200k_base_singleton()); } #[track_caller] diff --git a/crates/bpe/Cargo.toml b/crates/bpe/Cargo.toml index 9e10955..fdf26aa 100644 --- a/crates/bpe/Cargo.toml +++ b/crates/bpe/Cargo.toml @@ -26,7 +26,7 @@ serde = { version = "1", features = ["derive"] } [dev-dependencies] bpe = { path = "." } -tiktoken-rs = "0.6" +tiktoken-rs = "0.7" [package.metadata.docs.rs] all-features = true diff --git a/crates/bpe/benchmarks/Cargo.toml b/crates/bpe/benchmarks/Cargo.toml index e2f0234..19216f1 100644 --- a/crates/bpe/benchmarks/Cargo.toml +++ b/crates/bpe/benchmarks/Cargo.toml @@ -22,5 +22,5 @@ bpe = { path = "../../bpe", features = ["rand", "tiktoken"] } bpe-openai = { path = "../../bpe-openai" } criterion = "0.6" rand = "0.9" -tiktoken-rs = "0.6" +tiktoken-rs = "0.7" tokenizers = { version = "0.21", features = ["http"] } diff --git a/crates/bpe/tests/Cargo.toml b/crates/bpe/tests/Cargo.toml index fa0b0f0..e061e95 100644 --- a/crates/bpe/tests/Cargo.toml +++ b/crates/bpe/tests/Cargo.toml @@ -7,4 +7,4 @@ bpe = { path = "../../bpe", features = ["rand"] } bpe-openai = { path = "../../bpe-openai" } itertools = "0.14" rand = "0.9" -tiktoken-rs = "0.6" +tiktoken-rs = "0.7" diff --git a/crates/bpe/tests/src/lib.rs b/crates/bpe/tests/src/lib.rs index 9c85b4a..6ea1322 100644 --- a/crates/bpe/tests/src/lib.rs +++ b/crates/bpe/tests/src/lib.rs @@ -89,7 +89,6 @@ mod tests { .unwrap(); let bpe = &cl100k_base().bpe; let encoded1 = cl100k_base_singleton() - .lock() .encode_ordinary(input) .into_iter() .collect_vec();