Merge branch 'main' into backtrack

Author: ArthurZucker

.github/workflows/python-release-conda.yml

@@ -37,7 +37,7 @@ jobs:
             platform: linux
           - os: windows
             ls: dir
-            interpreter: 3.9 3.10 3.11 3.12 3.13 pypy3.8 pypy3.9 pypy3.10
+            interpreter: 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10
           - os: windows
             ls: dir
             target: x86_64
@@ -54,7 +54,7 @@ jobs:
               #   interpreter: 3.11 3.12
           - os: macos
             target: aarch64
-            interpreter: 3.9 3.10 3.11 3.12 3.13 pypy3.8 pypy3.9 pypy3.10
+            interpreter: 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10
           - os: ubuntu
             platform: linux
             target: i686

.github/workflows/python-release.yml

@@ -55,6 +55,7 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        
 
       - name: Install Rust
         uses: actions-rs/toolchain@v1
@@ -76,7 +77,7 @@ jobs:
 
 
       - name: Cache Cargo Registry
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.cargo/registry
           key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
@@ -107,7 +108,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: audit
-          args: -D warnings -f ./bindings/python/Cargo.lock
+          args: -D warnings -f ./bindings/python/Cargo.lock  --ignore RUSTSEC-2024-0436 --ignore RUSTSEC-2025-0014
 
       - name: Install
         working-directory: ./bindings/python

.github/workflows/python.yml

@@ -10,7 +10,7 @@ jobs:
   build:
     runs-on: ${{ matrix.os }}
     env:
-      MACOSX_DEPLOYMENT_TARGET: 10.11
+      MACOSX_DEPLOYMENT_TARGET: 10.12
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
@@ -94,7 +94,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: audit
-          args: -D warnings -f ./tokenizers/Cargo.lock
+          args: -D warnings -f ./tokenizers/Cargo.lock --ignore RUSTSEC-2024-0436 --ignore RUSTSEC-2025-0014
 
       # Verify that Readme.md is up to date.
       - name: Make sure, Readme generated from lib.rs matches actual Readme

.github/workflows/rust.yml

@@ -12,4 +12,7 @@ jobs:
       with:
         fetch-depth: 0
     - name: Secret Scanning
-      uses: trufflesecurity/trufflehog@main
+      uses: trufflesecurity/trufflehog@853e1e8d249fd1e29d0fcc7280d29b03df3d643d
+      with:
+        # exclude buggy postgres detector that is causing false positives and not relevant to our codebase
+        extra_args: --results=verified,unknown --exclude-detectors=postgres

.github/workflows/trufflehog.yml

@@ -2,7 +2,7 @@
 authors = ["Nicolas Patry <nicolas@huggingface.co>"]
 edition = "2021"
 name    = "node"
-version = "0.21.0-dev.0"
+version = "0.21.2-dev.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

bindings/node/Cargo.toml

@@ -122,7 +122,7 @@ describe('Encoding', () => {
       expect(indexes).toEqual([3, 5])
     })
 
-    it('returns the corrent indexes with pair sequences', () => {
+    it('returns the correct indexes with pair sequences', () => {
       expect(encodingDual.wordToTokens(3, 0)).toEqual([3, 5])
       expect(encodingDual.wordToTokens(3, 1)).toEqual([8, 9])
     })

bindings/node/lib/bindings/encoding.test.ts

@@ -1835,12 +1835,12 @@ __metadata:
   linkType: hard
 
 "brace-expansion@npm:^1.1.7":
-  version: 1.1.11
-  resolution: "brace-expansion@npm:1.1.11"
+  version: 1.1.12
+  resolution: "brace-expansion@npm:1.1.12"
   dependencies:
     balanced-match: ^1.0.0
     concat-map: 0.0.1
-  checksum: faf34a7bb0c3fcf4b59c7808bc5d2a96a40988addf2e7e09dfbb67a2251800e0d14cd2bfc1aa79174f2f5095c54ff27f46fb1289fe2d77dac755b5eb3434cc07
+  checksum: 12cb6d6310629e3048cadb003e1aca4d8c9bb5c67c3c321bafdd7e7a50155de081f78ea3e0ed92ecc75a9015e784f301efc8132383132f4f7904ad1ac529c562
   languageName: node
   linkType: hard
 

bindings/node/yarn.lock

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers-python"
-version = "0.21.0-dev.0"
+version = "0.21.2-dev.0"
 authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
 edition = "2021"
 
@@ -14,17 +14,17 @@ serde = { version = "1.0", features = ["rc", "derive"] }
 serde_json = "1.0"
 libc = "0.2"
 env_logger = "0.11"
-pyo3 = { version = "0.23", features = ["abi3", "abi3-py39", "py-clone"] }
-numpy = "0.23"
+pyo3 = { version = "0.25", features = ["abi3", "abi3-py39", "py-clone"] }
+numpy = "0.25"
 ndarray = "0.16"
-itertools = "0.12"
+itertools = "0.14"
 
 [dependencies.tokenizers]
 path = "../../tokenizers"
 
 [dev-dependencies]
 tempfile = "3.10"
-pyo3 = { version = "0.23", features = ["auto-initialize"] }
+pyo3 = { version = "0.25", features = ["auto-initialize"] }
 
 [features]
-defaut = ["pyo3/extension-module"]
+default = ["pyo3/extension-module"]

bindings/python/Cargo.toml

@@ -33,7 +33,7 @@ class BPEDecoder(Decoder):
 
     Args:
         suffix (:obj:`str`, `optional`, defaults to :obj:`</w>`):
-            The suffix that was used to caracterize an end-of-word. This suffix will
+            The suffix that was used to characterize an end-of-word. This suffix will
             be replaced by whitespaces during the decoding
     """
     def __init__(self, suffix="</w>"):