resolved conflicts with main

Author: nico-martin

.github/workflows/publish.yml

@@ -0,0 +1,72 @@
+name: Publish to NPM
+
+permissions:
+  contents: write
+  id-token: write
+  packages: write
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version bump type"
+        required: true
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+        default: patch
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 24.x
+
+      - name: Log NPM version
+        run: npm -v
+
+      - name: Configure git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run format check
+        run: npm run format:check
+
+      - name: Run linter
+        run: npm run lint
+
+      - name: Build application
+        run: npm run build
+
+      - name: Run tests
+        run: npm test
+
+      - name: Bump version
+        run: npm version ${{ inputs.version }} -m "🔖 @huggingface/tokenizers@%s"
+
+      - name: Get new version
+        id: package-version
+        run: echo "version=$(node -p "require('./package.json').version")" >> $GITHUB_OUTPUT
+
+      - name: Push changes
+        run: |
+          git push
+          git push --tags
+
+      - name: Publish to NPM
+        run: npm publish

.gitignore

@@ -13,6 +13,7 @@ dist
 dist-ssr
 *.local
 types
+tests/data
 
 # Editor directories and files
 .vscode/*

.prettierrc

@@ -1,7 +1,7 @@
 {
   "overrides": [
     {
-      "files": ["tests/**/*.ts"],
+      "files": ["README.md", "tests/**/*.ts"],
       "options": {
         "printWidth": 10000000
       }

README.md

@@ -30,28 +30,37 @@ Run today's most used tokenizers directly in your browser or Node.js application
 npm install @huggingface/tokenizers
 ```
 
+Alternatively, you can use it via a CDN as follows:
+
+```html
+<script type="module">
+  import { Tokenizer } from "https://cdn.jsdelivr.net/npm/@huggingface/tokenizers";
+</script>
+```
+
 ## Usage
 
 ```javascript
-import { Tokenizer } from '@huggingface/tokenizers';
+import { Tokenizer } from "@huggingface/tokenizers";
 
-// Load files from the Hugging Face Hub 
+// Load files from the Hugging Face Hub
 const modelId = "HuggingFaceTB/SmolLM3-3B";
-const tokenizerJson = await fetch(`https://huggingface.co/${modelId}/resolve/main/tokenizer.json`).then(res => res.json());
-const tokenizerConfig = await fetch(`https://huggingface.co/${modelId}/resolve/main/tokenizer_config.json`).then(res => res.json());
+const tokenizerJson = await fetch(`https://huggingface.co/${modelId}/resolve/main/tokenizer.json`).then((res) => res.json());
+const tokenizerConfig = await fetch(`https://huggingface.co/${modelId}/resolve/main/tokenizer_config.json`).then((res) => res.json());
 
 // Create tokenizer
 const tokenizer = new Tokenizer(tokenizerJson, tokenizerConfig);
 
 // Tokenize text
-const tokens = tokenizer.tokenize('Hello World');  // ['Hello', 'ĠWorld']
-const encoded = tokenizer.encode('Hello World');   // { ids: [9906, 4435], tokens: ['Hello', 'ĠWorld'], attention_mask: [1, 1] }
-const decoded = tokenizer.decode(encoded.ids);     // 'Hello World'
+const tokens = tokenizer.tokenize("Hello World"); // ['Hello', 'ĠWorld']
+const encoded = tokenizer.encode("Hello World"); // { ids: [9906, 4435], tokens: ['Hello', 'ĠWorld'], attention_mask: [1, 1] }
+const decoded = tokenizer.decode(encoded.ids); // 'Hello World'
 ```
 
 ## Requirements
 
 This library expects two files from Hugging Face models:
+
 - `tokenizer.json` - Contains the tokenizer configuration
 - `tokenizer_config.json` - Contains additional metadata
 
@@ -60,6 +69,7 @@ This library expects two files from Hugging Face models:
 Tokenizers.js supports [Hugging Face tokenizer components](https://huggingface.co/docs/tokenizers/components):
 
 ### Normalizers
+
 - NFD
 - NFKC
 - NFC
@@ -73,6 +83,7 @@ Tokenizers.js supports [Hugging Face tokenizer components](https://huggingface.c
 - Sequence
 
 ### Pre-tokenizers
+
 - BERT
 - ByteLevel
 - Whitespace
@@ -84,19 +95,22 @@ Tokenizers.js supports [Hugging Face tokenizer components](https://huggingface.c
 - Digits
 
 ### Models
+
 - BPE (Byte-Pair Encoding)
 - WordPiece
 - Unigram
 - Legacy
 
 ### Post-processors
+
 - ByteLevel
 - TemplateProcessing
 - RobertaProcessing
 - BertProcessing
 - Sequence
 
 ### Decoders
+
 - ByteLevel
 - WordPiece
 - Metaspace

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@huggingface/tokenizers",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "🤗 Tokenizers.js: A pure JS/TS implementation of today's most used tokenizers",
   "type": "module",
   "main": "dist/tokenizers.min.mjs",
@@ -32,7 +32,11 @@
     "dev": "npm run clean && node scripts/dev.mjs",
     "lint": "eslint src --ext .ts,.tsx",
     "test": "node --experimental-vm-modules --expose-gc --max-old-space-size=4096 node_modules/jest/bin/jest.js --verbose --logHeapUsage --maxWorkers=10%",
-    "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\""
+    "format": "prettier --write .",
+    "format:check": "prettier --check ."
+  },
+  "publishConfig": {
+    "access": "public"
   },
   "devDependencies": {
     "@types/jest": "^30.0.0",

package.json

@@ -43,7 +43,7 @@ const build = async (outfile) => {
     ...minifyOptions,
   });
   reportSize(outfile);
-}
+};
 
 await build("dist/tokenizers.mjs");
 await build("dist/tokenizers.cjs");

scripts/build.mjs

@@ -79,7 +79,7 @@ export interface TokenizerJSON {
   added_tokens?: AddedToken[];
   normalizer?: TokenizerConfigNormalizer;
   pre_tokenizer?: TokenizerConfigPreTokenizer;
-  post_processor?: TokenConfigPostProcessor;
+  post_processor?: TokenizerConfigPostProcessor;
   decoder?: TokenizerConfigDecoder;
   model: TokenizerModelConfig;
 }

src/static/tokenizer.d.ts

@@ -6,7 +6,6 @@
     "jsx": "react-jsx",
     "moduleResolution": "bundler",
     "esModuleInterop": true,
-    "skipLibCheck": true,
     "declaration": true,
     "declarationDir": "types",
     "outDir": "types",