fix: link time version errors instead of runtime checks

Author: daulet

Cargo.lock

@@ -1,8 +1,6 @@
 [package]
 name = "tokenizers"
-# Bump minor.patch version every time we bump tokenizers dependency version.
-# Can't bump major version because Go doesn't like major version >= 2.
-version = "1.22.0"
+version = "1.22.1"
 edition = "2021"
 
 [lib]

Cargo.toml

@@ -2,4 +2,4 @@ module github.com/daulet/tokenizers/example
 
 go 1.22
 
-require github.com/daulet/tokenizers v1.21.1
+require github.com/daulet/tokenizers v1.22.0

example/go.mod

@@ -1,5 +1,5 @@
-github.com/daulet/tokenizers v1.21.1 h1:Y+cxx+6K06bCN+bTPQW72zZTTVlJl/8Hq7afVRKmNok=
-github.com/daulet/tokenizers v1.21.1/go.mod h1:tGnMdZthXdcWY6DGD07IygpwJqiPvG85FQUnhs/wSCs=
+github.com/daulet/tokenizers v1.22.0 h1:Bj7sRW99FpfGU4tNIp0s0j/IKr26pPHqYLb6i1oT+kA=
+github.com/daulet/tokenizers v1.22.0/go.mod h1:tGnMdZthXdcWY6DGD07IygpwJqiPvG85FQUnhs/wSCs=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

example/go.sum

@@ -6,7 +6,13 @@ use tokenizers::tokenizer::Tokenizer;
 use serde::{Deserialize, Serialize};
 use tiktoken_rs;
 
-const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION");
+// Version-specific symbol that will cause link failure if version doesn't match
+// Bump minor.patch version every time we bump tokenizers dependency version.
+// Can't bump major version because Go doesn't like major version >= 2.
+#[no_mangle]
+pub extern "C" fn tokenizers_version_1_22_1() {
+    // This function exists purely as a link-time version check
+}
 
 /// Truncation direction for tokenizer truncation
 #[repr(u8)]
@@ -168,11 +174,6 @@ pub struct tokenizers_buffer {
     len: usize,
 }
 
-#[no_mangle]
-pub extern "C" fn tokenizers_version() -> *const libc::c_char {
-    std::ffi::CString::new(CARGO_PKG_VERSION).unwrap().into_raw()
-}
-
 #[no_mangle]
 pub extern "C" fn tokenizers_from_bytes(bytes: *const u8, len: u32, opts: &tokenizers_options, error: *mut *mut libc::c_char) -> *mut libc::c_void {
     if bytes.is_null() {

src/lib.rs

@@ -6,6 +6,11 @@ package tokenizers
 #cgo LDFLAGS: -ltokenizers -ldl -lm -lstdc++
 #include <stdlib.h>
 #include "tokenizers.h"
+
+// Link-time version check: this will fail to link if the library version doesn't match
+// Using a global variable that references the function ensures the linker must resolve it
+extern void tokenizers_version_1_22_1(void);
+void (*tokenizers_version_check)(void) = &tokenizers_version_1_22_1;
 */
 import "C"
 
@@ -21,11 +26,7 @@ import (
 	"unsafe"
 )
 
-const (
-	WANT_VERSION = "1.22.0"
-
-	baseURL = "https://huggingface.co"
-)
+const baseURL = "https://huggingface.co"
 
 // List of necessary tokenizer files and their mandatory status.
 // True means mandatory, false means optional.
@@ -37,14 +38,6 @@ var tokenizerFiles = map[string]bool{
 	"added_tokens.json":       false,
 }
 
-func init() {
-	version := C.tokenizers_version()
-	got := C.GoString(version)
-	if got != WANT_VERSION {
-		panic(fmt.Errorf("tokenizers library version mismatch, want: %s, got: %s", WANT_VERSION, got))
-	}
-}
-
 type Tokenizer struct {
 	tokenizer unsafe.Pointer
 }