Adopt Sendable protocol for tokenizer protocols and conforming types

Author: mattt

Sources/Tokenizers/BPETokenizer.swift

@@ -9,7 +9,7 @@
 import Foundation
 import Hub
 
-struct BytePair: Hashable {
+struct BytePair: Hashable, Sendable {
     let a: String
     let b: String
     init(_ a: String, _ b: String) {
@@ -32,7 +32,7 @@ struct BytePair: Hashable {
     }
 }
 
-class BPETokenizer: PreTrainedTokenizerModel {
+class BPETokenizer: PreTrainedTokenizerModel, @unchecked Sendable {
     let bpeRanks: [BytePair: Int]
     private let tokensToIds: [NSString: Int]
     private let idsToTokens: [Int: NSString]

Sources/Tokenizers/BertTokenizer.swift

@@ -9,7 +9,7 @@
 import Foundation
 import Hub
 
-public class BertTokenizer {
+public class BertTokenizer: @unchecked Sendable {
     private let basicTokenizer: BasicTokenizer
     private let wordpieceTokenizer: WordpieceTokenizer
     private let maxLen = 512

Sources/Tokenizers/Tokenizer.swift

@@ -150,7 +150,7 @@ public enum ChatTemplateArgument {
     case name(String)
 }
 
-public protocol Tokenizer {
+public protocol Tokenizer: Sendable {
     func tokenize(text: String) -> [String]
 
     /// Main entry point
@@ -268,7 +268,7 @@ let specialTokenAttributes: [String] = [
     "additional_special_tokens",
 ]
 
-public class PreTrainedTokenizer: Tokenizer {
+public class PreTrainedTokenizer: @unchecked Sendable, Tokenizer {
     let model: TokenizingModel
 
     public var bosToken: String? { model.bosToken }
@@ -652,7 +652,7 @@ public extension AutoTokenizer {
 
 // MARK: - Tokenizer model classes
 
-class T5Tokenizer: UnigramTokenizer {}
+class T5Tokenizer: UnigramTokenizer, @unchecked Sendable {}
 
 // MARK: - PreTrainedTokenizer classes
 
@@ -701,7 +701,7 @@ func maybeUpdatePostProcessor(tokenizerConfig: Config, processorConfig: Config?)
 }
 
 /// See https://github.com/xenova/transformers.js/blob/1a9964fb09b8f54fcbeac46dc6aae8d76795809d/src/tokenizers.js#L3203 for these exceptions
-class LlamaPreTrainedTokenizer: PreTrainedTokenizer {
+class LlamaPreTrainedTokenizer: PreTrainedTokenizer, @unchecked Sendable {
     let isLegacy: Bool
 
     required init(tokenizerConfig: Config, tokenizerData: Config, strict: Bool = true) throws {

Sources/Tokenizers/UnigramTokenizer.swift

@@ -9,7 +9,7 @@
 import Foundation
 import Hub
 
-class UnigramTokenizer: PreTrainedTokenizerModel {
+class UnigramTokenizer: PreTrainedTokenizerModel, @unchecked Sendable {
     struct SentencePieceToken {
         var token: String
         var score: Float