Conform tokenizers to Sendable without @unchecked

Author: mattt

Sources/Tokenizers/BertTokenizer.swift

@@ -9,7 +9,7 @@
 import Foundation
 import Hub
 
-public class BertTokenizer: @unchecked Sendable {
+public final class BertTokenizer: Sendable {
     private let basicTokenizer: BasicTokenizer
     private let wordpieceTokenizer: WordpieceTokenizer
     private let maxLen = 512
@@ -18,10 +18,10 @@ public class BertTokenizer: @unchecked Sendable {
     private let vocab: [String: Int]
     private let ids_to_tokens: [Int: String]
 
-    public var bosToken: String?
-    public var bosTokenId: Int?
-    public var eosToken: String?
-    public var eosTokenId: Int?
+    public let bosToken: String?
+    public let bosTokenId: Int?
+    public let eosToken: String?
+    public let eosTokenId: Int?
 
     public let fuseUnknownTokens: Bool
 
@@ -177,7 +177,7 @@ extension BertTokenizer: PreTrainedTokenizerModel {
     }
 }
 
-class BasicTokenizer {
+final class BasicTokenizer: Sendable {
     let doLowerCase: Bool
 
     init(doLowerCase: Bool = true) {
@@ -243,7 +243,7 @@ private extension Character {
     }
 }
 
-class WordpieceTokenizer {
+final class WordpieceTokenizer: Sendable {
     let unkToken = "[UNK]"
     private let maxInputCharsPerWord = 100
     private let vocab: [String: Int]