Skip config.json when loading tokenizers

Author: pcuenca

Sources/Hub/Hub.swift

@@ -80,7 +80,8 @@ public final class LanguageModelConfigurationFromHub: Sendable {
     public init(
         modelName: String,
         revision: String = "main",
-        hubApi: HubApi = .shared
+        hubApi: HubApi = .shared,
+        skipModelConfig: Bool = false
     ) {
         configPromise = Task.init {
             try await Self.loadConfig(modelName: modelName, revision: revision, hubApi: hubApi)
@@ -89,7 +90,8 @@ public final class LanguageModelConfigurationFromHub: Sendable {
 
     public init(
         modelFolder: URL,
-        hubApi: HubApi = .shared
+        hubApi: HubApi = .shared,
+        skipModelConfig: Bool = false
     ) {
         configPromise = Task {
             try await Self.loadConfig(modelFolder: modelFolder, hubApi: hubApi)
@@ -142,9 +144,11 @@ public final class LanguageModelConfigurationFromHub: Sendable {
     static func loadConfig(
         modelName: String,
         revision: String,
-        hubApi: HubApi = .shared
+        hubApi: HubApi = .shared,
+        skipModelConfig: Bool = false,
     ) async throws -> Configurations {
-        let filesToDownload = ["config.json", "tokenizer_config.json", "chat_template.jinja", "chat_template.json", "tokenizer.json"]
+        let modelConfig = skipModelConfig ? [] : ["config.json"]
+        let filesToDownload = modelConfig + ["tokenizer_config.json", "chat_template.jinja", "chat_template.json", "tokenizer.json"]
         let repo = Hub.Repo(id: modelName)
 
         do {

Sources/Tokenizers/Tokenizer.swift

@@ -630,7 +630,7 @@ public extension AutoTokenizer {
         hubApi: HubApi = .shared,
         strict: Bool = true
     ) async throws -> Tokenizer {
-        let config = LanguageModelConfigurationFromHub(modelName: model, hubApi: hubApi)
+        let config = LanguageModelConfigurationFromHub(modelName: model, hubApi: hubApi, skipModelConfig: true)
         guard let tokenizerConfig = try await config.tokenizerConfig else { throw TokenizerError.missingConfig }
         let tokenizerData = try await config.tokenizerData
 
@@ -642,7 +642,7 @@ public extension AutoTokenizer {
         hubApi: HubApi = .shared,
         strict: Bool = true
     ) async throws -> Tokenizer {
-        let config = LanguageModelConfigurationFromHub(modelFolder: modelFolder, hubApi: hubApi)
+        let config = LanguageModelConfigurationFromHub(modelFolder: modelFolder, hubApi: hubApi, skipModelConfig: true)
         guard let tokenizerConfig = try await config.tokenizerConfig else { throw TokenizerError.missingConfig }
         let tokenizerData = try await config.tokenizerData