Add support for loading modules as json or yaml

src/k16/kl/api/fs.clj

@@ -19,7 +19,14 @@
   (from-config-dir "config.edn"))
 
 (defn get-root-module-file ^java.io.File [group-name]
-  (from-modules-dir group-name "module.edn"))
+  (let [dir (from-modules-dir group-name)
+        file-name
+        (->> (.listFiles dir)
+             (map (fn [^java.io.File file] (.getName file)))
+             (filter (fn [file-name]
+                       (re-matches #"module\.(edn|yaml|yml|json)" file-name)))
+             first)]
+    (from-modules-dir group-name file-name)))
 
 (defn get-lock-file ^java.io.File [group-name]
   (from-modules-dir group-name "module.lock.edn"))

src/k16/kl/api/module.clj

@@ -1,11 +1,7 @@
 (ns k16.kl.api.module
   (:require
-   [clj-yaml.core :as yaml]
-   [clojure.edn :as edn]
-   [clojure.string :as str]
-   [jsonista.core :as json]
    [k16.kl.api.fs :as api.fs]
-   [k16.kl.api.module.parse :as module.parse]
+   [k16.kl.api.module.loader :as module.loader]
    [k16.kl.api.module.schema :as module.schema]
    [k16.kl.api.state :as api.state]
    [malli.core :as m]
@@ -14,27 +10,6 @@
 
 (set! *warn-on-reflection* true)
 
-(defn read-module-file [^java.io.File file]
-  (let [name (.getName file)
-        contents (slurp file)
-
-        ext (-> name (str/split #"\.") last keyword)
-
-        data (cond
-               (= ext :edn)
-               (edn/read-string contents)
-
-               (= ext :json)
-               (json/read-value contents)
-
-               (or (= ext :yml)
-                   (= ext :yaml))
-               (yaml/parse-string contents)
-
-               :else (throw (ex-info "Failed to read module file. Unsupported file format" {:file-name name})))]
-
-    (module.parse/parse-module-data data)))
-
 (defn- filter-by-known [left right]
   (->> left
        (filter (fn [[container-name]]
@@ -79,5 +54,5 @@
     final))
 
 (defn get-resolved-module [module-name modules]
-  (let [root-module (read-module-file (api.fs/get-root-module-file module-name))]
+  (let [root-module (module.loader/read-module-file (api.fs/get-root-module-file module-name))]
     (dissoc (merge-modules module-name root-module modules) :modules)))

src/k16/kl/api/resolver/downloader.clj → src/k16/kl/api/module/downloader.clj

@@ -1,11 +1,11 @@
-(ns k16.kl.api.resolver.downloader
+(ns k16.kl.api.module.downloader
   (:require
    [cli-matic.utils :as cli.util]
-   [clojure.edn :as edn]
    [clojure.string :as str]
+   [jsonista.core :as json]
    [k16.kl.api.fs :as api.fs]
    [k16.kl.api.github :as api.github]
-   [k16.kl.api.module.parse :as module.parse]
+   [k16.kl.api.module.loader :as module.loader]
    [k16.kl.log :as log]
    [promesa.core :as p]))
 
@@ -26,7 +26,17 @@
 
     (slurp (:body res))))
 
-(defn- replace-vars [contents vars]
+(defn- list-repo-files [identifier sha path]
+  (let [res (api.github/request {:path (str "/repos/" identifier "/contents/" path "?ref=" sha)
+                                 :headers {"Accept" "application/vnd.github.raw"}})]
+
+    (when (not= 200 (:status res))
+      (log/info (str "Failed to list files in " identifier "@" sha "/" path))
+      (cli.util/exit! (:body res) 1))
+
+    (json/read-value (:body res) json/keyword-keys-object-mapper)))
+
+(defn- replace-vars [vars contents]
   (->> vars
        (reduce (fn [acc [key value]]
                  (str/replace acc (str "{{" (name key) "}}") value))
@@ -35,10 +45,18 @@
 (defn download-module-config
   ([module-ref] (download-module-config module-ref {}))
   ([{:keys [url sha subdir] :or {subdir ".kl"}} vars]
-   (-> (read-repo-file url sha (relative-to subdir "module.edn"))
-       (replace-vars vars)
-       edn/read-string
-       module.parse/parse-module-data)))
+   (let [file-name (->> (list-repo-files url sha subdir)
+                        (map (fn [file]
+                               (-> (:path file)
+                                   (str/split #"\/")
+                                   last)))
+                        (filter (fn [file]
+                                  (re-matches #"module\.(edn|yaml|yml|json)" file)))
+                        first)]
+
+     (->> (read-repo-file url sha (relative-to subdir file-name))
+          (replace-vars vars)
+          (module.loader/parse-module-contents file-name)))))
 
 (defn download-remote-module! [{:keys [module-name submodule-name module-ref]}]
   (let [{:keys [sha url subdir]
@@ -61,8 +79,8 @@
              (map (fn [file]
                     (p/vthread
                      (log/info (str "Downloading " file " [" submodule-name "]"))
-                     (let [contents (-> (read-repo-file url sha (relative-to subdir file))
-                                        (replace-vars vars))]
+                     (let [contents (->> (read-repo-file url sha (relative-to subdir file))
+                                         (replace-vars vars))]
                        (spit (api.fs/from-submodule-dir module-name submodule-name file) contents)))))))
 
       (api.fs/write-edn (api.fs/from-submodule-dir module-name submodule-name "module.edn") module))))

src/k16/kl/api/module/loader.clj

@@ -0,0 +1,44 @@
+(ns k16.kl.api.module.loader
+  (:require
+   [clj-yaml.core :as yaml]
+   [clojure.edn :as edn]
+   [clojure.string :as str]
+   [clojure.walk :as walk]
+   [jsonista.core :as json]
+   [k16.kl.api.module.schema :as module.schema]
+   [malli.core :as m]
+   [malli.error :as me]
+   [malli.transform :as mt]))
+
+(defn parse-module-contents [file-name module-contents]
+  (let [ext (-> file-name (str/split #"\.") last keyword)
+
+        module-data
+        (cond
+          (= ext :edn)
+          (edn/read-string module-contents)
+
+          (= ext :json)
+          (json/read-value module-contents)
+
+          (or (= ext :yml)
+              (= ext :yaml))
+          (yaml/parse-string module-contents)
+
+          :else (throw (ex-info "Failed to parse module. Unsupported file format" file-name)))]
+
+    (try
+      (m/coerce module.schema/?PartialModule
+                (walk/keywordize-keys module-data)
+                mt/json-transformer)
+      (catch Exception e
+        (let [{:keys [type data]} (ex-data e)]
+          (when-not (= type :malli.core/invalid-input)
+            (throw e))
+
+          (throw (ex-info "Module invalid" {:reason (me/humanize (:explain data))})))))))
+
+(defn read-module-file [^java.io.File file]
+  (let [name (.getName file)
+        contents (slurp file)]
+    (parse-module-contents name contents)))

src/k16/kl/api/resolver.clj

@@ -4,7 +4,8 @@
    [jsonista.core :as json]
    [k16.kl.api.fs :as api.fs]
    [k16.kl.api.github :as api.github]
-   [k16.kl.api.resolver.downloader :as resolver.downloader]
+   [k16.kl.api.module.loader :as module.loader]
+   [k16.kl.api.module.downloader :as module.downloader]
    [k16.kl.log :as log]
    [promesa.core :as p]))
 
@@ -32,7 +33,7 @@
 
 (defn- resolve-module [partial-reference]
   (let [module-ref (resolve-module-ref partial-reference)
-        module (resolver.downloader/download-module-config module-ref)]
+        module (module.downloader/download-module-config module-ref)]
     {:ref module-ref
      :module module}))
 
@@ -112,7 +113,7 @@
      :modules (tree->modules tree')}))
 
 (defn pull! [module-name {:keys [update-lockfile? force?]}]
-  (let [module (api.fs/read-edn (api.fs/get-root-module-file module-name))
+  (let [module (module.loader/read-module-file (api.fs/get-root-module-file module-name))
         current-lock (api.fs/read-edn (api.fs/get-lock-file module-name))
 
         {:keys [lock modules]}
@@ -129,7 +130,7 @@
       (->> modules
            (map (fn [[submodule-name module-ref]]
                   (p/vthread
-                   (resolver.downloader/download-remote-module!
+                   (module.downloader/download-remote-module!
                     {:module-name module-name
                      :submodule-name (name submodule-name)
                      :module-ref module-ref}))))