Add module coercion and validation

Author: julienvincent

deps.edn

@@ -2,6 +2,7 @@
 
  :paths ["src" "resources"]
  :deps {org.clojure/clojure {:mvn/version "1.11.1"}
+        metosin/malli {:mvn/version "0.13.0"}
         meta-merge/meta-merge {:mvn/version "1.0.0"}
         metosin/jsonista {:mvn/version "0.3.7"}
         clj-commons/clj-yaml {:mvn/version "1.0.27"}

src/k16/kl/api/module.clj

@@ -1,11 +1,40 @@
 (ns k16.kl.api.module
   (:require
+   [clj-yaml.core :as yaml]
+   [clojure.edn :as edn]
+   [clojure.string :as str]
+   [jsonista.core :as json]
    [k16.kl.api.fs :as api.fs]
+   [k16.kl.api.module.parse :as module.parse]
+   [k16.kl.api.module.schema :as module.schema]
    [k16.kl.api.state :as api.state]
+   [malli.core :as m]
+   [malli.error :as me]
    [meta-merge.core :as metamerge]))
 
 (set! *warn-on-reflection* true)
 
+(defn read-module-file [^java.io.File file]
+  (let [name (.getName file)
+        contents (slurp file)
+
+        ext (-> name (str/split #"\.") last keyword)
+
+        data (cond
+               (= ext :edn)
+               (edn/read-string contents)
+
+               (= ext :json)
+               (json/read-value contents)
+
+               (or (= ext :yml)
+                   (= ext :yaml))
+               (yaml/parse-string contents)
+
+               :else (throw (ex-info "Failed to read module file. Unsupported file format" {:file-name name})))]
+
+    (module.parse/parse-module-data data)))
+
 (defn- filter-by-known [left right]
   (->> left
        (filter (fn [[container-name]]
@@ -35,12 +64,20 @@
 
         state-routes
         (filter-by-known (get-in state [:network :routes])
-                         (get-in merged-root [:network :routes]))]
+                         (get-in merged-root [:network :routes]))
+
+        final
+        (metamerge/meta-merge merged-root {:containers state-containers
+                                           :network {:services state-services
+                                                     :routes state-routes}})]
+
+    (when-not (m/validate module.schema/?Module final)
+      (throw (ex-info "Module invalid" {:reason (->> final
+                                                     (m/explain module.schema/?Module)
+                                                     me/humanize)})))
 
-    (metamerge/meta-merge merged-root {:containers state-containers
-                                       :network {:services state-services
-                                                 :routes state-routes}})))
+    final))
 
 (defn get-resolved-module [module-name modules]
-  (let [root-module (api.fs/read-edn (api.fs/get-root-module-file module-name))]
+  (let [root-module (read-module-file (api.fs/get-root-module-file module-name))]
     (dissoc (merge-modules module-name root-module modules) :modules)))

src/k16/kl/api/module/parse.clj

@@ -0,0 +1,19 @@
+(ns k16.kl.api.module.parse
+  (:require
+   [clojure.walk :as walk]
+   [k16.kl.api.module.schema :as module.schema]
+   [malli.core :as m]
+   [malli.error :as me]
+   [malli.transform :as mt]))
+
+(defn parse-module-data [module-data]
+  (try
+    (m/coerce module.schema/?PartialModule
+              (walk/keywordize-keys module-data)
+              mt/json-transformer)
+    (catch Exception e
+      (let [{:keys [type data]} (ex-data e)]
+        (when-not (= type :malli.core/invalid-input)
+          (throw e))
+
+        (throw (ex-info "Module invalid" {:reason (me/humanize (:explain data))}))))))

src/k16/kl/api/module/schema.clj

@@ -0,0 +1,65 @@
+(ns k16.kl.api.module.schema
+  (:require
+   [malli.core :as m]
+   [malli.util :as mu]))
+
+(set! *warn-on-reflection* true)
+
+(defn- make-recursively-optional [schema]
+  (let [modifier
+        (fn [schema]
+          (let [type (m/type schema)
+                properties (m/properties schema)]
+            (cond
+              (:kl/locked properties)
+              schema
+
+              (= :map type)
+              (mu/optional-keys schema)
+
+              :else
+              schema)))]
+
+    (m/walk schema (m/schema-walker modifier))))
+
+(def ?ModuleRef
+  [:map {:kl/locked true}
+   [:url :string]
+   [:sha {:optional true} :string]
+   [:ref {:optional true} :string]
+   [:subdir {:optional true} :string]])
+
+(def ?Endpoint
+  [:map
+   [:url :string]])
+
+(def ?Service
+  [:map
+   [:endpoints [:map-of :keyword ?Endpoint]]
+   [:default-endpoint :keyword]])
+
+(def ?Route
+  [:map
+   [:host :string]
+   [:path-prefix {:optional true} :string]
+   [:service :keyword]
+   [:endpoint {:optional true} :keyword]])
+
+(def ?Network
+  [:map
+   [:services {:optional true} [:map-of :keyword ?Service]]
+   [:routes {:optional true} [:map-of :keyword ?Route]]])
+
+(def ?Module
+  [:map
+   [:modules {:optional true} [:map-of :keyword ?ModuleRef]]
+
+   [:include {:optional true} [:sequential :string]]
+
+   [:network {:optional true} ?Network]
+
+   [:volumes {:optional true} [:map-of :keyword [:map]]]
+   [:containers {:optional true} [:map-of :keyword [:map]]]])
+
+(def ?PartialModule
+  (make-recursively-optional ?Module))

src/k16/kl/api/resolver/downloader.clj

@@ -5,6 +5,7 @@
    [clojure.string :as str]
    [k16.kl.api.fs :as api.fs]
    [k16.kl.api.github :as api.github]
+   [k16.kl.api.module.parse :as module.parse]
    [k16.kl.log :as log]
    [promesa.core :as p]))
 
@@ -36,7 +37,8 @@
   ([{:keys [url sha subdir] :or {subdir ".kl"}} vars]
    (-> (read-repo-file url sha (relative-to subdir "module.edn"))
        (replace-vars vars)
-       edn/read-string)))
+       edn/read-string
+       module.parse/parse-module-data)))
 
 (defn download-remote-module! [{:keys [module-name submodule-name module-ref]}]
   (let [{:keys [sha url subdir]