Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for loading local manifests #58

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Add support for loading local manifests #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
189548c
Add support for installing pods from local manifests
cap10morgan Oct 7, 2022
6d99ecf
Update name of manifest load fn
cap10morgan Oct 10, 2022
951a629
Merge branch 'master' into feature/local-manifests
cap10morgan Oct 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 48 additions & 34 deletions src/babashka/pods/impl/resolver.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,13 @@
"https://raw.githubusercontent.com/babashka/pod-registry/master/manifests/%s/%s/manifest.edn"
qsym version))

(defn pod-manifest-file
[qsym version]
(io/file @pods-repo-dir (str qsym) (str version) "manifest.edn"))

(defn pod-manifest
[qsym version force?]
(let [f (io/file @pods-repo-dir (str qsym) (str version) "manifest.edn")]
(let [f (pod-manifest-file qsym version)]
(if (and (not force?)
(.exists f))
(edn/read-string (slurp f))
Expand Down Expand Up @@ -182,41 +186,51 @@
(.digest digest))
(String. "UTF-8"))))

(defn install-pod-artifacts
[artifacts {cdir :cache-dir, ddir :data-dir, :keys [:force? :pod/options]}]
(let [execs (mapv (fn [artifact]
(let [url (:artifact/url artifact)
file-name (last (str/split url #"/"))
cache-file (io/file cdir file-name)
exe-file-name (:artifact/executable artifact)
executable (io/file ddir exe-file-name)]
(when (or force? (not (.exists executable)))
(warn (format "Downloading pod %s" url))
(download url cache-file false)
(when-let [expected-sha (:artifact/hash artifact)]
(let [sha (sha256 cache-file)]
(when-not (= (str/replace expected-sha #"^sha256:" "")
sha)
(throw (ex-info (str "Wrong SHA-256 for file" (str cache-file))
{:sha sha
:expected-sha expected-sha})))))
(let [filename (.getName cache-file)]
(cond (str/ends-with? filename ".zip")
(unzip {:zip-file cache-file
:destination-dir ddir
:verbose false})
(or (str/ends-with? filename ".tgz")
(str/ends-with? filename ".tar.gz"))
(un-tgz cache-file ddir
false))
(.delete cache-file))
(make-executable ddir [exe-file-name] false)
(warn (format "Successfully installed pod %s" exe-file-name))
(io/file ddir exe-file-name))
(io/file ddir exe-file-name)))
artifacts)]
{:executable (.getAbsolutePath ^java.io.File (first execs))
:options options}))

(defn resolve [qsym version force?]
(when-not (string? version)
(throw (IllegalArgumentException. "Version must be provided for resolving from pod registry!")))
(when-let [manifest (pod-manifest qsym version force?)]
(let [artifacts (match-artifacts manifest)
cdir (cache-dir manifest)
(let [cdir (cache-dir manifest)
ddir (data-dir manifest)
execs (mapv (fn [artifact]
(let [url (:artifact/url artifact)
file-name (last (str/split url #"/"))
cache-file (io/file cdir file-name)
executable (io/file ddir (:artifact/executable artifact))]
(when (or force? (not (.exists executable)))
(warn (format "Downloading pod %s (%s)" qsym version))
(download url cache-file false)
(when-let [expected-sha (:artifact/hash artifact)]
(let [sha (sha256 cache-file)]
(when-not (= (str/replace expected-sha #"^sha256:" "")
sha)
(throw (ex-info (str "Wrong SHA-256 for file" (str cache-file))
{:sha sha
:expected-sha expected-sha})))))
(let [filename (.getName cache-file)]
(cond (str/ends-with? filename ".zip")
(unzip {:zip-file cache-file
:destination-dir ddir
:verbose false})
(or (str/ends-with? filename ".tgz")
(str/ends-with? filename ".tar.gz"))
(un-tgz cache-file ddir
false))
(.delete cache-file))
(make-executable ddir [(:artifact/executable artifact)] false)
(warn (format "Successfully installed pod %s (%s)" qsym version))
(io/file ddir (:artifact/executable artifact)))
(io/file ddir (:artifact/executable artifact)))) artifacts)]
{:executable (.getAbsolutePath ^java.io.File (first execs))
:options (:pod/options manifest)})))
artifacts (match-artifacts manifest)
pod-options (:pod/options manifest)]
(install-pod-artifacts artifacts {:cache-dir cdir
:data-dir ddir
:force? force?
:pod/options pod-options}))))
55 changes: 44 additions & 11 deletions src/babashka/pods/sci.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,17 +41,32 @@
(with-open [r (PushbackInputStream. (io/input-stream cache-file))]
(impl/read r)))))

(defn load-pod-metadata* [bb-edn-file pod-spec {:keys [:version :cache] :as opts}]
(let [metadata (impl/load-pod-metadata pod-spec opts)
cache-file (when (and metadata cache)
(metadata-cache-file bb-edn-file pod-spec opts))]
(defn cache-pod-metadata! [metadata {:keys [:bb-edn-file :pod-name :pod-version]}]
(let [cache-file (metadata-cache-file bb-edn-file pod-name
{:version pod-version})]
(when cache-file
(io/make-parents cache-file)
(when (fs/writable? (fs/parent cache-file))
(with-open [w (io/output-stream cache-file)]
(impl/write w metadata))))
(impl/write w metadata))))))

(defn load-pod-metadata* [bb-edn-file pod-spec {:keys [:version :cache] :as opts}]
(let [metadata (impl/load-pod-metadata pod-spec opts)]
(when (and metadata cache)
(cache-pod-metadata! metadata {:bb-edn-file bb-edn-file
:pod-name pod-spec
:pod-version version}))
metadata))

(defn pod-namespaces
[pod-name metadata opts]
(reduce
(fn [pod-namespaces ns]
(let [ns-sym (-> ns (get "name") impl/bytes->string symbol)]
(assoc pod-namespaces ns-sym {:pod-spec pod-name
:opts (assoc opts :metadata metadata)})))
{} (get metadata "namespaces")))

(defn load-pod-metadata
([pod-spec opts] (load-pod-metadata nil pod-spec opts))
([bb-edn-file pod-spec {:keys [:cache] :as opts}]
Expand All @@ -62,12 +77,7 @@
opts))]
cached-metadata
(load-pod-metadata* bb-edn-file pod-spec opts))]
(reduce
(fn [pod-namespaces ns]
(let [ns-sym (-> ns (get "name") impl/bytes->string symbol)]
(assoc pod-namespaces ns-sym {:pod-spec pod-spec
:opts (assoc opts :metadata metadata)})))
{} (get metadata "namespaces")))))
(pod-namespaces pod-spec metadata opts))))

(defn load-pod
([ctx pod-spec] (load-pod ctx pod-spec nil))
Expand Down Expand Up @@ -122,6 +132,29 @@
(sci/future (impl/processor pod))
{:pod/id (:pod-id pod)})))

(defn load-pod-metadata-from-manifest
[manifest {:keys [:bb-edn-file]}]
(let [artifacts (resolver/match-artifacts manifest)
pod-name (:pod/name manifest)]
(when artifacts
(let [cdir (resolver/cache-dir manifest)
ddir (resolver/data-dir manifest)
pod (resolver/install-pod-artifacts
artifacts {:cache-dir cdir
:data-dir ddir
:pod/options (:pod/options manifest)})
metadata (impl/run-pod-for-metadata [(:executable pod)] nil)]
(let [pod-version (:pod/version manifest)
opts {:bb-edn-file bb-edn-file
:pod-name pod-name
:pod-version pod-version}]
(cache-pod-metadata! metadata opts)
(pod-namespaces pod-name metadata {:version pod-version}))))))

(defn pod-manifest-file
[manifest]
(resolver/pod-manifest-file (-> manifest :pod/name symbol) (:pod/version manifest)))

(defn unload-pod
([pod-id] (unload-pod pod-id {}))
([pod-id _opts]
Expand Down