diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 054e574..5747f55 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,20 +9,21 @@ jobs: build: strategy: matrix: - os: [ubuntu-latest, macos-latest, macos-latest-xlarge] include: - - os: ubuntu-latest - binary-name: kl-linux-amd64 - - os: macos-latest - binary-name: kl-macos-amd64 - - os: macos-latest-xlarge - binary-name: kl-macos-arm64 + - arch: linux-amd64 + runs-on: ubuntu-latest + - arch: linux-arm64 + runs-on: arm-c8 + - arch: macos-amd64 + runs-on: macos-latest + - arch: macos-arm64 + runs-on: macos-latest-xlarge env: GITHUB_USERNAME: ${{ secrets.ORG_GITHUB_ACTOR }} GITHUB_TOKEN: ${{ secrets.ORG_GITHUB_TOKEN }} - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.runs-on }} steps: - name: Checkout git repo uses: actions/checkout@v2 @@ -32,24 +33,20 @@ jobs: - name: Fetch Tags run: git fetch --tags origin - - uses: actions/setup-java@v2 + - uses: graalvm/setup-graalvm@v1 with: - distribution: 'temurin' - java-version: 21 + java-version: '22' + distribution: 'graalvm' + github-token: ${{ secrets.GITHUB_TOKEN }} - uses: extractions/setup-just@v1 - uses: DeLaGuardo/setup-clojure@9.5 with: cli: latest - - uses: graalvm/setup-graalvm@v1 - with: - java-version: '21' - distribution: 'graalvm' - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Configure Auth run: | + mkdir -p ~/.m2/ cp ./build/settings.xml ~/.m2/settings.xml - name: Build Native Image @@ -57,13 +54,13 @@ jobs: - name: Pack binary run: | - tar -czf ${{ matrix.binary-name }}.tar.gz -C ./target kl + tar -czf kl-${{ matrix.arch }}.tar.gz -C ./target kl - uses: actions/upload-artifact@v3 with: - name: kl-native-images + name: binaries if-no-files-found: error - path: ${{ matrix.binary-name }}.tar.gz + path: kl-${{ matrix.arch }}.tar.gz release: runs-on: ubuntu-latest @@ -75,7 +72,7 @@ jobs: - name: Download Binary Artifacts uses: actions/download-artifact@v2 with: - name: kl-native-images + name: binaries path: bin - name: Calculate checksums @@ -86,11 +83,6 @@ jobs: mv checksums.txt bin/checksums.txt - - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: 21 - - name: Release uses: softprops/action-gh-release@v1 if: startsWith(github.ref, 'refs/tags/') diff --git a/LICENCE b/LICENCE index 8aa2645..5a77d8e 100644 --- a/LICENCE +++ b/LICENCE @@ -1,6 +1,6 @@ MIT License -Copyright (c) [year] [fullname] +Copyright (c) Kepler16 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index a25cfe9..e3e3c79 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ # KL -This is a development tool which provides a well defined and flexible means of running and developing your technology stack locally on your machine. +This is a development tool which provides a well-defined and flexible means of running and developing your technology +stack locally on your machine. -This tool is mainly beneficial to technology stacks which are distributed in nature and require lots of independent modules to run and speak to each other. +This tool is mainly beneficial to technology stacks which are distributed in nature and require lots of independent +modules to run and speak to each other. -KL tries to assume as little as possible about systems being run in order to make it easy to use with any technology stack so long as the stack is or can be dockerized. +KL tries to assume as little as possible about systems being run in order to make it easy to use with any technology +stack so long as the stack is or can be dockerized. --- @@ -44,11 +47,14 @@ Or you can get the binaries directly from the GitHub releases page and put them ## Initial Setup -If this is your first time using this tool on your machine then you will need to configure your system to resolve `.test` domains to `127.0.0.1:80`. This is essential for the networking components to work. How this should be configured depends on your operating system. +If this is your first time using this tool on your machine then you will need to configure your system to resolve +`.test` domains to `127.0.0.1:80`. This is essential for the networking components to work. How this should be +configured depends on your operating system. ### `Macos` -On Macos you need to configure the native resolver to use the nameserver at `127.0.0.1:53` when resolving `.test` domains. This is done by creating a file at `/etc/resolver/test` with the contents: +On macOS, you need to configure the native resolver to use the nameserver at `127.0.0.1:53` when resolving `.test` +domains. This is done by creating a file at `/etc/resolver/test` with the contents: ``` nameserver 127.0.0.1 @@ -61,7 +67,8 @@ sudo mkdir -p /etc/resolver echo 'nameserver 127.0.0.1' | sudo tee -a /etc/resolver/test > /dev/null ``` -KL runs this DNS server for you as one of the networking containers. See the [Network Topology](./docs/network-topology.md) document for more information on the networking containers. +KL runs this DNS server for you as one of the networking containers. See the [Network +Topology](./docs/network-topology.md) document for more information on the networking containers. Now you can start the docker network and proxy containers: @@ -71,7 +78,10 @@ kl network start ### `Linux` -Linux doesn't have a completely standard way of handling DNS and so this setup will depend a bit on your particular setup. A very common/standard DNS resolver setup on linux is `systemd-resolved` and so below is a guide on how to set get setup using this. If you don't use `systemd-resolved` then you will need to configure your system to route `.test` domains to `127.0.0.1` however is appropriate for you. +Linux doesn't have a completely standard way of handling DNS and so this setup will depend a bit on your particular +setup. A very common/standard DNS resolver setup on Linux is `systemd-resolved` and so below is a guide on how to set +get setup using this. If you don't use `systemd-resolved` then you will need to configure your system to route `.test` +domains to `127.0.0.1` however is appropriate for you. Edit your `/etc/systemd/resolved.conf` file and add the following to the `[Resolve]` section: @@ -81,7 +91,9 @@ DNS=127.0.0.1:5343 Domains=~test ``` -This configures `systemd-resolved` to use the DNS server running at `127.0.0.1:5343` to resolve `.test` domains. KL runs this DNS server for you as one of the networking containers. See the [Network Topology](./docs/network-topology.md) document for more information on the networking containers. +This configures `systemd-resolved` to use the DNS server running at `127.0.0.1:5343` to resolve `.test` domains. KL runs +this DNS server for you as one of the networking containers. See the [Network Topology](./docs/network-topology.md) +document for more information on the networking containers. Apply the changes by restarting the `systemd-resolved` service @@ -96,7 +108,10 @@ kl network start ``` > [!NOTE] -> On linux the host DNS network container's port defaults to binding to `5343`. If you would like to change this you can start the networking components with a different port by running `kl network start --host-dns-port=`. You will then also need to update your `/etc/systemd/resolved.conf` file to match. +> +> On Linux the host DNS network container's port defaults to binding to `5343`. If you would like to change this +> you can start the networking components with a different port by running `kl network start --host-dns-port=`. +> You will then also need to update your `/etc/systemd/resolved.conf` file to match. --- @@ -104,29 +119,45 @@ kl network start #### Networking -The core idea behind KL is to enable identifying services with stable `.test` domains and then having a flexible means of swapping out which endpoint the domain routes traffic to. This is typically how we run our services in production environments, there is no reason not to do it locally too. +The core idea behind KL is to enable identifying services with stable `.test` domains and then having a flexible means +of swapping out which endpoint the domain routes traffic to. This is typically how we run our services in production +environments, there is no reason not to do it locally too. -One of the main benefits that we draw from this is the significant simplification of service configuration. Services can be configured statically (committed to code/configuration) with the domains of other services they need to communicate with regardless of where those services are running or what their ip:port combinations are. +One of the main benefits that we draw from this is the significant simplification of service configuration. Services can +be configured statically (committed to code/configuration) with the domains of other services they need to communicate +with regardless of where those services are running or what their ip:port combinations are. -These domains are stable across all developers' machines and don't change when a developer needs to alter how or where they are running any particular service. +These domains are stable across all developers' machines and don't change when a developer needs to alter how or where +they are running any particular service. -For an overview of the network topology constructed by kl head over to the [Network Topology](./docs/network-topology.md) document. +For an overview of the network topology constructed by kl head over to the [Network +Topology](./docs/network-topology.md) document. #### Containers -While the benefits of the stable network addresses are significant it can also become quite tedious to run ones entire tech stack locally, especially when it is comprised of multiple individual services. Tools like docker-compose help a lot in this regard but even that starts to become very unwieldy as the number of services grow and the rate at which they change increases. +While the benefits of the stable network addresses are significant it can also become quite tedious to run one's entire +tech stack locally, especially when it consists of multiple individual services. Tools like docker-compose help a lot in +this regard, but even that starts to become very unwieldy as the number of services grow and the rate at which they +change increases. -Managing large docker-compose configurations and keeping them up-to-date with changes happening in upstream services does not scale well. +Managing large docker-compose configurations and keeping them up-to-date with changes happening in upstream services +does not scale well. -KL provides a mechanism of defining container configurations alongside the services they relate to (in their respective repositories) and composing them together on each developers local machine. Because the configuration lives alongside the service it is simpler to keep in sync as changes are made to the service itself. Developers' can then use kl to pull down any upstream changes to run. +KL provides a mechanism of defining container configurations alongside the services they relate to (in their respective +repositories) and composing them together on each developers' local machine. Because the configuration lives alongside +the service it is simpler to keep in sync as changes are made to the service itself. Developers' can then use kl to pull +down any upstream changes to run. --- ## Modules -A module is the primary unit of configuration in kl and is comprised of [containers](#containers), [services](#services), [endpoints](#endpoints) and [routes](#routes). A module can also contain sub-modules which should reference externally defined module configurations. Sub-modules are recursively resolved and merged into the root module. +A module is the primary unit of configuration in kl and consists of [containers](#containers), [services](#services), +[endpoints](#endpoints), and [routes](#routes). A module can also contain submodules which should reference externally +defined module configurations. Submodules are recursively resolved and merged into the root module. -A module is a directory located in `~/.config/kl/modules/` that must contain at least a file called `module.(edn|json|yaml|yml)`. Here is an example of a module: +A module is a directory located in `~/.config/kl/modules/` that must contain at least a file called +`module.(edn|json|yaml|yml)`. Here is an example of a module: ```clj {:include ["config.toml"] @@ -148,14 +179,21 @@ A module is a directory located in `~/.config/kl/modules/` that must contain at #### Module Resolution -Any sub-modules defined in a module will be recursively resolved and deep-merged with it's parent module. When there are module conflicts (more than one module with the same name) then they are resolved as follows: +Any submodules defined in a module will be recursively resolved and deep-merged with its parent module. When there are +module conflicts (more than one module with the same name) then they are resolved as follows: + If the conflict is in a child module, the parent will be used. -+ If there are no matching parent modules then the chosen module is essentially random - the first module to resolve will be used. ++ If there are no matching parent modules then the chosen module is essentially random - the first module to resolve +will be used. -The module resolution is performed once and the result stored in a lockfile called `module.lock.edn`. This allows a developers local dev-setup to be unaffected by any upstream changes _by default_ until the developer decides to explicitly pull down upstream changes using `kl module update`. It is highly recommended to commit your local `module.edn` and `module.lock.edn` files to make it easy to roll back when upstream changes cause unintended effects. +The module resolution is performed once and the result stored in a lockfile called `module.lock.edn`. This allows a +developers' local dev-setup to be unaffected by any upstream changes _by default_ until the developer decides to +explicitly pull down upstream changes using `kl module update`. It is highly recommended to commit your local +`module.edn` and `module.lock.edn` files to make it easy to roll back when upstream changes cause unintended effects. -Because module resolution is implemented as a full deep-merge, this allows developers to override locally any component of the fully resolved module. For example if a developer wants to swap out the container image being used for a particular container they can do so as follows: +Because module resolution is implemented as a full deep-merge, this allows developers to override locally any component +of the fully resolved module. For example if a developer wants to swap out the container image being used for a +particular container they can do so as follows: ```clj {:modules {...} @@ -163,17 +201,17 @@ Because module resolution is implemented as a full deep-merge, this allows devel :containers {:example {:image "example-image:replaced-tag"}}} ``` -Note how only the property being changed needed to be specified. This is assuming the container `:example` was already defined by one of the referenced sub-modules. +Note how only the property being changed needed to be specified. This is assuming the container `:example` was already +defined by one of the referenced submodules. #### Variable Substitution -When a module is resolved kl will perform some basic variable substitution to allow for simple templating. Variables should be in the form `{{VAR_NAME}}` where `VAR_NAME` can be one of the following: +When a module is resolved kl will perform some basic variable substitution to allow for simple templating. Variables +should be in the form `{{VAR_NAME}}` where `VAR_NAME` can be one of the following: -Variable | Description ----|--- -SHA | The full git sha that the module was resolved as -SHA_SHORT | The short version of `SHA` - only the first 7 chars -DIR | This is the local module directory on the host machine. This can be used for referencing config files that are included as part of the module. +Variable | Description ---|--- SHA | The full git SHA that the module was resolved as SHA_SHORT | The short version of +`SHA` - only the first 7 chars DIR | This is the local module directory on the host machine. This can be used for +referencing config files that are included as part of the module. Here is an example module that makes use of templating: @@ -190,17 +228,26 @@ See the [Module Spec](./docs/module-spec.md) ## Containers -A container is a docker-compose `service` snippet. It supports all of the same fields and will be converted into a docker-compose file when run. When running containers with `kl containers run` you will be given a choice of which containers to start. If any containers that were running are deselected, they will be stopped. +A container is a docker-compose `service` snippet. It supports all the same fields and will be converted into a +docker-compose file when run. When running containers with `kl containers run` you will be given a choice of which +containers to start. If any containers that were running are deselected, they will be stopped. ## Services -A service is a stable container around a set of [endpoints](#endpoints). A service is configured with a `:default-endpoint` to which all traffic will be routed to by default. Services are generally referenced to by [routes](#routes). This configuration allows swapping which endpoint a service is routing to without having to reconfigure individual [routes](#routes). +A service is a stable container around a set of [endpoints](#endpoints). A service is configured with a +`:default-endpoint` to which all traffic will be routed to by default. Services are generally referenced to by +[routes](#routes). This configuration allows swapping which endpoint a service is routing to without having to +reconfigure individual [routes](#routes). ## Endpoints -An endpoint is defined as part of a [service](#services) and represent some process/service running somewhere that is reachable over HTTP or any scheme supported by [Traefik](https://doc.traefik.io/traefik/). This can be a container, a process on the host machine or some remote service in the cloud. +An endpoint is defined as part of a [service](#services) and represent some process/service running somewhere that is +reachable over HTTP or any scheme supported by [Traefik](https://doc.traefik.io/traefik/). This can be a container, a +process on the host machine or some remote service in the cloud. -An important detail to keep in mind is that endpoints are always resolved from the context of the proxy container which is running inside of the docker network. This means that any endpoint URL's need to be reachable from the proxy container/docker network. This also means that `localhost` or `127.0.0.1` does _not_ address the host. +An important detail to keep in mind is that endpoints are always resolved from the context of the proxy container which +is running inside the docker network. This means that any endpoint URL's need to be reachable from the proxy +container/docker network. This also means that `localhost` or `127.0.0.1` does _not_ address the host. Below are some common ways of addressing services running in different contexts: @@ -208,35 +255,43 @@ Below are some common ways of addressing services running in different contexts: + `http://example:8080` + `http://:8080` + A process running on the host and bound to port `8080` could be addressed as - + On macos and linux* - `http://host.docker.internal:8080` - + On linux - `http://172.17.0.1:8080` + + On macOS and Linux* - `http://host.docker.internal:8080` + + On Linux - `http://172.17.0.1:8080` > [!NOTE] > -> On **linux\*** docker does not configure the `host.docker.internal` domain which is typically only available on macos when using something like Docker Desktop. +> On **Linux\*** docker does not configure the `host.docker.internal` domain which is typically only available on macOS +> when using something like Docker Desktop. > -> To allow for a consistent way of addressing the host that works across all operating systems kl manually adds the `host.docker.internal:172.17.0.1` host to to the proxy container. +> To allow for a consistent way of addressing the host that works across all operating systems' kl manually adds the +> `host.docker.internal:172.17.0.1` host to the proxy container. > -> If you don't want this behaviour you can disable it by running `kl network start --add-host "host.docker.internal:"` (note the empty ip after the ':'). +> If you don't want this behaviour you can disable it by running `kl network start --add-host "host.docker.internal:"` +> (note the empty ip after the ':'). ## Routes -A route is an HTTP routing rule made up of a combination of `host` and `prefix`. A route points to a [service](#services) and will route to whichever `default-endpoint` is configured on the service. A route may also specify a specific [endpoint](#endpoint) on the service to route through which would override the default endpoint configured on the service. +A route is an HTTP routing rule made up of a combination of `host` and `prefix`. A route points to a +[service](#services) and will route to whichever `default-endpoint` is configured on the service. A route may also +specify a specific [endpoint](#endpoint) on the service to route through which would override the default endpoint +configured on the service. --- ## CLI Prompts -KL has built-in support for [fzf](https://github.com/junegunn/fzf) and [gum](https://github.com/charmbracelet/gum) for prompt interfaces and selections. If these programs are installed and on your `PATH` then they will automatically be used. Alternatively kl will fallback to using a native prompt implementation if neither gum nor fzf can be found. +KL has built-in support for [fzf](https://github.com/junegunn/fzf) and [gum](https://github.com/charmbracelet/gum) for +prompt interfaces and selections. If these programs are installed and on your `PATH` then they will automatically be +used. Alternatively kl will fall back to using a native prompt implementation if neither gum nor fzf can be found. For a better prompt experience it is highly recommended to have these programs installed on your PATH. ## FAQ -### Why does the host dnsmasq container bind a different port on Macos vs Linux? - -On macos there is no way to specify the port to use when configuring the system resolver via the `/etc/resolver/test` path. Therefore we need to use port `53`. - -On Linux there is typically already some dns components binding port `53` and so we default to using a different, non-standard port - `5343`. +### Why does the host dnsmasq container bind a different port on macOS vs Linux? +On macOS there is no way to specify the port to use when configuring the system resolver via the `/etc/resolver/test` +path. Therefore, we need to use port `53`. +On Linux there is typically already some DNS components binding port `53`, and so we default to using a different, +non-standard port - `5343`. diff --git a/build.clj b/build.clj index bd2a546..5793c4c 100644 --- a/build.clj +++ b/build.clj @@ -5,12 +5,12 @@ (def basis (b/create-basis {:project "deps.edn" :aliases [:native]})) (def class-dir "target/classes") -(def uber-file "target/kl.jar") +(def jar-file "target/kl.jar") (defn clean [_] (b/delete {:path "target"})) -(defn uber [_] +(defn build [_] (clean nil) (b/copy-dir {:src-dirs ["src" "resources"] :target-dir class-dir}) @@ -22,6 +22,6 @@ "-Dclojure.spec.skip-macros=true"]}) (b/uber {:class-dir class-dir - :uber-file uber-file + :uber-file jar-file :basis basis :main 'k16.kl.cli})) diff --git a/deps.edn b/deps.edn index 7ebc114..4c82021 100644 --- a/deps.edn +++ b/deps.edn @@ -1,19 +1,19 @@ {:mvn/repos {"github-kepler" {:url "https://maven.pkg.github.com/kepler16/*"}} :paths ["src" "resources"] - :deps {org.clojure/clojure {:mvn/version "1.11.1"} - metosin/malli {:mvn/version "0.13.0"} + :deps {org.clojure/clojure {:mvn/version "1.11.3"} + metosin/malli {:mvn/version "0.16.2"} meta-merge/meta-merge {:mvn/version "1.0.0"} - metosin/jsonista {:mvn/version "0.3.7"} + metosin/jsonista {:mvn/version "0.3.9"} clj-commons/clj-yaml {:mvn/version "1.0.27"} - babashka/process {:mvn/version "0.5.21"} - jansi-clj/jansi-clj {:mvn/version "1.0.1"} + babashka/process {:mvn/version "0.5.22"} + jansi-clj/jansi-clj {:mvn/version "1.0.3"} cli-matic/cli-matic {:mvn/version "0.5.4"} org.clojars.civa86/pretty.cli {:mvn/version "1.0.1"} funcool/promesa {:mvn/version "11.0.678"} - http-kit/http-kit {:mvn/version "2.7.0"}} + http-kit/http-kit {:mvn/version "2.8.0"}} - :aliases {:build {:deps {io.github.clojure/tools.build {:mvn/version "0.9.5"}} + :aliases {:build {:deps {io.github.clojure/tools.build {:mvn/version "0.10.5"}} :ns-default build} :native {:extra-deps {com.github.clj-easy/graal-build-time {:mvn/version "1.0.5"}}} diff --git a/justfile b/justfile index 6d265af..bcb58d9 100644 --- a/justfile +++ b/justfile @@ -1,8 +1,5 @@ -clean: - clojure -T:build clean - -build: clean - clojure -T:build uber +build: + clojure -T:build build native-image: $GRAALVM_HOME/bin/native-image -jar target/kl.jar target/kl diff --git a/resources/META-INF/native-image/k16.kl/native-image.properties b/resources/META-INF/native-image/k16.kl/native-image.properties index 0e5223a..1ec4d8f 100644 --- a/resources/META-INF/native-image/k16.kl/native-image.properties +++ b/resources/META-INF/native-image/k16.kl/native-image.properties @@ -2,6 +2,7 @@ Args = --no-fallback \ --features=clj_easy.graal_build_time.InitClojureClasses \ --report-unsupported-elements-at-runtime \ -H:+UnlockExperimentalVMOptions \ + --initialize-at-build-time=org.fusesource.jansi.Ansi$Attribute,org.fusesource.jansi.Ansi$Color \ -H:ReflectionConfigurationResources=${.}/reflect-config.json \ -H:ResourceConfigurationResources=${.}/resource-config.json \ -H:+ReportExceptionStackTraces diff --git a/resources/k16/kl/module/network/module.edn b/resources/k16/kl/module/network/module.edn index 1bedff6..6585c2a 100644 --- a/resources/k16/kl/module/network/module.edn +++ b/resources/k16/kl/module/network/module.edn @@ -40,10 +40,12 @@ :networks {:kl {:ipv4_address "172.5.0.100"}}} :proxy ^:replace - {:image "traefik:v2.6" + {:image "traefik:v3.1" :restart "unless-stopped" :networks {:kl {:ipv4_address "172.5.0.101"}} :command ["--api.insecure=true" + "--log.level=INFO" + "--accesslog=true" "--providers.file.directory=/proxy-config" "--providers.file.watch=true"] :ports ["80:80"] diff --git a/src/k16/kl/api/proxy.clj b/src/k16/kl/api/proxy.clj index faad816..8b58100 100644 --- a/src/k16/kl/api/proxy.clj +++ b/src/k16/kl/api/proxy.clj @@ -7,7 +7,7 @@ (api.fs/from-config-dir "proxy/" (str module-name ".yaml"))) (defn- route->traefik-rule [{:keys [host path-prefix]}] - (cond-> (str "Host(`" host "`)") + (cond-> (str "HostRegexp(`" host "`)") path-prefix (str " && PathPrefix(`" path-prefix "`)"))) (defn- build-routes [module] diff --git a/src/k16/kl/prompt/editor.clj b/src/k16/kl/prompt/editor.clj deleted file mode 100644 index cfc1a2c..0000000 --- a/src/k16/kl/prompt/editor.clj +++ /dev/null @@ -1,21 +0,0 @@ -(ns k16.kl.prompt.editor - (:require - [babashka.process :as proc] - [clojure.java.io :as io])) - -(set! *warn-on-reflection* true) - -(defn gen-hash [n] - (->> (repeatedly n #(rand-int 256)) - (map #(format "%02x" %)) - (apply str))) - -(defn open-editor [{:keys [contents filetype]}] - (let [tmp-file (io/file (System/getProperty "java.io.tmpdir") (str "tmp-" (gen-hash 5) (if filetype filetype "")))] - - (when contents - (spit tmp-file contents)) - - (proc/shell ["nvim" (.toString tmp-file)]) - - (slurp tmp-file)))