Merge pull request #43 from kudulab/dc-fixes

Fixes for the latest docker-compose versions

Author: xmik

.circleci/config.yml

@@ -47,7 +47,7 @@ jobs:
       - run:
           name: Install dojo
           command: |
-            version="0.10.5"
+            version="0.13.0"
             wget -O /tmp/dojo https://github.com/kudulab/dojo/releases/download/${version}/dojo_linux_amd64
             chmod +x /tmp/dojo
             sudo mv /tmp/dojo /usr/bin/dojo
@@ -69,7 +69,7 @@ jobs:
       - run:
           name: Install dojo
           command: |
-            version="0.10.5"
+            version="0.13.0"
             wget -O /tmp/dojo https://github.com/kudulab/dojo/releases/download/${version}/dojo_linux_amd64
             chmod +x /tmp/dojo
             sudo mv /tmp/dojo /usr/bin/dojo
@@ -87,7 +87,7 @@ jobs:
       - run:
           name: Install dojo
           command: |
-            version="0.10.5"
+            version="0.13.0"
             wget -O /tmp/dojo https://github.com/kudulab/dojo/releases/download/${version}/dojo_linux_amd64
             chmod +x /tmp/dojo
             sudo mv /tmp/dojo /usr/bin/dojo

CHANGELOG.md

@@ -1,3 +1,11 @@
+### 0.13.1 (2024-Dec-29)
+
+* docker-compose: support it when there is no version statement in the top of the yaml file; tested with docker-compose v2.24.5 and v2.31.0
+* docker-compose: fix json decoding on mac - the Publishers field was resulting in decoding errors
+* bump dojo image scripts base images
+  * alpine:3.21
+  * ubuntu:24.10
+
 ### 0.13.0 (2024-Feb-05)
 
 * Support multiple levels of log output produced by Dojo: `silent`, `error`, `warn`, `info`, `debug`

README.md

@@ -61,7 +61,7 @@ brew install kudulab/homebrew-dojo-osx/dojo
 ```
 * a manual install:
 ```sh
-version="0.13.0"
+version="0.13.1"
 # on Linux:
 wget -O /tmp/dojo https://github.com/kudulab/dojo/releases/download/${version}/dojo_linux_amd64
 # or on Mac:
@@ -91,7 +91,7 @@ dojo "gradle test jar"
 The beginning of the output shows the docker command executed by Dojo:
 ```console
 /tmp/gocd-yaml-config-plugin$ dojo "gradle test jar"
-2020/12/09 07:40:28 [ 1]  INFO: (main.main) Dojo version 0.10.5
+2020/12/09 07:40:28 [ 1]  INFO: (main.main) Dojo version 0.13.1
 2020/12/09 07:40:28 [ 4]  INFO: (main.DockerDriver.HandleRun) docker command will be:
  docker run --rm -v /tmp/gocd-yaml-config-plugin:/dojo/work -v /home/tomzo:/dojo/identity:ro --env-file=/tmp/dojo-environment-dojo-gocd-yaml-config-plugin-2020-12-09_07-40-50-60121947 -v /tmp/.X11-unix:/tmp/.X11-unix -ti --name=dojo-gocd-yaml-config-plugin-2020-12-09_07-40-50-60121947 kudulab/openjdk-dojo:1.4.1 "gradle test jar"
 Unable to find image 'kudulab/openjdk-dojo:1.4.1' locally
@@ -192,13 +192,13 @@ Docker version 19.03.5, build 633a0ea838
 $ docker info | grep "Server Version"
  Server Version: 19.03.5
 
-$ docker run --rm alpine:3.15 whoami
-Unable to find image 'alpine:3.15' locally
+$ docker run --rm alpine:3.21 whoami
+Unable to find image 'alpine:3.21' locally
 # pulling image messages
 root
 dojo@ebc220b9655f(inception-dojo):/dojo/work$ docker images
 REPOSITORY          TAG                 IMAGE ID            CREATED             SIZE
-alpine              3.9                 78a2ce922f86        7 months ago        5.55MB
+alpine              3.21                 78a2ce922f86        7 months ago        5.55MB
 ```
 
 There is also an Alpine dind Docker image. Use it in the following way:
@@ -216,7 +216,7 @@ You can use Dojo with any Docker image, it does not have to be a Dojo Docker ima
 ```
 $ nano Dojofile
 $ cat Dojofile
-DOJO_DOCKER_IMAGE="alpine:3.16"
+DOJO_DOCKER_IMAGE="alpine:3.21"
 $ dojo
 # now we run interactively in the Dojo Docker container
 / # whoami
@@ -322,7 +322,7 @@ We have also established several **best practices** for dojo image development:
 Dojo provides [several scripts](image_scripts/src) to be used inside dojo images to meet most of above requirements. Scripts can be installed in a `Dockerfile` with:
 
 ```dockerfile
-ENV DOJO_VERSION=0.10.5
+ENV DOJO_VERSION=0.13.1
 RUN git clone --depth 1 -b ${DOJO_VERSION} https://github.com/kudulab/dojo.git /tmp/dojo_git &&\
   /tmp/dojo_git/image_scripts/src/install.sh && \
   rm -r /tmp/dojo_git
@@ -352,7 +352,7 @@ ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
 RUN chmod +x /tini
 
 # Install common Dojo scripts
-ENV DOJO_VERSION=0.10.5
+ENV DOJO_VERSION=0.13.1
 RUN apt-get update && \
   DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
   sudo git ca-certificates && \
@@ -397,10 +397,10 @@ cd "${dojo_work}"
 For alpine images a typical dockerfile has following structure:
 
 ```dockerfile
-FROM alpine:3.15
+FROM alpine:3.21
 
 # Install common Dojo scripts
-ENV DOJO_VERSION=0.10.5
+ENV DOJO_VERSION=0.13.1
 RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories && \
   apk add --no-cache tini bash shadow sudo git && \
   git clone --depth 1 -b ${DOJO_VERSION} https://github.com/kudulab/dojo.git /tmp/dojo_git &&\
@@ -663,7 +663,6 @@ DOJO_DOCKER_COMPOSE_FILE="docker-compose.yml"
 Next to the `Dojofile`, you must create a docker-compose.yml according the [official reference](https://docs.docker.com/compose/compose-file/).
 Example docker-compose file:
 ```yaml
-version: '2.2'
 services:
   default:
     links:
@@ -684,7 +683,7 @@ You can try creating above 2 files in any directory and run `dojo`. The output s
 
 ```console
 tomzo@073c1c477b1f:/tmp/compose-example$ dojo
-2019/04/28 18:38:17 [ 1]  INFO: (main.main) Dojo version 0.3.2
+2019/04/28 18:38:17 [ 1]  INFO: (main.main) Dojo version 0.13.1
 2019/04/28 18:38:18 [20]  INFO: (main.DockerComposeDriver.HandleRun) docker-compose run command will be:
  docker-compose -f docker-compose.yml -f docker-compose.yml.dojo -p dojo-compose-example-2019-04-28_18-38-17-33362956 run --rm default
 Creating network "dojo-compose-example-2019-04-28_18-38-17-33362956_default" with the default driver
@@ -750,7 +749,7 @@ Usage of dojo <flags> [--] <CMD>:
   -identity-dir-outer string
     	Directory on host, to be mounted into a docker container to /dojo/identity. Default: $HOME
   -image string
-    	Docker image name and tag, e.g. alpine:3.15
+    	Docker image name and tag, e.g. alpine:3.21
   -interactive string
     	Set to false if you want to force not interactive docker run
   -ll string
@@ -818,15 +817,15 @@ is printed, press Ctrl+C. You may also test pressing Ctrl+C more times.
 
 1. driver: docker, container's PID 1 process **not** preserving signals:
 ```
-dojo --image=alpine:3.15 -i=false sh -c "echo 'will sleep' && sleep 1d"
+dojo --image=alpine:3.21 -i=false sh -c "echo 'will sleep' && sleep 1d"
 ```
 2. driver: docker, container's PID 1 process preserving signals:
 ```
-dojo --docker-options="--init" --image=alpine:3.15 -i=false sh -c "echo 'will sleep' && sleep 1d"
+dojo --docker-options="--init" --image=alpine:3.21 -i=false sh -c "echo 'will sleep' && sleep 1d"
 ```
 3. driver: docker-compose:
 ```
-dojo --driver=docker-compose --dcf=./test/test-files/itest-dc.yaml -i=false --image=alpine:3.15 sh -c "echo 'will sleep' && sleep 1d"
+dojo --driver=docker-compose --dcf=./test/test-files/itest-dc.yaml -i=false --image=alpine:3.21 sh -c "echo 'will sleep' && sleep 1d"
 ```
 
 ### Preserving exported Bash functions [#17](https://github.com/kudulab/dojo/issues/17)
@@ -972,7 +971,7 @@ $ dojo -c Dojofile.build
 
 4. Run end to end tests:
 ```
-./tasks e2e ubuntu18
+./tasks e2e ubuntu
 ./tasks e2e alpine
 ```
 
@@ -996,7 +995,7 @@ $ dojo -c Dojofile.build
 
 ## License
 
-Copyright 2019-2022 Ava Czechowska, Tom Setkowski
+Copyright 2019-2025 Ava Czechowska, Tom Setkowski
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

config.go

@@ -91,7 +91,7 @@ func getCLIConfig() Config {
 	flagSet.StringVar(&driver, "d", "", usageDriver+" (shorthand)")
 
 	var image string
-	const usageImage = "Docker image name and tag, e.g. alpine:3.15"
+	const usageImage = "Docker image name and tag, e.g. alpine:3.21"
 	flagSet.StringVar(&image, "image", "", usageImage)
 
 	var logLevel string

docker_compose_driver.go

@@ -60,7 +60,7 @@ func isDCVersionLaterThan2(dcVersion string) bool {
 func (dc DockerComposeDriver) parseDCFileVersion(contents string) (float64, error) {
 	firstLine := strings.Split(contents, "\n")[0]
 	if !strings.HasPrefix(firstLine, "version") {
-		return 0, fmt.Errorf("First line of docker-compose file did not start with: version")
+		return -1, nil
 	}
 	versionQuoted := strings.Split(firstLine, ":")[1]
 	versionQuoted = strings.Trim(versionQuoted, " ")
@@ -78,7 +78,7 @@ func (dc DockerComposeDriver) verifyDCFile(fileContents string, filePath string)
 	if err != nil {
 		return 0, err
 	}
-	if version < 2 || version >= 3 {
+	if version != -1 && (version < 2 || version >= 3) {
 		return 0, fmt.Errorf("docker-compose file: %s should contain version number >=2 and <3, current version: %v", filePath, version)
 	}
 	requiredStr := "default:"
@@ -129,12 +129,18 @@ func (dc DockerComposeDriver) generateDCFileContentsWithEnv(expContainers []stri
 }
 
 func (dc DockerComposeDriver) generateInitialDCFile(config Config, version float64) string {
-	contents := fmt.Sprintf(
+	if version == -1 {
+		// version was not set
+		return fmt.Sprintf(
+			`services:
+  default:
+    image: "%s"`, config.DockerImage)
+	}
+	return fmt.Sprintf(
 		`version: '%v'
 services:
   default:
-    image: %s`, version, config.DockerImage)
-	return contents
+    image: "%s"`, version, config.DockerImage)
 }
 
 func (dc DockerComposeDriver) ConstructDockerComposeCommandPart1(config Config, projectName string) string {
@@ -701,24 +707,24 @@ func (dc DockerComposeDriver) getDCContainersNames(mergedConfig Config, projectN
 }
 
 type DC2PSOutput struct {
-	Command    string `json:"Command"`
-	CreatedAt  string `json:"CreatedAt"`
-	ExitCode   int    `json:"ExitCode"`
-	Health     string `json:"Health"`
-	ID         string `json:"ID"`
-	Image      string `json:"Image"`
-	Labels     string `json:"Labels"`
-	Name       string `json:"Name"`
-	Names      string `json:"Names"`
-	Networks   string `json:"Networks"`
-	Ports      string `json:"Ports"`
-	Project    string `json:"Project"`
-	Publishers string `json:"Publishers"`
-	RunningFor string `json:"RunningFor"`
-	Service    string `json:"Service"`
-	Size       string `json:"Size"`
-	State      string `json:"State"`
-	Status     string `json:"Status"`
+	Command    string   `json:"Command"`
+	CreatedAt  string   `json:"CreatedAt"`
+	ExitCode   int      `json:"ExitCode"`
+	Health     string   `json:"Health"`
+	ID         string   `json:"ID"`
+	Image      string   `json:"Image"`
+	Labels     string   `json:"Labels"`
+	Name       string   `json:"Name"`
+	Names      string   `json:"Names"`
+	Networks   string   `json:"Networks"`
+	Ports      string   `json:"Ports"`
+	Project    string   `json:"Project"`
+	Publishers []string `json:"Publishers"`
+	RunningFor string   `json:"RunningFor"`
+	Service    string   `json:"Service"`
+	Size       string   `json:"Size"`
+	State      string   `json:"State"`
+	Status     string   `json:"Status"`
 }
 
 // Parses the output of the `docker-compose ps --format json --all` command, into json.
@@ -728,6 +734,9 @@ type DC2PSOutput struct {
 // {"Command":"\"/bin/sh -c 'while t…\"","CreatedAt":"2024-02-03 21:03:46 +0000 UTC","ExitCode":0,"Health":"","ID":"2d5c5b0343d0","Image":"alpine:3.19","Labels":"com.docker.compose.depends_on=,com.docker.compose.image=sha256:05455a08881ea9cf0e752bc48e61bbd71a34c029bb13df01e40e3e70e0d007bd,com.docker.compose.version=2.24.5,com.docker.compose.service=abc,com.docker.compose.config-hash=270e27422cb1e6a4c1713ae22a3ffca0e8aa50ec0f06fe493fa4f83a17bd29e9,com.docker.compose.container-number=1,com.docker.compose.oneoff=False,com.docker.compose.project=testdojorunid,com.docker.compose.project.config_files=/dojo/work/test/test-files/itest-dc.yaml,/dojo/work/test/test-files/itest-dc.yaml.dojo,com.docker.compose.project.working_dir=/dojo/work/test/test-files","LocalVolumes":"0","Mounts":"/tmp/test-dojo…,/tmp/test-dojo…","Name":"testdojorunid-abc-1","Names":"testdojorunid-abc-1","Networks":"testdojorunid_default","Ports":"","Project":"testdojorunid","Publishers":null,"RunningFor":"3 seconds ago","Service":"abc","Size":"0B","State":"running","Status":"Up 2 seconds"}
 // {"Command":"\"/bin/sh -c 'while t…\"","CreatedAt":"2024-02-03 21:03:46 +0000 UTC","ExitCode":0,"Health":"","ID":"b2ed210567c3","Image":"alpine:3.19","Labels":"com.docker.compose.project=testdojorunid,com.docker.compose.project.config_files=/dojo/work/test/test-files/itest-dc.yaml,/dojo/work/test/test-files/itest-dc.yaml.dojo,com.docker.compose.project.working_dir=/dojo/work/test/test-files,com.docker.compose.depends_on=,com.docker.compose.container-number=1,com.docker.compose.image=sha256:05455a08881ea9cf0e752bc48e61bbd71a34c029bb13df01e40e3e70e0d007bd,com.docker.compose.oneoff=False,com.docker.compose.service=def,com.docker.compose.version=2.24.5,com.docker.compose.config-hash=270e27422cb1e6a4c1713ae22a3ffca0e8aa50ec0f06fe493fa4f83a17bd29e9","LocalVolumes":"0","Mounts":"/tmp/test-dojo…,/tmp/test-dojo…","Name":"testdojorunid-def-1","Names":"testdojorunid-def-1","Networks":"testdojorunid_default","Ports":"","Project":"testdojorunid","Publishers":null,"RunningFor":"3 seconds ago","Service":"def","Size":"0B","State":"running","Status":"Up 2 seconds"}
 // {"Command":"\"sh -c 'sleep 10'\"","CreatedAt":"2024-02-03 21:03:47 +0000 UTC","ExitCode":0,"Health":"","ID":"af4817fede41","Image":"alpine:3.15","Labels":"com.docker.compose.version=2.24.5,com.docker.compose.container-number=1,com.docker.compose.depends_on=abc:service_started:true,def:service_started:true,com.docker.compose.oneoff=True,com.docker.compose.project=testdojorunid,com.docker.compose.slug=742bcbb0e4bc05b21928a8d17be4ea9bb12a6775fd40692dd59c74a460279eb8,com.docker.compose.config-hash=462afacb4521d13580c2096c7b00b98970f07fe841e408c4c5a95a4a46839eaa,com.docker.compose.image=sha256:32b91e3161c8fc2e3baf2732a594305ca5093c82ff4e0c9f6ebbd2a879468e1d,com.docker.compose.project.config_files=/dojo/work/test/test-files/itest-dc.yaml,/dojo/work/test/test-files/itest-dc.yaml.dojo,com.docker.compose.project.working_dir=/dojo/work/test/test-files,com.docker.compose.service=default","LocalVolumes":"0","Mounts":"/home/dojo,/dojo/work,/tmp/test-dojo…,/tmp/test-dojo…,/tmp/.X11-unix,/tmp/dojo-ites…","Name":"testdojorunid-default-run-742bcbb0e4bc","Names":"testdojorunid-default-run-742bcbb0e4bc","Networks":"testdojorunid_default","Ports":"","Project":"testdojorunid","Publishers":null,"RunningFor":"2 seconds ago","Service":"default","Size":"0B","State":"running","Status":"Up 1 second"}
+//
+// when using docker-compose: 2.31m the field with Publishers is set to []:
+// {"Command":"\"/bin/sh -c 'while t…\"","CreatedAt":"2024-12-20 15:58:00 +1300 NZDT","ExitCode":143,"Health":"","ID":"f543828473a7","Image":"alpine:3.19","Labels":"com.docker.compose.image=sha256:7a85bf5dc56c949be827f84f9185161265c58f589bb8b2a6b6bb6d3076c1be21,desktop.docker.io/binds/0/Source=/tmp/dojo-environment-multiline-dojo-test-files-2024-12-2015-58-00-6660523964295751458,desktop.docker.io/binds/0/SourceKind=hostFile,desktop.docker.io/binds/0/Target=/etc/dojo.d/variables/00-multiline-vars.sh,desktop.docker.io/binds/1/Target=/etc/dojo.d/variables/01-bash-functions.sh,com.docker.compose.oneoff=False,com.docker.compose.project.config_files=/Users/ava.czechowska/code/dojo/test/test-files/itest-dc.yaml,/Users/ava.czechowska/code/dojo/test/test-files/itest-dc.yaml.dojo,com.docker.compose.service=abc,desktop.docker.io/binds/1/SourceKind=hostFile,com.docker.compose.container-number=1,com.docker.compose.project.working_dir=/Users/ava.czechowska/code/dojo/test/test-files,com.docker.compose.config-hash=4439e404114c9d0f183d756084f3e0e80c56b731e1f1e14e6db1844134294969,com.docker.compose.depends_on=,com.docker.compose.project=dojo-test-files-2024-12-2015-58-00-6660523964295751458,com.docker.compose.version=2.31.0,desktop.docker.io/binds/1/Source=/tmp/dojo-environment-bash-functions-dojo-test-files-2024-12-2015-58-00-6660523964295751458","LocalVolumes":"0","Mounts":"/host_mnt/priv…,/host_mnt/priv…","Name":"dojo-test-files-2024-12-2015-58-00-6660523964295751458-abc-1","Names":"dojo-test-files-2024-12-2015-58-00-6660523964295751458-abc-1","Networks":"dojo-test-files-2024-12-2015-58-00-6660523964295751458_default","Ports":"","Project":"dojo-test-files-2024-12-2015-58-00-6660523964295751458","Publishers":[],"RunningFor":"36 seconds ago","Service":"abc","Size":"0B","State":"exited","Status":"Exited (143) 10 seconds ago"}
 func ParseDCPSOutPut_DCVersion2(output string) ([]DC2PSOutput, string) {
 	var output_as_jsons []DC2PSOutput
 
@@ -740,7 +749,7 @@ func ParseDCPSOutPut_DCVersion2(output string) ([]DC2PSOutput, string) {
 		err := json.Unmarshal([]byte(line), &one_output_as_json)
 		if err != nil {
 			return []DC2PSOutput{},
-				fmt.Errorf("Error when decoding the JSON response from docker-compose ps command: %s", err).Error()
+				fmt.Errorf("Error when decoding the JSON response from docker-compose ps command: %s; line %s", err, line).Error()
 		}
 		if one_output_as_json.State == "" {
 			// This means that something went wrong, e.g. docker-compose ps output is now different

docker_compose_driver_test.go

@@ -17,7 +17,7 @@ func Test_parseDCFileVersion(t *testing.T) {
 	mytestsObj := []mytests{
 		mytests{"version: '4.55'", 4.55, ""},
 		mytests{"version: \"4.55\"", 4.55, ""},
-		mytests{"", 0, "First line of docker-compose file did not start with: version"},
+		mytests{"", -1, ""},
 	}
 	logger := NewLogger("debug")
 	dc := NewDockerComposeDriver(NewMockedShellServiceNotInteractive(logger), NewMockedFileService(logger), logger, "")

image_scripts/DockerfileAlpine

@@ -1,4 +1,4 @@
-FROM alpine:3.19
+FROM alpine:3.21
 
 # For Dojo:
 # * entrypoint requires sudo and bash

image_scripts/DockerfileUbuntu

@@ -1,4 +1,4 @@
-FROM ubuntu:23.10
+FROM ubuntu:24.10
 
 # the Dojo scripts create a user with ID==1000,
 # previous ubuntu images (e.g. 20.04) seem to not have the ubuntu user,

image_scripts/README.md

@@ -47,7 +47,7 @@ The `src/install.sh` script:
 ## Development
 Build and test Dojo docker images:
 ```
-cd dojo_image_scripts
+cd image_scripts
 ./tasks build
 ./tasks test_scripts
 ./tasks e2e

test/signal/signals-tests.sh

@@ -118,7 +118,7 @@ set -x; rm -f "${test_file}"; touch "${test_file}"; set +x;
 
 wait_for_the_docker_daemon_to_be_running
 this_test_exit_status=0
-run_test_process_and_send_signal "./bin/dojo --debug=true --test=true --image=alpine:3.19 -i=false sh -c \"echo 'will sleep' && sleep 1d\"" "will sleep"
+run_test_process_and_send_signal "./bin/dojo --debug=true --test=true --image=alpine:3.21 -i=false sh -c \"echo 'will sleep' && sleep 1d\"" "will sleep"
 
 ## Test checks
 log_test "----------------------------------------------------------"