Merge pull request #3750 from uselagoon/make-export-refactoring

refactor: create make command for external repos and add image repo tags to docker-compose yaml
uselagoon · Sep 16, 2024 · 35966e5 · 35966e5
2 parents 8687dbf + 0adbf6b
commit 35966e5
Show file tree

Hide file tree

Showing 3 changed files with 85 additions and 55 deletions.
diff --git a/Makefile b/Makefile
@@ -273,16 +273,16 @@ webhooks-test-services = webhook-handler webhooks2tasks backup-handler
 # These targets are used as dependencies to bring up containers in the right order.
 .PHONY: main-test-services-up
 main-test-services-up: $(foreach image,$(main-test-services),build/$(image))
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d $(main-test-services)
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.local-dev.yaml --compatibility up -d $(main-test-services)
 	$(MAKE) wait-for-keycloak
 
 .PHONY: drupaltest-services-up
 drupaltest-services-up: main-test-services-up $(foreach image,$(drupal-test-services),build/$(image))
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d $(drupal-test-services)
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.local-dev.yaml --compatibility up -d $(drupal-test-services)
 
 .PHONY: webhooks-test-services-up
 webhooks-test-services-up: main-test-services-up $(foreach image,$(webhooks-test-services),build/$(image))
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d $(webhooks-test-services)
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.local-dev.yaml --compatibility up -d $(webhooks-test-services)
 
 #######
 ####### Publishing Images
@@ -329,13 +329,13 @@ logs:
 # Start all Lagoon Services
 up:
 ifeq ($(ARCH), darwin)
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.development.yaml --compatibility up -d
 else
 	# once this docker issue is fixed we may be able to do away with this
 	# linux-specific workaround: https://github.com/docker/cli/issues/2290
 	KEYCLOAK_URL=$$(docker network inspect -f '{{(index .IPAM.Config 0).Gateway}}' bridge):8088 \
 		IMAGE_REPO=$(CI_BUILD_TAG) \
-		docker compose -p $(CI_BUILD_TAG) --compatibility up -d
+		docker compose -p $(CI_BUILD_TAG) -f docker-compose.development.yaml --compatibility up -d
 endif
 	$(MAKE) wait-for-keycloak
 
@@ -363,23 +363,32 @@ local-dev-yarn-stop:
 
 .PHONY: ui-development
 ui-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.local-dev.yaml --compatibility up -d api api-db api-sidecar-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis
 	$(MAKE) wait-for-keycloak
 
 .PHONY: api-development
 api-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis
+	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.local-dev.yaml --compatibility up -d api api-db api-sidecar-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis
 	$(MAKE) wait-for-keycloak
 
 .PHONY: ui-logs-development
 ui-logs-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler actions-handler local-api-data-watcher-pusher ui keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog
-	$(MAKE) wait-for-keycloak
+	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="ui" $(MAKE) compose-api-logs-development
 
 .PHONY: api-logs-development
 api-logs-development: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility up -d api api-db api-sidecar-handler actions-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog
-	$(MAKE) wait-for-keycloak
+	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="" $(MAKE) compose-api-logs-development
+
+# compose-api-logs-development can be consumed by other repositories to start a local api
+# supported make variable passthrough are
+# IMAGE_REPO - the docker repostory to use (uselagoon, testlagoon, other)
+# IMAGE_REPO_TAG - the tag to use for (main, pr-1234, vX.X.X, other) (each service also has a specific tag override, see the docker-compose.yaml file)
+# COMPOSE_STACK_NAME - the name of the stack that will be started
+# ADDITIONAL_SERVICES - a way to pass through additional services ("ui", "ui ssh", etc..)
+.PHONY: compose-api-logs-development
+compose-api-logs-development:
+	docker compose -p $(COMPOSE_STACK_NAME) $(ADDITIONAL_FLAGS) --compatibility up -d $(ADDITIONAL_SERVICES) api api-db api-sidecar-handler actions-handler local-api-data-watcher-pusher keycloak keycloak-db broker api-redis logs2notifications local-minio mailhog
+	$(MAKE) CI_BUILD_TAG=$(COMPOSE_STACK_NAME) wait-for-keycloak
 
 ## CI targets
 

diff --git a/docker-compose.local-dev.yaml b/docker-compose.local-dev.yaml
@@ -0,0 +1,46 @@
+version: '3.2'
+
+volumes:
+  broker: {}
+
+services:
+  webhook-handler:
+    command: yarn run dev
+    volumes:
+      - ./services/webhook-handler/src:/app/services/webhook-handler/src
+      - ./node-packages:/app/node-packages:delegated
+    environment:
+      - CONSOLE_LOGGING_LEVEL=trace
+  webhooks2tasks:
+    command: yarn run dev
+    volumes:
+      - ./services/webhooks2tasks/src:/app/services/webhooks2tasks/src
+      - ./node-packages:/app/node-packages:delegated
+    environment:
+      - CONSOLE_LOGGING_LEVEL=trace
+  api-db-init:
+    volumes:
+      - ./services/api/database:/app/services/api/database
+      - ./node-packages:/app/node-packages:delegated
+      - /app/node-packages/commons/dist
+  api-lagoon-migrations:
+    volumes:
+      - ./services/api/src:/app/services/api/src
+  api:
+    command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
+    volumes:
+      - ./services/api/src:/app/services/api/src
+      - ./node-packages:/app/node-packages:delegated
+      - /app/node-packages/commons/dist
+  auth-server:
+    command: yarn run dev
+    volumes:
+      - ./services/auth-server/src:/app/services/auth-server/src
+  keycloak:
+    volumes:
+      - "./services/keycloak/startup-scripts:/opt/keycloak/startup-scripts"
+      - "./services/keycloak/themes/lagoon:/opt/keycloak/themes/lagoon"
+      - "./local-dev/keycloak:/lagoon/keycloak"
+  local-api-data-watcher-pusher:
+    volumes:
+      - ./local-dev/api-data-watcher-pusher:/home
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -5,12 +5,11 @@ volumes:
 
 services:
   api-db:
-    image: ${IMAGE_REPO:-lagoon}/api-db
+    image: ${IMAGE_REPO:-lagoon}/api-db:${IMAGE_REPO_API_DB_TAG:-${IMAGE_REPO_TAG:-latest}}
     ports:
       - '3366:3306'
   webhook-handler:
-    image: ${IMAGE_REPO:-lagoon}/webhook-handler
-    command: yarn run dev
+    image: ${IMAGE_REPO:-lagoon}/webhook-handler:${IMAGE_REPO_WEBHOOK_HANDLER_TAG:-${IMAGE_REPO_TAG:-latest}}
     ports:
       - '7777:3000'
     volumes:
@@ -19,22 +18,22 @@ services:
     environment:
       - CONSOLE_LOGGING_LEVEL=trace
   backup-handler:
-    image: ${IMAGE_REPO:-lagoon}/backup-handler
+    image: ${IMAGE_REPO:-lagoon}/backup-handler:${IMAGE_REPO_BACKUP_HANDLER_TAG:-${IMAGE_REPO_TAG:-latest}}
     restart: on-failure
     ports:
       - '7778:3000'
     depends_on:
       - broker
   broker:
-    image: ${IMAGE_REPO:-lagoon}/broker
+    image: ${IMAGE_REPO:-lagoon}/broker:${IMAGE_REPO_BROKER_TAG:-${IMAGE_REPO_TAG:-latest}}
     ports:
       - '15672:15672'
       - '5672:5672'
     hostname: broker
     volumes:
       - broker:/var/lib/rabbitmq
   broker-ff-enable:
-    image: ${IMAGE_REPO:-lagoon}/broker
+    image: ${IMAGE_REPO:-lagoon}/broker:${IMAGE_REPO_BROKER_TAG:-${IMAGE_REPO_TAG:-latest}}
     entrypoint: /enable-feature-flags.sh
     environment:
       - SERVICE_NAME=broker
@@ -44,11 +43,11 @@ services:
   api-sidecar-handler:
     # this is neded for the internal dns references
     container_name: apisidecarhandler
-    image: ${IMAGE_REPO:-lagoon}/api-sidecar-handler
+    image: ${IMAGE_REPO:-lagoon}/api-sidecar-handler:${IMAGE_REPO_BROKER_TAG:-${IMAGE_REPO_TAG:-latest}}
     ports:
       - '3333:3333'
   logs2notifications:
-    image: ${IMAGE_REPO:-lagoon}/logs2notifications
+    image: ${IMAGE_REPO:-lagoon}/logs2notifications:${IMAGE_REPO_LOGS2NOTIFICATIONS_TAG:-${IMAGE_REPO_TAG:-latest}}
     environment:
       - S3_FILES_HOST=http://172.17.0.1:9000
   mailhog:
@@ -57,33 +56,23 @@ services:
     ports:
       - '32025:8025'
   webhooks2tasks:
-    image: ${IMAGE_REPO:-lagoon}/webhooks2tasks
-    command: yarn run dev
-    volumes:
-      - ./services/webhooks2tasks/src:/app/services/webhooks2tasks/src
-      - ./node-packages:/app/node-packages:delegated
+    image: ${IMAGE_REPO:-lagoon}/webhooks2tasks:${IMAGE_REPO_WEBHOOKS2TASKS_TAG:-${IMAGE_REPO_TAG:-latest}}
     environment:
       - CONSOLE_LOGGING_LEVEL=trace
       - SIDECAR_HANDLER_HOST=apisidecarhandler
   api-db-init:
-    image: ${IMAGE_REPO:-lagoon}/api
+    image: ${IMAGE_REPO:-lagoon}/api:${IMAGE_REPO_API_TAG:-${IMAGE_REPO_TAG:-latest}}
     command: >
       sh -c "./node_modules/.bin/knex migrate:list --cwd /app/services/api/database
       && ./node_modules/.bin/knex migrate:latest --cwd /app/services/api/database"
-    volumes:
-      - ./services/api/database:/app/services/api/database
-      - ./node-packages:/app/node-packages:delegated
-      - /app/node-packages/commons/dist
     depends_on:
       api-db:
         condition: service_started
       keycloak:
         condition: service_started
   api-lagoon-migrations:
-    image: ${IMAGE_REPO:-lagoon}/api
+    image: ${IMAGE_REPO:-lagoon}/api:${IMAGE_REPO_API_TAG:-${IMAGE_REPO_TAG:-latest}}
     command: sh -c "./node_modules/.bin/tsc && node -r dotenv-extended/config dist/migrations/lagoon/migration.js"
-    volumes:
-      - ./services/api/src:/app/services/api/src
     environment:
       - NODE_ENV=development
       - CONSOLE_LOGGING_LEVEL=trace
@@ -95,12 +84,7 @@ services:
       api-sidecar-handler:
         condition: service_started
   api:
-    image: ${IMAGE_REPO:-lagoon}/api
-    command: ./node_modules/.bin/tsc-watch --build --incremental --onSuccess "node -r dotenv-extended/config dist/index"
-    volumes:
-      - ./services/api/src:/app/services/api/src
-      - ./node-packages:/app/node-packages:delegated
-      - /app/node-packages/commons/dist
+    image: ${IMAGE_REPO:-lagoon}/api:${IMAGE_REPO_API_TAG:-${IMAGE_REPO_TAG:-latest}}
     environment:
       - KEYCLOAK_FRONTEND_URL=http://localhost:8088/
       - NODE_ENV=development
@@ -138,12 +122,12 @@ services:
     ports:
       - '8888:3000'
   actions-handler:
-    image: ${IMAGE_REPO:-lagoon}/actions-handler
+    image: ${IMAGE_REPO:-lagoon}/actions-handler:${IMAGE_REPO_ACTIONS_HANDLER_TAG:-${IMAGE_REPO_TAG:-latest}}
     restart: on-failure
     depends_on:
       - broker
   ssh:
-    image: ${IMAGE_REPO:-lagoon}/ssh
+    image: ${IMAGE_REPO:-lagoon}/ssh:${IMAGE_REPO_SSH_TAG:-${IMAGE_REPO_TAG:-latest}}
     depends_on:
       - api
       - auth-server
@@ -154,18 +138,15 @@ services:
       - ./services/ssh/home/command.sh:/home/command.sh
       - ./services/ssh/home/rsh.sh:/home/rsh.sh
   auth-server:
-    image: ${IMAGE_REPO:-lagoon}/auth-server
-    command: yarn run dev
+    image: ${IMAGE_REPO:-lagoon}/auth-server:${IMAGE_REPO_AUTH_SERVER_TAG:-${IMAGE_REPO_TAG:-latest}}
     environment:
       - JWTISSUER=auth-server.dev
       - JWTAUDIENCE=api.dev
       - CONSOLE_LOGGING_LEVEL=trace
-    volumes:
-      - ./services/auth-server/src:/app/services/auth-server/src
     ports:
       - '3001:3000'
   keycloak:
-    image: ${IMAGE_REPO:-lagoon}/keycloak
+    image: ${IMAGE_REPO:-lagoon}/keycloak:${IMAGE_REPO_KEYCLOAK_TAG:-${IMAGE_REPO_TAG:-latest}}
     user: '111111111'
     depends_on:
       - keycloak-db
@@ -178,16 +159,12 @@ services:
       # Uncomment for local new relic tracking
       # - NEW_RELIC_LICENSE_KEY=
       # - NEW_RELIC_APP_NAME=keycloak-local
-    volumes:
-      - "./services/keycloak/startup-scripts:/opt/keycloak/startup-scripts"
-      - "./services/keycloak/themes/lagoon:/opt/keycloak/themes/lagoon"
-      - "./local-dev/keycloak:/lagoon/keycloak"
   keycloak-db:
-    image: ${IMAGE_REPO:-lagoon}/keycloak-db
+    image: ${IMAGE_REPO:-lagoon}/keycloak-db:${IMAGE_REPO_KEYCLOAK_DB_TAG:-${IMAGE_REPO_TAG:-latest}}
     ports:
       - '3336:3306'
   tests-kubernetes:
-    image: ${IMAGE_REPO:-lagoon}/tests
+    image: ${IMAGE_REPO:-lagoon}/tests:${IMAGE_REPO_TESTS_TAG:-${IMAGE_REPO_TAG:-latest}}
     environment:
       - CLUSTER_TYPE=control-k8s
       - WEBHOOK_HOST=webhook-handler
@@ -212,11 +189,9 @@ services:
   local-api-data-watcher-pusher:
     depends_on:
       - api
-    image: ${IMAGE_REPO:-lagoon}/local-api-data-watcher-pusher
+    image: ${IMAGE_REPO:-lagoon}/local-api-data-watcher-pusher:${IMAGE_REPO_API_DATA_WATCHER_TAG:-${IMAGE_REPO_TAG:-latest}}
     environment:
       - CLEAR_API_DATA=false
-    volumes:
-      - ./local-dev/api-data-watcher-pusher:/home
   local-minio:
     image: minio/minio
     entrypoint: sh
@@ -234,4 +209,4 @@ services:
     labels:
       version: '2'
   api-redis:
-    image: ${IMAGE_REPO:-lagoon}/api-redis
+    image: ${IMAGE_REPO:-lagoon}/api-redis:${IMAGE_REPO_API_REDIS_TAG:-${IMAGE_REPO_TAG:-latest}}