Merge pull request #56 from CDLUC3/codebuild

Codebuild modifications

Author: terrywbrady

.gitmodules

@@ -34,14 +34,6 @@
 	path = mrt-services/dep_cloud/mrt-cloud
 	url = [email protected]:cdluc3/mrt-cloud
 	branch = main
-[submodule "mrt-services/dep_zoo/mrt-zoo"]
-	path = mrt-services/dep_zoo/mrt-zoo
-	url = [email protected]:cdluc3/mrt-zoo
-	branch = main
-[submodule "mrt-services/dep_cdlzk/cdl-zk-queue"]
-	path = mrt-services/dep_cdlzk/cdl-zk-queue
-	url = [email protected]:cdluc3/cdl-zk-queue
-	branch = main
 [submodule "mrt-integ-tests"]
 	path = mrt-integ-tests
 	url = [email protected]:cdluc3/mrt-integ-tests.git

bin/fresh_build.sh

@@ -239,8 +239,6 @@ git_repo_submodules() {
   checkout_build_config 'mrt-services/dep_core/mrt-core2' 'mrt-core'
   checkout_build_config 'mrt-services/dep_cloud/mrt-cloud' 'mrt-cloud'
   checkout_build_config 'mrt-services/dep_zk/mrt-zk' 'mrt-zk'
-  # checkout_build_config 'mrt-services/dep_cdlzk/cdl-zk-queue' 'cdl-zk-queue'
-  # checkout_build_config 'mrt-services/dep_zoo/mrt-zoo' 'mrt-zoo'
 
   if [[ "$MAVEN_PROFILE" == "-P inventory" ]] && [[ "$CHECK_REPO_TAG" != "" ]]
   then
@@ -293,6 +291,10 @@ build_integration_test_images() {
   build_it_image mrt-minio-it-with-content/docker-compose.yml ${ECR_REGISTRY}/mrt-minio-it-with-content:dev
   build_it_image merritt-tomcat/docker-compose.yml ${ECR_REGISTRY}/merritt-tomcat:dev
   build_it_image merritt-maven/docker-compose.yml ${ECR_REGISTRY}/merritt-maven:dev
+  if [[ -f fakesmtp ]]
+  then
+    build_it_image fakesmtp/docker-compose.yml ${ECR_REGISTRY}/fakesmtp:dev
+  fi
 }
 
 check_maven_profile() {
@@ -339,35 +341,35 @@ build_maven_artifacts() {
       if check_maven_profile 'store'
       then
         mkdir -p $ARTIFACTS/mrt-store
-        cp $WKDIR/mrt-services/store/mrt-store/store-war/target/mrt-storewar-1.0-SNAPSHOT.war $ARTIFACTS/mrt-store/mrt-store-${TAG_PUB}.war
+        cp $WKDIR/mrt-services/store/mrt-store/store-war/target/mrt-storewar-*-SNAPSHOT.war $ARTIFACTS/mrt-store/mrt-store-${TAG_PUB}.war
         jar uf $ARTIFACTS/mrt-store/mrt-store-${TAG_PUB}.war -C ${WKDIR_PAR} ${BUILD_TXT_FILE}
       fi
 
       if check_maven_profile 'replic'
       then
         mkdir -p $ARTIFACTS/mrt-replic
-        cp $WKDIR/mrt-services/replic/mrt-replic/replication-war/target/mrt-replicationwar-1.0-SNAPSHOT.war $ARTIFACTS/mrt-replic/mrt-replic-${TAG_PUB}.war
+        cp $WKDIR/mrt-services/replic/mrt-replic/replication-war/target/mrt-replicationwar-*-SNAPSHOT.war $ARTIFACTS/mrt-replic/mrt-replic-${TAG_PUB}.war
         jar uf $ARTIFACTS/mrt-replic/mrt-replic-${TAG_PUB}.war -C ${WKDIR_PAR} ${BUILD_TXT_FILE}
       fi
 
       if check_maven_profile 'ingest'
       then
         mkdir -p $ARTIFACTS/mrt-ingest
-        cp $WKDIR/mrt-services/ingest/mrt-ingest/ingest-war/target/mrt-ingestwar-1.0-SNAPSHOT.war $ARTIFACTS/mrt-ingest/mrt-ingest-${TAG_PUB}.war
+        cp $WKDIR/mrt-services/ingest/mrt-ingest/ingest-war/target/mrt-ingestwar-*-SNAPSHOT.war $ARTIFACTS/mrt-ingest/mrt-ingest-${TAG_PUB}.war
         jar uf $ARTIFACTS/mrt-ingest/mrt-ingest-${TAG_PUB}.war -C ${WKDIR_PAR} ${BUILD_TXT_FILE}
       fi
 
       if check_maven_profile 'audit'
       then
         mkdir -p $ARTIFACTS/mrt-audit
-        cp $WKDIR/mrt-services/audit/mrt-audit/audit-war/target/mrt-auditwarpub-1.0-SNAPSHOT.war $ARTIFACTS/mrt-audit/mrt-audit-${TAG_PUB}.war
+        cp $WKDIR/mrt-services/audit/mrt-audit/audit-war/target/mrt-auditwarpub-*-SNAPSHOT.war $ARTIFACTS/mrt-audit/mrt-audit-${TAG_PUB}.war
         jar uf $ARTIFACTS/mrt-audit/mrt-audit-${TAG_PUB}.war -C ${WKDIR_PAR} ${BUILD_TXT_FILE}
       fi
 
       if check_maven_profile 'inventory'
       then
         mkdir -p $ARTIFACTS/mrt-inventory
-        cp $WKDIR/mrt-services/inventory/mrt-inventory/inv-war/target/mrt-invwar-1.0-SNAPSHOT.war $ARTIFACTS/mrt-inventory/mrt-inventory-${TAG_PUB}.war
+        cp $WKDIR/mrt-services/inventory/mrt-inventory/inv-war/target/mrt-invwar-*-SNAPSHOT.war $ARTIFACTS/mrt-inventory/mrt-inventory-${TAG_PUB}.war
         jar uf $ARTIFACTS/mrt-inventory/mrt-inventory-${TAG_PUB}.war -C ${WKDIR_PAR} ${BUILD_TXT_FILE}
       fi
     fi
@@ -381,13 +383,23 @@ build_microservice_images() {
 
   cd $WKDIR/mrt-services
 
-  build_image_push ${ECR_REGISTRY}/mrt-ingest:dev ingest
-  build_image_push ${ECR_REGISTRY}/mrt-inventory:dev inventory
-  build_image_push ${ECR_REGISTRY}/mrt-store:dev store
-  build_image_push ${ECR_REGISTRY}/mrt-audit:dev audit
-  build_image_push ${ECR_REGISTRY}/mrt-replic:dev replic
-
-  build_image_push ${ECR_REGISTRY}/mrt-dashboard ui
+  if [[ -f ingest/Dockerfile ]]
+  then
+    echo "using old Dockerfiles..."
+    build_image_push ${ECR_REGISTRY}/mrt-ingest:dev ingest
+    build_image_push ${ECR_REGISTRY}/mrt-inventory:dev inventory
+    build_image_push ${ECR_REGISTRY}/mrt-store:dev store
+    build_image_push ${ECR_REGISTRY}/mrt-audit:dev audit
+    build_image_push ${ECR_REGISTRY}/mrt-replic:dev replic
+    build_image_push ${ECR_REGISTRY}/mrt-dashboard ui
+  else
+    build_image_push ${ECR_REGISTRY}/mrt-ingest:dev ingest/mrt-ingest
+    build_image_push ${ECR_REGISTRY}/mrt-inventory:dev inventory/mrt-inventory
+    build_image_push ${ECR_REGISTRY}/mrt-store:dev store/mrt-store
+    build_image_push ${ECR_REGISTRY}/mrt-audit:dev audit/mrt-audit
+    build_image_push ${ECR_REGISTRY}/mrt-replic:dev replic/mrt-replic
+    build_image_push ${ECR_REGISTRY}/mrt-dashboard ui/mrt-dashboard
+  fi
 }
 
 build_docker_stack_support_images(){

bin/it_build.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+docker-compose \
+  -f mrt-inttest-services/fakesmtp/docker-compose.yml \
+  -f mrt-inttest-services/merritt-maven/docker-compose.yml \
+  -f mrt-inttest-services/merritt-tomcat/docker-compose.yml \
+  -f mrt-inttest-services/mock-merritt-it/docker-compose.yml \
+  -f mrt-inttest-services/mrt-it-database/docker-compose.yml \
+  -f mrt-inttest-services/mrt-it-database/docker-compose-audit-replic.yml \
+  -f mrt-inttest-services/mrt-minio-it/docker-compose.yml \
+  -f mrt-inttest-services/mrt-minio-it-with-content/docker-compose.yml build

build-config.yml

@@ -66,6 +66,16 @@ flagsets:
     run-maven: true
     run-maven-tests: false
     build-support: false
+  codebuildflags: &codebuildflags
+    description: "Run maven builds only.  No tests."
+    build-it: true
+    build-stack: true
+    push: false
+    scan-unfixable: false
+    scan-fixable: false
+    run-maven: true
+    run-maven-tests: false
+    build-support: true
   sbox: &sbox
     description: "Run maven builds and tests. Pull IT images from ECR"
     build-it: false
@@ -122,6 +132,19 @@ build-config:
       mrt-replic: main
       mrt-dashboard: main
       mrt-integ-tests: main
+  codebuild:
+    <<: *codebuildflags
+    tags: &codebuild
+      mrt-core: codebuild
+      mrt-cloud: codebuild
+      mrt-zk: codebuild
+      mrt-inventory: codebuild2
+      mrt-store: codebuild
+      mrt-ingest: codebuild
+      mrt-audit: codebuild
+      mrt-replic: codebuild
+      mrt-dashboard: codebuild
+      mrt-integ-tests: main
   main-docker:
     <<: *docker
     tags:

buildspec.yml

@@ -18,39 +18,74 @@ phases:
       - aws ecr create-repository --repository-name mrt-it-database-audit-replic || echo 'ignore if exists...'
       - aws ecr create-repository --repository-name mrt-minio-it || echo 'ignore if exists...'
       - aws ecr create-repository --repository-name mrt-minio-it-with-content || echo 'ignore if exists...'
+      - aws ecr create-repository --repository-name fakesmtp || echo 'ignore if exists...'
 
       - aws ecr create-repository --repository-name callback || echo 'ignore if exists...'
       - aws ecr create-repository --repository-name mrt-opendj || echo 'ignore if exists...'
       - aws ecr create-repository --repository-name mrt-init || echo 'ignore if exists...'
       - aws ecr create-repository --repository-name mrt-database || echo 'ignore if exists...'
 
+      # - docker buildx create --use 
+      # - BUILDCMD="buildx build --platform linux/arm64,linux/amd64 --push --quiet"
+      - BUILDCMD="build --push --quiet"
 
   build:
     on-failure: ABORT
     commands:
-      - docker-compose -f mrt-inttest-services/merritt-tomcat/docker-compose.yml build
-      - docker-compose -f mrt-inttest-services/merritt-tomcat/docker-compose.yml push
-      - docker-compose -f mrt-inttest-services/merritt-maven/docker-compose.yml build
-      - docker-compose -f mrt-inttest-services/merritt-maven/docker-compose.yml push
-      - docker-compose -f mrt-inttest-services/mock-merritt-it/docker-compose.yml build
-      - docker-compose -f mrt-inttest-services/mock-merritt-it/docker-compose.yml push
-      - docker-compose -f mrt-inttest-services/mrt-it-database/docker-compose.yml build
-      - docker-compose -f mrt-inttest-services/mrt-it-database/docker-compose.yml push
-      - docker-compose -f mrt-inttest-services/mrt-it-database/docker-compose-audit-replic.yml build
-      - docker-compose -f mrt-inttest-services/mrt-it-database/docker-compose-audit-replic.yml push
-      - docker-compose -f mrt-inttest-services/mrt-minio-it/docker-compose.yml build
-      - docker-compose -f mrt-inttest-services/mrt-minio-it/docker-compose.yml push
-      - docker-compose -f mrt-inttest-services/mrt-minio-it-with-content/docker-compose.yml build
-      - docker-compose -f mrt-inttest-services/mrt-minio-it-with-content/docker-compose.yml push
-
-      - docker build --quiet --build-arg ECR_REGISTRY=${ECR_REGISTRY} --no-cache --force-rm -t ${ECR_REGISTRY}/mrt-database mrt-services/mysql
-      - docker push ${ECR_REGISTRY}/mrt-database
-      - docker build --quiet --build-arg ECR_REGISTRY=${ECR_REGISTRY} --no-cache --force-rm -t ${ECR_REGISTRY}/mrt-opendj mrt-services/ldap
-      - docker push ${ECR_REGISTRY}/mrt-opendj
-      - docker build --quiet --build-arg ECR_REGISTRY=${ECR_REGISTRY} --no-cache --force-rm -t ${ECR_REGISTRY}/mrt-init mrt-services/merritt-init
-      - docker push ${ECR_REGISTRY}/mrt-init
-      - docker build --quiet --build-arg ECR_REGISTRY=${ECR_REGISTRY} --no-cache --force-rm -t ${ECR_REGISTRY}/callback mrt-services/callback
-      - docker push ${ECR_REGISTRY}/callback
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/merritt-maven:dev \
+          --build-arg JAVA_RELEASE_ARG=${JAVA_RELEASE:-8} \
+          mrt-inttest-services/merritt-maven
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/merritt-tomcat:dev \
+          mrt-inttest-services/merritt-tomcat
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/mock-merritt-it:dev \
+          mrt-inttest-services/mock-merritt-it
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/mrt-it-database:dev \
+          mrt-inttest-services/mrt-it-database
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/mrt-it-database-audit-replic:dev \
+          --build-arg LOAD_SQL=/tmp/audit_replic_data.sql \
+          mrt-inttest-services/mrt-it-database
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/mrt-minio-it:dev \
+          mrt-inttest-services/mrt-minio-it
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/mrt-minio-it-with-content:dev \
+          --build-arg ECR_REGISTRY=${ECR_REGISTRY} \
+          mrt-inttest-services/mrt-minio-it-with-content
+
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/mrt-database \
+          mrt-services/mysql
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/mrt-opendj \
+          mrt-services/ldap
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/mrt-init \
+          mrt-services/merritt-init
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/callback \
+          mrt-services/callback
+
   post_build:
+    # this build may fail due to rate limits
+    # on-failure: ABORT
     commands:
-      #- aws s3 rm s3://${S3CFBUCKET}/api --recursive
+      - |
+        docker ${BUILDCMD} \
+          -t ${ECR_REGISTRY}/fakesmtp:dev \
+          --push mrt-inttest-services/fakesmtp

mrt-integ-tests

@@ -0,0 +1 @@
+FROM docker.io/ghusta/fakesmtp

mrt-inttest-services/fakesmtp/Dockerfile

@@ -0,0 +1,8 @@
+version: '3.3'
+services:
+  fakesmtp:
+    container_name: fakesmtp
+    image: ${ECR_REGISTRY}/fakesmtp:dev
+    build:
+      context: .
+      dockerfile: Dockerfile

mrt-inttest-services/fakesmtp/docker-compose.yml

@@ -1,14 +1,10 @@
 version: '3.3'
 services:
-  minio-server:
+  merritt-maven:
     container_name: merritt-maven
     image: ${ECR_REGISTRY}/merritt-maven:dev
     build:
       context: .
       dockerfile: Dockerfile
       args:
-        ECR_REGISTRY: ${ECR_REGISTRY}
-        JAVA_RELEASE_ARG: ${JAVA_RELEASE:-8}
-    ports:
-    - published: 8080
-      target: 8080
+        JAVA_RELEASE_ARG: ${JAVA_RELEASE:-8}

mrt-inttest-services/merritt-maven/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '3.3'
 services:
-  minio-server:
+  merritt-tomcat:
     container_name: merritt-tomcat
     image: ${ECR_REGISTRY}/merritt-tomcat:dev
     build: