task/GITHUB-2414 GitHub action update

task/GITHUB-2414 GitHub action update task/GITHUB-2414 GitHub action update task/GITHUB-2414 GitHub action update task/GITHUB-2414 GitHub action task/GITHUB-2414 GitHub action task/GITHUB-2414 GitHub action task/GITHUB-2414 GitHub action task/GITHUB-2414 GitHub action task/GITHUB-2414 GitHub action task/GITHUB-2414 GitHub action task/GITHUB-2414 GitHub action Revert "task/GITHUB-2414 GitHub action" This reverts commit 66bb527. task/GITHUB-2414 GitHub action github action update github action update
NationalSecurityAgency · Jun 27, 2024 · 5a0136f · 5a0136f
1 parent 0f34220
commit 5a0136f
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 21 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -9,7 +9,7 @@ on:
   pull_request:
     paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE']
   workflow_dispatch:
-  
+
 env:
   JAVA_VERSION: '11'
   JAVA_DISTRIBUTION: 'zulu' #This is the default on v1 of the action for 1.8
@@ -49,7 +49,7 @@ jobs:
         else
           echo "Nothing to do"
         fi
-        
+
   # Build the code and run the unit/integration tests.
   build-and-test:
     runs-on: ubuntu-latest
@@ -73,11 +73,6 @@ jobs:
           || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \
           || { echo "***** TESTS FAILED. Attempting final retry."; $RUN_TESTS; }
 
-  # This step can be uncommented to support feature development.  When developing a 
-  # feature which spans multiple repos, you can update the poms to use the latest 
-  # SNAPSHOT across the board, and run this action as a sanity check to ensure 
-  # everything is working in unison.
-
   # # Build the code and run the microservice unit/integration tests.
   # build-and-test-microservices:
   #   runs-on: ubuntu-latest
@@ -117,6 +112,65 @@ jobs:
   #           || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \
   #           || { echo "***** TESTS FAILED. Attempting final retry."; $RUN_TESTS; }
 
+  compose-build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free up some space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf $AGENT_TOOLSDIRECTORY
+      - name: Checkout Code
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+      - name: Set up JDK ${{env.JAVA_VERSION}}
+        uses: actions/setup-java@v3
+        with:
+          distribution: ${{env.JAVA_DISTRIBUTION}}
+          java-version: ${{env.JAVA_VERSION}}
+      - uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-build-
+            ${{ runner.os }}-maven-format-
+            ${{ runner.os }}-maven-
+      # Builds the quickstart and microservice docker images and runs a query test
+      - name: Docker Compose Query Tests
+        env:
+          USER_NAME: ${{ secrets.USER_NAME }}
+          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+        run: |
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean install
+          # free up some space so that we don't run out
+          docker system prune -f
+          mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean
+          cd docker
+          ./bootstrap.sh
+          attempt=0
+          max_attempts=20
+          while [ $attempt -lt $max_attempts ]; do
+            attempt=$((attempt+1))
+            echo "Starting docker compose (Attempt ${attempt}/${max_attempts})"
+            nohup docker compose up -d --no-recreate < /dev/null > compose.out 2>&1 &
+            sleep 60s
+            cat compose.out
+            # check to see if the query service is running
+            QUERY="$(docker compose ps --status running --services | grep query || true)"
+            if [ "$QUERY" == "query" ] ; then
+              echo "Docker compose started successfully"
+              break
+            elif [ $attempt -eq $max_attempts ] ; then
+              echo "Failed to start docker compose"
+              exit 1
+            fi
+          done
+          cd scripts
+          ./testAll.sh
+
   quickstart-build-and-test:
     runs-on: ubuntu-latest
     steps:

diff --git a/README.md b/README.md
@@ -3,7 +3,6 @@
 </p>
 
 [![Apache License][li]][ll] ![Build Status](https://github.com/NationalSecurityAgency/datawave/actions/workflows/tests.yml/badge.svg)
-
 DataWave is a Java-based ingest and query framework that leverages [Apache Accumulo](http://accumulo.apache.org/) to provide fast, secure access to your data. DataWave supports a wide variety of use cases, including but not limited to...
 
 * Data fusion across structured and unstructured datasets

diff --git a/docker/config/application-federation.yml b/docker/config/application-federation.yml
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -9,7 +9,7 @@ services:
       - quickstart
     # To run the wildfly webservice, change `--accumulo` to `--web`
     command: ["datawave-bootstrap.sh", "--accumulo"]
-    image: datawave/quickstart-compose:7.0.1-SNAPSHOT
+    image: datawave/quickstart-compose
     environment:
       - DW_CONTAINER_HOST=quickstart
       - DW_DATAWAVE_WEB_JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:8787 -Duser.timezone=GMT -Dfile.encoding=UTF-8 -Djava.net.preferIPv4Stack=true
@@ -122,7 +122,7 @@ services:
 
   configuration:
     entrypoint: [ "java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5009","-jar","app.jar" ]
-    image: datawave/config-service:4.0.1-SNAPSHOT
+    image: datawave/config-service
     command:
       - --spring.output.ansi.enabled=ALWAYS
       - --spring.profiles.active=consul,native,open_actuator
@@ -146,7 +146,7 @@ services:
         condition: service_started
 
   cache:
-    image: datawave/hazelcast-service:4.0.1-SNAPSHOT
+    image: datawave/hazelcast-service
     scale: 1
     command:
       - --spring.profiles.active=consul,compose,remoteauth
@@ -174,7 +174,7 @@ services:
 
   authorization:
     entrypoint: [ "java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5008","-jar","app.jar" ]
-    image: datawave/authorization-service:4.0.1-SNAPSHOT
+    image: datawave/authorization-service
     command:
       - --spring.output.ansi.enabled=ALWAYS
       - --spring.profiles.active=consul,mock,compose,federation
@@ -203,7 +203,7 @@ services:
     profiles:
       - accumulo
       - full
-    image: datawave/accumulo-service:4.0.1-SNAPSHOT
+    image: datawave/accumulo-service
     command:
       - --spring.output.ansi.enabled=ALWAYS
       - --spring.profiles.active=consul,compose,remoteauth
@@ -236,7 +236,7 @@ services:
         condition: service_healthy
 
   audit:
-    image: datawave/audit-service:4.0.1-SNAPSHOT
+    image: datawave/audit-service
     command:
       - --spring.output.ansi.enabled=ALWAYS
       - --spring.profiles.active=consul,compose,remoteauth
@@ -270,7 +270,7 @@ services:
 
   metrics:
     entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5007","-jar","app.jar"]
-    image: datawave/query-metric-service:4.0.2-SNAPSHOT
+    image: datawave/query-metric-service
     command:
       - --spring.output.ansi.enabled=ALWAYS
       - --spring.profiles.active=consul,compose,remoteauth
@@ -307,7 +307,7 @@ services:
     profiles:
       - dictionary
       - full
-    image: datawave/dictionary-service:4.0.1-SNAPSHOT
+    image: datawave/dictionary-service
     command:
       - --spring.output.ansi.enabled=ALWAYS
       - --spring.profiles.active=consul,compose,remoteauth
@@ -355,7 +355,7 @@ services:
   # If you want to test cached results, set the CACHED_RESULTS environment variable to 'true'
   query:
     entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005","-jar","app.jar"]
-    image: datawave/query-service:1.0.1-SNAPSHOT
+    image: datawave/query-service
     command:
       - --spring.output.ansi.enabled=ALWAYS
       - --spring.profiles.active=consul,compose,remoteauth,querymessaging,metricssource,query,mrquery,cachedresults,federation
@@ -399,7 +399,7 @@ services:
     profiles:
       - full
     entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005","-jar","app.jar"]
-    image: datawave/mapreduce-query-service:1.0.1-SNAPSHOT
+    image: datawave/mapreduce-query-service
     command:
       - --spring.output.ansi.enabled=ALWAYS
       - --spring.profiles.active=consul,compose,remoteauth,query,mrquery,federation
@@ -437,7 +437,7 @@ services:
 
   executor-pool1:
     entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006","-jar","app.jar"]
-    image: datawave/query-executor-service:1.0.1-SNAPSHOT
+    image: datawave/query-executor-service
     command:
       - --spring.application.name=executor-pool1
       - --spring.cloud.config.name=executor
@@ -486,7 +486,7 @@ services:
     profiles:
       - pool2
       - full
-    image: datawave/query-executor-service:1.0.1-SNAPSHOT
+    image: datawave/query-executor-service
     command:
       - --spring.application.name=executor-pool2
       - --spring.cloud.config.name=executor
@@ -533,7 +533,7 @@ services:
     profiles:
       - modification
       - full
-    image: datawave/modification-service:1.0.1-SNAPSHOT
+    image: datawave/modification-service
     command:
       - --spring.output.ansi.enabled=ALWAYS
       - --spring.profiles.active=consul,compose,remoteauth,query