diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce2cb2b9723..b630b405a04 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,7 +9,7 @@ on: pull_request: paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE'] workflow_dispatch: - + env: JAVA_VERSION: '11' JAVA_DISTRIBUTION: 'zulu' #This is the default on v1 of the action for 1.8 @@ -49,7 +49,7 @@ jobs: else echo "Nothing to do" fi - + # Build the code and run the unit/integration tests. build-and-test: runs-on: ubuntu-latest @@ -73,11 +73,6 @@ jobs: || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \ || { echo "***** TESTS FAILED. Attempting final retry."; $RUN_TESTS; } - # This step can be uncommented to support feature development. When developing a - # feature which spans multiple repos, you can update the poms to use the latest - # SNAPSHOT across the board, and run this action as a sanity check to ensure - # everything is working in unison. - # # Build the code and run the microservice unit/integration tests. # build-and-test-microservices: # runs-on: ubuntu-latest @@ -117,6 +112,65 @@ jobs: # || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \ # || { echo "***** TESTS FAILED. Attempting final retry."; $RUN_TESTS; } + compose-build-and-test: + runs-on: ubuntu-latest + steps: + - name: Free up some space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + sudo rm -rf $AGENT_TOOLSDIRECTORY + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: Set up JDK ${{env.JAVA_VERSION}} + uses: actions/setup-java@v3 + with: + distribution: ${{env.JAVA_DISTRIBUTION}} + java-version: ${{env.JAVA_VERSION}} + - uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-build-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-build- + ${{ runner.os }}-maven-format- + ${{ runner.os }}-maven- + # Builds the quickstart and microservice docker images and runs a query test + - name: Docker Compose Query Tests + env: + USER_NAME: ${{ secrets.USER_NAME }} + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + run: | + mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean install + # free up some space so that we don't run out + docker system prune -f + mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean + cd docker + ./bootstrap.sh + attempt=0 + max_attempts=20 + while [ $attempt -lt $max_attempts ]; do + attempt=$((attempt+1)) + echo "Starting docker compose (Attempt ${attempt}/${max_attempts})" + nohup docker compose up -d --no-recreate < /dev/null > compose.out 2>&1 & + sleep 60s + cat compose.out + # check to see if the query service is running + QUERY="$(docker compose ps --status running --services | grep query || true)" + if [ "$QUERY" == "query" ] ; then + echo "Docker compose started successfully" + break + elif [ $attempt -eq $max_attempts ] ; then + echo "Failed to start docker compose" + exit 1 + fi + done + cd scripts + ./testAll.sh + quickstart-build-and-test: runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index 2b154209b7b..04c7e4c8460 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@

[![Apache License][li]][ll] ![Build Status](https://github.com/NationalSecurityAgency/datawave/actions/workflows/tests.yml/badge.svg) - DataWave is a Java-based ingest and query framework that leverages [Apache Accumulo](http://accumulo.apache.org/) to provide fast, secure access to your data. DataWave supports a wide variety of use cases, including but not limited to... * Data fusion across structured and unstructured datasets diff --git a/docker/config/application-federation.yml b/docker/config/application-federation.yml old mode 100644 new mode 100755 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 690950b145c..d1842371c68 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -9,7 +9,7 @@ services: - quickstart # To run the wildfly webservice, change `--accumulo` to `--web` command: ["datawave-bootstrap.sh", "--accumulo"] - image: datawave/quickstart-compose:7.0.1-SNAPSHOT + image: datawave/quickstart-compose environment: - DW_CONTAINER_HOST=quickstart - DW_DATAWAVE_WEB_JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:8787 -Duser.timezone=GMT -Dfile.encoding=UTF-8 -Djava.net.preferIPv4Stack=true @@ -122,7 +122,7 @@ services: configuration: entrypoint: [ "java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5009","-jar","app.jar" ] - image: datawave/config-service:4.0.1-SNAPSHOT + image: datawave/config-service command: - --spring.output.ansi.enabled=ALWAYS - --spring.profiles.active=consul,native,open_actuator @@ -146,7 +146,7 @@ services: condition: service_started cache: - image: datawave/hazelcast-service:4.0.1-SNAPSHOT + image: datawave/hazelcast-service scale: 1 command: - --spring.profiles.active=consul,compose,remoteauth @@ -174,7 +174,7 @@ services: authorization: entrypoint: [ "java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5008","-jar","app.jar" ] - image: datawave/authorization-service:4.0.1-SNAPSHOT + image: datawave/authorization-service command: - --spring.output.ansi.enabled=ALWAYS - --spring.profiles.active=consul,mock,compose,federation @@ -203,7 +203,7 @@ services: profiles: - accumulo - full - image: datawave/accumulo-service:4.0.1-SNAPSHOT + image: datawave/accumulo-service command: - --spring.output.ansi.enabled=ALWAYS - --spring.profiles.active=consul,compose,remoteauth @@ -236,7 +236,7 @@ services: condition: service_healthy audit: - image: datawave/audit-service:4.0.1-SNAPSHOT + image: datawave/audit-service command: - --spring.output.ansi.enabled=ALWAYS - --spring.profiles.active=consul,compose,remoteauth @@ -270,7 +270,7 @@ services: metrics: entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5007","-jar","app.jar"] - image: datawave/query-metric-service:4.0.2-SNAPSHOT + image: datawave/query-metric-service command: - --spring.output.ansi.enabled=ALWAYS - --spring.profiles.active=consul,compose,remoteauth @@ -307,7 +307,7 @@ services: profiles: - dictionary - full - image: datawave/dictionary-service:4.0.1-SNAPSHOT + image: datawave/dictionary-service command: - --spring.output.ansi.enabled=ALWAYS - --spring.profiles.active=consul,compose,remoteauth @@ -355,7 +355,7 @@ services: # If you want to test cached results, set the CACHED_RESULTS environment variable to 'true' query: entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005","-jar","app.jar"] - image: datawave/query-service:1.0.1-SNAPSHOT + image: datawave/query-service command: - --spring.output.ansi.enabled=ALWAYS - --spring.profiles.active=consul,compose,remoteauth,querymessaging,metricssource,query,mrquery,cachedresults,federation @@ -399,7 +399,7 @@ services: profiles: - full entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005","-jar","app.jar"] - image: datawave/mapreduce-query-service:1.0.1-SNAPSHOT + image: datawave/mapreduce-query-service command: - --spring.output.ansi.enabled=ALWAYS - --spring.profiles.active=consul,compose,remoteauth,query,mrquery,federation @@ -437,7 +437,7 @@ services: executor-pool1: entrypoint: ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006","-jar","app.jar"] - image: datawave/query-executor-service:1.0.1-SNAPSHOT + image: datawave/query-executor-service command: - --spring.application.name=executor-pool1 - --spring.cloud.config.name=executor @@ -486,7 +486,7 @@ services: profiles: - pool2 - full - image: datawave/query-executor-service:1.0.1-SNAPSHOT + image: datawave/query-executor-service command: - --spring.application.name=executor-pool2 - --spring.cloud.config.name=executor @@ -533,7 +533,7 @@ services: profiles: - modification - full - image: datawave/modification-service:1.0.1-SNAPSHOT + image: datawave/modification-service command: - --spring.output.ansi.enabled=ALWAYS - --spring.profiles.active=consul,compose,remoteauth,query