Pulsar CI #2019

Workflow file for this run

.github/workflows/pulsar-ci.yaml at 2715853

	#
	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.
	#

	name: Pulsar CI
	on:
	pull_request:
	branches:
	- master
	- branch-*
	- pulsar-*
	schedule:
	# scheduled job with JDK 21
	- cron: '0 12 * * *'
	# scheduled job with JDK 17
	# if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job
	- cron: '0 6 * * *'
	workflow_dispatch:
	inputs:
	collect_coverage:
	description: 'Collect test coverage and upload to Codecov'
	required: true
	type: boolean
	default: true
	jdk_major_version:
	description: 'JDK major version to use for the build'
	required: true
	type: choice
	options:
	- '17'
	- '21'
	default: '21'
	trace_test_resource_cleanup:
	description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.'
	required: true
	type: choice
	options:
	- 'off'
	- 'on'
	- 'full'
	default: 'off'
	thread_leak_detector_wait_millis:
	description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.'
	required: true
	type: number
	default: 10000

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version \|\| '' }}
	cancel-in-progress: true

	env:
	MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
	# defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
	# it's possible to rerun individual failed jobs when the build artifacts are available
	# if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
	ARTIFACT_RETENTION_DAYS: 3
	JDK_DISTRIBUTION: corretto

	jobs:
	preconditions:
	name: Preconditions
	runs-on: ubuntu-22.04
	outputs:
	docs_only: ${{ steps.check_changes.outputs.docs_only }}
	changed_tests: ${{ steps.changes.outputs.tests_files }}
	need_owasp: ${{ steps.changes.outputs.need_owasp }}
	collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }}
	jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }}
	java_non_tests: ${{ steps.changes.outputs.java_non_tests }}
	steps:
	- name: Cancel scheduled jobs in forks by default
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }}
	uses: actions/github-script@v6
	with:
	script: \|
	await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId});
	process.exit(1);

	- name: Select JDK major version
	id: jdk_major_version
	run: \|
	# use JDK 17 for the scheduled build with cron expression '0 6 * * *'
	if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' \|\| 'false' }}" == "true" ]]; then
	echo "jdk_major_version=17" >> $GITHUB_OUTPUT
	exit 0
	fi
	# use JDK 21 for build unless overridden with workflow_dispatch input
	echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version \|\| '21'}}" >> $GITHUB_OUTPUT

	- name: checkout
	if: ${{ github.event_name == 'pull_request' }}
	uses: actions/checkout@v4

	- name: Detect changed files
	if: ${{ github.event_name == 'pull_request' }}
	id: changes
	uses: apache/pulsar-test-infra/paths-filter@master
	with:
	filters: .github/changes-filter.yaml
	list-files: csv

	- name: Check changed files
	if: ${{ github.event_name == 'pull_request' }}
	id: check_changes
	run: \|
	if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then
	echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
	else
	echo docs_only=false >> $GITHUB_OUTPUT
	fi

	- name: Check if coverage should be collected
	id: check_coverage
	run: \|
	echo "collect_coverage=${{
	(steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch'
	&& (github.base_ref == 'master' \|\| github.ref_name == 'master'))
	\|\| (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true')
	}}" >> $GITHUB_OUTPUT

	- name: Check if the PR has been approved for testing
	if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
	env:
	GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
	GITHUB_TOKEN: ${{ github.token }}
	run: \|
	build/pulsar_ci_tool.sh check_ready_to_test

	build-and-license-check:
	needs: preconditions
	name: Build and License check
	env:
	JOB_NAME: Build and License check
	DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Cache local Maven repository
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

	- name: Check source code license headers
	run: mvn -B -T 8 -ntp initialize apache-rat:check license:check

	- name: Check source code style
	run: mvn -B -T 8 -ntp initialize checkstyle:check

	- name: Build core-modules
	run: \|
	mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true

	- name: Check binary licenses
	run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Save maven build results to Github artifact cache so that the results can be reused
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \
	tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \
	-I zstd -cf - .m2/repository/org/apache/pulsar
	cd $GITHUB_WORKSPACE
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \
	tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait


	unit-tests:
	name: CI - Unit - ${{ matrix.name }}
	env:
	JOB_NAME: CI - Unit - ${{ matrix.name }}
	COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
	DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup \|\| 'off' }}
	TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
	THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis \|\| 10000 }}
	THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
	runs-on: ubuntu-22.04
	timeout-minutes: ${{ matrix.timeout \|\| 60 }}
	needs: ['preconditions', 'build-and-license-check']
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- name: Other
	group: OTHER
	timeout: 75
	- name: Brokers - Broker Group 1
	group: BROKER_GROUP_1
	- name: Brokers - Broker Group 2
	group: BROKER_GROUP_2
	- name: Brokers - Broker Group 3
	group: BROKER_GROUP_3
	- name: Brokers - Broker Group 4
	group: BROKER_GROUP_4
	- name: Brokers - Client Api
	group: BROKER_CLIENT_API
	- name: Brokers - Client Impl
	group: BROKER_CLIENT_IMPL
	- name: Proxy
	group: PROXY
	- name: Pulsar IO
	group: PULSAR_IO
	timeout: 75
	- name: Pulsar IO - Elastic Search
	group: PULSAR_IO_ELASTIC
	- name: Pulsar IO - Kafka Connect Adaptor
	group: PULSAR_IO_KAFKA_CONNECT
	- name: Pulsar Client
	group: CLIENT
	- name: Pulsar Metadata
	group: METADATA

	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Clean Disk when tracing test resource cleanup
	if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
	uses: ./.github/actions/clean-disk

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Cache Maven dependencies
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ matrix.jdk \|\| env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ matrix.jdk \|\| env.CI_JDK_MAJOR_VERSION }}

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Restore maven build results from Github artifact cache
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

	- name: Run setup commands
	run: \|
	${{ matrix.setup }}

	- name: Run unit test group '${{ matrix.group }}'
	run: \|
	CHANGED_TESTS="${{ needs.preconditions.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }}

	- name: Upload coverage to build artifacts
	if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
	run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_unittest_coverage_files ${{ matrix.group }}

	- name: print JVM thread dumps when cancelled
	if: cancelled()
	run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

	- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
	if: ${{ always() }}
	uses: ./.github/actions/copy-test-reports

	- name: Publish Test Report
	uses: apache/pulsar-test-infra/action-junit-report@master
	if: ${{ always() }}
	with:
	report_paths: 'test-reports/TEST-*.xml'
	annotate_only: 'true'

	- name: Report detected thread leaks
	if: ${{ always() }}
	run: \|
	if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then
	cd "$THREAD_LEAK_DETECTOR_DIR"
	cat threadleak*.txt \| awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}'
	fi

	- name: Upload Surefire reports
	uses: actions/upload-artifact@v4
	if: ${{ !success() \|\| env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
	with:
	name: Unit-${{ matrix.group }}-surefire-reports
	path: surefire-reports
	retention-days: 7

	- name: Upload possible heap dump, core dump or crash files
	uses: actions/upload-artifact@v4
	if: ${{ always() }}
	with:
	name: Unit-${{ matrix.group }}-dumps
	path: \|
	/tmp/*.hprof
	*/hs_err_.log
	*/core.
	${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/*
	${{ env.THREAD_LEAK_DETECTOR_DIR }}/*
	retention-days: 7
	if-no-files-found: ignore

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait


	unit-tests-upload-coverage:
	name: CI - Unit - Upload Coverage
	runs-on: ubuntu-22.04
	timeout-minutes: 30
	needs: ['preconditions', 'unit-tests']
	env:
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}

	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Cache Maven dependencies
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ matrix.jdk \|\| env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ matrix.jdk \|\| env.CI_JDK_MAJOR_VERSION }}

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Restore maven build results from Github artifact cache
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

	- name: Restore coverage files from build artifacts and create Jacoco reports
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
	cd $GITHUB_WORKSPACE/target
	zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report \|\| true

	- name: Upload Jacoco report files to build artifacts
	uses: actions/upload-artifact@v4
	with:
	name: Jacoco-coverage-report-unittests
	path: target/jacoco_test_coverage_report_unittests.zip
	retention-days: 3

	- name: Upload to Codecov
	uses: ./.github/actions/upload-coverage
	with:
	flags: unittests

	- name: Delete coverage files from build artifacts
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait

	pulsar-java-test-image:
	name: Build Pulsar java-test-image docker image
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	needs: ['preconditions', 'build-and-license-check']
	if: ${{ needs.preconditions.outputs.docs_only != 'true'}}
	strategy:
	fail-fast: false
	matrix:
	platform:
	- linux/amd64
	- linux/arm64
	env:
	DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Cache Maven dependencies
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: restore maven build results from Github artifact cache
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

	- name: Set up QEMU
	uses: docker/setup-qemu-action@v3
	with:
	platforms: arm64

	- name: Build java-test-image docker image - ${{ matrix.platform }}
	run: \|
	# build docker image
	DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
	-Ddocker.platforms=${{ matrix.platform }} \
	-Dmaven.test.skip=true -DskipSourceReleaseAssembly=true \
	-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true

	- name: save docker image apachepulsar/java-test-image:latest to Github artifact cache
	if: ${{ matrix.platform == 'linux/amd64' }}
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait

	integration-tests:
	name: CI - Integration - ${{ matrix.name }}
	runs-on: ubuntu-22.04
	timeout-minutes: ${{ matrix.timeout \|\| 60 }}
	needs: ['preconditions', 'pulsar-java-test-image']
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	env:
	JOB_NAME: CI - Integration - ${{ matrix.name }}
	PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
	DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- name: Backwards Compatibility
	group: BACKWARDS_COMPAT
	no_coverage: true

	- name: Cli
	group: CLI

	- name: Messaging
	group: MESSAGING

	- name: LoadBalance
	group: LOADBALANCE
	no_coverage: true

	- name: Shade on Java 8
	group: SHADE_RUN
	runtime_jdk: 8
	setup: ./build/run_integration_group.sh SHADE_BUILD
	no_coverage: true

	- name: Shade on Java 11
	group: SHADE_RUN
	runtime_jdk: 11
	setup: ./build/run_integration_group.sh SHADE_BUILD
	no_coverage: true

	- name: Shade on Java 17
	group: SHADE_RUN
	runtime_jdk: 17
	setup: ./build/run_integration_group.sh SHADE_BUILD
	no_coverage: true

	- name: Shade on Java 21
	group: SHADE_RUN
	runtime_jdk: 21
	setup: ./build/run_integration_group.sh SHADE_BUILD
	no_coverage: true

	- name: Standalone
	group: STANDALONE

	- name: Transaction
	group: TRANSACTION

	- name: Metrics
	group: METRICS

	- name: Upgrade
	group: UPGRADE

	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Cache Maven dependencies
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Restore maven build results from Github artifact cache
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

	- name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image

	- name: Run setup commands
	if: ${{ matrix.setup }}
	run: \|
	${{ matrix.setup }}

	- name: Set up runtime JDK ${{ matrix.runtime_jdk }}
	uses: actions/setup-java@v4
	if: ${{ matrix.runtime_jdk }}
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ matrix.runtime_jdk }}

	- name: Run integration test group '${{ matrix.group }}'
	run: \|
	if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then
	coverage_args="--coverage"
	fi
	./build/run_integration_group.sh ${{ matrix.group }} $coverage_args

	- name: Upload coverage to build artifacts
	if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }}
	run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }}

	- name: print JVM thread dumps when cancelled
	if: cancelled()
	run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

	- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
	if: ${{ always() }}
	uses: ./.github/actions/copy-test-reports

	- name: Publish Test Report
	uses: apache/pulsar-test-infra/action-junit-report@master
	if: ${{ always() }}
	with:
	report_paths: 'test-reports/TEST-*.xml'
	annotate_only: 'true'

	- name: Upload Surefire reports
	uses: actions/upload-artifact@v4
	if: ${{ !success() }}
	with:
	name: Integration-${{ matrix.group }}-surefire-reports
	path: surefire-reports
	retention-days: 7

	- name: Upload container logs
	uses: actions/upload-artifact@v4
	if: ${{ !success() }}
	continue-on-error: true
	with:
	name: Integration-${{ matrix.group }}-container-logs
	path: tests/integration/target/container-logs
	retention-days: 7

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait

	integration-tests-upload-coverage:
	name: CI - Integration - Upload Coverage
	runs-on: ubuntu-22.04
	timeout-minutes: 30
	needs: ['preconditions', 'integration-tests']
	if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
	env:
	PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Cache Maven dependencies
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Restore maven build results from Github artifact cache
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

	- name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image

	- name: Restore coverage files from build artifacts and create Jacoco reports
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
	cd $GITHUB_WORKSPACE/target
	zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report \|\| true

	- name: Upload Jacoco report files to build artifacts
	uses: actions/upload-artifact@v4
	with:
	name: Jacoco-coverage-report-inttests
	path: target/jacoco_test_coverage_report_inttests.zip
	retention-days: 3

	- name: Upload to Codecov
	uses: ./.github/actions/upload-coverage
	with:
	flags: inttests

	- name: Delete coverage files from build artifacts
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait

	delete-integration-test-docker-image-artifact:
	name: "Delete integration test docker image artifact"
	runs-on: ubuntu-22.04
	timeout-minutes: 10
	needs: [
	'preconditions',
	'integration-tests',
	'integration-tests-upload-coverage'
	]
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Delete docker image from GitHub Actions Artifacts
	run: \|
	gh-actions-artifact-client.js delete pulsar-java-test-image.zst

	pulsar-test-latest-version-image:
	name: Build Pulsar docker image
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	needs: ['preconditions', 'build-and-license-check']
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	env:
	DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Clean Disk
	uses: ./.github/actions/clean-disk
	with:
	mode: full

	- name: Cache local Maven repository
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: restore maven build results from Github artifact cache
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

	- name: Build latest-version-image docker image
	run: \|
	# build docker image
	# include building of Connectors, Offloaders and server distros
	DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
	-Pmain,docker -Dmaven.test.skip=true \
	-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true

	# check full build artifacts licenses
	- name: Check binary licenses
	run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar--bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell--bin.tar.gz

	- name: Run Trivy container scan
	id: trivy_scan
	uses: aquasecurity/[email protected]
	if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
	continue-on-error: true
	with:
	image-ref: "apachepulsar/pulsar:latest"
	scanners: vuln
	severity: CRITICAL,HIGH,MEDIUM,LOW
	limit-severities-for-sarif: true
	format: 'sarif'
	output: 'trivy-results.sarif'

	- name: Upload Trivy scan results to GitHub Security tab
	uses: github/codeql-action/upload-sarif@v3
	if: ${{ steps.trivy_scan.outcome == 'success' && github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
	continue-on-error: true
	with:
	sarif_file: 'trivy-results.sarif'

	- name: Clean up disk space
	run: \|
	# release disk space since saving docker image consumes local disk space
	#
	echo "::group::Available diskspace before cleaning"
	time df -BM / /mnt
	echo "::endgroup::"
	echo "::group::Clean build directory"
	# docker build changes some files to root ownership, fix this before deleting files
	sudo chown -R $USER:$GROUP .
	# clean build directories
	time git clean -fdx
	echo "::endgroup::"
	echo "::group::Available diskspace after cleaning build directory"
	time df -BM / /mnt
	echo "::endgroup::"
	echo "::group::Delete maven repository"
	# delete maven repository
	time rm -rf ~/.m2/repository
	echo "::endgroup::"
	echo "::group::Available diskspace after cleaning maven repository"
	time df -BM / /mnt
	echo "::endgroup::"

	- name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait

	system-tests:
	name: CI - System - ${{ matrix.name }}
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	needs: ['preconditions', 'pulsar-test-latest-version-image']
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	env:
	JOB_NAME: CI - System - ${{ matrix.name }}
	PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
	DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- name: Tiered FileSystem
	group: TIERED_FILESYSTEM

	- name: Tiered JCloud
	group: TIERED_JCLOUD

	- name: Function
	group: FUNCTION

	- name: Schema
	group: SCHEMA

	- name: Pulsar Connectors - Thread
	group: PULSAR_CONNECTORS_THREAD

	- name: Pulsar Connectors - Process
	group: PULSAR_CONNECTORS_PROCESS

	- name: Pulsar IO
	group: PULSAR_IO
	clean_disk: true

	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Clean Disk when needed
	if: ${{ matrix.clean_disk }}
	uses: ./.github/actions/clean-disk

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Cache local Maven repository
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Restore maven build results from Github artifact cache
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

	- name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

	- name: Run setup commands
	if: ${{ matrix.setup }}
	run: \|
	${{ matrix.setup }}

	- name: Run system test group '${{ matrix.group }}'
	run: \|
	if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then
	coverage_args="--coverage"
	fi
	./build/run_integration_group.sh ${{ matrix.group }} $coverage_args

	- name: Upload coverage to build artifacts
	if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }}
	run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }}

	- name: print JVM thread dumps when cancelled
	if: cancelled()
	run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

	- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
	if: ${{ always() }}
	uses: ./.github/actions/copy-test-reports

	- name: Publish Test Report
	uses: apache/pulsar-test-infra/action-junit-report@master
	if: ${{ always() }}
	with:
	report_paths: 'test-reports/TEST-*.xml'
	annotate_only: 'true'

	- name: Upload container logs
	uses: actions/upload-artifact@v4
	if: ${{ !success() }}
	continue-on-error: true
	with:
	name: System-${{ matrix.group }}-container-logs
	path: tests/integration/target/container-logs
	retention-days: 7

	- name: Upload Surefire reports
	uses: actions/upload-artifact@v4
	if: ${{ !success() }}
	with:
	name: System-${{ matrix.name }}-surefire-reports
	path: surefire-reports
	retention-days: 7

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait

	system-tests-upload-coverage:
	name: CI - System - Upload Coverage
	runs-on: ubuntu-22.04
	timeout-minutes: 30
	needs: ['preconditions', 'system-tests']
	if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
	env:
	PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}

	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Cache local Maven repository
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Restore maven build results from Github artifact cache
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

	- name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

	- name: Restore coverage files from build artifacts and create Jacoco reports
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
	cd $GITHUB_WORKSPACE/target
	zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report \|\| true

	- name: Upload Jacoco report files to build artifacts
	uses: actions/upload-artifact@v4
	with:
	name: Jacoco-coverage-report-systests
	path: target/jacoco_test_coverage_report_systests.zip
	retention-days: 3

	- name: Upload to Codecov
	uses: ./.github/actions/upload-coverage
	with:
	flags: systests

	- name: Delete coverage files from build artifacts
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait

	flaky-system-tests:
	name: CI Flaky - System - ${{ matrix.name }}
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	needs: [ 'preconditions', 'pulsar-test-latest-version-image' ]
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	env:
	JOB_NAME: CI Flaky - System - ${{ matrix.name }}
	PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
	DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- name: Plugin
	group: PLUGIN

	- name: Pulsar IO - Oracle
	group: PULSAR_IO_ORA
	clean_disk: true

	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Clean Disk when needed
	if: ${{ matrix.clean_disk }}
	uses: ./.github/actions/clean-disk

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Cache local Maven repository
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Restore maven build results from Github artifact cache
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

	- name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
	run: \|
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

	- name: Run setup commands
	if: ${{ matrix.setup }}
	run: \|
	${{ matrix.setup }}

	- name: Run system test group '${{ matrix.group }}'
	run: \|
	./build/run_integration_group.sh ${{ matrix.group }}

	- name: print JVM thread dumps when cancelled
	if: cancelled()
	run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

	- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
	if: ${{ always() }}
	uses: ./.github/actions/copy-test-reports

	- name: Publish Test Report
	uses: apache/pulsar-test-infra/action-junit-report@master
	if: ${{ always() }}
	with:
	report_paths: 'test-reports/TEST-*.xml'
	annotate_only: 'true'

	- name: Upload container logs
	uses: actions/upload-artifact@v4
	if: ${{ !success() }}
	continue-on-error: true
	with:
	name: System-${{ matrix.group }}-container-logs
	path: tests/integration/target/container-logs
	retention-days: 7

	- name: Upload Surefire reports
	uses: actions/upload-artifact@v4
	if: ${{ !success() }}
	with:
	name: System-${{ matrix.name }}-surefire-reports
	path: surefire-reports
	retention-days: 7

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait

	delete-system-test-docker-image-artifact:
	name: "Delete system test docker image artifact"
	runs-on: ubuntu-22.04
	timeout-minutes: 10
	needs: [
	'preconditions',
	'system-tests',
	'system-tests-upload-coverage',
	'flaky-system-tests'
	]
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Delete docker image from GitHub Actions Artifacts
	run: \|
	gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst

	macos-build:
	name: Build Pulsar on MacOS
	runs-on: macos-latest
	timeout-minutes: 120
	needs: ['preconditions', 'integration-tests']
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	env:
	DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Cache Maven dependencies
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-all-

	- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

	- name: build package
	run: mvn -B clean package -DskipTests -T 1C -ntp

	codeql:
	name: Run CodeQL Analysis
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	needs: ['preconditions', 'unit-tests']
	if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' \|\| github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') \|\| (github.event_name != 'pull_request' && github.ref_name == 'master')) }}
	permissions:
	actions: read
	contents: read
	security-events: write
	env:
	DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	CODEQL_LANGUAGE: java-kotlin
	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Clean Disk when needed
	if: ${{ matrix.clean_disk }}
	uses: ./.github/actions/clean-disk

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Cache local Maven repository
	uses: actions/cache@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

	- name: Initialize CodeQL
	uses: github/codeql-action/init@v3
	with:
	languages: ${{ env.CODEQL_LANGUAGE }}

	- name: Build Java code
	run: \|
	mvn -B -ntp -Pcore-modules,-main install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true

	- name: Perform CodeQL Analysis
	uses: github/codeql-action/analyze@v3
	with:
	category: "/language:${{ env.CODEQL_LANGUAGE }}"

	owasp-dep-check:
	name: OWASP dependency check
	runs-on: ubuntu-22.04
	timeout-minutes: 120
	needs: [ 'preconditions', 'integration-tests' ]
	if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
	env:
	DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
	NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
	steps:
	- name: checkout
	uses: actions/checkout@v4

	- name: Tune Runner VM
	uses: ./.github/actions/tune-runner-vm

	- name: Setup ssh access to build runner VM
	# ssh access is enabled for builds in own forks
	if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	uses: ./.github/actions/ssh-access
	continue-on-error: true
	with:
	limit-access-to-actor: true

	- name: Restore Maven repository cache
	uses: actions/cache/restore@v4
	timeout-minutes: 5
	with:
	path: \|
	~/.m2/repository///*
	!~/.m2/repository/org/apache/pulsar
	key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-m2-dependencies-core-modules-

	- name: Set up JDK ${{ matrix.jdk \|\| env.CI_JDK_MAJOR_VERSION }}
	uses: actions/setup-java@v4
	with:
	distribution: ${{ env.JDK_DISTRIBUTION }}
	java-version: ${{ matrix.jdk \|\| env.CI_JDK_MAJOR_VERSION }}

	- name: Clean Disk
	uses: ./.github/actions/clean-disk

	- name: Install gh-actions-artifact-client.js
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Restore maven build results from Github artifact cache
	run: \|
	cd $HOME
	$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

	- name: OWASP cache key weeknum
	id: get-weeknum
	run: \|
	echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT
	shell: bash

	- name: Restore OWASP Dependency Check data
	id: restore-owasp-dependency-check-data
	uses: actions/cache/restore@v4
	timeout-minutes: 5
	with:
	path: ~/.m2/repository/org/owasp/dependency-check-data
	key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }}
	enableCrossOsArchive: true
	restore-keys: \|
	owasp-dependency-check-data-

	# Projects dependent on flume, hdfs, and hbase currently excluded from the scan.
	- name: trigger dependency check
	run: \|
	mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \
	-pl '!distribution/server,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'

	- name: Upload report
	uses: actions/upload-artifact@v4
	if: ${{ cancelled() \|\| failure() }}
	continue-on-error: true
	with:
	name: dependency report
	path: target/dependency-check-report.html

	- name: Wait for ssh connection when build fails
	# ssh access is enabled for builds in own forks
	uses: ./.github/actions/ssh-access
	if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
	continue-on-error: true
	with:
	action: wait

	# This job is required for pulls to be merged. This job is referenced by name in .asf.yaml file in the
	# protected_branches section for master branch required_status_checks.
	# It depends on all other jobs in this workflow.
	# This job also cleans up the binaries at the end of the workflow.
	pulsar-ci-checks-completed:
	name: "Pulsar CI checks completed"
	# run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled
	# this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled
	# jobs are not enabled for other than apache/pulsar repository.
	if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }}
	runs-on: ubuntu-22.04
	timeout-minutes: 10
	needs: [
	'preconditions',
	'unit-tests',
	'integration-tests',
	'system-tests',
	'flaky-system-tests',
	'macos-build',
	'unit-tests-upload-coverage',
	'integration-tests-upload-coverage',
	'system-tests-upload-coverage',
	'owasp-dep-check',
	'codeql'
	]
	steps:
	- name: Check that all required jobs were completed successfully
	if: ${{ needs.preconditions.result != 'success' \|\| needs.preconditions.outputs.docs_only != 'true' }}
	run: \|
	if [[ ! ( \
	"${{ needs.preconditions.result }}" == "success" \
	&& "${{ needs.unit-tests.result }}" == "success" \
	&& "${{ needs.integration-tests.result }}" == "success" \
	&& "${{ needs.system-tests.result }}" == "success" \
	&& "${{ needs.macos-build.result }}" == "success" \
	&& ( "${{ needs.codeql.result }}" == "success" \|\| "${{ needs.codeql.result }}" == "skipped" ) \
	) ]]; then
	echo "Required jobs haven't been completed successfully."
	exit 1
	fi

	- name: checkout
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	uses: actions/checkout@v4

	- name: Tune Runner VM
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	uses: ./.github/actions/tune-runner-vm

	- name: Install gh-actions-artifact-client.js
	if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
	uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

	- name: Delete maven repository binaries from GitHub Actions Artifacts
	if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs..result, 'failure') && !contains(needs..result, 'cancelled') }}
	run: \|
	gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst \|\| true
	gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst \|\| true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Pulsar CI #2019

Workflow file

Pulsar CI #2019

Jobs

Run details

Workflow file for this run