diff --git a/.github/ISSUE_TEMPLATE/flaky-test.yml b/.github/ISSUE_TEMPLATE/flaky-test.yml
index 44ff64197822c..e7b57e1aeda87 100644
--- a/.github/ISSUE_TEMPLATE/flaky-test.yml
+++ b/.github/ISSUE_TEMPLATE/flaky-test.yml
@@ -18,7 +18,7 @@
name: Flaky test
title: "Flaky-test: test_class.test_method"
description: Report a flaky test failure
-labels: [ "component/test", "flaky-tests" ]
+labels: [ "area/test", "type/flaky-tests" ]
body:
- type: markdown
attributes:
diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml
index 7e5f77f9a83fe..ab0f65767a62d 100644
--- a/.github/actions/tune-runner-vm/action.yml
+++ b/.github/actions/tune-runner-vm/action.yml
@@ -77,12 +77,6 @@ runs:
# stop Azure Linux agent to save RAM
sudo systemctl stop walinuxagent.service || true
- # enable docker experimental mode which is
- # required for using "docker build --squash" / "-Ddocker.squash=true"
- daemon_json="$(sudo cat /etc/docker/daemon.json | jq '.experimental = true')"
- echo "$daemon_json" | sudo tee /etc/docker/daemon.json
- # restart docker daemon
- sudo systemctl restart docker
echo '::endgroup::'
# show memory
diff --git a/.github/labeler.yml b/.github/labeler.yml
index 4fc556900926a..851dd2ed27219 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -16,4 +16,5 @@
# under the License.
PIP:
-- 'pip/**'
+ - changed-files:
+ - any-glob-to-any-file: 'pip/**'
diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml
index 9aa2c896547a9..655503849b1c3 100644
--- a/.github/workflows/ci-go-functions.yaml
+++ b/.github/workflows/ci-go-functions.yaml
@@ -85,7 +85,7 @@ jobs:
uses: ./.github/actions/tune-runner-vm
- name: Set up Go
- uses: actions/setup-go@v2
+ uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
id: go
diff --git a/.github/workflows/ci-maven-cache-update.yaml b/.github/workflows/ci-maven-cache-update.yaml
index 53dff03c248cc..a673a30843417 100644
--- a/.github/workflows/ci-maven-cache-update.yaml
+++ b/.github/workflows/ci-maven-cache-update.yaml
@@ -50,7 +50,7 @@ jobs:
name: Update Maven dependency cache for ${{ matrix.name }}
env:
JOB_NAME: Update Maven dependency cache for ${{ matrix.name }}
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
runs-on: ${{ matrix.runs-on }}
timeout-minutes: 45
@@ -64,7 +64,7 @@ jobs:
mvn_arguments: ''
- name: all modules - macos
- runs-on: macos-11
+ runs-on: macos-latest
cache_name: 'm2-dependencies-all'
- name: core-modules
diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
index a273e902c88d2..a1c6dd594d3a2 100644
--- a/.github/workflows/ci-owasp-dependency-check.yaml
+++ b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -24,8 +24,9 @@ on:
workflow_dispatch:
env:
- MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+ MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
JDK_DISTRIBUTION: corretto
+ NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
jobs:
run-owasp-dependency-check:
@@ -33,7 +34,7 @@ jobs:
name: Check ${{ matrix.branch }}
env:
JOB_NAME: Check ${{ matrix.branch }}
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
runs-on: ubuntu-22.04
timeout-minutes: 75
strategy:
@@ -42,12 +43,9 @@ jobs:
matrix:
include:
- branch: master
+ - branch: branch-3.3
- branch: branch-3.2
- - branch: branch-3.1
- branch: branch-3.0
- - branch: branch-2.11
- - branch: branch-2.10
- jdk: 11
steps:
- name: checkout
@@ -58,16 +56,14 @@ jobs:
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- - name: Cache local Maven repository
- uses: actions/cache@v4
+ - name: Restore Maven repository cache
+ uses: actions/cache/restore@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
- !~/.m2/repository/org/owasp/dependency-check-data
key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- lookup-only: true
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
${{ runner.os }}-m2-dependencies-core-modules-
@@ -79,7 +75,7 @@ jobs:
java-version: ${{ matrix.jdk || '17' }}
- name: run install by skip tests
- run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true
+ run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none -pl '!distribution/io,!distribution/offloaders'
- name: OWASP cache key weeknum
id: get-weeknum
@@ -89,7 +85,7 @@ jobs:
- name: Restore OWASP Dependency Check data
id: restore-owasp-dependency-check-data
- uses: actions/cache/restore@v3
+ uses: actions/cache/restore@v4
timeout-minutes: 5
with:
path: ~/.m2/repository/org/owasp/dependency-check-data
@@ -105,7 +101,7 @@ jobs:
- name: Save OWASP Dependency Check data
if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }}
- uses: actions/cache/save@v3
+ uses: actions/cache/save@v4
timeout-minutes: 5
with:
path: ~/.m2/repository/org/owasp/dependency-check-data
diff --git a/.github/workflows/ci-semantic-pull-request.yml b/.github/workflows/ci-semantic-pull-request.yml
index ba421405d5790..15ac85090243c 100644
--- a/.github/workflows/ci-semantic-pull-request.yml
+++ b/.github/workflows/ci-semantic-pull-request.yml
@@ -34,7 +34,7 @@ jobs:
name: Check pull request title
runs-on: ubuntu-latest
steps:
- - uses: amannn/action-semantic-pull-request@v5.0.2
+ - uses: amannn/action-semantic-pull-request@v5.5.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index 94b148a743443..f10e61c8fd20e 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -26,4 +26,4 @@ jobs:
pull-requests: write
runs-on: ubuntu-latest
steps:
- - uses: actions/labeler@v4
+ - uses: actions/labeler@v5
diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml
index a92e5cd26c35b..bfc5140943172 100644
--- a/.github/workflows/pulsar-ci-flaky.yaml
+++ b/.github/workflows/pulsar-ci-flaky.yaml
@@ -148,7 +148,7 @@ jobs:
env:
JOB_NAME: Flaky tests suite
COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
index db1c175bc6221..bf44c51b6ad02 100644
--- a/.github/workflows/pulsar-ci.yaml
+++ b/.github/workflows/pulsar-ci.yaml
@@ -25,9 +25,9 @@ on:
- branch-*
- pulsar-*
schedule:
- # scheduled job with JDK 17
- - cron: '0 12 * * *'
# scheduled job with JDK 21
+ - cron: '0 12 * * *'
+ # scheduled job with JDK 17
# if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job
- cron: '0 6 * * *'
workflow_dispatch:
@@ -44,7 +44,7 @@ on:
options:
- '17'
- '21'
- default: '17'
+ default: '21'
trace_test_resource_cleanup:
description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.'
required: true
@@ -95,13 +95,13 @@ jobs:
- name: Select JDK major version
id: jdk_major_version
run: |
- # use JDK 21 for the scheduled build with cron expression '0 6 * * *'
+ # use JDK 17 for the scheduled build with cron expression '0 6 * * *'
if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then
- echo "jdk_major_version=21" >> $GITHUB_OUTPUT
+ echo "jdk_major_version=17" >> $GITHUB_OUTPUT
exit 0
fi
- # use JDK 17 for build unless overridden with workflow_dispatch input
- echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '17'}}" >> $GITHUB_OUTPUT
+ # use JDK 21 for build unless overridden with workflow_dispatch input
+ echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '21'}}" >> $GITHUB_OUTPUT
- name: checkout
if: ${{ github.event_name == 'pull_request' }}
@@ -147,7 +147,7 @@ jobs:
name: Build and License check
env:
JOB_NAME: Build and License check
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
runs-on: ubuntu-22.04
timeout-minutes: 60
@@ -224,7 +224,7 @@ jobs:
env:
JOB_NAME: CI - Unit - ${{ matrix.name }}
COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
@@ -465,8 +465,14 @@ jobs:
timeout-minutes: 60
needs: ['preconditions', 'build-and-license-check']
if: ${{ needs.preconditions.outputs.docs_only != 'true'}}
+ strategy:
+ fail-fast: false
+ matrix:
+ platform:
+ - linux/amd64
+ - linux/arm64
env:
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
@@ -509,14 +515,21 @@ jobs:
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- - name: Build java-test-image docker image
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+ with:
+ platforms: arm64
+
+ - name: Build java-test-image docker image - ${{ matrix.platform }}
run: |
# build docker image
- mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
- -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \
+ DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
+ -Ddocker.platforms=${{ matrix.platform }} \
+ -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true \
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
- name: save docker image apachepulsar/java-test-image:latest to Github artifact cache
+ if: ${{ matrix.platform == 'linux/amd64' }}
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image
@@ -537,7 +550,7 @@ jobs:
env:
JOB_NAME: CI - Integration - ${{ matrix.name }}
PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
strategy:
fail-fast: false
@@ -590,6 +603,9 @@ jobs:
- name: Metrics
group: METRICS
+ - name: Upgrade
+ group: UPGRADE
+
steps:
- name: checkout
uses: actions/checkout@v4
@@ -815,7 +831,7 @@ jobs:
needs: ['preconditions', 'build-and-license-check']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
env:
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
@@ -868,14 +884,34 @@ jobs:
run: |
# build docker image
# include building of Connectors, Offloaders and server distros
- mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
- -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \
+ DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
+ -Pmain,docker -Dmaven.test.skip=true \
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
# check full build artifacts licenses
- name: Check binary licenses
run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz
+ - name: Run Trivy container scan
+ id: trivy_scan
+ uses: aquasecurity/trivy-action@0.26.0
+ if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
+ continue-on-error: true
+ with:
+ image-ref: "apachepulsar/pulsar:latest"
+ scanners: vuln
+ severity: CRITICAL,HIGH,MEDIUM,LOW
+ limit-severities-for-sarif: true
+ format: 'sarif'
+ output: 'trivy-results.sarif'
+
+ - name: Upload Trivy scan results to GitHub Security tab
+ uses: github/codeql-action/upload-sarif@v3
+ if: ${{ steps.trivy_scan.outcome == 'success' && github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
+ continue-on-error: true
+ with:
+ sarif_file: 'trivy-results.sarif'
+
- name: Clean up disk space
run: |
# release disk space since saving docker image consumes local disk space
@@ -921,7 +957,7 @@ jobs:
env:
JOB_NAME: CI - System - ${{ matrix.name }}
PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
strategy:
fail-fast: false
@@ -1151,7 +1187,7 @@ jobs:
env:
JOB_NAME: CI Flaky - System - ${{ matrix.name }}
PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
strategy:
fail-fast: false
@@ -1289,12 +1325,12 @@ jobs:
macos-build:
name: Build Pulsar on MacOS
- runs-on: macos-11
+ runs-on: macos-latest
timeout-minutes: 120
needs: ['preconditions', 'integration-tests']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
env:
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
- name: checkout
@@ -1334,7 +1370,7 @@ jobs:
contents: read
security-events: write
env:
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
CODEQL_LANGUAGE: java-kotlin
steps:
@@ -1395,8 +1431,9 @@ jobs:
needs: [ 'preconditions', 'integration-tests' ]
if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
env:
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+ DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
+ NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
steps:
- name: checkout
uses: actions/checkout@v4
@@ -1412,16 +1449,14 @@ jobs:
with:
limit-access-to-actor: true
- - name: Cache Maven dependencies
- uses: actions/cache@v4
+ - name: Restore Maven repository cache
+ uses: actions/cache/restore@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
- !~/.m2/repository/org/owasp/dependency-check-data
key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- lookup-only: true
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-
@@ -1450,7 +1485,7 @@ jobs:
- name: Restore OWASP Dependency Check data
id: restore-owasp-dependency-check-data
- uses: actions/cache/restore@v3
+ uses: actions/cache/restore@v4
timeout-minutes: 5
with:
path: ~/.m2/repository/org/owasp/dependency-check-data
@@ -1481,12 +1516,16 @@ jobs:
with:
action: wait
- # This job is required for pulls to be merged.
+ # This job is required for pulls to be merged. This job is referenced by name in .asf.yaml file in the
+ # protected_branches section for master branch required_status_checks.
# It depends on all other jobs in this workflow.
- # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing.
+ # This job also cleans up the binaries at the end of the workflow.
pulsar-ci-checks-completed:
name: "Pulsar CI checks completed"
- if: ${{ always() && needs.preconditions.result == 'success' }}
+ # run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled
+ # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled
+ # jobs are not enabled for other than apache/pulsar repository.
+ if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }}
runs-on: ubuntu-22.04
timeout-minutes: 10
needs: [
@@ -1504,10 +1543,11 @@ jobs:
]
steps:
- name: Check that all required jobs were completed successfully
- if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
+ if: ${{ needs.preconditions.result != 'success' || needs.preconditions.outputs.docs_only != 'true' }}
run: |
if [[ ! ( \
- "${{ needs.unit-tests.result }}" == "success" \
+ "${{ needs.preconditions.result }}" == "success" \
+ && "${{ needs.unit-tests.result }}" == "success" \
&& "${{ needs.integration-tests.result }}" == "success" \
&& "${{ needs.system-tests.result }}" == "success" \
&& "${{ needs.macos-build.result }}" == "success" \
diff --git a/.gitignore b/.gitignore
index cd00c44200059..80d760cd29df7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -97,3 +97,5 @@ test-reports/
# Gradle Enterprise
.mvn/.gradle-enterprise/
+# Gradle Develocity
+.mvn/.develocity/
diff --git a/.mvn/gradle-enterprise-custom-user-data.groovy b/.mvn/develocity-custom-user-data.groovy
similarity index 100%
rename from .mvn/gradle-enterprise-custom-user-data.groovy
rename to .mvn/develocity-custom-user-data.groovy
diff --git a/.mvn/gradle-enterprise.xml b/.mvn/develocity.xml
similarity index 67%
rename from .mvn/gradle-enterprise.xml
rename to .mvn/develocity.xml
index b9ae41d579944..5c0fbb47c7217 100644
--- a/.mvn/gradle-enterprise.xml
+++ b/.mvn/develocity.xml
@@ -19,25 +19,21 @@
under the License.
-->
-
-
- #{env['GRADLE_ENTERPRISE_ACCESS_KEY']?.trim() > '' and !(env['GITHUB_HEAD_REF']?.matches('(?i).*(experiment|wip|private).*') or env['GITHUB_REPOSITORY']?.matches('(?i).*(experiment|wip|private).*'))}
+ #{(env['GRADLE_ENTERPRISE_ACCESS_KEY']?.trim() > '' or env['DEVELOCITY_ACCESS_KEY']?.trim() > '') and !(env['GITHUB_HEAD_REF']?.matches('(?i).*(experiment|wip|private).*') or env['GITHUB_REPOSITORY']?.matches('(?i).*(experiment|wip|private).*'))}
https://ge.apache.org
false
- true
true
true
#{isFalse(env['GITHUB_ACTIONS'])}
- ALWAYS
- true
#{{'0.0.0.0'}}
@@ -50,4 +46,4 @@
false
-
+
\ No newline at end of file
diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml
index bc051debf614c..eb998dc3471b8 100644
--- a/.mvn/extensions.xml
+++ b/.mvn/extensions.xml
@@ -23,12 +23,12 @@
xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
com.gradle
- gradle-enterprise-maven-extension
- 1.20.1
+ develocity-maven-extension
+ 1.21.6
com.gradle
common-custom-user-data-maven-extension
- 1.12.5
+ 2.0
diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties
index db95c131dde6f..d58dfb70bab56 100644
--- a/.mvn/wrapper/maven-wrapper.properties
+++ b/.mvn/wrapper/maven-wrapper.properties
@@ -5,14 +5,15 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.5/apache-maven-3.8.5-bin.zip
-wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar
+wrapperVersion=3.3.2
+distributionType=only-script
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip
diff --git a/README.md b/README.md
index e84265f91a717..1d53af9f08149 100644
--- a/README.md
+++ b/README.md
@@ -141,8 +141,6 @@ components in the Pulsar ecosystem, including connectors, adapters, and other la
>
> This project includes a [Maven Wrapper](https://maven.apache.org/wrapper/) that can be used instead of a system-installed Maven.
> Use it by replacing `mvn` by `./mvnw` on Linux and `mvnw.cmd` on Windows in the commands below.
->
-> It's better to use CMD rather than Powershell on Windows. Because maven will activate the `windows` profile which runs `rename-netty-native-libs.cmd`.
### Build
@@ -192,6 +190,10 @@ Check https://pulsar.apache.org for documentation and examples.
## Build custom docker images
+The commands used in the Apache Pulsar release process can be found in the [release process documentation](https://pulsar.apache.org/contribute/release-process/#stage-docker-images).
+
+Here are some general instructions for building custom docker images:
+
* Docker images must be built with Java 8 for `branch-2.7` or previous branches because of [ISSUE-8445](https://github.com/apache/pulsar/issues/8445).
* Java 11 is the recommended JDK version in `branch-2.8`, `branch-2.9` and `branch-2.10`.
* Java 17 is the recommended JDK version in `master`.
@@ -200,6 +202,8 @@ The following command builds the docker images `apachepulsar/pulsar-all:latest`
```bash
mvn clean install -DskipTests
+# setting DOCKER_CLI_EXPERIMENTAL=enabled is required in some environments with older docker versions
+export DOCKER_CLI_EXPERIMENTAL=enabled
mvn package -Pdocker,-main -am -pl docker/pulsar-all -DskipTests
```
diff --git a/bin/bookkeeper b/bin/bookkeeper
index 0cc07dd49aba5..ac7b622ac2363 100755
--- a/bin/bookkeeper
+++ b/bin/bookkeeper
@@ -69,6 +69,29 @@ else
JAVA=$JAVA_HOME/bin/java
fi
+# JAVA_MAJOR_VERSION should get set by conf/bkenv.sh, just in case it's not
+if [[ -z $JAVA_MAJOR_VERSION ]]; then
+ for token in $("$JAVA" -version 2>&1 | grep 'version "'); do
+ if [[ $token =~ \"([[:digit:]]+)\.([[:digit:]]+)(.*)\" ]]; then
+ if [[ ${BASH_REMATCH[1]} == "1" ]]; then
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[2]}
+ else
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[1]}
+ fi
+ break
+ elif [[ $token =~ \"([[:digit:]]+)(.*)\" ]]; then
+ # Process the java versions without dots, such as `17-internal`.
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[1]}
+ break
+ fi
+ done
+fi
+
+if [[ $JAVA_MAJOR_VERSION -lt 17 ]]; then
+ echo "Error: Bookkeeper included in Pulsar requires Java 17 or later." 1>&2
+ exit 1
+fi
+
# exclude tests jar
RELEASE_JAR=`ls $BK_HOME/bookkeeper-server-*.jar 2> /dev/null | grep -v tests | tail -1`
if [ $? == 0 ]; then
@@ -168,17 +191,12 @@ OPTS="$OPTS -Dlog4j.configurationFile=`basename $BOOKIE_LOG_CONF`"
# Allow Netty to use reflection access
OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
-IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
-# Start --add-opens options
-# '--add-opens' option is not supported in jdk8
-if [[ -z "$IS_JAVA_8" ]]; then
- # BookKeeper: enable posix_fadvise usage and DirectMemoryCRC32Digest (https://github.com/apache/bookkeeper/pull/3234)
- OPTS="$OPTS --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED"
- # Netty: enable java.nio.DirectByteBuffer
- # https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java
- # https://github.com/netty/netty/issues/12265
- OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
-fi
+# BookKeeper: enable posix_fadvise usage and DirectMemoryCRC32Digest (https://github.com/apache/bookkeeper/pull/3234)
+OPTS="$OPTS --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED"
+# Netty: enable java.nio.DirectByteBuffer
+# https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java
+# https://github.com/netty/netty/issues/12265
+OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
OPTS="-cp $BOOKIE_CLASSPATH $OPTS"
@@ -214,20 +232,20 @@ OPTS="$OPTS $BK_METADATA_OPTIONS"
#Change to BK_HOME to support relative paths
cd "$BK_HOME"
if [ $COMMAND == "bookie" ]; then
- exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.server.Main --conf $BOOKIE_CONF $@
+ exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.server.Main --conf $BOOKIE_CONF "$@"
elif [ $COMMAND == "autorecovery" ]; then
- exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.replication.AutoRecoveryMain --conf $BOOKIE_CONF $@
+ exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.replication.AutoRecoveryMain --conf $BOOKIE_CONF "$@"
elif [ $COMMAND == "localbookie" ]; then
NUMBER=$1
shift
- exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.util.LocalBookKeeper $NUMBER $BOOKIE_CONF $@
+ exec $JAVA $OPTS $JMX_ARGS org.apache.bookkeeper.util.LocalBookKeeper $NUMBER $BOOKIE_CONF "$@"
elif [ $COMMAND == "upgrade" ]; then
- exec $JAVA $OPTS org.apache.bookkeeper.bookie.FileSystemUpgrade --conf $BOOKIE_CONF $@
+ exec $JAVA $OPTS org.apache.bookkeeper.bookie.FileSystemUpgrade --conf $BOOKIE_CONF "$@"
elif [ $COMMAND == "shell" ]; then
ENTRY_FORMATTER_ARG="-DentryFormatterClass=${ENTRY_FORMATTER_CLASS:-org.apache.bookkeeper.util.StringEntryFormatter}"
- exec $JAVA $OPTS $ENTRY_FORMATTER_ARG org.apache.bookkeeper.bookie.BookieShell -conf $BOOKIE_CONF $@
+ exec $JAVA $OPTS $ENTRY_FORMATTER_ARG org.apache.bookkeeper.bookie.BookieShell -conf $BOOKIE_CONF "$@"
elif [ $COMMAND == "help" -o $COMMAND == "--help" -o $COMMAND == "-h" ]; then
bookkeeper_help;
else
- exec $JAVA $OPTS $COMMAND $@
+ exec $JAVA $OPTS $COMMAND "$@"
fi
diff --git a/bin/function-localrunner b/bin/function-localrunner
index 2e0aa0f6dffe2..a47f3efa48609 100755
--- a/bin/function-localrunner
+++ b/bin/function-localrunner
@@ -34,21 +34,46 @@ else
JAVA=$JAVA_HOME/bin/java
fi
+for token in $("$JAVA" -version 2>&1 | grep 'version "'); do
+ if [[ $token =~ \"([[:digit:]]+)\.([[:digit:]]+)(.*)\" ]]; then
+ if [[ ${BASH_REMATCH[1]} == "1" ]]; then
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[2]}
+ else
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[1]}
+ fi
+ break
+ elif [[ $token =~ \"([[:digit:]]+)(.*)\" ]]; then
+ # Process the java versions without dots, such as `17-internal`.
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[1]}
+ break
+ fi
+done
+
PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"}
# Garbage collection options
-PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"}
+if [ -z "$PULSAR_GC" ]; then
+ PULSAR_GC="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"
+ if [[ $JAVA_MAJOR_VERSION -ge 21 ]]; then
+ PULSAR_GC="-XX:+UseZGC -XX:+ZGenerational ${PULSAR_GC}"
+ else
+ PULSAR_GC="-XX:+UseZGC ${PULSAR_GC}"
+ fi
+fi
# Garbage collection log.
-IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
-if [[ -z "$IS_JAVA_8" ]]; then
- # >= JDK 9
- PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xlog:gc:logs/pulsar_gc_%p.log:time,uptime:filecount=10,filesize=20M"}
- # '--add-opens' option is not supported in JDK 1.8
- OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED"
-else
- # == JDK 1.8
- PULSAR_GC_LOG=${PULSAR_GC_LOG:-"-Xloggc:logs/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M"}
+PULSAR_GC_LOG_DIR=${PULSAR_GC_LOG_DIR:-logs}
+if [[ -z "$PULSAR_GC_LOG" ]]; then
+ if [[ $JAVA_MAJOR_VERSION -gt 8 ]]; then
+ PULSAR_GC_LOG="-Xlog:gc*,safepoint:${PULSAR_GC_LOG_DIR}/pulsar_gc_%p.log:time,uptime,tags:filecount=10,filesize=20M"
+ if [[ $JAVA_MAJOR_VERSION -ge 17 ]]; then
+ # Use async logging on Java 17+ https://bugs.openjdk.java.net/browse/JDK-8264323
+ PULSAR_GC_LOG="-Xlog:async ${PULSAR_GC_LOG}"
+ fi
+ else
+ # Java 8 gc log options
+ PULSAR_GC_LOG="-Xloggc:${PULSAR_GC_LOG_DIR}/pulsar_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=20M"
+ fi
fi
# Extra options to be passed to the jvm
@@ -90,13 +115,16 @@ OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
# Allow Netty to use reflection access
OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
+OPTS="$OPTS -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true"
+
+if [[ $JAVA_MAJOR_VERSION -gt 8 ]]; then
+ # Required by Pulsar client optimized checksum calculation on other than Linux x86_64 platforms
+ # reflection access to java.util.zip.CRC32C
+ OPTS="$OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED"
+fi
-# Start --add-opens options
-# '--add-opens' option is not supported in jdk8
-if [[ -z "$IS_JAVA_8" ]]; then
- # Netty: enable java.nio.DirectByteBuffer
- # https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java
- # https://github.com/netty/netty/issues/12265
+if [[ $JAVA_MAJOR_VERSION -ge 11 ]]; then
+ # Required by Netty for optimized direct byte buffer access
OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
fi
diff --git a/bin/pulsar b/bin/pulsar
index ab0029af5b0da..0125dd81e8bb4 100755
--- a/bin/pulsar
+++ b/bin/pulsar
@@ -129,6 +129,29 @@ else
JAVA=$JAVA_HOME/bin/java
fi
+# JAVA_MAJOR_VERSION should get set by conf/pulsar_env.sh, just in case it's not
+if [[ -z $JAVA_MAJOR_VERSION ]]; then
+ for token in $("$JAVA" -version 2>&1 | grep 'version "'); do
+ if [[ $token =~ \"([[:digit:]]+)\.([[:digit:]]+)(.*)\" ]]; then
+ if [[ ${BASH_REMATCH[1]} == "1" ]]; then
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[2]}
+ else
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[1]}
+ fi
+ break
+ elif [[ $token =~ \"([[:digit:]]+)(.*)\" ]]; then
+ # Process the java versions without dots, such as `17-internal`.
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[1]}
+ break
+ fi
+ done
+fi
+
+if [[ $JAVA_MAJOR_VERSION -lt 17 ]]; then
+ echo "Error: Pulsar requires Java 17 or later." 1>&2
+ exit 1
+fi
+
# exclude tests jar
RELEASE_JAR=`ls $PULSAR_HOME/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1`
if [ $? == 0 ]; then
@@ -254,27 +277,21 @@ OPTS="$OPTS -Djute.maxbuffer=10485760 -Djava.net.preferIPv4Stack=true"
# Enable TCP keepalive for all Zookeeper client connections
OPTS="$OPTS -Dzookeeper.clientTcpKeepAlive=true"
+# BookKeeper: enable posix_fadvise usage and DirectMemoryCRC32Digest (https://github.com/apache/bookkeeper/pull/3234)
+OPTS="$OPTS --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED"
+# Required by JvmDefaultGCMetricsLogger & MBeanStatsGenerator
+OPTS="$OPTS --add-opens java.management/sun.management=ALL-UNNAMED"
+# Required by MBeanStatsGenerator
+OPTS="$OPTS --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED"
# Allow Netty to use reflection access
OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
-IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
-# Start --add-opens options
-# '--add-opens' option is not supported in jdk8
-if [[ -z "$IS_JAVA_8" ]]; then
- # BookKeeper: enable posix_fadvise usage and DirectMemoryCRC32Digest (https://github.com/apache/bookkeeper/pull/3234)
- OPTS="$OPTS --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util.zip=ALL-UNNAMED"
- # Netty: enable java.nio.DirectByteBuffer
- # https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java
- # https://github.com/netty/netty/issues/12265
- OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
- # netty.DnsResolverUtil
- OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED"
- # JvmDefaultGCMetricsLogger & MBeanStatsGenerator
- OPTS="$OPTS --add-opens java.management/sun.management=ALL-UNNAMED"
- # MBeanStatsGenerator
- OPTS="$OPTS --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED"
- # LinuxInfoUtils
- OPTS="$OPTS --add-opens java.base/jdk.internal.platform=ALL-UNNAMED"
-fi
+OPTS="$OPTS -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true"
+# Netty: enable java.nio.DirectByteBuffer
+# https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java
+# https://github.com/netty/netty/issues/12265
+OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
+# Required by LinuxInfoUtils
+OPTS="$OPTS --add-opens java.base/jdk.internal.platform=ALL-UNNAMED"
OPTS="-cp $PULSAR_CLASSPATH $OPTS"
@@ -329,56 +346,56 @@ fi
cd "$PULSAR_HOME"
if [ $COMMAND == "broker" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-broker.log"}
- exec $JAVA $LOG4J2_SHUTDOWN_HOOK_DISABLED $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.PulsarBrokerStarter --broker-conf $PULSAR_BROKER_CONF $@
+ exec $JAVA $LOG4J2_SHUTDOWN_HOOK_DISABLED $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.PulsarBrokerStarter --broker-conf $PULSAR_BROKER_CONF "$@"
elif [ $COMMAND == "bookie" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"bookkeeper.log"}
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.bookkeeper.server.Main --conf $PULSAR_BOOKKEEPER_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.bookkeeper.server.Main --conf $PULSAR_BOOKKEEPER_CONF "$@"
elif [ $COMMAND == "zookeeper" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"zookeeper.log"}
- exec $JAVA ${ZK_OPTS} $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_ZK_CONF $@
+ exec $JAVA ${ZK_OPTS} $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_ZK_CONF "$@"
elif [ $COMMAND == "global-zookeeper" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"global-zookeeper.log"}
# Allow global ZK to turn into read-only mode when it cannot reach the quorum
OPTS="${OPTS} ${ZK_OPTS} -Dreadonlymode.enabled=true"
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_GLOBAL_ZK_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_GLOBAL_ZK_CONF "$@"
elif [ $COMMAND == "configuration-store" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"configuration-store.log"}
# Allow global ZK to turn into read-only mode when it cannot reach the quorum
OPTS="${OPTS} ${ZK_OPTS} -Dreadonlymode.enabled=true"
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_CONFIGURATION_STORE_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.zookeeper.server.quorum.QuorumPeerMain $PULSAR_CONFIGURATION_STORE_CONF "$@"
elif [ $COMMAND == "proxy" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-proxy.log"}
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.proxy.server.ProxyServiceStarter --config $PULSAR_PROXY_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.proxy.server.ProxyServiceStarter --config $PULSAR_PROXY_CONF "$@"
elif [ $COMMAND == "websocket" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-websocket.log"}
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.websocket.service.WebSocketServiceStarter $PULSAR_WEBSOCKET_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.websocket.service.WebSocketServiceStarter $PULSAR_WEBSOCKET_CONF "$@"
elif [ $COMMAND == "functions-worker" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-functions-worker.log"}
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.functions.worker.FunctionWorkerStarter -c $PULSAR_WORKER_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.pulsar.functions.worker.FunctionWorkerStarter -c $PULSAR_WORKER_CONF "$@"
elif [ $COMMAND == "standalone" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-standalone.log"}
- exec $JAVA $LOG4J2_SHUTDOWN_HOOK_DISABLED $OPTS ${ZK_OPTS} -Dpulsar.log.file=$PULSAR_LOG_FILE -Dpulsar.config.file=$PULSAR_STANDALONE_CONF org.apache.pulsar.PulsarStandaloneStarter $@
+ exec $JAVA $LOG4J2_SHUTDOWN_HOOK_DISABLED $OPTS ${ZK_OPTS} -Dpulsar.log.file=$PULSAR_LOG_FILE -Dpulsar.config.file=$PULSAR_STANDALONE_CONF org.apache.pulsar.PulsarStandaloneStarter "$@"
elif [ ${COMMAND} == "autorecovery" ]; then
PULSAR_LOG_FILE=${PULSAR_LOG_FILE:-"pulsar-autorecovery.log"}
- exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.bookkeeper.replication.AutoRecoveryMain --conf $PULSAR_BOOKKEEPER_CONF $@
+ exec $JAVA $OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE org.apache.bookkeeper.replication.AutoRecoveryMain --conf $PULSAR_BOOKKEEPER_CONF "$@"
elif [ $COMMAND == "initialize-cluster-metadata" ]; then
- exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataSetup $@
+ exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataSetup "$@"
elif [ $COMMAND == "delete-cluster-metadata" ]; then
- exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataTeardown $@
+ exec $JAVA $OPTS org.apache.pulsar.PulsarClusterMetadataTeardown "$@"
elif [ $COMMAND == "initialize-transaction-coordinator-metadata" ]; then
- exec $JAVA $OPTS org.apache.pulsar.PulsarTransactionCoordinatorMetadataSetup $@
+ exec $JAVA $OPTS org.apache.pulsar.PulsarTransactionCoordinatorMetadataSetup "$@"
elif [ $COMMAND == "initialize-namespace" ]; then
- exec $JAVA $OPTS org.apache.pulsar.PulsarInitialNamespaceSetup $@
+ exec $JAVA $OPTS org.apache.pulsar.PulsarInitialNamespaceSetup "$@"
elif [ $COMMAND == "zookeeper-shell" ]; then
- exec $JAVA $OPTS org.apache.zookeeper.ZooKeeperMain $@
+ exec $JAVA $OPTS org.apache.zookeeper.ZooKeeperMain "$@"
elif [ $COMMAND == "broker-tool" ]; then
- exec $JAVA $OPTS org.apache.pulsar.broker.tools.BrokerTool $@
+ exec $JAVA $OPTS org.apache.pulsar.broker.tools.BrokerTool "$@"
elif [ $COMMAND == "compact-topic" ]; then
- exec $JAVA $OPTS org.apache.pulsar.compaction.CompactorTool --broker-conf $PULSAR_BROKER_CONF $@
+ exec $JAVA $OPTS org.apache.pulsar.compaction.CompactorTool --broker-conf $PULSAR_BROKER_CONF "$@"
elif [ $COMMAND == "tokens" ]; then
- exec $JAVA $OPTS org.apache.pulsar.utils.auth.tokens.TokensCliUtils $@
+ exec $JAVA $OPTS org.apache.pulsar.utils.auth.tokens.TokensCliUtils "$@"
elif [ $COMMAND == "version" ]; then
- exec $JAVA $OPTS org.apache.pulsar.PulsarVersionStarter $@
+ exec $JAVA $OPTS org.apache.pulsar.PulsarVersionStarter "$@"
elif [ $COMMAND == "help" -o $COMMAND == "--help" -o $COMMAND == "-h" ]; then
pulsar_help;
else
diff --git a/bin/pulsar-admin-common.cmd b/bin/pulsar-admin-common.cmd
index c52bc1389f68a..c59f0e9b424d3 100644
--- a/bin/pulsar-admin-common.cmd
+++ b/bin/pulsar-admin-common.cmd
@@ -19,7 +19,7 @@
@echo off
-if "%JAVA_HOME%" == "" (
+if not defined JAVA_HOME (
for %%i in (java.exe) do set "JAVACMD=%%~$PATH:i"
) else (
set "JAVACMD=%JAVA_HOME%\bin\java.exe"
@@ -28,16 +28,28 @@ if "%JAVA_HOME%" == "" (
if not exist "%JAVACMD%" (
echo The JAVA_HOME environment variable is not defined correctly, so Pulsar CLI cannot be started. >&2
echo JAVA_HOME is set to "%JAVA_HOME%", but "%JAVACMD%" does not exist. >&2
- goto error
+ exit /b 1
)
+set JAVA_MAJOR_VERSION=0
+REM Requires "setlocal enabledelayedexpansion" to work
+for /f tokens^=3 %%g in ('"!JAVACMD!" -version 2^>^&1 ^| findstr /i version') do (
+ set JAVA_MAJOR_VERSION=%%g
+)
+set JAVA_MAJOR_VERSION=%JAVA_MAJOR_VERSION:"=%
+for /f "delims=.-_ tokens=1-2" %%v in ("%JAVA_MAJOR_VERSION%") do (
+ if /I "%%v" EQU "1" (
+ set JAVA_MAJOR_VERSION=%%w
+ ) else (
+ set JAVA_MAJOR_VERSION=%%v
+ )
+)
for %%i in ("%~dp0.") do SET "SCRIPT_PATH=%%~fi"
set "PULSAR_HOME_DIR=%SCRIPT_PATH%\..\"
for %%i in ("%PULSAR_HOME_DIR%.") do SET "PULSAR_HOME=%%~fi"
set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_HOME%\lib\*"
-
if "%PULSAR_CLIENT_CONF%" == "" set "PULSAR_CLIENT_CONF=%PULSAR_HOME%\conf\client.conf"
if "%PULSAR_LOG_CONF%" == "" set "PULSAR_LOG_CONF=%PULSAR_HOME%\conf\log4j2.yaml"
@@ -50,18 +62,21 @@ set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_LOG_CONF_DIR%"
set "OPTS=%OPTS% -Dlog4j.configurationFile="%PULSAR_LOG_CONF_BASENAME%""
set "OPTS=%OPTS% -Djava.net.preferIPv4Stack=true"
-set "isjava8=false"
-FOR /F "tokens=*" %%g IN ('"java -version 2>&1"') do (
- echo %%g|find "version" >nul
- if errorlevel 0 (
- echo %%g|find "1.8" >nul
- if errorlevel 0 (
- set "isjava8=true"
- )
- )
+REM Allow Netty to use reflection access
+set "OPTS=%OPTS% -Dio.netty.tryReflectionSetAccessible=true"
+set "OPTS=%OPTS% -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true"
+
+if %JAVA_MAJOR_VERSION% GTR 8 (
+ set "OPTS=%OPTS% --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED"
+ REM Required by Pulsar client optimized checksum calculation on other than Linux x86_64 platforms
+ REM reflection access to java.util.zip.CRC32C
+ set "OPTS=%OPTS% --add-opens java.base/java.util.zip=ALL-UNNAMED"
)
-if "%isjava8%" == "false" set "OPTS=%OPTS% --add-opens java.base/sun.net=ALL-UNNAMED"
+if %JAVA_MAJOR_VERSION% GEQ 11 (
+ REM Required by Netty for optimized direct byte buffer access
+ set "OPTS=%OPTS% --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
+)
set "OPTS=-cp "%PULSAR_CLASSPATH%" %OPTS%"
set "OPTS=%OPTS% %PULSAR_EXTRA_OPTS%"
@@ -78,7 +93,4 @@ set "OPTS=%OPTS% -Dpulsar.log.dir=%PULSAR_LOG_DIR%"
set "OPTS=%OPTS% -Dpulsar.log.level=%PULSAR_LOG_LEVEL%"
set "OPTS=%OPTS% -Dpulsar.log.root.level=%PULSAR_LOG_ROOT_LEVEL%"
set "OPTS=%OPTS% -Dpulsar.log.immediateFlush=%PULSAR_LOG_IMMEDIATE_FLUSH%"
-set "OPTS=%OPTS% -Dpulsar.routing.appender.default=%PULSAR_ROUTING_APPENDER_DEFAULT%"
-
-:error
-exit /b 1
+set "OPTS=%OPTS% -Dpulsar.routing.appender.default=%PULSAR_ROUTING_APPENDER_DEFAULT%"
\ No newline at end of file
diff --git a/bin/pulsar-admin-common.sh b/bin/pulsar-admin-common.sh
index 8aa21c00f634d..336ff43c1a861 100755
--- a/bin/pulsar-admin-common.sh
+++ b/bin/pulsar-admin-common.sh
@@ -37,6 +37,21 @@ else
JAVA=$JAVA_HOME/bin/java
fi
+for token in $("$JAVA" -version 2>&1 | grep 'version "'); do
+ if [[ $token =~ \"([[:digit:]]+)\.([[:digit:]]+)(.*)\" ]]; then
+ if [[ ${BASH_REMATCH[1]} == "1" ]]; then
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[2]}
+ else
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[1]}
+ fi
+ break
+ elif [[ $token =~ \"([[:digit:]]+)(.*)\" ]]; then
+ # Process the java versions without dots, such as `17-internal`.
+ JAVA_MAJOR_VERSION=${BASH_REMATCH[1]}
+ break
+ fi
+done
+
# exclude tests jar
RELEASE_JAR=`ls $PULSAR_HOME/pulsar-*.jar 2> /dev/null | grep -v tests | tail -1`
if [ $? == 0 ]; then
@@ -91,11 +106,20 @@ PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF`"
OPTS="$OPTS -Djava.net.preferIPv4Stack=true"
-IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
-# Start --add-opens options
-# '--add-opens' option is not supported in jdk8
-if [[ -z "$IS_JAVA_8" ]]; then
+# Allow Netty to use reflection access
+OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
+OPTS="$OPTS -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true"
+
+if [[ $JAVA_MAJOR_VERSION -gt 8 ]]; then
OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED"
+ # Required by Pulsar client optimized checksum calculation on other than Linux x86_64 platforms
+ # reflection access to java.util.zip.CRC32C
+ OPTS="$OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED"
+fi
+
+if [[ $JAVA_MAJOR_VERSION -ge 11 ]]; then
+ # Required by Netty for optimized direct byte buffer access
+ OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
fi
OPTS="-cp $PULSAR_CLASSPATH $OPTS"
diff --git a/bin/pulsar-admin.cmd b/bin/pulsar-admin.cmd
index 45bd8d4541fed..e29d804d70c45 100644
--- a/bin/pulsar-admin.cmd
+++ b/bin/pulsar-admin.cmd
@@ -18,7 +18,7 @@
@REM
@echo off
-
+setlocal enabledelayedexpansion
for %%i in ("%~dp0.") do SET "SCRIPT_PATH=%%~fi"
set "PULSAR_HOME_DIR=%SCRIPT_PATH%\..\"
for %%i in ("%PULSAR_HOME_DIR%.") do SET "PULSAR_HOME=%%~fi"
@@ -27,4 +27,5 @@ if ERRORLEVEL 1 (
exit /b 1
)
cd "%PULSAR_HOME%"
-"%JAVACMD%" %OPTS% org.apache.pulsar.admin.cli.PulsarAdminTool %PULSAR_CLIENT_CONF% %*
\ No newline at end of file
+"%JAVACMD%" %OPTS% org.apache.pulsar.admin.cli.PulsarAdminTool %PULSAR_CLIENT_CONF% %*
+endlocal
\ No newline at end of file
diff --git a/bin/pulsar-client.cmd b/bin/pulsar-client.cmd
index 9e3cef45a25a7..9cacf71cc3a79 100644
--- a/bin/pulsar-client.cmd
+++ b/bin/pulsar-client.cmd
@@ -18,7 +18,7 @@
@REM
@echo off
-
+setlocal enabledelayedexpansion
for %%i in ("%~dp0.") do SET "SCRIPT_PATH=%%~fi"
set "PULSAR_HOME_DIR=%SCRIPT_PATH%\..\"
for %%i in ("%PULSAR_HOME_DIR%.") do SET "PULSAR_HOME=%%~fi"
@@ -27,4 +27,5 @@ if ERRORLEVEL 1 (
exit /b 1
)
cd "%PULSAR_HOME%"
-"%JAVACMD%" %OPTS% org.apache.pulsar.client.cli.PulsarClientTool %PULSAR_CLIENT_CONF% %*
\ No newline at end of file
+"%JAVACMD%" %OPTS% org.apache.pulsar.client.cli.PulsarClientTool %PULSAR_CLIENT_CONF% %*
+endlocal
\ No newline at end of file
diff --git a/bin/pulsar-daemon b/bin/pulsar-daemon
index 210162b6a2190..2c05cb5c49dab 100755
--- a/bin/pulsar-daemon
+++ b/bin/pulsar-daemon
@@ -157,7 +157,7 @@ start ()
echo starting $command, logging to $logfile
echo Note: Set immediateFlush to true in conf/log4j2.yaml will guarantee the logging event is flushing to disk immediately. The default behavior is switched off due to performance considerations.
pulsar=$PULSAR_HOME/bin/pulsar
- nohup $pulsar $command "$1" > "$out" 2>&1 < /dev/null &
+ nohup $pulsar $command "$@" > "$out" 2>&1 < /dev/null &
echo $! > $pid
sleep 1; head $out
sleep 2;
@@ -216,7 +216,7 @@ stop ()
case $startStop in
(start)
- start "$*"
+ start "$@"
;;
(stop)
@@ -224,21 +224,20 @@ case $startStop in
;;
(restart)
- forceStopFlag=$(echo "$*"|grep "\-force")
- if [[ "$forceStopFlag" != "" ]]
+ if [[ "$1" == "-force" ]]
then
- stop "-force"
+ stop -force
+ # remove "-force" from the arguments
+ shift
else
stop
fi
if [ "$?" == 0 ]
then
- sleep 3
- paramaters="$*"
- startParamaters=${paramaters//-force/}
- start "$startParamaters"
+ sleep 3
+ start "$@"
else
- echo "WARNNING : $command failed restart, for $command is not stopped completely."
+ echo "WARNNING : $command failed restart, for $command is not stopped completely."
fi
;;
diff --git a/bin/pulsar-perf b/bin/pulsar-perf
index bdc1dc1ed8b8c..9108a42ef994f 100755
--- a/bin/pulsar-perf
+++ b/bin/pulsar-perf
@@ -84,37 +84,6 @@ add_maven_deps_to_classpath() {
fi
PULSAR_CLASSPATH=${CLASSPATH}:`cat "${f}"`
}
-pulsar_help() {
- cat <
-where command is one of:
- produce Run a producer
- consume Run a consumer
- transaction Run a transaction repeatedly
- read Run a topic reader
-
- websocket-producer Run a websocket producer
-
- managed-ledger Write directly on managed-ledgers
- monitor-brokers Continuously receive broker data and/or load reports
- simulation-client Run a simulation server acting as a Pulsar client
- simulation-controller Run a simulation controller to give commands to servers
-
- gen-doc Generate documentation automatically.
-
- help This help message
-
-or command is the full name of a class with a defined main() method.
-
-Environment variables:
- PULSAR_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF)
- PULSAR_CLIENT_CONF Configuration file for client (default: $DEFAULT_CLIENT_CONF)
- PULSAR_EXTRA_OPTS Extra options to be passed to the jvm
- PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath
-
-These variable can also be set in conf/pulsar_env.sh
-EOF
-}
if [ -d "$PULSAR_HOME/lib" ]; then
PULSAR_CLASSPATH="$PULSAR_CLASSPATH:$PULSAR_HOME/lib/*"
@@ -134,11 +103,20 @@ PULSAR_CLASSPATH="$PULSAR_JAR:$PULSAR_CLASSPATH:$PULSAR_EXTRA_CLASSPATH"
PULSAR_CLASSPATH="`dirname $PULSAR_LOG_CONF`:$PULSAR_CLASSPATH"
OPTS="$OPTS -Dlog4j.configurationFile=`basename $PULSAR_LOG_CONF` -Djava.net.preferIPv4Stack=true"
-IS_JAVA_8=$( $JAVA -version 2>&1 | grep version | grep '"1\.8' )
-# Start --add-opens options
-# '--add-opens' option is not supported in jdk8
-if [[ -z "$IS_JAVA_8" ]]; then
+# Allow Netty to use reflection access
+OPTS="$OPTS -Dio.netty.tryReflectionSetAccessible=true"
+OPTS="$OPTS -Dorg.apache.pulsar.shade.io.netty.tryReflectionSetAccessible=true"
+
+if [[ $JAVA_MAJOR_VERSION -gt 8 ]]; then
OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED"
+ # Required by Pulsar client optimized checksum calculation on other than Linux x86_64 platforms
+ # reflection access to java.util.zip.CRC32C
+ OPTS="$OPTS --add-opens java.base/java.util.zip=ALL-UNNAMED"
+fi
+
+if [[ $JAVA_MAJOR_VERSION -ge 11 ]]; then
+ # Required by Netty for optimized direct byte buffer access
+ OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
fi
OPTS="-cp $PULSAR_CLASSPATH $OPTS"
@@ -162,36 +140,4 @@ OPTS="$OPTS -Dpulsar.log.file=$PULSAR_LOG_FILE"
#Change to PULSAR_HOME to support relative paths
cd "$PULSAR_HOME"
-# if no args specified, show usage
-if [ $# = 0 ]; then
- pulsar_help;
- exit 1;
-fi
-
-# get arguments
-COMMAND=$1
-shift
-
-if [ "$COMMAND" == "produce" ]; then
- exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceProducer --conf-file $PULSAR_PERFTEST_CONF "$@"
-elif [ "$COMMAND" == "consume" ]; then
- exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceConsumer --conf-file $PULSAR_PERFTEST_CONF "$@"
-elif [ "$COMMAND" == "transaction" ]; then
- exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceTransaction --conf-file $PULSAR_PERFTEST_CONF "$@"
-elif [ "$COMMAND" == "read" ]; then
- exec $JAVA $OPTS org.apache.pulsar.testclient.PerformanceReader --conf-file $PULSAR_PERFTEST_CONF "$@"
-elif [ "$COMMAND" == "monitor-brokers" ]; then
- exec $JAVA $OPTS org.apache.pulsar.testclient.BrokerMonitor "$@"
-elif [ "$COMMAND" == "simulation-client" ]; then
- exec $JAVA $OPTS org.apache.pulsar.testclient.LoadSimulationClient "$@"
-elif [ "$COMMAND" == "simulation-controller" ]; then
- exec $JAVA $OPTS org.apache.pulsar.testclient.LoadSimulationController "$@"
-elif [ "$COMMAND" == "websocket-producer" ]; then
- exec $JAVA $OPTS org.apache.pulsar.proxy.socket.client.PerformanceClient "$@"
-elif [ "$COMMAND" == "managed-ledger" ]; then
- exec $JAVA $OPTS org.apache.pulsar.testclient.ManagedLedgerWriter "$@"
-elif [ "$COMMAND" == "gen-doc" ]; then
- exec $JAVA $OPTS org.apache.pulsar.testclient.CmdGenerateDocumentation "$@"
-else
- pulsar_help;
-fi
+exec $JAVA $OPTS org.apache.pulsar.testclient.PulsarPerfTestTool $PULSAR_PERFTEST_CONF "$@"
diff --git a/bin/pulsar-perf.cmd b/bin/pulsar-perf.cmd
index cf6c25b77e59d..aaeaa7a21856b 100644
--- a/bin/pulsar-perf.cmd
+++ b/bin/pulsar-perf.cmd
@@ -18,149 +18,17 @@
@REM
@echo off
-
-if "%JAVA_HOME%" == "" (
- for %%i in (java.exe) do set "JAVACMD=%%~$PATH:i"
-) else (
- set "JAVACMD=%JAVA_HOME%\bin\java.exe"
-)
-
-if not exist "%JAVACMD%" (
- echo The JAVA_HOME environment variable is not defined correctly, so Pulsar CLI cannot be started. >&2
- echo JAVA_HOME is set to "%JAVA_HOME%", but "%JAVACMD%" does not exist. >&2
- exit /B 1
-)
-
+setlocal enabledelayedexpansion
for %%i in ("%~dp0.") do SET "SCRIPT_PATH=%%~fi"
set "PULSAR_HOME_DIR=%SCRIPT_PATH%\..\"
for %%i in ("%PULSAR_HOME_DIR%.") do SET "PULSAR_HOME=%%~fi"
-set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_HOME%\lib\*"
-
-
-if "%PULSAR_CLIENT_CONF%" == "" set "PULSAR_CLIENT_CONF=%PULSAR_HOME%\conf\client.conf"
-if "%PULSAR_LOG_CONF%" == "" set "PULSAR_LOG_CONF=%PULSAR_HOME%\conf\log4j2.yaml"
-
-set "PULSAR_LOG_CONF_DIR1=%PULSAR_LOG_CONF%\..\"
-for %%i in ("%PULSAR_LOG_CONF_DIR1%.") do SET "PULSAR_LOG_CONF_DIR=%%~fi"
-for %%a in ("%PULSAR_LOG_CONF%") do SET "PULSAR_LOG_CONF_BASENAME=%%~nxa"
-
-set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_LOG_CONF_DIR%"
-if not "%PULSAR_EXTRA_CLASSPATH%" == "" set "PULSAR_CLASSPATH=%PULSAR_CLASSPATH%;%PULSAR_EXTRA_CLASSPATH%"
-
-
-if "%PULSAR_PERFTEST_CONF%" == "" set "PULSAR_PERFTEST_CONF=%PULSAR_CLIENT_CONF%"
-
-
-set "OPTS=%OPTS% -Dlog4j.configurationFile="%PULSAR_LOG_CONF_BASENAME%""
-set "OPTS=%OPTS% -Djava.net.preferIPv4Stack=true"
-
-
-set "OPTS=-cp "%PULSAR_CLASSPATH%" %OPTS%"
-set "OPTS=%OPTS% %PULSAR_EXTRA_OPTS%"
-
-if "%PULSAR_LOG_DIR%" == "" set "PULSAR_LOG_DIR=%PULSAR_HOME%\logs"
-if "%PULSAR_LOG_FILE%" == "" set "PULSAR_LOG_FILE=pulsar-perftest.log"
if "%PULSAR_LOG_APPENDER%" == "" set "PULSAR_LOG_APPENDER=Console"
-if "%PULSAR_LOG_LEVEL%" == "" set "PULSAR_LOG_LEVEL=info"
-if "%PULSAR_LOG_ROOT_LEVEL%" == "" set "PULSAR_LOG_ROOT_LEVEL=%PULSAR_LOG_LEVEL%"
-if "%PULSAR_LOG_IMMEDIATE_FLUSH%" == "" set "PULSAR_LOG_IMMEDIATE_FLUSH=false"
-
-
-set "OPTS=%OPTS% -Dpulsar.log.appender=%PULSAR_LOG_APPENDER%"
-set "OPTS=%OPTS% -Dpulsar.log.dir=%PULSAR_LOG_DIR%"
-set "OPTS=%OPTS% -Dpulsar.log.level=%PULSAR_LOG_LEVEL%"
-set "OPTS=%OPTS% -Dpulsar.log.root.level=%PULSAR_LOG_ROOT_LEVEL%"
-set "OPTS=%OPTS% -Dpulsar.log.immediateFlush=%PULSAR_LOG_IMMEDIATE_FLUSH%"
-
-set "COMMAND=%1"
-
-for /f "tokens=1,* delims= " %%a in ("%*") do set "_args=%%b"
-
-if "%COMMAND%" == "produce" (
- call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceProducer
- exit /B %ERROR_CODE%
-)
-if "%COMMAND%" == "consume" (
- call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceConsumer
- exit /B %ERROR_CODE%
-)
-if "%COMMAND%" == "transaction" (
- call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceTransaction
- exit /B %ERROR_CODE%
-)
-if "%COMMAND%" == "read" (
- call :execCmdWithConfigFile org.apache.pulsar.testclient.PerformanceReader
- exit /B %ERROR_CODE%
-)
-if "%COMMAND%" == "monitor-brokers" (
- call :execCmd org.apache.pulsar.testclient.BrokerMonitor
- exit /B %ERROR_CODE%
-)
-if "%COMMAND%" == "simulation-client" (
- call :execCmd org.apache.pulsar.testclient.LoadSimulationClient
- exit /B %ERROR_CODE%
-)
-if "%COMMAND%" == "simulation-controller" (
- call :execCmd org.apache.pulsar.testclient.LoadSimulationController
- exit /B %ERROR_CODE%
-)
-if "%COMMAND%" == "websocket-producer" (
- call :execCmd org.apache.pulsar.proxy.socket.client.PerformanceClient
- exit /B %ERROR_CODE%
-)
-if "%COMMAND%" == "managed-ledger" (
- call :execCmd org.apache.pulsar.testclient.ManagedLedgerWriter
- exit /B %ERROR_CODE%
-)
-if "%COMMAND%" == "gen-doc" (
- call :execCmd org.apache.pulsar.testclient.CmdGenerateDocumentation
- exit /B %ERROR_CODE%
-)
-
-call :usage
-exit /B %ERROR_CODE%
-
-:execCmdWithConfigFile
-"%JAVACMD%" %OPTS% %1 --conf-file "%PULSAR_PERFTEST_CONF%" %_args%
-if ERRORLEVEL 1 (
- call :error
-)
-goto :eof
-
-:execCmd
-"%JAVACMD%" %OPTS% %1 %_args%
+if "%PULSAR_LOG_FILE%" == "" set "PULSAR_LOG_FILE=pulsar-perftest.log"
+call "%PULSAR_HOME%\bin\pulsar-admin-common.cmd"
if ERRORLEVEL 1 (
- call :error
+ exit /b 1
)
-goto :eof
-
-
-
-:error
-set ERROR_CODE=1
-goto :eof
-
-
-
+if "%PULSAR_PERFTEST_CONF%" == "" set "PULSAR_PERFTEST_CONF=%PULSAR_CLIENT_CONF%"
+"%JAVACMD%" %OPTS% org.apache.pulsar.testclient.PulsarPerfTestTool "%PULSAR_PERFTEST_CONF%" %*
+endlocal
-:usage
-echo Usage: pulsar-perf COMMAND
-echo where command is one of:
-echo produce Run a producer
-echo consume Run a consumer
-echo transaction Run a transaction repeatedly
-echo read Run a topic reader
-echo websocket-producer Run a websocket producer
-echo managed-ledger Write directly on managed-ledgers
-echo monitor-brokers Continuously receive broker data and/or load reports
-echo simulation-client Run a simulation server acting as a Pulsar client
-echo simulation-controller Run a simulation controller to give commands to servers
-echo gen-doc Generate documentation automatically.
-echo help This help message
-echo or command is the full name of a class with a defined main() method.
-echo Environment variables:
-echo PULSAR_LOG_CONF Log4j configuration file (default %PULSAR_HOME%\logs)
-echo PULSAR_CLIENT_CONF Configuration file for client (default: %PULSAR_HOME%\conf\client.conf)
-echo PULSAR_EXTRA_OPTS Extra options to be passed to the jvm
-echo PULSAR_EXTRA_CLASSPATH Add extra paths to the pulsar classpath
-goto error
diff --git a/bin/pulsar-shell.cmd b/bin/pulsar-shell.cmd
index c339d34289572..615408f9c7a6e 100644
--- a/bin/pulsar-shell.cmd
+++ b/bin/pulsar-shell.cmd
@@ -18,7 +18,7 @@
@REM
@echo off
-
+setlocal enabledelayedexpansion
for %%i in ("%~dp0.") do SET "SCRIPT_PATH=%%~fi"
set "PULSAR_HOME_DIR=%SCRIPT_PATH%\..\"
for %%i in ("%PULSAR_HOME_DIR%.") do SET "PULSAR_HOME=%%~fi"
@@ -26,9 +26,9 @@ call "%PULSAR_HOME%\bin\pulsar-admin-common.cmd"
if ERRORLEVEL 1 (
exit /b 1
)
-
set "OPTS=%OPTS% -Dorg.jline.terminal.jansi=false"
set "OPTS=%OPTS% -Dpulsar.shell.config.default=%cd%"
set "DEFAULT_CONFIG=-Dpulsar.shell.config.default="%PULSAR_CLIENT_CONF%""
cd "%PULSAR_HOME%"
-"%JAVACMD%" %OPTS% %DEFAULT_CONFIG% org.apache.pulsar.shell.PulsarShell %*
\ No newline at end of file
+"%JAVACMD%" %OPTS% %DEFAULT_CONFIG% org.apache.pulsar.shell.PulsarShell %*
+endlocal
\ No newline at end of file
diff --git a/bouncy-castle/bc/LICENSE b/bouncy-castle/bc/LICENSE
index dae8f16df5b82..c95d33d3d1ffb 100644
--- a/bouncy-castle/bc/LICENSE
+++ b/bouncy-castle/bc/LICENSE
@@ -205,6 +205,5 @@
This projects includes binary packages with the following licenses:
Bouncy Castle License
* Bouncy Castle -- licenses/LICENSE-bouncycastle.txt
- - org.bouncycastle-bcpkix-jdk18on-1.75.jar
- - org.bouncycastle-bcprov-jdk18on-1.75.jar
- - org.bouncycastle-bcprov-ext-jdk18on-1.75.jar
+ - org.bouncycastle-bcpkix-jdk18on-1.78.1.jar
+ - org.bouncycastle-bcprov-jdk18on-1.78.1.jar
diff --git a/bouncy-castle/bc/pom.xml b/bouncy-castle/bc/pom.xml
index a20f9146b76b4..e440923af6de5 100644
--- a/bouncy-castle/bc/pom.xml
+++ b/bouncy-castle/bc/pom.xml
@@ -25,8 +25,7 @@
org.apache.pulsar
bouncy-castle-parent
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
bouncy-castle-bc
diff --git a/bouncy-castle/bcfips-include-test/pom.xml b/bouncy-castle/bcfips-include-test/pom.xml
index 99495aa987c5a..f4478174b86dd 100644
--- a/bouncy-castle/bcfips-include-test/pom.xml
+++ b/bouncy-castle/bcfips-include-test/pom.xml
@@ -24,8 +24,7 @@
org.apache.pulsar
bouncy-castle-parent
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
bcfips-include-test
diff --git a/bouncy-castle/bcfips/pom.xml b/bouncy-castle/bcfips/pom.xml
index 410dd14a260dd..250b3db6b9b08 100644
--- a/bouncy-castle/bcfips/pom.xml
+++ b/bouncy-castle/bcfips/pom.xml
@@ -25,8 +25,7 @@
org.apache.pulsar
bouncy-castle-parent
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
bouncy-castle-bcfips
diff --git a/bouncy-castle/pom.xml b/bouncy-castle/pom.xml
index c78b8fd6d8ea1..4d85a163104a2 100644
--- a/bouncy-castle/pom.xml
+++ b/bouncy-castle/pom.xml
@@ -25,8 +25,7 @@
org.apache.pulsar
pulsar
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
diff --git a/build/build_java_test_image.sh b/build/build_java_test_image.sh
index 0747e6dacb82a..3869b6688051f 100755
--- a/build/build_java_test_image.sh
+++ b/build/build_java_test_image.sh
@@ -20,12 +20,6 @@
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$SCRIPT_DIR/.."
-SQUASH_PARAM=""
-# check if docker experimental mode is enabled which is required for
-# using "docker build --squash" for squashing all intermediate layers of the build to a single layer
-if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]]; then
- SQUASH_PARAM="-Ddocker.squash=true"
-fi
mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \
- -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \
+ -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true \
"$@" install
\ No newline at end of file
diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh
index 034b2ce60cf37..3d63f104cd500 100755
--- a/build/pulsar_ci_tool.sh
+++ b/build/pulsar_ci_tool.sh
@@ -353,6 +353,7 @@ _ci_upload_coverage_files() {
--transform="flags=r;s|\\(/jacoco.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \
--transform="flags=r;s|\\(/tmp/jacocoDir/.*\\).exec$|\\1_${testtype}_${testgroup}.exec|" \
--exclude="*/META-INF/bundled-dependencies/*" \
+ --exclude="*/META-INF/versions/*" \
$GITHUB_WORKSPACE/target/classpath_* \
$(find "$GITHUB_WORKSPACE" -path "*/target/jacoco*.exec" -printf "%p\n%h/classes\n" | sort | uniq) \
$([ -d /tmp/jacocoDir ] && echo "/tmp/jacocoDir" ) \
@@ -494,11 +495,11 @@ ci_create_test_coverage_report() {
local classfilesArgs="--classfiles $({
{
for classpathEntry in $(cat $completeClasspathFile | { grep -v -f $filterArtifactsFile || true; } | sort | uniq | { grep -v -E "$excludeJarsPattern" || true; }); do
- if [[ -f $classpathEntry && -n "$(unzip -Z1C $classpathEntry 'META-INF/bundled-dependencies/*' 2>/dev/null)" ]]; then
- # file must be processed by removing META-INF/bundled-dependencies
+ if [[ -f $classpathEntry && -n "$(unzip -Z1C $classpathEntry 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' 2>/dev/null)" ]]; then
+ # file must be processed by removing META-INF/bundled-dependencies and META-INF/versions
local jartempfile=$(mktemp -t jarfile.XXXX --suffix=.jar)
cp $classpathEntry $jartempfile
- zip -q -d $jartempfile 'META-INF/bundled-dependencies/*' &> /dev/null
+ zip -q -d $jartempfile 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' &> /dev/null
echo $jartempfile
else
echo $classpathEntry
@@ -560,7 +561,7 @@ ci_create_inttest_coverage_report() {
# remove jar file that causes duplicate classes issue
rm /tmp/jacocoDir/pulsar_lib/org.apache.pulsar-bouncy-castle* || true
# remove any bundled dependencies as part of .jar/.nar files
- find /tmp/jacocoDir/pulsar_lib '(' -name "*.jar" -or -name "*.nar" ')' -exec echo "Processing {}" \; -exec zip -q -d {} 'META-INF/bundled-dependencies/*' \; |grep -E -v "Nothing to do|^$" || true
+ find /tmp/jacocoDir/pulsar_lib '(' -name "*.jar" -or -name "*.nar" ')' -exec echo "Processing {}" \; -exec zip -q -d {} 'META-INF/bundled-dependencies/*' 'META-INF/versions/*' \; |grep -E -v "Nothing to do|^$" || true
fi
# projects that aren't considered as production code and their own src/main/java source code shouldn't be analysed
local excludeProjectsPattern="testmocks|testclient|buildtools"
diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh
index 2d82fce08878d..63b92d4e0a798 100755
--- a/build/run_integration_group.sh
+++ b/build/run_integration_group.sh
@@ -177,6 +177,10 @@ test_group_standalone() {
mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-standalone.xml -DintegrationTests
}
+test_group_upgrade() {
+ mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-upgrade.xml -DintegrationTests
+}
+
test_group_transaction() {
mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-transaction.xml -DintegrationTests
}
diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh
index 76931f4a44370..cdaf69e351b6d 100755
--- a/build/run_unit_group.sh
+++ b/build/run_unit_group.sh
@@ -80,11 +80,13 @@ function test_group_broker_group_1() {
}
function test_group_broker_group_2() {
- mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other'
+ mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,broker-replication,websocket,other'
}
function test_group_broker_group_3() {
mvn_test -pl pulsar-broker -Dgroups='broker-admin'
+ # run AdminApiTransactionMultiBrokerTest independently with a larger heap size
+ mvn_test -pl pulsar-broker -DtestMaxHeapSize=1500M -Dtest=org.apache.pulsar.broker.admin.v3.AdminApiTransactionMultiBrokerTest -DtestForkCount=1 -DtestReuseFork=false
}
function test_group_broker_group_4() {
@@ -139,13 +141,21 @@ function print_testng_failures() {
function test_group_broker_flaky() {
echo "::endgroup::"
echo "::group::Running quarantined tests"
- mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='quarantine' -DexcludedGroups='' -DfailIfNoTests=false \
+ mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='quarantine' -DexcludedGroups='flaky' -DfailIfNoTests=false \
-DtestForkCount=2 ||
print_testng_failures pulsar-broker/target/surefire-reports/testng-failed.xml "Quarantined test failure in" "Quarantined test failures"
echo "::endgroup::"
echo "::group::Running flaky tests"
- mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='flaky' -DtestForkCount=2
+ mvn_test --no-fail-fast -pl pulsar-broker -Dgroups='flaky' -DexcludedGroups='quarantine' -DtestForkCount=2
echo "::endgroup::"
+ local modules_with_flaky_tests=$(git grep -l '@Test.*"flaky"' | grep '/src/test/java/' | \
+ awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker' | sort | uniq | \
+ perl -0777 -p -e 's/\n(\S)/,$1/g')
+ if [ -n "${modules_with_flaky_tests}" ]; then
+ echo "::group::Running flaky tests in modules '${modules_with_flaky_tests}'"
+ mvn_test --no-fail-fast -pl "${modules_with_flaky_tests}" -Dgroups='flaky' -DexcludedGroups='quarantine' -DfailIfNoTests=false
+ echo "::endgroup::"
+ fi
}
function test_group_proxy() {
@@ -179,7 +189,7 @@ function test_group_other() {
perl -0777 -p -e 's/\n(\S)/,$1/g')
if [ -n "${modules_with_quarantined_tests}" ]; then
echo "::group::Running quarantined tests outside of pulsar-broker & pulsar-proxy (if any)"
- mvn_test --no-fail-fast -pl "${modules_with_quarantined_tests}" test -Dgroups='quarantine' -DexcludedGroups='' \
+ mvn_test --no-fail-fast -pl "${modules_with_quarantined_tests}" test -Dgroups='quarantine' -DexcludedGroups='flaky' \
-DfailIfNoTests=false || \
echo "::warning::There were test failures in the 'quarantine' test group."
echo "::endgroup::"
diff --git a/buildtools/pom.xml b/buildtools/pom.xml
index d1e6e5f5ce42c..b1ae0cd9b73f3 100644
--- a/buildtools/pom.xml
+++ b/buildtools/pom.xml
@@ -31,17 +31,17 @@
org.apache.pulsar
buildtools
- 3.3.0-SNAPSHOT
+ 4.0.0-SNAPSHOT
jar
Pulsar Build Tools
- 2023-12-28T19:33:08Z
+ 2024-08-09T08:42:01Z
1.8
1.8
3.1.0
- 2.18.0
- 1.7.32
+ 2.23.1
+ 2.0.13
7.7.1
3.11
4.1
@@ -63,6 +63,13 @@
+
+ org.slf4j
+ slf4j-bom
+ ${slf4j.version}
+ pom
+ import
+
org.apache.logging.log4j
log4j-bom
@@ -100,6 +107,12 @@
org.testng
testng
${testng.version}
+
+
+ org.slf4j
+ *
+
+
org.apache.logging.log4j
@@ -111,12 +124,11 @@
org.apache.logging.log4j
- log4j-slf4j-impl
+ log4j-slf4j2-impl
org.slf4j
jcl-over-slf4j
- ${slf4j.version}
org.apache.commons
diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java b/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java
index 38cd2a1747a63..0c464fd97a970 100644
--- a/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/AnnotationListener.java
@@ -32,6 +32,10 @@ public class AnnotationListener implements IAnnotationTransformer {
private static final long DEFAULT_TEST_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5);
private static final String OTHER_GROUP = "other";
+ private static final String FLAKY_GROUP = "flaky";
+
+ private static final String QUARANTINE_GROUP = "quarantine";
+
public AnnotationListener() {
System.out.println("Created annotation listener");
}
@@ -51,9 +55,27 @@ public void transform(ITestAnnotation annotation,
annotation.setTimeOut(DEFAULT_TEST_TIMEOUT_MILLIS);
}
+ replaceGroupsIfFlakyOrQuarantineGroupIsIncluded(annotation);
addToOtherGroupIfNoGroupsSpecified(annotation);
}
+ // A test method will inherit the test groups from the class level and this solution ensures that a test method
+ // added to the flaky or quarantine group will not be executed as part of other groups.
+ private void replaceGroupsIfFlakyOrQuarantineGroupIsIncluded(ITestAnnotation annotation) {
+ if (annotation.getGroups() != null && annotation.getGroups().length > 1) {
+ for (String group : annotation.getGroups()) {
+ if (group.equals(QUARANTINE_GROUP)) {
+ annotation.setGroups(new String[]{QUARANTINE_GROUP});
+ return;
+ }
+ if (group.equals(FLAKY_GROUP)) {
+ annotation.setGroups(new String[]{FLAKY_GROUP});
+ return;
+ }
+ }
+ }
+ }
+
private void addToOtherGroupIfNoGroupsSpecified(ITestOrConfiguration annotation) {
// Add test to "other" group if there's no specified group
if (annotation.getGroups() == null || annotation.getGroups().length == 0) {
diff --git a/conf/bkenv.sh b/conf/bkenv.sh
index b41532d3a0c91..8beea47cee312 100644
--- a/conf/bkenv.sh
+++ b/conf/bkenv.sh
@@ -37,9 +37,6 @@ BOOKIE_LOG_DIR=${BOOKIE_LOG_DIR:-"${PULSAR_LOG_DIR}"}
# Memory size options
BOOKIE_MEM=${BOOKIE_MEM:-${PULSAR_MEM:-"-Xms2g -Xmx2g -XX:MaxDirectMemorySize=2g"}}
-# Garbage collection options
-BOOKIE_GC=${BOOKIE_GC:-${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"}}
-
if [ -z "$JAVA_HOME" ]; then
JAVA_BIN=java
else
@@ -60,6 +57,17 @@ for token in $("$JAVA_BIN" -version 2>&1 | grep 'version "'); do
fi
done
+# Garbage collection options
+BOOKIE_GC="${BOOKIE_GC:-${PULSAR_GC}}"
+if [ -z "$BOOKIE_GC" ]; then
+ BOOKIE_GC="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"
+ if [[ $JAVA_MAJOR_VERSION -ge 21 ]]; then
+ BOOKIE_GC="-XX:+UseZGC -XX:+ZGenerational ${BOOKIE_GC}"
+ else
+ BOOKIE_GC="-XX:+UseZGC ${BOOKIE_GC}"
+ fi
+fi
+
if [[ -z "$BOOKIE_GC_LOG" ]]; then
# fallback to PULSAR_GC_LOG if it is set
BOOKIE_GC_LOG="$PULSAR_GC_LOG"
diff --git a/conf/bookkeeper.conf b/conf/bookkeeper.conf
index 548ece01b842d..4058d787e2c00 100644
--- a/conf/bookkeeper.conf
+++ b/conf/bookkeeper.conf
@@ -658,7 +658,7 @@ diskCheckInterval=10000
#############################################################################
# Metadata service uri that bookkeeper uses for loading the corresponding metadata driver and resolving its metadata service location
-# Examples:
+# Examples:
# - metadataServiceUri=zk+hierarchical://my-zk-1:2181/ledgers
# - metadataServiceUri=etcd+hierarchical:http://my-etcd:2379
# - metadataServiceUri=metadata-store:zk:my-zk-1:2281
@@ -761,13 +761,20 @@ dbStorage_readAheadCacheBatchSize=1000
## DbLedgerStorage uses RocksDB to store the indexes from
## (ledgerId, entryId) -> (entryLog, offset)
+# These settings are ignored since Pulsar 2.11 / Bookkeeper 4.15
+# NOTICE: The settings in conf/default_rocksdb.conf, conf/entry_location_rocksdb.conf and
+# conf/ledger_metadata_rocksdb.conf files are primarily used to configure RocksDB
+# settings. dbStorage_rocksDB_* config keys are ignored.
+
# Size of RocksDB block-cache. For best performance, this cache
# should be big enough to hold a significant portion of the index
# database which can reach ~2GB in some cases
# Default is to use 10% of the direct memory size
+# These settings are ignored since Pulsar 2.11 / Bookkeeper 4.15
dbStorage_rocksDB_blockCacheSize=
# Other RocksDB specific tunables
+# These settings are ignored since Pulsar 2.11 / Bookkeeper 4.15
dbStorage_rocksDB_writeBufferSizeMB=64
dbStorage_rocksDB_sstSizeInMB=64
dbStorage_rocksDB_blockSize=65536
diff --git a/conf/broker.conf b/conf/broker.conf
index fd6bba0f45d2c..e745fcb2b0a8f 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -88,6 +88,16 @@ advertisedAddress=
# If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data.
haProxyProtocolEnabled=false
+# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
+webServiceHaProxyProtocolEnabled=false
+
+# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false.
+webServiceTrustXForwardedFor=false
+
+# Add detailed client/remote and server/local addresses and ports to http/https request logging.
+# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled.
+webServiceLogDetailedAddresses=
+
# Number of threads to config Netty Acceptor. Default is 1
numAcceptorThreads=
@@ -149,6 +159,14 @@ skipBrokerShutdownOnOOM=false
# Factory class-name to create topic with custom workflow
topicFactoryClassName=
+# Max capacity of the topic name cache. -1 means unlimited cache; 0 means broker will clear all cache
+# per "maxSecondsToClearTopicNameCache", it does not mean broker will not cache TopicName.
+topicNameCacheMaxCapacity=100000
+
+# A Specifies the minimum number of seconds that the topic name stays in memory, to avoid clear cache frequently when
+# there are too many topics are in use.
+maxSecondsToClearTopicNameCache=7200
+
# Enable backlog quota check. Enforces action on topic when the quota is reached
backlogQuotaCheckEnabled=true
@@ -170,6 +188,10 @@ backlogQuotaDefaultRetentionPolicy=producer_request_hold
# Default ttl for namespaces if ttl is not already configured at namespace policies. (disable default-ttl with value 0)
ttlDurationDefaultInSeconds=0
+# Additional system subscriptions that will be ignored by ttl check. The cursor names are comma separated.
+# Default is empty.
+# additionalSystemCursorNames=
+
# Enable topic auto creation if new producer or consumer connected (disable auto creation with value false)
allowAutoTopicCreation=true
@@ -333,6 +355,25 @@ maxUnackedMessagesPerBroker=0
# limit/2 messages
maxUnackedMessagesPerSubscriptionOnBrokerBlocked=0.16
+# For Key_Shared subscriptions, if messages cannot be dispatched to consumers due to a slow consumer
+# or a blocked key hash (because of ordering constraints), the broker will continue reading more
+# messages from the backlog and attempt to dispatch them to consumers until the number of replay
+# messages reaches the calculated threshold.
+# Formula: threshold = min(keySharedLookAheadMsgInReplayThresholdPerConsumer *
+# connected consumer count, keySharedLookAheadMsgInReplayThresholdPerSubscription).
+# Setting this value to 0 will disable the limit calculated per consumer.
+keySharedLookAheadMsgInReplayThresholdPerConsumer=2000
+
+# For Key_Shared subscriptions, if messages cannot be dispatched to consumers due to a slow consumer
+# or a blocked key hash (because of ordering constraints), the broker will continue reading more
+# messages from the backlog and attempt to dispatch them to consumers until the number of replay
+# messages reaches the calculated threshold.
+# Formula: threshold = min(keySharedLookAheadMsgInReplayThresholdPerConsumer *
+# connected consumer count, keySharedLookAheadMsgInReplayThresholdPerSubscription).
+# This value should be set to a value less than 2 * managedLedgerMaxUnackedRangesToPersist.
+# Setting this value to 0 will disable the limit calculated per subscription.
+keySharedLookAheadMsgInReplayThresholdPerSubscription=20000
+
# Broker periodically checks if subscription is stuck and unblock if flag is enabled. (Default is disabled)
unblockStuckSubscriptionEnabled=false
@@ -445,6 +486,16 @@ dispatcherReadFailureBackoffMaxTimeInMs=60000
# The read failure backoff mandatory stop time in milliseconds. By default it is 0s.
dispatcherReadFailureBackoffMandatoryStopTimeInMs=0
+# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered
+# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff
+# delay. This parameter sets the initial backoff delay in milliseconds.
+dispatcherRetryBackoffInitialTimeInMs=1
+
+# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered
+# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff
+# delay. This parameter sets the maximum backoff delay in milliseconds.
+dispatcherRetryBackoffMaxTimeInMs=10
+
# Precise dispatcher flow control according to history message number of each entry
preciseDispatcherFlowControl=false
@@ -541,6 +592,12 @@ brokerServiceCompactionPhaseOneLoopTimeInSeconds=30
# Whether retain null-key message during topic compaction
topicCompactionRetainNullKey=false
+# Class name of the factory that implements the topic compaction service.
+# If value is "org.apache.pulsar.compaction.EventTimeCompactionServiceFactory",
+# will create topic compaction service based on message eventTime.
+# By default compaction service is based on message publishing order.
+compactionServiceFactoryClassName=org.apache.pulsar.compaction.PulsarCompactionServiceFactory
+
# Whether to enable the delayed delivery for messages.
# If disabled, messages will be immediately delivered and there will
# be no tracking overhead.
@@ -1197,6 +1254,9 @@ managedLedgerDataReadPriority=tiered-storage-first
# (default is -1, which is disabled)
managedLedgerOffloadThresholdInSeconds=-1
+# Trigger offload on topic load or not. Default is false.
+# triggerOffloadOnTopicLoad=false
+
# Max number of entries to append to a cursor ledger
managedLedgerCursorMaxEntriesPerLedger=50000
@@ -1376,20 +1436,16 @@ loadBalancerMsgThroughputMultiplierDifferenceShedderThreshold=4
# It only takes effect in the ThresholdShedder strategy.
loadBalancerHistoryResourcePercentage=0.9
-# The BandWithIn usage weight when calculating new resource usage.
-# It only takes effect in the ThresholdShedder strategy.
-loadBalancerBandwithInResourceWeight=1.0
+# The BandWidthIn usage weight when calculating new resource usage.
+loadBalancerBandwidthInResourceWeight=1.0
-# The BandWithOut usage weight when calculating new resource usage.
-# It only takes effect in the ThresholdShedder strategy.
-loadBalancerBandwithOutResourceWeight=1.0
+# The BandWidthOut usage weight when calculating new resource usage.
+loadBalancerBandwidthOutResourceWeight=1.0
# The CPU usage weight when calculating new resource usage.
-# It only takes effect in the ThresholdShedder strategy.
loadBalancerCPUResourceWeight=1.0
# The direct memory usage weight when calculating new resource usage.
-# It only takes effect in the ThresholdShedder strategy.
# Direct memory usage cannot accurately reflect the machine's load,
# and it is not recommended to use it to score the machine's load.
loadBalancerDirectMemoryResourceWeight=0
@@ -1401,6 +1457,25 @@ loadBalancerBundleUnloadMinThroughputThreshold=10
# Time to wait for the unloading of a namespace bundle
namespaceBundleUnloadingTimeoutMs=60000
+# configuration for AvgShedder, a new shedding and placement strategy
+# The low threshold for the difference between the highest and lowest loaded brokers.
+loadBalancerAvgShedderLowThreshold = 15
+
+# The high threshold for the difference between the highest and lowest loaded brokers.
+loadBalancerAvgShedderHighThreshold = 40
+
+# The number of times the low threshold is triggered before the bundle is unloaded.
+loadBalancerAvgShedderHitCountLowThreshold = 8
+
+# The number of times the high threshold is triggered before the bundle is unloaded.
+loadBalancerAvgShedderHitCountHighThreshold = 2
+
+# In the UniformLoadShedder and AvgShedder strategy, the maximum unload ratio.
+# For AvgShedder, recommend to set to 0.5, so that it will distribute the load evenly
+# between the highest and lowest brokers.
+maxUnloadPercentage = 0.2
+
+
### --- Load balancer extension --- ###
# Option to enable the debug mode for the load balancer logics.
@@ -1412,14 +1487,14 @@ loadBalancerDebugModeEnabled=false
# (100% resource usage is 1.0 load).
# The shedder logic tries to distribute bundle load across brokers to meet this target std.
# The smaller value will incur load balancing more frequently.
-# (only used in load balancer extension TransferSheddeer)
+# (only used in load balancer extension TransferShedder)
loadBalancerBrokerLoadTargetStd=0.25
# Threshold to the consecutive count of fulfilled shedding(unload) conditions.
# If the unload scheduler consecutively finds bundles that meet unload conditions
# many times bigger than this threshold, the scheduler will shed the bundles.
# The bigger value will incur less bundle unloading/transfers.
-# (only used in load balancer extension TransferSheddeer)
+# (only used in load balancer extension TransferShedder)
loadBalancerSheddingConditionHitCountThreshold=3
# Option to enable the bundle transfer mode when distributing bundle loads.
@@ -1427,18 +1502,18 @@ loadBalancerSheddingConditionHitCountThreshold=3
# -- pre-assigns the destination broker upon unloading).
# Off: unload bundles from overloaded brokers
# -- post-assigns the destination broker upon lookups).
-# (only used in load balancer extension TransferSheddeer)
+# (only used in load balancer extension TransferShedder)
loadBalancerTransferEnabled=true
# Maximum number of brokers to unload bundle load for each unloading cycle.
# The bigger value will incur more unloading/transfers for each unloading cycle.
-# (only used in load balancer extension TransferSheddeer)
+# (only used in load balancer extension TransferShedder)
loadBalancerMaxNumberOfBrokerSheddingPerCycle=3
# Delay (in seconds) to the next unloading cycle after unloading.
# The logic tries to give enough time for brokers to recompute load after unloading.
# The bigger value will delay the next unloading cycle longer.
-# (only used in load balancer extension TransferSheddeer)
+# (only used in load balancer extension TransferShedder)
loadBalanceSheddingDelayInSeconds=180
# Broker load data time to live (TTL in seconds).
@@ -1446,14 +1521,17 @@ loadBalanceSheddingDelayInSeconds=180
# and those brokers will be ignored in the load computation.
# When tuning this value, please consider loadBalancerReportUpdateMaxIntervalMinutes.
#The current default is loadBalancerReportUpdateMaxIntervalMinutes * 2.
-# (only used in load balancer extension TransferSheddeer)
+# (only used in load balancer extension TransferShedder)
loadBalancerBrokerLoadDataTTLInSeconds=1800
# Max number of bundles in bundle load report from each broker.
# The load balancer distributes bundles across brokers,
# based on topK bundle load data and other broker load data.
# The bigger value will increase the overhead of reporting many bundles in load data.
-# (only used in load balancer extension logics)
+# Used for ExtensibleLoadManagerImpl and ModularLoadManagerImpl, default value is 10.
+# User can disable the bundle filtering feature of ModularLoadManagerImpl by setting this value to -1.
+# Enabling this feature can reduce the pressure on the zookeeper when doing load report.
+# WARNING: too small value could result in a long load balance time.
loadBalancerMaxNumberOfBundlesInBundleLoadReport=10
# Service units'(bundles) split interval. Broker periodically checks whether
@@ -1485,6 +1563,15 @@ loadBalancerNamespaceBundleSplitConditionHitCountThreshold=3
# (only used in load balancer extension logics)
loadBalancerServiceUnitStateTombstoneDelayTimeInSeconds=3600
+# Name of ServiceUnitStateTableView implementation class to use
+loadManagerServiceUnitStateTableViewClassName=org.apache.pulsar.broker.loadbalance.extensions.channel.ServiceUnitStateTableViewImpl
+
+# Specify ServiceUnitTableViewSyncer to sync service unit(bundle) states between metadata store and
+# system topic table views during migration from one to the other. One could enable this
+# syncer before migration and disable it after the migration finishes.
+# It accepts `MetadataStoreToSystemTopicSyncer` or `SystemTopicToMetadataStoreSyncer` to
+# enable it. It accepts `None` to disable it."
+loadBalancerServiceUnitTableViewSyncer=None
### --- Replication --- ###
@@ -1506,6 +1593,16 @@ replicatorPrefix=pulsar.repl
# due to missing ZooKeeper watch (disable with value 0)
replicationPolicyCheckDurationSeconds=600
+# Whether the internal replication of the local cluster will trigger topic auto-creation on the remote cluster.
+# 1. After enabling namespace-level Geo-Replication: whether the local broker will create topics on the remote
+# cluster automatically when calling `pulsar-admin topics create-partitioned-topic`.
+# 2. When enabling topic-level Geo-Replication on a partitioned topic: whether the local broker will create topics on
+# the remote cluster.
+# 3. Whether the internal Geo-Replicator in the local cluster will trigger non-persistent topic auto-creation for
+# remote clusters.
+# It is not a dynamic config, the default value is "true" to preserve backward-compatible behavior.
+createTopicToRemoteClusterForReplication=true
+
# Default message retention time.
# 0 means retention is disabled. -1 means data is not removed by time quota.
defaultRetentionTimeInMinutes=0
@@ -1534,6 +1631,9 @@ webSocketNumServiceThreads=
# Number of connections per Broker in Pulsar Client used in WebSocket proxy
webSocketConnectionsPerBroker=
+# Memory limit in MBs for direct memory in Pulsar Client used in WebSocket proxy
+webSocketPulsarClientMemoryLimitInMB=0
+
# Time in milliseconds that idle WebSocket session times out
webSocketSessionIdleTimeoutMillis=300000
@@ -1589,6 +1689,8 @@ exposePublisherStats=true
statsUpdateFrequencyInSecs=60
statsUpdateInitialDelayInSecs=60
+healthCheckMetricsUpdateTimeInSeconds=-1
+
# Enable expose the precise backlog stats.
# Set false to use published counter and consumed counter to calculate, this would be more efficient but may be inaccurate.
# Default is false.
@@ -1660,10 +1762,10 @@ s3ManagedLedgerOffloadBucket=
# For Amazon S3 ledger offload, Alternative endpoint to connect to (useful for testing)
s3ManagedLedgerOffloadServiceEndpoint=
-# For Amazon S3 ledger offload, Max block size in bytes. (64MB by default, 5MB minimum)
+# For Amazon S3 ledger offload, Max block size in bytes. (64MiB by default, 5MiB minimum)
s3ManagedLedgerOffloadMaxBlockSizeInBytes=67108864
-# For Amazon S3 ledger offload, Read buffer size in bytes (1MB by default)
+# For Amazon S3 ledger offload, Read buffer size in bytes (1MiB by default)
s3ManagedLedgerOffloadReadBufferSizeInBytes=1048576
# For Google Cloud Storage ledger offload, region where offload bucket is located.
@@ -1673,10 +1775,11 @@ gcsManagedLedgerOffloadRegion=
# For Google Cloud Storage ledger offload, Bucket to place offloaded ledger into
gcsManagedLedgerOffloadBucket=
-# For Google Cloud Storage ledger offload, Max block size in bytes. (64MB by default, 5MB minimum)
-gcsManagedLedgerOffloadMaxBlockSizeInBytes=67108864
+# For Google Cloud Storage ledger offload, Max block size in bytes. (128MiB by default, 5MiB minimum)
+# Since JClouds limits the maximum number of blocks to 32, the maximum size of a ledger is 32 times the block size.
+gcsManagedLedgerOffloadMaxBlockSizeInBytes=134217728
-# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MB by default)
+# For Google Cloud Storage ledger offload, Read buffer size in bytes (1MiB by default)
gcsManagedLedgerOffloadReadBufferSizeInBytes=1048576
# For Google Cloud Storage, path to json file containing service account credentials.
@@ -1790,9 +1893,8 @@ strictBookieAffinityEnabled=false
# These settings are left here for compatibility
# The heap memory usage weight when calculating new resource usage.
-# It only takes effect in the ThresholdShedder strategy.
# Deprecated: Memory is no longer used as a load balancing item
-loadBalancerMemoryResourceWeight=1.0
+loadBalancerMemoryResourceWeight=0
# Zookeeper quorum connection string
# Deprecated: use metadataStoreUrl instead
diff --git a/conf/client.conf b/conf/client.conf
index 8a485e5676c7b..25d65c3947e39 100644
--- a/conf/client.conf
+++ b/conf/client.conf
@@ -41,7 +41,7 @@ authPlugin=
# authParams=tlsCertFile:/path/to/client-cert.pem,tlsKeyFile:/path/to/client-key.pem
authParams=
-# Allow TLS connections to servers whose certificate cannot be
+# Allow TLS connections to servers whose certificate cannot
# be verified to have been signed by a trusted certificate
# authority.
tlsAllowInsecureConnection=false
diff --git a/conf/default_rocksdb.conf b/conf/default_rocksdb.conf
index e1a21bb845222..74e3005ba6687 100644
--- a/conf/default_rocksdb.conf
+++ b/conf/default_rocksdb.conf
@@ -24,7 +24,14 @@
info_log_level=INFO_LEVEL
# set by jni: options.setKeepLogFileNum
keep_log_file_num=30
+ # set by jni: options.setLogFileTimeToRoll
+ log_file_time_to_roll=86400
[CFOptions "default"]
- # set by jni: options.setLogFileTimeToRoll
- log_file_time_to_roll=86400
\ No newline at end of file
+ #no default setting in CFOptions
+
+[TableOptions/BlockBasedTable "default"]
+ # set by jni: tableOptions.setFormatVersion
+ format_version=5
+ # set by jni: tableOptions.setChecksumType
+ checksum=kxxHash
\ No newline at end of file
diff --git a/conf/entry_location_rocksdb.conf b/conf/entry_location_rocksdb.conf
index 42d916ded378f..9c675554b24ae 100644
--- a/conf/entry_location_rocksdb.conf
+++ b/conf/entry_location_rocksdb.conf
@@ -27,7 +27,7 @@
# set by jni: options.setLogFileTimeToRoll
log_file_time_to_roll=86400
# set by jni: options.setMaxBackgroundJobs or options.setIncreaseParallelism
- max_background_jobs=2
+ max_background_jobs=32
# set by jni: options.setMaxSubcompactions
max_subcompactions=1
# set by jni: options.setMaxTotalWalSize
@@ -61,10 +61,10 @@
# set by jni: tableOptions.setBlockCache
block_cache=206150041
# set by jni: tableOptions.setFormatVersion
- format_version=2
+ format_version=5
# set by jni: tableOptions.setChecksumType
checksum=kxxHash
# set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder]
filter_policy=rocksdb.BloomFilter:10:false
# set by jni: tableOptions.setCacheIndexAndFilterBlocks
- cache_index_and_filter_blocks=true
+ cache_index_and_filter_blocks=true
\ No newline at end of file
diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml
index 3871c74a88778..6f995576ebd64 100644
--- a/conf/functions_worker.yml
+++ b/conf/functions_worker.yml
@@ -27,6 +27,16 @@ workerHostname: localhost
workerPort: 6750
workerPortTls: 6751
+# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
+webServiceHaProxyProtocolEnabled: false
+
+# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false.
+webServiceTrustXForwardedFor: false
+
+# Add detailed client/remote and server/local addresses and ports to http/https request logging.
+# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled.
+webServiceLogDetailedAddresses: null
+
# The Configuration metadata store url
# Examples:
# * zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181
diff --git a/conf/ledger_metadata_rocksdb.conf b/conf/ledger_metadata_rocksdb.conf
index e1a21bb845222..74e3005ba6687 100644
--- a/conf/ledger_metadata_rocksdb.conf
+++ b/conf/ledger_metadata_rocksdb.conf
@@ -24,7 +24,14 @@
info_log_level=INFO_LEVEL
# set by jni: options.setKeepLogFileNum
keep_log_file_num=30
+ # set by jni: options.setLogFileTimeToRoll
+ log_file_time_to_roll=86400
[CFOptions "default"]
- # set by jni: options.setLogFileTimeToRoll
- log_file_time_to_roll=86400
\ No newline at end of file
+ #no default setting in CFOptions
+
+[TableOptions/BlockBasedTable "default"]
+ # set by jni: tableOptions.setFormatVersion
+ format_version=5
+ # set by jni: tableOptions.setChecksumType
+ checksum=kxxHash
\ No newline at end of file
diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml
index 9c261a6b89a50..0e49503581c48 100644
--- a/conf/log4j2.yaml
+++ b/conf/log4j2.yaml
@@ -19,7 +19,7 @@
Configuration:
- status: INFO
+ status: ERROR
monitorInterval: 30
name: pulsar
packages: io.prometheus.client.log4j2
diff --git a/conf/proxy.conf b/conf/proxy.conf
index 8285e1cb75320..6e6c960e8009e 100644
--- a/conf/proxy.conf
+++ b/conf/proxy.conf
@@ -63,6 +63,16 @@ advertisedAddress=
# If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data.
haProxyProtocolEnabled=false
+# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
+webServiceHaProxyProtocolEnabled=false
+
+# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false.
+webServiceTrustXForwardedFor=false
+
+# Add detailed client/remote and server/local addresses and ports to http/https request logging.
+# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled.
+webServiceLogDetailedAddresses=
+
# Enables zero-copy transport of data across network interfaces using the splice system call.
# Zero copy mode cannot be used when TLS is enabled or when proxyLogLevel is > 0.
proxyZeroCopyModeEnabled=true
@@ -376,5 +386,7 @@ zooKeeperCacheExpirySeconds=-1
enableProxyStatsEndpoints=true
# Whether the '/metrics' endpoint requires authentication. Defaults to true
authenticateMetricsEndpoint=true
-# Enable cache metrics data, default value is false
-metricsBufferResponse=false
+# Time in milliseconds that metrics endpoint would time out. Default is 30s.
+# Set it to 0 to disable timeout.
+metricsServletTimeoutMs=30000
+
diff --git a/conf/pulsar_env.sh b/conf/pulsar_env.sh
index c7bba23c234d9..f95d0ac83c13a 100755
--- a/conf/pulsar_env.sh
+++ b/conf/pulsar_env.sh
@@ -44,9 +44,6 @@
# Extra options to be passed to the jvm
PULSAR_MEM=${PULSAR_MEM:-"-Xms2g -Xmx2g -XX:MaxDirectMemorySize=4g"}
-# Garbage collection options
-PULSAR_GC=${PULSAR_GC:-"-XX:+UseZGC -XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"}
-
if [ -z "$JAVA_HOME" ]; then
JAVA_BIN=java
else
@@ -67,6 +64,16 @@ for token in $("$JAVA_BIN" -version 2>&1 | grep 'version "'); do
fi
done
+# Garbage collection options
+if [ -z "$PULSAR_GC" ]; then
+ PULSAR_GC="-XX:+PerfDisableSharedMem -XX:+AlwaysPreTouch"
+ if [[ $JAVA_MAJOR_VERSION -ge 21 ]]; then
+ PULSAR_GC="-XX:+UseZGC -XX:+ZGenerational ${PULSAR_GC}"
+ else
+ PULSAR_GC="-XX:+UseZGC ${PULSAR_GC}"
+ fi
+fi
+
PULSAR_GC_LOG_DIR=${PULSAR_GC_LOG_DIR:-"${PULSAR_LOG_DIR}"}
if [[ -z "$PULSAR_GC_LOG" ]]; then
@@ -94,3 +101,7 @@ PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS:-" -Dpulsar.allocator.exit_on_oom=true -D
#Wait time before forcefully kill the pulsar server instance, if the stop is not successful
#PULSAR_STOP_TIMEOUT=
+# Enable semantically stable telemetry for JVM metrics, unless otherwise overridden by the user.
+if [ -z "$OTEL_SEMCONV_STABILITY_OPT_IN" ]; then
+ export OTEL_SEMCONV_STABILITY_OPT_IN=jvm
+fi
diff --git a/conf/pulsar_tools_env.sh b/conf/pulsar_tools_env.sh
index 9d22b73905df3..96ee304bf0b3a 100755
--- a/conf/pulsar_tools_env.sh
+++ b/conf/pulsar_tools_env.sh
@@ -57,9 +57,6 @@ if [ -n "$PULSAR_MEM" ]; then
fi
PULSAR_MEM=${PULSAR_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"}
-# Garbage collection options
-PULSAR_GC=${PULSAR_GC:-" -client "}
-
# Extra options to be passed to the jvm
PULSAR_EXTRA_OPTS="${PULSAR_MEM} ${PULSAR_GC} ${PULSAR_GC_LOG} -Dio.netty.leakDetectionLevel=disabled ${PULSAR_EXTRA_OPTS}"
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 5c94d63817a12..535800a43f3e0 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -51,6 +51,16 @@ advertisedAddress=
# If true, the real IP addresses of consumers and producers can be obtained when getting topic statistics data.
haProxyProtocolEnabled=false
+# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
+webServiceHaProxyProtocolEnabled=false
+
+# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false.
+webServiceTrustXForwardedFor=false
+
+# Add detailed client/remote and server/local addresses and ports to http/https request logging.
+# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled.
+webServiceLogDetailedAddresses=
+
# Number of threads to use for Netty IO. Default is set to 2 * Runtime.getRuntime().availableProcessors()
numIOThreads=
@@ -111,6 +121,10 @@ backlogQuotaDefaultLimitSecond=-1
# Default ttl for namespaces if ttl is not already configured at namespace policies. (disable default-ttl with value 0)
ttlDurationDefaultInSeconds=0
+# Additional system subscriptions that will be ignored by ttl check. The cursor names are comma separated.
+# Default is empty.
+# additionalSystemCursorNames=
+
# Enable the deletion of inactive topics. This parameter need to cooperate with the allowAutoTopicCreation parameter.
# If brokerDeleteInactiveTopicsEnabled is set to true, we should ensure that allowAutoTopicCreation is also set to true.
brokerDeleteInactiveTopicsEnabled=true
@@ -218,6 +232,25 @@ maxUnackedMessagesPerBroker=0
# limit/2 messages
maxUnackedMessagesPerSubscriptionOnBrokerBlocked=0.16
+# For Key_Shared subscriptions, if messages cannot be dispatched to consumers due to a slow consumer
+# or a blocked key hash (because of ordering constraints), the broker will continue reading more
+# messages from the backlog and attempt to dispatch them to consumers until the number of replay
+# messages reaches the calculated threshold.
+# Formula: threshold = min(keySharedLookAheadMsgInReplayThresholdPerConsumer *
+# connected consumer count, keySharedLookAheadMsgInReplayThresholdPerSubscription).
+# Setting this value to 0 will disable the limit calculated per consumer.
+keySharedLookAheadMsgInReplayThresholdPerConsumer=2000
+
+# For Key_Shared subscriptions, if messages cannot be dispatched to consumers due to a slow consumer
+# or a blocked key hash (because of ordering constraints), the broker will continue reading more
+# messages from the backlog and attempt to dispatch them to consumers until the number of replay
+# messages reaches the calculated threshold.
+# Formula: threshold = min(keySharedLookAheadMsgInReplayThresholdPerConsumer *
+# connected consumer count, keySharedLookAheadMsgInReplayThresholdPerSubscription).
+# This value should be set to a value less than 2 * managedLedgerMaxUnackedRangesToPersist.
+# Setting this value to 0 will disable the limit calculated per subscription.
+keySharedLookAheadMsgInReplayThresholdPerSubscription=20000
+
# Tick time to schedule task that checks topic publish rate limiting across all topics
# Reducing to lower value can give more accuracy while throttling publish but
# it uses more CPU to perform frequent check. (Disable publish throttling with value 0)
@@ -269,6 +302,16 @@ dispatcherReadFailureBackoffMaxTimeInMs=60000
# The read failure backoff mandatory stop time in milliseconds. By default it is 0s.
dispatcherReadFailureBackoffMandatoryStopTimeInMs=0
+# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered
+# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff
+# delay. This parameter sets the initial backoff delay in milliseconds.
+dispatcherRetryBackoffInitialTimeInMs=1
+
+# On Shared and KeyShared subscriptions, if all available messages in the subscription are filtered
+# out and not dispatched to any consumer, message dispatching will be rescheduled with a backoff
+# delay. This parameter sets the maximum backoff delay in milliseconds.
+dispatcherRetryBackoffMaxTimeInMs=10
+
# Precise dispatcher flow control according to history message number of each entry
preciseDispatcherFlowControl=false
@@ -821,6 +864,9 @@ managedLedgerPrometheusStatsLatencyRolloverSeconds=60
# Whether trace managed ledger task execution time
managedLedgerTraceTaskExecution=true
+# Trigger offload on topic load or not. Default is false.
+# triggerOffloadOnTopicLoad=false
+
# If you want to custom bookie ID or use a dynamic network address for the bookie,
# you can set this option.
# Bookie advertises itself using bookieId rather than
@@ -893,25 +939,20 @@ loadBalancerBrokerThresholdShedderPercentage=10
# It only takes effect in the ThresholdShedder strategy.
loadBalancerHistoryResourcePercentage=0.9
-# The BandWithIn usage weight when calculating new resource usage.
-# It only takes effect in the ThresholdShedder strategy.
-loadBalancerBandwithInResourceWeight=1.0
+# The BandWidthIn usage weight when calculating new resource usage.
+loadBalancerBandwidthInResourceWeight=1.0
-# The BandWithOut usage weight when calculating new resource usage.
-# It only takes effect in the ThresholdShedder strategy.
-loadBalancerBandwithOutResourceWeight=1.0
+# The BandWidthOut usage weight when calculating new resource usage.
+loadBalancerBandwidthOutResourceWeight=1.0
# The CPU usage weight when calculating new resource usage.
-# It only takes effect in the ThresholdShedder strategy.
loadBalancerCPUResourceWeight=1.0
# The heap memory usage weight when calculating new resource usage.
-# It only takes effect in the ThresholdShedder strategy.
-loadBalancerMemoryResourceWeight=1.0
+loadBalancerMemoryResourceWeight=0
# The direct memory usage weight when calculating new resource usage.
-# It only takes effect in the ThresholdShedder strategy.
-loadBalancerDirectMemoryResourceWeight=1.0
+loadBalancerDirectMemoryResourceWeight=0
# Bundle unload minimum throughput threshold (MB), avoiding bundle unload frequently.
# It only takes effect in the ThresholdShedder strategy.
@@ -937,6 +978,16 @@ replicationProducerQueueSize=1000
# due to missing ZooKeeper watch (disable with value 0)
replicationPolicyCheckDurationSeconds=600
+# Whether the internal replication of the local cluster will trigger topic auto-creation on the remote cluster.
+# 1. After enabling namespace-level Geo-Replication: whether the local broker will create topics on the remote
+# cluster automatically when calling `pulsar-admin topics create-partitioned-topic`.
+# 2. When enabling topic-level Geo-Replication on a partitioned topic: whether the local broker will create topics on
+# the remote cluster.
+# 3. Whether the internal Geo-Replicator in the local cluster will trigger non-persistent topic auto-creation for
+# remote clusters.
+# It is not a dynamic config, the default value is "true" to preserve backward-compatible behavior.
+createTopicToRemoteClusterForReplication=true
+
# Default message retention time. 0 means retention is disabled. -1 means data is not removed by time quota
defaultRetentionTimeInMinutes=0
@@ -957,6 +1008,9 @@ webSocketNumIoThreads=8
# Number of connections per Broker in Pulsar Client used in WebSocket proxy
webSocketConnectionsPerBroker=8
+# Memory limit in MBs for direct memory in Pulsar Client used in WebSocket proxy
+webSocketPulsarClientMemoryLimitInMB=0
+
# Time in milliseconds that idle WebSocket session times out
webSocketSessionIdleTimeoutMillis=300000
@@ -1293,3 +1347,9 @@ disableBrokerInterceptors=true
# Whether retain null-key message during topic compaction
topicCompactionRetainNullKey=false
+
+# Class name of the factory that implements the topic compaction service.
+# If value is "org.apache.pulsar.compaction.EventTimeCompactionServiceFactory",
+# will create topic compaction service based on message eventTime.
+# By default compaction service is based on message publishing order.
+compactionServiceFactoryClassName=org.apache.pulsar.compaction.PulsarCompactionServiceFactory
\ No newline at end of file
diff --git a/conf/websocket.conf b/conf/websocket.conf
index 490cff2722ee5..91f7f7d4c23bb 100644
--- a/conf/websocket.conf
+++ b/conf/websocket.conf
@@ -46,6 +46,16 @@ statusFilePath=
# Hostname or IP address the service binds on, default is 0.0.0.0.
bindAddress=0.0.0.0
+# Enable or disable the use of HA proxy protocol for resolving the client IP for http/https requests.
+webServiceHaProxyProtocolEnabled=false
+
+# Trust X-Forwarded-For header for resolving the client IP for http/https requests. Default is false.
+webServiceTrustXForwardedFor=false
+
+# Add detailed client/remote and server/local addresses and ports to http/https request logging.
+# Defaults to true when either webServiceHaProxyProtocolEnabled or webServiceTrustXForwardedFor is enabled.
+webServiceLogDetailedAddresses=
+
# Name of the pulsar cluster to connect to
clusterName=
@@ -61,6 +71,9 @@ numHttpServerThreads=
# Number of connections per Broker in Pulsar Client used in WebSocket proxy
webSocketConnectionsPerBroker=
+# Memory limit in MBs for direct memory in Pulsar Client used in WebSocket proxy
+webSocketPulsarClientMemoryLimitInMB=0
+
# Time in milliseconds that idle WebSocket session times out
webSocketSessionIdleTimeoutMillis=300000
diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf
index fe3bae6bb153b..ff3677174024c 100644
--- a/deployment/terraform-ansible/templates/broker.conf
+++ b/deployment/terraform-ansible/templates/broker.conf
@@ -320,7 +320,7 @@ dispatcherMinReadBatchSize=1
# Max number of entries to dispatch for a shared subscription. By default it is 20 entries.
dispatcherMaxRoundRobinBatchSize=20
-# Precise dispathcer flow control according to history message number of each entry
+# Precise dispatcher flow control according to history message number of each entry
preciseDispatcherFlowControl=false
# Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic
@@ -638,7 +638,7 @@ bookkeeperMetadataServiceUri=
# Authentication plugin to use when connecting to bookies
bookkeeperClientAuthenticationPlugin=
-# BookKeeper auth plugin implementatation specifics parameters name and values
+# BookKeeper auth plugin implementation specifics parameters name and values
bookkeeperClientAuthenticationParametersName=
bookkeeperClientAuthenticationParameters=
@@ -944,7 +944,7 @@ defaultNamespaceBundleSplitAlgorithm=range_equally_divide
loadBalancerLoadSheddingStrategy=org.apache.pulsar.broker.loadbalance.impl.ThresholdShedder
# The broker resource usage threshold.
-# When the broker resource usage is gratter than the pulsar cluster average resource usge,
+# When the broker resource usage is greater than the pulsar cluster average resource usge,
# the threshold shedder will be triggered to offload bundles from the broker.
# It only take effect in ThresholdShedder strategy.
loadBalancerBrokerThresholdShedderPercentage=10
@@ -953,27 +953,27 @@ loadBalancerBrokerThresholdShedderPercentage=10
# It only take effect in ThresholdShedder strategy.
loadBalancerHistoryResourcePercentage=0.9
-# The BandWithIn usage weight when calculating new resourde usage.
+# The BandWidthIn usage weight when calculating new resourde usage.
# It only take effect in ThresholdShedder strategy.
-loadBalancerBandwithInResourceWeight=1.0
+loadBalancerBandwidthInResourceWeight=1.0
-# The BandWithOut usage weight when calculating new resourde usage.
+# The BandWidthOut usage weight when calculating new resourde usage.
# It only take effect in ThresholdShedder strategy.
-loadBalancerBandwithOutResourceWeight=1.0
+loadBalancerBandwidthOutResourceWeight=1.0
-# The CPU usage weight when calculating new resourde usage.
+# The CPU usage weight when calculating new resource usage.
# It only take effect in ThresholdShedder strategy.
loadBalancerCPUResourceWeight=1.0
-# The heap memory usage weight when calculating new resourde usage.
+# The heap memory usage weight when calculating new resource usage.
# It only take effect in ThresholdShedder strategy.
-loadBalancerMemoryResourceWeight=1.0
+loadBalancerMemoryResourceWeight=0
-# The direct memory usage weight when calculating new resourde usage.
+# The direct memory usage weight when calculating new resource usage.
# It only take effect in ThresholdShedder strategy.
-loadBalancerDirectMemoryResourceWeight=1.0
+loadBalancerDirectMemoryResourceWeight=0
-# Bundle unload minimum throughput threshold (MB), avoding bundle unload frequently.
+# Bundle unload minimum throughput threshold (MB), avoiding bundle unload frequently.
# It only take effect in ThresholdShedder strategy.
loadBalancerBundleUnloadMinThroughputThreshold=10
@@ -995,7 +995,7 @@ replicatorPrefix=pulsar.repl
# Duration to check replication policy to avoid replicator inconsistency
# due to missing ZooKeeper watch (disable with value 0)
-replicatioPolicyCheckDurationSeconds=600
+replicationPolicyCheckDurationSeconds=600
# Default message retention time. 0 means retention is disabled. -1 means data is not removed by time quota
defaultRetentionTimeInMinutes=0
diff --git a/deployment/terraform-ansible/templates/client.conf b/deployment/terraform-ansible/templates/client.conf
index ba1d396bf8423..755577cf38e03 100644
--- a/deployment/terraform-ansible/templates/client.conf
+++ b/deployment/terraform-ansible/templates/client.conf
@@ -41,7 +41,7 @@ authPlugin=
# authParams=tlsCertFile:/path/to/client-cert.pem,tlsKeyFile:/path/to/client-key.pem
authParams=
-# Allow TLS connections to servers whose certificate cannot be
+# Allow TLS connections to servers whose certificate cannot
# be verified to have been signed by a trusted certificate
# authority.
tlsAllowInsecureConnection=false
diff --git a/distribution/io/pom.xml b/distribution/io/pom.xml
index 22f8de5e15497..813c4d26d9391 100644
--- a/distribution/io/pom.xml
+++ b/distribution/io/pom.xml
@@ -25,8 +25,7 @@
org.apache.pulsar
distribution
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
pulsar-io-distribution
@@ -137,7 +136,6 @@
org.owasp
dependency-check-maven
- ${dependency-check-maven.version}
diff --git a/distribution/offloaders/pom.xml b/distribution/offloaders/pom.xml
index 6e489ceb81b75..38beeacde8ba4 100644
--- a/distribution/offloaders/pom.xml
+++ b/distribution/offloaders/pom.xml
@@ -25,8 +25,7 @@
org.apache.pulsar
distribution
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
pulsar-offloader-distribution
diff --git a/distribution/pom.xml b/distribution/pom.xml
index f248b49f1f32a..67604e145dd73 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -25,8 +25,7 @@
org.apache.pulsar
pulsar
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
distribution
diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
index 599a9755f9155..36641dea20f0c 100644
--- a/distribution/server/pom.xml
+++ b/distribution/server/pom.xml
@@ -25,8 +25,7 @@
org.apache.pulsar
distribution
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
pulsar-server-distribution
@@ -40,6 +39,19 @@
${project.version}
+
+ ${project.groupId}
+ pulsar-metadata
+ ${project.version}
+
+
+
+ ${project.groupId}
+ jetcd-core-shaded
+ ${project.version}
+ shaded
+
+
${project.groupId}
pulsar-docs-tools
@@ -180,7 +192,7 @@
org.apache.logging.log4j
- log4j-slf4j-impl
+ log4j-slf4j2-impl
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index cb99d62edfeb7..271f6dc6cebf7 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -249,20 +249,21 @@ The Apache Software License, Version 2.0
- info.picocli-picocli-shell-jline3-4.7.5.jar
* High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar
* Jackson
- - com.fasterxml.jackson.core-jackson-annotations-2.14.2.jar
- - com.fasterxml.jackson.core-jackson-core-2.14.2.jar
- - com.fasterxml.jackson.core-jackson-databind-2.14.2.jar
- - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.14.2.jar
- - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.14.2.jar
- - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.14.2.jar
- - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.14.2.jar
- - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.14.2.jar
- - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.14.2.jar
- - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.14.2.jar
- - com.fasterxml.jackson.module-jackson-module-parameter-names-2.14.2.jar
+ - com.fasterxml.jackson.core-jackson-annotations-2.17.2.jar
+ - com.fasterxml.jackson.core-jackson-core-2.17.2.jar
+ - com.fasterxml.jackson.core-jackson-databind-2.17.2.jar
+ - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.17.2.jar
+ - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.17.2.jar
+ - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.17.2.jar
+ - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.17.2.jar
+ - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.17.2.jar
+ - com.fasterxml.jackson.datatype-jackson-datatype-jdk8-2.17.2.jar
+ - com.fasterxml.jackson.datatype-jackson-datatype-jsr310-2.17.2.jar
+ - com.fasterxml.jackson.module-jackson-module-parameter-names-2.17.2.jar
* Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar
* Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar
- * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.9.0.jar
+ * Fastutil -- it.unimi.dsi-fastutil-8.5.14.jar
+ * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.17.0.jar
* Bitbucket -- org.bitbucket.b_c-jose4j-0.9.4.jar
* Gson
- com.google.code.gson-gson-2.8.9.jar
@@ -284,7 +285,7 @@ The Apache Software License, Version 2.0
- commons-cli-commons-cli-1.5.0.jar
- commons-codec-commons-codec-1.15.jar
- commons-configuration-commons-configuration-1.10.jar
- - commons-io-commons-io-2.8.0.jar
+ - commons-io-commons-io-2.14.0.jar
- commons-lang-commons-lang-2.6.jar
- commons-logging-commons-logging-1.1.1.jar
- org.apache.commons-commons-collections4-4.4.jar
@@ -292,34 +293,33 @@ The Apache Software License, Version 2.0
- org.apache.commons-commons-lang3-3.11.jar
- org.apache.commons-commons-text-1.10.0.jar
* Netty
- - io.netty-netty-buffer-4.1.105.Final.jar
- - io.netty-netty-codec-4.1.105.Final.jar
- - io.netty-netty-codec-dns-4.1.105.Final.jar
- - io.netty-netty-codec-http-4.1.105.Final.jar
- - io.netty-netty-codec-http2-4.1.105.Final.jar
- - io.netty-netty-codec-socks-4.1.105.Final.jar
- - io.netty-netty-codec-haproxy-4.1.105.Final.jar
- - io.netty-netty-common-4.1.105.Final.jar
- - io.netty-netty-handler-4.1.105.Final.jar
- - io.netty-netty-handler-proxy-4.1.105.Final.jar
- - io.netty-netty-resolver-4.1.105.Final.jar
- - io.netty-netty-resolver-dns-4.1.105.Final.jar
- - io.netty-netty-resolver-dns-classes-macos-4.1.105.Final.jar
- - io.netty-netty-resolver-dns-native-macos-4.1.105.Final-osx-aarch_64.jar
- - io.netty-netty-resolver-dns-native-macos-4.1.105.Final-osx-x86_64.jar
- - io.netty-netty-transport-4.1.105.Final.jar
- - io.netty-netty-transport-classes-epoll-4.1.105.Final.jar
- - io.netty-netty-transport-native-epoll-4.1.105.Final-linux-aarch_64.jar
- - io.netty-netty-transport-native-epoll-4.1.105.Final-linux-x86_64.jar
- - io.netty-netty-transport-native-unix-common-4.1.105.Final.jar
- - io.netty-netty-transport-native-unix-common-4.1.105.Final-linux-x86_64.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.61.Final.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar
- - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar
- - io.netty-netty-tcnative-classes-2.0.61.Final.jar
+ - io.netty-netty-buffer-4.1.113.Final.jar
+ - io.netty-netty-codec-4.1.113.Final.jar
+ - io.netty-netty-codec-dns-4.1.113.Final.jar
+ - io.netty-netty-codec-http-4.1.113.Final.jar
+ - io.netty-netty-codec-http2-4.1.113.Final.jar
+ - io.netty-netty-codec-socks-4.1.113.Final.jar
+ - io.netty-netty-codec-haproxy-4.1.113.Final.jar
+ - io.netty-netty-common-4.1.113.Final.jar
+ - io.netty-netty-handler-4.1.113.Final.jar
+ - io.netty-netty-handler-proxy-4.1.113.Final.jar
+ - io.netty-netty-resolver-4.1.113.Final.jar
+ - io.netty-netty-resolver-dns-4.1.113.Final.jar
+ - io.netty-netty-resolver-dns-classes-macos-4.1.113.Final.jar
+ - io.netty-netty-resolver-dns-native-macos-4.1.113.Final-osx-aarch_64.jar
+ - io.netty-netty-resolver-dns-native-macos-4.1.113.Final-osx-x86_64.jar
+ - io.netty-netty-transport-4.1.113.Final.jar
+ - io.netty-netty-transport-classes-epoll-4.1.113.Final.jar
+ - io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar
+ - io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar
+ - io.netty-netty-transport-native-unix-common-4.1.113.Final.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar
+ - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar
+ - io.netty-netty-tcnative-classes-2.0.66.Final.jar
- io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.24.Final.jar
- io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar
- io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar
@@ -338,80 +338,80 @@ The Apache Software License, Version 2.0
- io.prometheus-simpleclient_tracer_otel-0.16.0.jar
- io.prometheus-simpleclient_tracer_otel_agent-0.16.0.jar
* Prometheus exporter
- - io.prometheus-prometheus-metrics-config-1.1.0.jar
- - io.prometheus-prometheus-metrics-exporter-common-1.1.0.jar
- - io.prometheus-prometheus-metrics-exporter-httpserver-1.1.0.jar
- - io.prometheus-prometheus-metrics-exposition-formats-1.1.0.jar
- - io.prometheus-prometheus-metrics-model-1.1.0.jar
- - io.prometheus-prometheus-metrics-shaded-protobuf-1.1.0.jar
+ - io.prometheus-prometheus-metrics-config-1.2.1.jar
+ - io.prometheus-prometheus-metrics-exporter-common-1.2.1.jar
+ - io.prometheus-prometheus-metrics-exporter-httpserver-1.2.1.jar
+ - io.prometheus-prometheus-metrics-exposition-formats-1.2.1.jar
+ - io.prometheus-prometheus-metrics-model-1.2.1.jar
+ - io.prometheus-prometheus-metrics-shaded-protobuf-1.2.1.jar
* Jakarta Bean Validation API
- jakarta.validation-jakarta.validation-api-2.0.2.jar
- javax.validation-validation-api-1.1.0.Final.jar
* Log4J
- - org.apache.logging.log4j-log4j-api-2.18.0.jar
- - org.apache.logging.log4j-log4j-core-2.18.0.jar
- - org.apache.logging.log4j-log4j-slf4j-impl-2.18.0.jar
- - org.apache.logging.log4j-log4j-web-2.18.0.jar
+ - org.apache.logging.log4j-log4j-api-2.23.1.jar
+ - org.apache.logging.log4j-log4j-core-2.23.1.jar
+ - org.apache.logging.log4j-log4j-slf4j2-impl-2.23.1.jar
+ - org.apache.logging.log4j-log4j-web-2.23.1.jar
* Java Native Access JNA
- net.java.dev.jna-jna-jpms-5.12.1.jar
- net.java.dev.jna-jna-platform-jpms-5.12.1.jar
* BookKeeper
- - org.apache.bookkeeper-bookkeeper-common-4.16.4.jar
- - org.apache.bookkeeper-bookkeeper-common-allocator-4.16.4.jar
- - org.apache.bookkeeper-bookkeeper-proto-4.16.4.jar
- - org.apache.bookkeeper-bookkeeper-server-4.16.4.jar
- - org.apache.bookkeeper-bookkeeper-tools-framework-4.16.4.jar
- - org.apache.bookkeeper-circe-checksum-4.16.4.jar
- - org.apache.bookkeeper-cpu-affinity-4.16.4.jar
- - org.apache.bookkeeper-statelib-4.16.4.jar
- - org.apache.bookkeeper-stream-storage-api-4.16.4.jar
- - org.apache.bookkeeper-stream-storage-common-4.16.4.jar
- - org.apache.bookkeeper-stream-storage-java-client-4.16.4.jar
- - org.apache.bookkeeper-stream-storage-java-client-base-4.16.4.jar
- - org.apache.bookkeeper-stream-storage-proto-4.16.4.jar
- - org.apache.bookkeeper-stream-storage-server-4.16.4.jar
- - org.apache.bookkeeper-stream-storage-service-api-4.16.4.jar
- - org.apache.bookkeeper-stream-storage-service-impl-4.16.4.jar
- - org.apache.bookkeeper.http-http-server-4.16.4.jar
- - org.apache.bookkeeper.http-vertx-http-server-4.16.4.jar
- - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.16.4.jar
- - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.16.4.jar
- - org.apache.distributedlog-distributedlog-common-4.16.4.jar
- - org.apache.distributedlog-distributedlog-core-4.16.4-tests.jar
- - org.apache.distributedlog-distributedlog-core-4.16.4.jar
- - org.apache.distributedlog-distributedlog-protocol-4.16.4.jar
- - org.apache.bookkeeper.stats-codahale-metrics-provider-4.16.4.jar
- - org.apache.bookkeeper-bookkeeper-slogger-api-4.16.4.jar
- - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.16.4.jar
- - org.apache.bookkeeper-native-io-4.16.4.jar
+ - org.apache.bookkeeper-bookkeeper-common-4.17.1.jar
+ - org.apache.bookkeeper-bookkeeper-common-allocator-4.17.1.jar
+ - org.apache.bookkeeper-bookkeeper-proto-4.17.1.jar
+ - org.apache.bookkeeper-bookkeeper-server-4.17.1.jar
+ - org.apache.bookkeeper-bookkeeper-tools-framework-4.17.1.jar
+ - org.apache.bookkeeper-circe-checksum-4.17.1.jar
+ - org.apache.bookkeeper-cpu-affinity-4.17.1.jar
+ - org.apache.bookkeeper-statelib-4.17.1.jar
+ - org.apache.bookkeeper-stream-storage-api-4.17.1.jar
+ - org.apache.bookkeeper-stream-storage-common-4.17.1.jar
+ - org.apache.bookkeeper-stream-storage-java-client-4.17.1.jar
+ - org.apache.bookkeeper-stream-storage-java-client-base-4.17.1.jar
+ - org.apache.bookkeeper-stream-storage-proto-4.17.1.jar
+ - org.apache.bookkeeper-stream-storage-server-4.17.1.jar
+ - org.apache.bookkeeper-stream-storage-service-api-4.17.1.jar
+ - org.apache.bookkeeper-stream-storage-service-impl-4.17.1.jar
+ - org.apache.bookkeeper.http-http-server-4.17.1.jar
+ - org.apache.bookkeeper.http-vertx-http-server-4.17.1.jar
+ - org.apache.bookkeeper.stats-bookkeeper-stats-api-4.17.1.jar
+ - org.apache.bookkeeper.stats-prometheus-metrics-provider-4.17.1.jar
+ - org.apache.distributedlog-distributedlog-common-4.17.1.jar
+ - org.apache.distributedlog-distributedlog-core-4.17.1-tests.jar
+ - org.apache.distributedlog-distributedlog-core-4.17.1.jar
+ - org.apache.distributedlog-distributedlog-protocol-4.17.1.jar
+ - org.apache.bookkeeper.stats-codahale-metrics-provider-4.17.1.jar
+ - org.apache.bookkeeper-bookkeeper-slogger-api-4.17.1.jar
+ - org.apache.bookkeeper-bookkeeper-slogger-slf4j-4.17.1.jar
+ - org.apache.bookkeeper-native-io-4.17.1.jar
* Apache HTTP Client
- org.apache.httpcomponents-httpclient-4.5.13.jar
- org.apache.httpcomponents-httpcore-4.4.15.jar
* AirCompressor
- - io.airlift-aircompressor-0.20.jar
+ - io.airlift-aircompressor-0.27.jar
* AsyncHttpClient
- org.asynchttpclient-async-http-client-2.12.1.jar
- org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar
* Jetty
- - org.eclipse.jetty-jetty-client-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-continuation-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-http-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-io-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-proxy-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-security-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-server-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-servlet-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-servlets-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-util-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-util-ajax-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-websocket-api-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-websocket-client-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-websocket-common-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-websocket-server-9.4.54.v20240208.jar
- - org.eclipse.jetty.websocket-websocket-servlet-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.54.v20240208.jar
- - org.eclipse.jetty-jetty-alpn-server-9.4.54.v20240208.jar
+ - org.eclipse.jetty-jetty-client-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-continuation-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-http-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-io-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-proxy-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-security-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-server-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-servlet-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-servlets-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-util-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-util-ajax-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-websocket-api-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-websocket-client-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-websocket-common-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-websocket-server-9.4.56.v20240826.jar
+ - org.eclipse.jetty.websocket-websocket-servlet-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.56.v20240826.jar
+ - org.eclipse.jetty-jetty-alpn-server-9.4.56.v20240826.jar
* SnakeYaml -- org.yaml-snakeyaml-2.0.jar
* RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar
* Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.24.0.jar
@@ -430,23 +430,23 @@ The Apache Software License, Version 2.0
- org.jetbrains.kotlin-kotlin-stdlib-jdk8-1.8.20.jar
- org.jetbrains-annotations-13.0.jar
* gRPC
- - io.grpc-grpc-all-1.55.3.jar
- - io.grpc-grpc-auth-1.55.3.jar
- - io.grpc-grpc-context-1.55.3.jar
- - io.grpc-grpc-core-1.55.3.jar
- - io.grpc-grpc-netty-1.55.3.jar
- - io.grpc-grpc-protobuf-1.55.3.jar
- - io.grpc-grpc-protobuf-lite-1.55.3.jar
- - io.grpc-grpc-stub-1.55.3.jar
- - io.grpc-grpc-alts-1.55.3.jar
- - io.grpc-grpc-api-1.55.3.jar
- - io.grpc-grpc-grpclb-1.55.3.jar
- - io.grpc-grpc-netty-shaded-1.55.3.jar
- - io.grpc-grpc-services-1.55.3.jar
- - io.grpc-grpc-xds-1.55.3.jar
- - io.grpc-grpc-rls-1.55.3.jar
- - io.grpc-grpc-servlet-1.55.3.jar
- - io.grpc-grpc-servlet-jakarta-1.55.3.jar
+ - io.grpc-grpc-all-1.56.1.jar
+ - io.grpc-grpc-auth-1.56.1.jar
+ - io.grpc-grpc-context-1.56.1.jar
+ - io.grpc-grpc-core-1.56.1.jar
+ - io.grpc-grpc-protobuf-1.56.1.jar
+ - io.grpc-grpc-protobuf-lite-1.56.1.jar
+ - io.grpc-grpc-stub-1.56.1.jar
+ - io.grpc-grpc-alts-1.56.1.jar
+ - io.grpc-grpc-api-1.56.1.jar
+ - io.grpc-grpc-grpclb-1.56.1.jar
+ - io.grpc-grpc-netty-shaded-1.56.1.jar
+ - io.grpc-grpc-services-1.56.1.jar
+ - io.grpc-grpc-xds-1.56.1.jar
+ - io.grpc-grpc-rls-1.56.1.jar
+ - io.grpc-grpc-servlet-1.56.1.jar
+ - io.grpc-grpc-servlet-jakarta-1.56.1.jar
+ - io.grpc-grpc-util-1.60.0.jar
* Perfmark
- io.perfmark-perfmark-api-0.26.0.jar
* OpenCensus
@@ -455,14 +455,14 @@ The Apache Software License, Version 2.0
- io.opencensus-opencensus-proto-0.2.0.jar
* Jodah
- net.jodah-typetools-0.5.0.jar
- - net.jodah-failsafe-2.4.4.jar
+ - dev.failsafe-failsafe-3.3.2.jar
* Byte Buddy
- net.bytebuddy-byte-buddy-1.14.12.jar
* zt-zip
- org.zeroturnaround-zt-zip-1.17.jar
* Apache Avro
- - org.apache.avro-avro-1.11.3.jar
- - org.apache.avro-avro-protobuf-1.11.3.jar
+ - org.apache.avro-avro-1.11.4.jar
+ - org.apache.avro-avro-protobuf-1.11.4.jar
* Apache Curator
- org.apache.curator-curator-client-5.1.0.jar
- org.apache.curator-curator-framework-5.1.0.jar
@@ -481,7 +481,10 @@ The Apache Software License, Version 2.0
* Prometheus
- io.prometheus-simpleclient_httpserver-0.16.0.jar
* Oxia
- - io.streamnative.oxia-oxia-client-0.1.0-shaded.jar
+ - io.streamnative.oxia-oxia-client-api-0.4.5.jar
+ - io.streamnative.oxia-oxia-client-0.4.5.jar
+ * OpenHFT
+ - net.openhft-zero-allocation-hashing-0.16.jar
* Java JSON WebTokens
- io.jsonwebtoken-jjwt-api-0.11.1.jar
- io.jsonwebtoken-jjwt-impl-0.11.1.jar
@@ -489,12 +492,11 @@ The Apache Software License, Version 2.0
* JCTools - Java Concurrency Tools for the JVM
- org.jctools-jctools-core-2.1.2.jar
* Vertx
- - io.vertx-vertx-auth-common-4.3.8.jar
- - io.vertx-vertx-bridge-common-4.3.8.jar
- - io.vertx-vertx-core-4.3.8.jar
- - io.vertx-vertx-web-4.3.8.jar
- - io.vertx-vertx-web-common-4.3.8.jar
- - io.vertx-vertx-grpc-4.3.5.jar
+ - io.vertx-vertx-auth-common-4.5.10.jar
+ - io.vertx-vertx-bridge-common-4.5.10.jar
+ - io.vertx-vertx-core-4.5.10.jar
+ - io.vertx-vertx-web-4.5.10.jar
+ - io.vertx-vertx-web-common-4.5.10.jar
* Apache ZooKeeper
- org.apache.zookeeper-zookeeper-3.9.2.jar
- org.apache.zookeeper-zookeeper-jute-3.9.2.jar
@@ -504,41 +506,39 @@ The Apache Software License, Version 2.0
* Google HTTP Client
- com.google.http-client-google-http-client-gson-1.41.0.jar
- com.google.http-client-google-http-client-1.41.0.jar
- - com.google.auto.value-auto-value-annotations-1.9.jar
- - com.google.re2j-re2j-1.6.jar
- * Jetcd
- - io.etcd-jetcd-api-0.7.5.jar
- - io.etcd-jetcd-common-0.7.5.jar
- - io.etcd-jetcd-core-0.7.5.jar
- - io.etcd-jetcd-grpc-0.7.5.jar
+ - com.google.auto.value-auto-value-annotations-1.10.1.jar
+ - com.google.re2j-re2j-1.7.jar
+ * Jetcd - shaded
* IPAddress
- - com.github.seancfoley-ipaddress-5.3.3.jar
+ - com.github.seancfoley-ipaddress-5.5.0.jar
* RxJava
- io.reactivex.rxjava3-rxjava-3.0.1.jar
* RoaringBitmap
- - org.roaringbitmap-RoaringBitmap-0.9.44.jar
- - org.roaringbitmap-shims-0.9.44.jar
+ - org.roaringbitmap-RoaringBitmap-1.2.0.jar
* OpenTelemetry
- - io.opentelemetry-opentelemetry-api-1.34.1.jar
- - io.opentelemetry-opentelemetry-api-events-1.34.1-alpha.jar
- - io.opentelemetry-opentelemetry-context-1.34.1.jar
- - io.opentelemetry-opentelemetry-exporter-common-1.34.1.jar
- - io.opentelemetry-opentelemetry-exporter-otlp-1.34.1.jar
- - io.opentelemetry-opentelemetry-exporter-otlp-common-1.34.1.jar
- - io.opentelemetry-opentelemetry-exporter-prometheus-1.34.1-alpha.jar
- - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.34.1.jar
- - io.opentelemetry-opentelemetry-extension-incubator-1.34.1-alpha.jar
- - io.opentelemetry-opentelemetry-sdk-1.34.1.jar
- - io.opentelemetry-opentelemetry-sdk-common-1.34.1.jar
- - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.34.1.jar
- - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.34.1.jar
- - io.opentelemetry-opentelemetry-sdk-logs-1.34.1.jar
- - io.opentelemetry-opentelemetry-sdk-metrics-1.34.1.jar
- - io.opentelemetry-opentelemetry-sdk-trace-1.34.1.jar
- - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.32.1.jar
- - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.32.1-alpha.jar
- - io.opentelemetry.instrumentation-opentelemetry-resources-1.32.1-alpha.jar
- - io.opentelemetry.semconv-opentelemetry-semconv-1.23.1-alpha.jar
+ - io.opentelemetry-opentelemetry-api-1.38.0.jar
+ - io.opentelemetry-opentelemetry-api-incubator-1.38.0-alpha.jar
+ - io.opentelemetry-opentelemetry-context-1.38.0.jar
+ - io.opentelemetry-opentelemetry-exporter-common-1.38.0.jar
+ - io.opentelemetry-opentelemetry-exporter-otlp-1.38.0.jar
+ - io.opentelemetry-opentelemetry-exporter-otlp-common-1.38.0.jar
+ - io.opentelemetry-opentelemetry-exporter-prometheus-1.38.0-alpha.jar
+ - io.opentelemetry-opentelemetry-exporter-sender-okhttp-1.38.0.jar
+ - io.opentelemetry-opentelemetry-sdk-1.38.0.jar
+ - io.opentelemetry-opentelemetry-sdk-common-1.38.0.jar
+ - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-1.38.0.jar
+ - io.opentelemetry-opentelemetry-sdk-extension-autoconfigure-spi-1.38.0.jar
+ - io.opentelemetry-opentelemetry-sdk-logs-1.38.0.jar
+ - io.opentelemetry-opentelemetry-sdk-metrics-1.38.0.jar
+ - io.opentelemetry-opentelemetry-sdk-trace-1.38.0.jar
+ - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.3.jar
+ - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.3-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-resources-1.33.3-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.3-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.3-alpha.jar
+ - io.opentelemetry.semconv-opentelemetry-semconv-1.25.0-alpha.jar
+ * Spotify completable-futures
+ - com.spotify-completable-futures-0.3.6.jar
BSD 3-clause "New" or "Revised" License
* Google auth library
@@ -555,8 +555,8 @@ BSD 2-Clause License
MIT License
* Java SemVer -- com.github.zafarkhaja-java-semver-0.9.0.jar -- ../licenses/LICENSE-SemVer.txt
* SLF4J -- ../licenses/LICENSE-SLF4J.txt
- - org.slf4j-slf4j-api-1.7.32.jar
- - org.slf4j-jcl-over-slf4j-1.7.32.jar
+ - org.slf4j-slf4j-api-2.0.13.jar
+ - org.slf4j-jcl-over-slf4j-2.0.13.jar
* The Checker Framework
- org.checkerframework-checker-qual-3.33.0.jar
* oshi
@@ -566,8 +566,8 @@ MIT License
- com.auth0-jwks-rsa-0.22.0.jar
Protocol Buffers License
* Protocol Buffers
- - com.google.protobuf-protobuf-java-3.19.6.jar -- ../licenses/LICENSE-protobuf.txt
- - com.google.protobuf-protobuf-java-util-3.19.6.jar -- ../licenses/LICENSE-protobuf.txt
+ - com.google.protobuf-protobuf-java-3.25.5.jar -- ../licenses/LICENSE-protobuf.txt
+ - com.google.protobuf-protobuf-java-util-3.25.5.jar -- ../licenses/LICENSE-protobuf.txt
CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
* Java Annotations API
@@ -583,15 +583,15 @@ CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
- org.glassfish.hk2-osgi-resource-locator-1.0.3.jar
- org.glassfish.hk2.external-aopalliance-repackaged-2.6.1.jar
* Jersey
- - org.glassfish.jersey.containers-jersey-container-servlet-2.41.jar
- - org.glassfish.jersey.containers-jersey-container-servlet-core-2.41.jar
- - org.glassfish.jersey.core-jersey-client-2.41.jar
- - org.glassfish.jersey.core-jersey-common-2.41.jar
- - org.glassfish.jersey.core-jersey-server-2.41.jar
- - org.glassfish.jersey.ext-jersey-entity-filtering-2.41.jar
- - org.glassfish.jersey.media-jersey-media-json-jackson-2.41.jar
- - org.glassfish.jersey.media-jersey-media-multipart-2.41.jar
- - org.glassfish.jersey.inject-jersey-hk2-2.41.jar
+ - org.glassfish.jersey.containers-jersey-container-servlet-2.42.jar
+ - org.glassfish.jersey.containers-jersey-container-servlet-core-2.42.jar
+ - org.glassfish.jersey.core-jersey-client-2.42.jar
+ - org.glassfish.jersey.core-jersey-common-2.42.jar
+ - org.glassfish.jersey.core-jersey-server-2.42.jar
+ - org.glassfish.jersey.ext-jersey-entity-filtering-2.42.jar
+ - org.glassfish.jersey.media-jersey-media-json-jackson-2.42.jar
+ - org.glassfish.jersey.media-jersey-media-multipart-2.42.jar
+ - org.glassfish.jersey.inject-jersey-hk2-2.42.jar
* Mimepull -- org.jvnet.mimepull-mimepull-1.9.15.jar
Eclipse Distribution License 1.0 -- ../licenses/LICENSE-EDL-1.0.txt
@@ -613,10 +613,9 @@ Creative Commons Attribution License
Bouncy Castle License
* Bouncy Castle -- ../licenses/LICENSE-bouncycastle.txt
- - org.bouncycastle-bcpkix-jdk18on-1.75.jar
- - org.bouncycastle-bcprov-ext-jdk18on-1.75.jar
- - org.bouncycastle-bcprov-jdk18on-1.75.jar
- - org.bouncycastle-bcutil-jdk18on-1.75.jar
+ - org.bouncycastle-bcpkix-jdk18on-1.78.1.jar
+ - org.bouncycastle-bcprov-jdk18on-1.78.1.jar
+ - org.bouncycastle-bcutil-jdk18on-1.78.1.jar
------------------------
diff --git a/distribution/shell/pom.xml b/distribution/shell/pom.xml
index 5f4fc549ccc62..45108aba68f87 100644
--- a/distribution/shell/pom.xml
+++ b/distribution/shell/pom.xml
@@ -25,8 +25,7 @@
org.apache.pulsar
distribution
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
pulsar-shell-distribution
@@ -51,7 +50,7 @@
org.apache.logging.log4j
- log4j-slf4j-impl
+ log4j-slf4j2-impl
diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
index 2b2f1c26be112..15b2a918b9ebc 100644
--- a/distribution/shell/src/assemble/LICENSE.bin.txt
+++ b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -309,22 +309,21 @@ pulsar-client-cpp/lib/checksum/crc32c_sw.cc
This projects includes binary packages with the following licenses:
The Apache Software License, Version 2.0
- * JCommander -- jcommander-1.82.jar
* Picocli
- picocli-4.7.5.jar
- picocli-shell-jline3-4.7.5.jar
* Jackson
- - jackson-annotations-2.14.2.jar
- - jackson-core-2.14.2.jar
- - jackson-databind-2.14.2.jar
- - jackson-dataformat-yaml-2.14.2.jar
- - jackson-jaxrs-base-2.14.2.jar
- - jackson-jaxrs-json-provider-2.14.2.jar
- - jackson-module-jaxb-annotations-2.14.2.jar
- - jackson-module-jsonSchema-2.14.2.jar
- - jackson-datatype-jdk8-2.14.2.jar
- - jackson-datatype-jsr310-2.14.2.jar
- - jackson-module-parameter-names-2.14.2.jar
+ - jackson-annotations-2.17.2.jar
+ - jackson-core-2.17.2.jar
+ - jackson-databind-2.17.2.jar
+ - jackson-dataformat-yaml-2.17.2.jar
+ - jackson-jaxrs-base-2.17.2.jar
+ - jackson-jaxrs-json-provider-2.17.2.jar
+ - jackson-module-jaxb-annotations-2.17.2.jar
+ - jackson-module-jsonSchema-2.17.2.jar
+ - jackson-datatype-jdk8-2.17.2.jar
+ - jackson-datatype-jsr310-2.17.2.jar
+ - jackson-module-parameter-names-2.17.2.jar
* Conscrypt -- conscrypt-openjdk-uber-2.5.2.jar
* Gson
- gson-2.8.9.jar
@@ -334,49 +333,49 @@ The Apache Software License, Version 2.0
- listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
* J2ObjC Annotations -- j2objc-annotations-1.3.jar
* Netty Reactive Streams -- netty-reactive-streams-2.0.6.jar
+ * Swagger -- swagger-annotations-1.6.2.jar
* DataSketches
- memory-0.8.3.jar
- sketches-core-0.8.3.jar
* Apache Commons
- commons-codec-1.15.jar
- commons-configuration-1.10.jar
- - commons-io-2.8.0.jar
+ - commons-io-2.14.0.jar
- commons-lang-2.6.jar
- commons-logging-1.2.jar
- commons-lang3-3.11.jar
- commons-text-1.10.0.jar
- commons-compress-1.26.0.jar
* Netty
- - netty-buffer-4.1.105.Final.jar
- - netty-codec-4.1.105.Final.jar
- - netty-codec-dns-4.1.105.Final.jar
- - netty-codec-http-4.1.105.Final.jar
- - netty-codec-socks-4.1.105.Final.jar
- - netty-codec-haproxy-4.1.105.Final.jar
- - netty-common-4.1.105.Final.jar
- - netty-handler-4.1.105.Final.jar
- - netty-handler-proxy-4.1.105.Final.jar
- - netty-resolver-4.1.105.Final.jar
- - netty-resolver-dns-4.1.105.Final.jar
- - netty-transport-4.1.105.Final.jar
- - netty-transport-classes-epoll-4.1.105.Final.jar
- - netty-transport-native-epoll-4.1.105.Final-linux-aarch_64.jar
- - netty-transport-native-epoll-4.1.105.Final-linux-x86_64.jar
- - netty-transport-native-unix-common-4.1.105.Final.jar
- - netty-transport-native-unix-common-4.1.105.Final-linux-x86_64.jar
- - netty-tcnative-boringssl-static-2.0.61.Final.jar
- - netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar
- - netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar
- - netty-tcnative-boringssl-static-2.0.61.Final-osx-aarch_64.jar
- - netty-tcnative-boringssl-static-2.0.61.Final-osx-x86_64.jar
- - netty-tcnative-boringssl-static-2.0.61.Final-windows-x86_64.jar
- - netty-tcnative-classes-2.0.61.Final.jar
+ - netty-buffer-4.1.113.Final.jar
+ - netty-codec-4.1.113.Final.jar
+ - netty-codec-dns-4.1.113.Final.jar
+ - netty-codec-http-4.1.113.Final.jar
+ - netty-codec-socks-4.1.113.Final.jar
+ - netty-codec-haproxy-4.1.113.Final.jar
+ - netty-common-4.1.113.Final.jar
+ - netty-handler-4.1.113.Final.jar
+ - netty-handler-proxy-4.1.113.Final.jar
+ - netty-resolver-4.1.113.Final.jar
+ - netty-resolver-dns-4.1.113.Final.jar
+ - netty-transport-4.1.113.Final.jar
+ - netty-transport-classes-epoll-4.1.113.Final.jar
+ - netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar
+ - netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar
+ - netty-transport-native-unix-common-4.1.113.Final.jar
+ - netty-tcnative-boringssl-static-2.0.66.Final.jar
+ - netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar
+ - netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar
+ - netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar
+ - netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar
+ - netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar
+ - netty-tcnative-classes-2.0.66.Final.jar
- netty-incubator-transport-classes-io_uring-0.0.24.Final.jar
- netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar
- netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar
- - netty-resolver-dns-classes-macos-4.1.105.Final.jar
- - netty-resolver-dns-native-macos-4.1.105.Final-osx-aarch_64.jar
- - netty-resolver-dns-native-macos-4.1.105.Final-osx-x86_64.jar
+ - netty-resolver-dns-classes-macos-4.1.113.Final.jar
+ - netty-resolver-dns-native-macos-4.1.113.Final-osx-aarch_64.jar
+ - netty-resolver-dns-native-macos-4.1.113.Final-osx-x86_64.jar
* Prometheus client
- simpleclient-0.16.0.jar
- simpleclient_log4j2-0.16.0.jar
@@ -384,35 +383,41 @@ The Apache Software License, Version 2.0
- simpleclient_tracer_otel-0.16.0.jar
- simpleclient_tracer_otel_agent-0.16.0.jar
* Log4J
- - log4j-api-2.18.0.jar
- - log4j-core-2.18.0.jar
- - log4j-slf4j-impl-2.18.0.jar
- - log4j-web-2.18.0.jar
+ - log4j-api-2.23.1.jar
+ - log4j-core-2.23.1.jar
+ - log4j-slf4j2-impl-2.23.1.jar
+ - log4j-web-2.23.1.jar
+ * OpenTelemetry
+ - opentelemetry-api-1.38.0.jar
+ - opentelemetry-api-incubator-1.38.0-alpha.jar
+ - opentelemetry-context-1.38.0.jar
* BookKeeper
- - bookkeeper-common-allocator-4.16.4.jar
- - cpu-affinity-4.16.4.jar
- - circe-checksum-4.16.4.jar
+ - bookkeeper-common-allocator-4.17.1.jar
+ - cpu-affinity-4.17.1.jar
+ - circe-checksum-4.17.1.jar
* AirCompressor
- - aircompressor-0.20.jar
+ - aircompressor-0.27.jar
* AsyncHttpClient
- async-http-client-2.12.1.jar
- async-http-client-netty-utils-2.12.1.jar
* Jetty
- - jetty-client-9.4.54.v20240208.jar
- - jetty-http-9.4.54.v20240208.jar
- - jetty-io-9.4.54.v20240208.jar
- - jetty-util-9.4.54.v20240208.jar
- - javax-websocket-client-impl-9.4.54.v20240208.jar
- - websocket-api-9.4.54.v20240208.jar
- - websocket-client-9.4.54.v20240208.jar
- - websocket-common-9.4.54.v20240208.jar
+ - jetty-client-9.4.56.v20240826.jar
+ - jetty-http-9.4.56.v20240826.jar
+ - jetty-io-9.4.56.v20240826.jar
+ - jetty-util-9.4.56.v20240826.jar
+ - javax-websocket-client-impl-9.4.56.v20240826.jar
+ - websocket-api-9.4.56.v20240826.jar
+ - websocket-client-9.4.56.v20240826.jar
+ - websocket-common-9.4.56.v20240826.jar
* SnakeYaml -- snakeyaml-2.0.jar
* Google Error Prone Annotations - error_prone_annotations-2.24.0.jar
* Javassist -- javassist-3.25.0-GA.jar
* Apache Avro
- - avro-1.11.3.jar
- - avro-protobuf-1.11.3.jar
+ - avro-1.11.4.jar
+ - avro-protobuf-1.11.4.jar
+ * RE2j -- re2j-1.7.jar
+ * Spotify completable-futures -- completable-futures-0.3.6.jar
BSD 3-clause "New" or "Revised" License
* JSR305 -- jsr305-3.0.2.jar -- ../licenses/LICENSE-JSR305.txt
@@ -420,13 +425,13 @@ BSD 3-clause "New" or "Revised" License
MIT License
* SLF4J -- ../licenses/LICENSE-SLF4J.txt
- - slf4j-api-1.7.32.jar
+ - slf4j-api-2.0.13.jar
* The Checker Framework
- checker-qual-3.33.0.jar
Protocol Buffers License
* Protocol Buffers
- - protobuf-java-3.19.6.jar -- ../licenses/LICENSE-protobuf.txt
+ - protobuf-java-3.25.5.jar -- ../licenses/LICENSE-protobuf.txt
CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
* Java Annotations API
@@ -442,12 +447,12 @@ CDDL-1.1 -- ../licenses/LICENSE-CDDL-1.1.txt
- aopalliance-repackaged-2.6.1.jar
- osgi-resource-locator-1.0.3.jar
* Jersey
- - jersey-client-2.41.jar
- - jersey-common-2.41.jar
- - jersey-entity-filtering-2.41.jar
- - jersey-media-json-jackson-2.41.jar
- - jersey-media-multipart-2.41.jar
- - jersey-hk2-2.41.jar
+ - jersey-client-2.42.jar
+ - jersey-common-2.42.jar
+ - jersey-entity-filtering-2.42.jar
+ - jersey-media-json-jackson-2.42.jar
+ - jersey-media-multipart-2.42.jar
+ - jersey-hk2-2.42.jar
* Mimepull -- mimepull-1.9.15.jar
Eclipse Distribution License 1.0 -- ../licenses/LICENSE-EDL-1.0.txt
@@ -470,10 +475,9 @@ Creative Commons Attribution License
Bouncy Castle License
* Bouncy Castle -- ../licenses/LICENSE-bouncycastle.txt
- - bcpkix-jdk18on-1.75.jar
- - bcprov-ext-jdk18on-1.75.jar
- - bcprov-jdk18on-1.75.jar
- - bcutil-jdk18on-1.75.jar
+ - bcpkix-jdk18on-1.78.1.jar
+ - bcprov-jdk18on-1.78.1.jar
+ - bcutil-jdk18on-1.78.1.jar
------------------------
diff --git a/docker/glibc-package/Dockerfile b/docker/glibc-package/Dockerfile
index f9c238cbdfc55..016e5c622365f 100644
--- a/docker/glibc-package/Dockerfile
+++ b/docker/glibc-package/Dockerfile
@@ -19,6 +19,7 @@
ARG GLIBC_VERSION=2.38
+ARG ALPINE_VERSION=3.20
FROM ubuntu:22.04 as build
ARG GLIBC_VERSION
@@ -51,7 +52,7 @@ RUN tar --dereference --hard-dereference -zcf /glibc-bin.tar.gz /usr/glibc-compa
################################################
## Build the APK package
-FROM alpine:3.19 as apk
+FROM alpine:$ALPINE_VERSION as apk
ARG GLIBC_VERSION
RUN apk add abuild sudo build-base
diff --git a/docker/pom.xml b/docker/pom.xml
index 21ed4de940826..ffcaec3ffdc30 100644
--- a/docker/pom.xml
+++ b/docker/pom.xml
@@ -26,7 +26,7 @@
org.apache.pulsar
pulsar
- 3.3.0-SNAPSHOT
+ 4.0.0-SNAPSHOT
docker-images
Apache Pulsar :: Docker Images
@@ -68,7 +68,6 @@
false
true
- true
false
diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
index 3da14ea84bcb3..b43121dd0f613 100644
--- a/docker/pulsar-all/pom.xml
+++ b/docker/pulsar-all/pom.xml
@@ -23,7 +23,7 @@
org.apache.pulsar
docker-images
- 3.3.0-SNAPSHOT
+ 4.0.0-SNAPSHOT
4.0.0
pulsar-all-docker-image
@@ -70,7 +70,7 @@
git-commit-id-no-git
- ${basedir}/../../.git/index
+ ${basedir}/../../.git
@@ -145,22 +145,22 @@
package
build
- tag
+ push
- ${docker.organization}/pulsar-all
+ ${docker.organization}/${docker.image}-all
${project.basedir}
- latest
+ ${docker.tag}
${project.version}-${git.commit.id.abbrev}
target/apache-pulsar-io-connectors-${project.version}-bin
target/pulsar-offloader-distribution-${project.version}-bin.tar.gz
- ${docker.organization}/pulsar:${project.version}-${git.commit.id.abbrev}
+ ${docker.organization}/${docker.image}:${project.version}-${git.commit.id.abbrev}
@@ -177,29 +177,12 @@
-
docker-push
-
-
-
- io.fabric8
- docker-maven-plugin
-
-
- default
- package
-
- build
- tag
- push
-
-
-
-
-
-
+
+ false
+ linux/amd64,linux/arm64
+
-
diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile
index 3b0b6322734c6..f8c22dc14a821 100644
--- a/docker/pulsar/Dockerfile
+++ b/docker/pulsar/Dockerfile
@@ -17,8 +17,11 @@
# under the License.
#
+ARG ALPINE_VERSION=3.20
+ARG IMAGE_JDK_MAJOR_VERSION=21
+
# First create a stage with just the Pulsar tarball and scripts
-FROM alpine as pulsar
+FROM alpine:$ALPINE_VERSION as pulsar
RUN apk add zip
@@ -39,7 +42,7 @@ COPY scripts/* /pulsar/bin/
# container when gid=0 is prohibited. In that case, the container must be run with uid 10000 with
# any group id != 0 (for example 10001).
# The file permissions are preserved when copying files from this builder image to the target image.
-RUN for SUBDIRECTORY in conf data download logs instances/deps; do \
+RUN for SUBDIRECTORY in conf data download logs instances/deps packages-storage; do \
mkdir -p /pulsar/$SUBDIRECTORY; \
chmod -R ug+rwx /pulsar/$SUBDIRECTORY; \
chown -R 10000:0 /pulsar/$SUBDIRECTORY; \
@@ -48,34 +51,13 @@ RUN for SUBDIRECTORY in conf data download logs instances/deps; do \
RUN chmod -R g+rx /pulsar/bin
RUN chmod -R o+rx /pulsar
-## Create 2nd stage to build the Python dependencies
-## Since it needs to have GCC available, we're doing it in a different layer
-FROM alpine:3.19 AS python-deps
-
-RUN apk add --no-cache \
- bash \
- python3-dev \
- g++ \
- musl-dev \
- libffi-dev \
- py3-pip \
- py3-grpcio \
- py3-yaml
-
-RUN pip3 install --break-system-packages \
- kazoo
-
-ARG PULSAR_CLIENT_PYTHON_VERSION
-RUN pip3 install --break-system-packages \
- pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION}
-
+# Enable snappy-java to use system lib
+RUN echo 'OPTS="$OPTS -Dorg.xerial.snappy.use.systemlib=true"' >> /pulsar/conf/bkenv.sh
### Create one stage to include JVM distribution
-FROM alpine AS jvm
+FROM amazoncorretto:${IMAGE_JDK_MAJOR_VERSION}-alpine AS jvm
-RUN wget -O /etc/apk/keys/amazoncorretto.rsa.pub https://apk.corretto.aws/amazoncorretto.rsa.pub
-RUN echo "https://apk.corretto.aws" >> /etc/apk/repositories
-RUN apk add --no-cache amazon-corretto-21 binutils
+RUN apk add --no-cache binutils
# Use JLink to create a slimmer JDK distribution (see: https://adoptium.net/blog/2021/10/jlink-to-produce-own-runtime/)
# This still includes all JDK modules, though in the future we could compile a list of required modules
@@ -83,21 +65,63 @@ RUN /usr/lib/jvm/default-jvm/bin/jlink --add-modules ALL-MODULE-PATH --compress
RUN echo networkaddress.cache.ttl=1 >> /opt/jvm/conf/security/java.security
RUN echo networkaddress.cache.negative.ttl=1 >> /opt/jvm/conf/security/java.security
+## Create one stage to include snappy-java native lib
+# Fix the issue when using snappy-java in x86 arch alpine
+# See https://github.com/xerial/snappy-java/issues/181 https://github.com/xerial/snappy-java/issues/579
+# We need to ensure that the version of the native library matches the version of snappy-java imported via Maven
+FROM alpine:$ALPINE_VERSION AS snappy-java
+ARG SNAPPY_VERSION
+RUN apk add git alpine-sdk util-linux cmake autoconf automake libtool openjdk17 maven curl bash tar
+ENV JAVA_HOME=/usr
+RUN curl -Ls https://github.com/xerial/snappy-java/archive/refs/tags/v$SNAPPY_VERSION.tar.gz | tar zxf - && cd snappy-java-$SNAPPY_VERSION && make clean-native native
FROM apachepulsar/glibc-base:2.38 as glibc
## Create final stage from Alpine image
## and add OpenJDK and Python dependencies (for Pulsar functions)
-FROM alpine:3.19.1
+FROM alpine:$ALPINE_VERSION
ENV LANG C.UTF-8
-# Install some utilities
+# Install some utilities, some are required by Pulsar scripts
RUN apk add --no-cache \
bash \
python3 \
py3-pip \
+ py3-grpcio \
+ py3-yaml \
gcompat \
- ca-certificates
+ ca-certificates \
+ procps \
+ curl \
+ bind-tools \
+ openssl
+
+# Upgrade all packages to get latest versions with security fixes
+RUN apk upgrade --no-cache
+
+# Python dependencies
+
+# The grpcio@1.59.3 is installed by apk, and Pulsar-client@3.4.0 requires grpcio>=1.60.0, which causes the grocio to be reinstalled by pip.
+# If pip cannot find the grpcio wheel that the doesn't match the OS, the grpcio will be compiled locally.
+# Once https://github.com/apache/pulsar-client-python/pull/211 is released, keep only the pulsar-client[all] and kazoo dependencies, and remove comments.
+ARG PULSAR_CLIENT_PYTHON_VERSION
+RUN echo -e "\
+#pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION}\n\
+pulsar-client==${PULSAR_CLIENT_PYTHON_VERSION}\n\
+# Zookeeper\n\
+kazoo\n\
+# functions\n\
+protobuf>=3.6.1,<=3.20.3\n\
+grpcio>=1.59.3\n\
+apache-bookkeeper-client>=4.16.1\n\
+prometheus_client\n\
+ratelimit\n\
+# avro\n\
+fastavro>=1.9.2\n\
+" > /requirements.txt
+
+RUN pip3 install --break-system-packages --no-cache-dir --only-binary grpcio -r /requirements.txt
+RUN rm /requirements.txt
# Install GLibc compatibility library
COPY --from=glibc /root/packages /root/packages
@@ -106,8 +130,10 @@ RUN apk add --allow-untrusted --force-overwrite /root/packages/glibc-*.apk
COPY --from=jvm /opt/jvm /opt/jvm
ENV JAVA_HOME=/opt/jvm
-# Copy Python depedencies from the other stage
-COPY --from=python-deps /usr/lib/python3.11/site-packages /usr/lib/python3.11/site-packages
+COPY --from=snappy-java /tmp/libsnappyjava.so /usr/lib/libsnappyjava.so
+
+# The default is /pulsat/bin and cannot be written.
+ENV PULSAR_PID_DIR=/pulsar/logs
ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE
@@ -115,8 +141,10 @@ COPY --from=pulsar /pulsar /pulsar
WORKDIR /pulsar
ENV PATH=$PATH:$JAVA_HOME/bin:/pulsar/bin
+# Use musl libc library for RocksDB
+ENV ROCKSDB_MUSL_LIBC=true
# The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely on its specific value.
ARG DEFAULT_USERNAME=pulsar
-RUN adduser ${DEFAULT_USERNAME} -u 10000 -G root -D
+RUN adduser ${DEFAULT_USERNAME} -u 10000 -G root -D -H -h /pulsar/data
USER 10000
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 79ff4bd33b10c..481fc319be732 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -23,7 +23,7 @@
org.apache.pulsar
docker-images
- 3.3.0-SNAPSHOT
+ 4.0.0-SNAPSHOT
4.0.0
pulsar-docker-image
@@ -52,7 +52,7 @@
git-commit-id-no-git
- ${basedir}/../../.git/index
+ ${basedir}/../../.git
@@ -72,20 +72,22 @@
package
build
- tag
+ push
- ${docker.organization}/pulsar
+ ${docker.organization}/${docker.image}
target/pulsar-server-distribution-${project.version}-bin.tar.gz
${pulsar.client.python.version}
+ ${snappy.version}
+ ${IMAGE_JDK_MAJOR_VERSION}
${project.basedir}
- latest
+ ${docker.tag}
${project.version}-${git.commit.id.abbrev}
@@ -121,29 +123,12 @@
-
docker-push
-
-
-
- io.fabric8
- docker-maven-plugin
-
-
- default
- package
-
- build
- tag
- push
-
-
-
-
-
-
+
+ false
+ linux/amd64,linux/arm64
+
-
diff --git a/docker/pulsar/scripts/gen-yml-from-env.py b/docker/pulsar/scripts/gen-yml-from-env.py
index aa40408ed5b1f..916b147f0cbba 100755
--- a/docker/pulsar/scripts/gen-yml-from-env.py
+++ b/docker/pulsar/scripts/gen-yml-from-env.py
@@ -50,6 +50,9 @@
'brokerClientTlsProtocols',
'webServiceTlsCiphers',
'webServiceTlsProtocols',
+ 'additionalJavaRuntimeArguments',
+ 'additionalEnabledConnectorUrlPatterns',
+ 'additionalEnabledFunctionsUrlPatterns'
]
PF_ENV_PREFIX = 'PF_'
diff --git a/docker/pulsar/scripts/pulsar-zookeeper-ruok.sh b/docker/pulsar/scripts/pulsar-zookeeper-ruok.sh
index 7a0228c2386bd..045258696ff0b 100755
--- a/docker/pulsar/scripts/pulsar-zookeeper-ruok.sh
+++ b/docker/pulsar/scripts/pulsar-zookeeper-ruok.sh
@@ -20,7 +20,7 @@
# Check ZK server status
-status=$(echo ruok | nc -q 1 localhost 2181)
+status=$({ echo ruok; sleep 1; } | nc 127.0.0.1 2181)
if [ "$status" == "imok" ]; then
exit 0
else
diff --git a/docker/pulsar/scripts/update-ini-from-env.py b/docker/pulsar/scripts/update-ini-from-env.py
new file mode 100755
index 0000000000000..6b0d7a795c3f8
--- /dev/null
+++ b/docker/pulsar/scripts/update-ini-from-env.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import sys
+import configparser
+import re
+
+def get_first_word(section_name):
+ # Split the section name by any non-word character and return the first word
+ return re.split(r'\W+', section_name)[0]
+
+def update_ini_file(ini_file_path, env_prefix):
+ # Read the existing INI file
+ config = configparser.ConfigParser()
+ config.read(ini_file_path)
+
+ # Flag to track if any updates were made
+ updated = False
+
+ # Iterate over environment variables
+ for key, value in os.environ.items():
+ if env_prefix and not key.startswith(env_prefix):
+ continue
+
+ stripped_key = key[len(env_prefix):] if env_prefix else key
+
+ # Iterate through sections
+ for section in config.sections():
+ first_word = get_first_word(section)
+ prefix = first_word + '_'
+ if stripped_key.startswith(prefix):
+ config.set(section, stripped_key[len(prefix):], value)
+ updated = True
+ break
+ elif config.has_option(section, stripped_key):
+ config.set(section, stripped_key, value)
+ updated = True
+ break
+
+ # Write the updated INI file only if there were updates
+ if updated:
+ with open(ini_file_path, 'w') as configfile:
+ config.write(configfile)
+
+if __name__ == "__main__":
+ if len(sys.argv) != 3:
+ print("Usage: python3 update-ini-from-env.py ")
+ sys.exit(1)
+
+ ini_file_path = sys.argv[1]
+ env_prefix = sys.argv[2]
+ update_ini_file(ini_file_path, env_prefix)
\ No newline at end of file
diff --git a/docker/pulsar/scripts/update-rocksdb-conf-from-env.py b/docker/pulsar/scripts/update-rocksdb-conf-from-env.py
new file mode 100755
index 0000000000000..2e55b455de3b7
--- /dev/null
+++ b/docker/pulsar/scripts/update-rocksdb-conf-from-env.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# allows tuning of RocksDB configuration via environment variables which were effective
+# before Pulsar 2.11 / BookKeeper 4.15 / https://github.com/apache/bookkeeper/pull/3056
+# the script should be applied to the `conf/entry_location_rocksdb.conf` file
+
+import os
+import sys
+import configparser
+
+# Constants for section keys
+DB_OPTIONS = "DBOptions"
+CF_OPTIONS = "CFOptions \"default\""
+TABLE_OPTIONS = "TableOptions/BlockBasedTable \"default\""
+
+def update_ini_file(ini_file_path):
+ config = configparser.ConfigParser()
+ config.read(ini_file_path)
+ updated = False
+
+ # Mapping of environment variables to INI sections and keys
+ env_to_ini_mapping = {
+ "dbStorage_rocksDB_logPath": (DB_OPTIONS, "log_path"),
+ "dbStorage_rocksDB_logLevel": (DB_OPTIONS, "info_log_level"),
+ "dbStorage_rocksDB_lz4CompressionEnabled": (CF_OPTIONS, "compression"),
+ "dbStorage_rocksDB_writeBufferSizeMB": (CF_OPTIONS, "write_buffer_size"),
+ "dbStorage_rocksDB_sstSizeInMB": (CF_OPTIONS, "target_file_size_base"),
+ "dbStorage_rocksDB_blockSize": (TABLE_OPTIONS, "block_size"),
+ "dbStorage_rocksDB_bloomFilterBitsPerKey": (TABLE_OPTIONS, "filter_policy"),
+ "dbStorage_rocksDB_blockCacheSize": (TABLE_OPTIONS, "block_cache"),
+ "dbStorage_rocksDB_numLevels": (CF_OPTIONS, "num_levels"),
+ "dbStorage_rocksDB_numFilesInLevel0": (CF_OPTIONS, "level0_file_num_compaction_trigger"),
+ "dbStorage_rocksDB_maxSizeInLevel1MB": (CF_OPTIONS, "max_bytes_for_level_base"),
+ "dbStorage_rocksDB_format_version": (TABLE_OPTIONS, "format_version")
+ }
+
+ # Type conversion functions
+ def mb_to_bytes(mb):
+ return str(int(mb) * 1024 * 1024)
+
+ def str_to_bool(value):
+ return True if value.lower() in ["true", "1", "yes"] else False
+
+ # Iterate over environment variables
+ for key, value in os.environ.items():
+ if key.startswith("PULSAR_PREFIX_"):
+ key = key[len("PULSAR_PREFIX_"):]
+
+ if key in env_to_ini_mapping:
+ section, option = env_to_ini_mapping[key]
+ if key in ["dbStorage_rocksDB_writeBufferSizeMB", "dbStorage_rocksDB_sstSizeInMB", "dbStorage_rocksDB_maxSizeInLevel1MB"]:
+ value = mb_to_bytes(value)
+ elif key == "dbStorage_rocksDB_lz4CompressionEnabled":
+ value = "kLZ4Compression" if str_to_bool(value) else "kNoCompression"
+ elif key == "dbStorage_rocksDB_bloomFilterBitsPerKey":
+ value = "rocksdb.BloomFilter:{}:false".format(value)
+ if config.get(section, option, fallback=None) != value:
+ config.set(section, option, value)
+ updated = True
+
+ # Write the updated INI file only if there were updates
+ if updated:
+ with open(ini_file_path, 'w') as configfile:
+ config.write(configfile)
+
+if __name__ == "__main__":
+ ini_file_path = sys.argv[1] if len(sys.argv) > 1 else "conf/entry_location_rocksdb.conf"
+ update_ini_file(ini_file_path)
\ No newline at end of file
diff --git a/jclouds-shaded/pom.xml b/jclouds-shaded/pom.xml
index 998fe98aa0f8d..dd19faad904bc 100644
--- a/jclouds-shaded/pom.xml
+++ b/jclouds-shaded/pom.xml
@@ -26,8 +26,7 @@
org.apache.pulsar
pulsar
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
jclouds-shaded
diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml
new file mode 100644
index 0000000000000..a0885f8509547
--- /dev/null
+++ b/jetcd-core-shaded/pom.xml
@@ -0,0 +1,198 @@
+
+
+
+ 4.0.0
+
+ org.apache.pulsar
+ pulsar
+ 4.0.0-SNAPSHOT
+
+
+ jetcd-core-shaded
+ Apache Pulsar :: jetcd-core shaded
+
+
+
+ io.etcd
+ jetcd-core
+
+
+ io.grpc
+ grpc-netty
+
+
+ io.netty
+ *
+
+
+
+
+ io.grpc
+ grpc-netty-shaded
+
+
+
+ dev.failsafe
+ failsafe
+
+
+ io.grpc
+ grpc-protobuf
+
+
+ io.grpc
+ grpc-stub
+
+
+ io.grpc
+ grpc-grpclb
+
+
+ io.grpc
+ grpc-util
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+ package
+
+ shade
+
+
+ true
+ true
+ false
+
+
+ io.etcd:*
+ io.vertx:*
+
+
+
+
+
+ io.vertx
+ org.apache.pulsar.jetcd.shaded.io.vertx
+
+
+
+ META-INF/versions/(\d+)/io/vertx/
+ META-INF/versions/$1/org/apache/pulsar/jetcd/shaded/io/vertx/
+ true
+
+
+
+ io.grpc.netty
+ io.grpc.netty.shaded.io.grpc.netty
+
+
+
+ io.netty
+ io.grpc.netty.shaded.io.netty
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+ META-INF/maven/${project.groupId}/${project.artifactId}/pom.xml
+
+
+
+
+
+
+ true
+
+
+
+
+
+ META-INF/maven/${project.groupId}/${project.artifactId}/pom.xml
+ ${project.basedir}/dependency-reduced-pom.xml
+
+
+
+ true
+ shaded
+
+
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+
+
+ attach-shaded-jar
+ package
+
+ attach-artifact
+
+
+
+
+ ${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar
+ jar
+ shaded
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-antrun-plugin
+ ${maven-antrun-plugin.version}
+
+
+ unpack-shaded-jar
+ package
+
+ run
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml
index 59bb82911f2b5..22b093f7aafd7 100644
--- a/managed-ledger/pom.xml
+++ b/managed-ledger/pom.xml
@@ -25,8 +25,7 @@
org.apache.pulsar
pulsar
- 3.3.0-SNAPSHOT
- ..
+ 4.0.0-SNAPSHOT
managed-ledger
@@ -72,6 +71,12 @@
${project.version}
+
+ ${project.groupId}
+ pulsar-opentelemetry
+ ${project.version}
+
+
com.google.guava
guava
@@ -104,6 +109,10 @@
+
+ org.roaringbitmap
+ RoaringBitmap
+
io.dropwizard.metrics
metrics-core
@@ -121,6 +130,12 @@
test
+
+ io.opentelemetry
+ opentelemetry-sdk-testing
+ test
+
+
org.slf4j
slf4j-api
@@ -147,7 +162,7 @@
-
+
org.apache.maven.plugins
maven-jar-plugin
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java
index dcf2c225e8b35..70db427afce4f 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/AsyncCallbacks.java
@@ -24,7 +24,6 @@
import java.util.Optional;
import org.apache.bookkeeper.common.annotation.InterfaceAudience;
import org.apache.bookkeeper.common.annotation.InterfaceStability;
-import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl;
/**
* Definition of all the callbacks used for the ManagedLedger asynchronous API.
@@ -48,7 +47,7 @@ interface OpenReadOnlyCursorCallback {
}
interface OpenReadOnlyManagedLedgerCallback {
- void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl managedLedger, Object ctx);
+ void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger managedLedger, Object ctx);
void openReadOnlyManagedLedgerFailed(ManagedLedgerException exception, Object ctx);
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java
index 868a8e4265365..11148ef1a59f5 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloader.java
@@ -230,5 +230,9 @@ default void scanLedgers(OffloadedLedgerMetadataConsumer consumer,
Map offloadDriverMetadata) throws ManagedLedgerException {
throw ManagedLedgerException.getManagedLedgerException(new UnsupportedOperationException());
}
+
+ default boolean isAppendable() {
+ return true;
+ }
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java
index 7ecb8f08d573d..9fbf9b73c057e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/LedgerOffloaderFactory.java
@@ -31,7 +31,7 @@
*/
@LimitedPrivate
@Evolving
-public interface LedgerOffloaderFactory {
+public interface LedgerOffloaderFactory extends AutoCloseable {
/**
* Check whether the provided driver driverName is supported.
@@ -111,4 +111,9 @@ default T create(OffloadPoliciesImpl offloadPolicies,
throws IOException {
return create(offloadPolicies, userMetadata, scheduler, offloaderStats);
}
+
+ @Override
+ default void close() throws Exception {
+ // no-op
+ }
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
index 8372592c851d1..042e03998696c 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursor.java
@@ -34,7 +34,7 @@
import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.SkipEntriesCallback;
-import org.apache.bookkeeper.mledger.impl.PositionImpl;
+import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats;
/**
* A ManagedCursor is a persisted cursor inside a ManagedLedger.
@@ -46,6 +46,8 @@
@InterfaceStability.Stable
public interface ManagedCursor {
+ String CURSOR_INTERNAL_PROPERTY_PREFIX = "#pulsar.internal.";
+
@SuppressWarnings("checkstyle:javadoctype")
enum FindPositionConstraint {
SearchActiveEntries, SearchAllAvailableEntries
@@ -152,7 +154,7 @@ enum IndividualDeletedEntries {
* max position can read
*/
void asyncReadEntries(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx,
- PositionImpl maxPosition);
+ Position maxPosition);
/**
@@ -165,7 +167,7 @@ void asyncReadEntries(int numberOfEntriesToRead, ReadEntriesCallback callback, O
* @param maxPosition max position can read
*/
void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback,
- Object ctx, PositionImpl maxPosition);
+ Object ctx, Position maxPosition);
/**
* Asynchronously read entries from the ManagedLedger.
@@ -178,7 +180,7 @@ void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesC
* @param skipCondition predicate of read filter out
*/
default void asyncReadEntriesWithSkip(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback,
- Object ctx, PositionImpl maxPosition, Predicate skipCondition) {
+ Object ctx, Position maxPosition, Predicate skipCondition) {
asyncReadEntries(numberOfEntriesToRead, maxSizeBytes, callback, ctx, maxPosition);
}
@@ -256,7 +258,7 @@ List readEntriesOrWait(int maxEntries, long maxSizeBytes)
* max position can read
*/
void asyncReadEntriesOrWait(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx,
- PositionImpl maxPosition);
+ Position maxPosition);
/**
* Asynchronously read entries from the ManagedLedger, up to the specified number and size.
@@ -277,7 +279,7 @@ void asyncReadEntriesOrWait(int numberOfEntriesToRead, ReadEntriesCallback callb
* max position can read
*/
void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallback callback, Object ctx,
- PositionImpl maxPosition);
+ Position maxPosition);
/**
* Asynchronously read entries from the ManagedLedger, up to the specified number and size.
@@ -298,7 +300,7 @@ void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallba
* predicate of read filter out
*/
default void asyncReadEntriesWithSkipOrWait(int maxEntries, ReadEntriesCallback callback, Object ctx,
- PositionImpl maxPosition, Predicate skipCondition) {
+ Position maxPosition, Predicate skipCondition) {
asyncReadEntriesOrWait(maxEntries, callback, ctx, maxPosition);
}
@@ -323,15 +325,15 @@ default void asyncReadEntriesWithSkipOrWait(int maxEntries, ReadEntriesCallback
* predicate of read filter out
*/
default void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallback callback,
- Object ctx, PositionImpl maxPosition,
- Predicate skipCondition) {
+ Object ctx, Position maxPosition,
+ Predicate skipCondition) {
asyncReadEntriesOrWait(maxEntries, maxSizeBytes, callback, ctx, maxPosition);
}
/**
* Cancel a previously scheduled asyncReadEntriesOrWait operation.
*
- * @see #asyncReadEntriesOrWait(int, ReadEntriesCallback, Object, PositionImpl)
+ * @see #asyncReadEntriesOrWait(int, ReadEntriesCallback, Object, Position)
* @return true if the read operation was canceled or false if there was no pending operation
*/
boolean cancelPendingReadRequest();
@@ -837,7 +839,7 @@ default void skipNonRecoverableLedger(long ledgerId){}
* Get last individual deleted range.
* @return range
*/
- Range getLastIndividualDeletedRange();
+ Range getLastIndividualDeletedRange();
/**
* Trim delete entries for the given entries.
@@ -847,7 +849,7 @@ default void skipNonRecoverableLedger(long ledgerId){}
/**
* Get deleted batch indexes list for a batch message.
*/
- long[] getDeletedBatchIndexesAsLongArray(PositionImpl position);
+ long[] getDeletedBatchIndexesAsLongArray(Position position);
/**
* @return the managed cursor stats MBean
@@ -870,4 +872,32 @@ default void skipNonRecoverableLedger(long ledgerId){}
default boolean isCursorDataFullyPersistable() {
return true;
}
+
+ /**
+ * Called by the system to trigger periodic rollover in absence of activity.
+ */
+ default boolean periodicRollover() {
+ return false;
+ }
+
+ /**
+ * Get the attributes associated with the cursor.
+ *
+ * @return the attributes associated with the cursor
+ */
+ default ManagedCursorAttributes getManagedCursorAttributes() {
+ return new ManagedCursorAttributes(this);
+ }
+
+ ManagedLedgerInternalStats.CursorStats getCursorStats();
+
+ boolean isMessageDeleted(Position position);
+
+ ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException;
+
+ long[] getBatchPositionAckSet(Position position);
+
+ int applyMaxSizeCap(int maxEntries, long maxSizeBytes);
+
+ void updateReadStats(int readEntriesCount, long readEntriesSize);
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorAttributes.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorAttributes.java
new file mode 100644
index 0000000000000..6c06e68d75e24
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedCursorAttributes.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger;
+
+import io.opentelemetry.api.common.Attributes;
+import lombok.Getter;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes;
+import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ManagedCursorOperationStatus;
+
+@Getter
+public class ManagedCursorAttributes {
+
+ private final Attributes attributes;
+ private final Attributes attributesOperationSucceed;
+ private final Attributes attributesOperationFailure;
+
+ public ManagedCursorAttributes(ManagedCursor cursor) {
+ var mlName = cursor.getManagedLedger().getName();
+ var topicName = TopicName.get(TopicName.fromPersistenceNamingEncoding(mlName));
+ attributes = Attributes.of(
+ OpenTelemetryAttributes.ML_CURSOR_NAME, cursor.getName(),
+ OpenTelemetryAttributes.ML_LEDGER_NAME, mlName,
+ OpenTelemetryAttributes.PULSAR_NAMESPACE, topicName.getNamespace()
+ );
+ attributesOperationSucceed = Attributes.builder()
+ .putAll(attributes)
+ .putAll(ManagedCursorOperationStatus.SUCCESS.attributes)
+ .build();
+ attributesOperationFailure = Attributes.builder()
+ .putAll(attributes)
+ .putAll(ManagedCursorOperationStatus.FAILURE.attributes)
+ .build();
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
index f91d9ec3f5a02..de69d97bb79aa 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedger.java
@@ -18,8 +18,10 @@
*/
package org.apache.bookkeeper.mledger;
+import com.google.common.collect.Range;
import io.netty.buffer.ByteBuf;
import java.util.Map;
+import java.util.NavigableMap;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.function.Predicate;
@@ -374,6 +376,8 @@ void asyncOpenCursor(String name, InitialPosition initialPosition, Map range);
+
/**
* Get the total number of active entries for this managed ledger.
*
@@ -631,6 +635,11 @@ void asyncSetProperties(Map properties, AsyncCallbacks.UpdatePro
*/
void trimConsumedLedgersInBackground(CompletableFuture promise);
+ /**
+ * Rollover cursors in background if needed.
+ */
+ default void rolloverCursorsInBackground() {}
+
/**
* If a ledger is lost, this ledger will be skipped after enabled "autoSkipNonRecoverableData", and the method is
* used to delete information about this ledger in the ManagedCursor.
@@ -691,4 +700,37 @@ default void skipNonRecoverableLedger(long ledgerId){}
* Check if managed ledger should cache backlog reads.
*/
void checkCursorsToCacheEntries();
+
+ /**
+ * Get managed ledger attributes.
+ */
+ default ManagedLedgerAttributes getManagedLedgerAttributes() {
+ return new ManagedLedgerAttributes(this);
+ }
+
+ void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx);
+
+ /**
+ * Get all the managed ledgers.
+ */
+ NavigableMap getLedgersInfo();
+
+ Position getNextValidPosition(Position position);
+
+ Position getPreviousPosition(Position position);
+
+ long getEstimatedBacklogSize(Position position);
+
+ Position getPositionAfterN(Position startPosition, long n, PositionBound startRange);
+
+ int getPendingAddEntriesCount();
+
+ long getCacheSize();
+
+ default CompletableFuture getLastDispatchablePosition(final Predicate predicate,
+ final Position startPosition) {
+ return CompletableFuture.completedFuture(PositionFactory.EARLIEST);
+ }
+
+ Position getFirstPosition();
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerAttributes.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerAttributes.java
new file mode 100644
index 0000000000000..c3759a533a571
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerAttributes.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger;
+
+import io.opentelemetry.api.common.Attributes;
+import lombok.Getter;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes;
+import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.ManagedLedgerOperationStatus;
+
+@Getter
+public class ManagedLedgerAttributes {
+
+ private final Attributes attributes;
+ private final Attributes attributesOperationSucceed;
+ private final Attributes attributesOperationFailure;
+
+ public ManagedLedgerAttributes(ManagedLedger ml) {
+ var mlName = ml.getName();
+ attributes = Attributes.of(
+ OpenTelemetryAttributes.ML_NAME, mlName,
+ OpenTelemetryAttributes.PULSAR_NAMESPACE, getNamespace(mlName)
+ );
+ attributesOperationSucceed = Attributes.builder()
+ .putAll(attributes)
+ .putAll(ManagedLedgerOperationStatus.SUCCESS.attributes)
+ .build();
+ attributesOperationFailure = Attributes.builder()
+ .putAll(attributes)
+ .putAll(ManagedLedgerOperationStatus.FAILURE.attributes)
+ .build();
+ }
+
+ private static String getNamespace(String mlName) {
+ try {
+ return TopicName.get(TopicName.fromPersistenceNamingEncoding(mlName)).getNamespace();
+ } catch (RuntimeException e) {
+ return null;
+ }
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
index 0c93a5b642cf6..a1e1deb503e20 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerConfig.java
@@ -33,7 +33,7 @@
import org.apache.bookkeeper.mledger.impl.NullLedgerOffloader;
import org.apache.bookkeeper.mledger.intercept.ManagedLedgerInterceptor;
import org.apache.commons.collections4.MapUtils;
-import org.apache.pulsar.common.util.collections.ConcurrentOpenLongPairRangeSet;
+import org.apache.pulsar.common.util.collections.OpenLongPairRangeSet;
/**
* Configuration class for a ManagedLedger.
@@ -64,6 +64,7 @@ public class ManagedLedgerConfig {
private long retentionTimeMs = 0;
private long retentionSizeInMB = 0;
private boolean autoSkipNonRecoverableData;
+ private boolean ledgerForceRecovery;
private boolean lazyCursorRecovery = false;
private long metadataOperationsTimeoutSeconds = 60;
private long readEntryTimeoutSeconds = 120;
@@ -85,7 +86,10 @@ public class ManagedLedgerConfig {
private int minimumBacklogCursorsForCaching = 0;
private int minimumBacklogEntriesForCaching = 1000;
private int maxBacklogBetweenCursorsForCaching = 1000;
-
+ private boolean triggerOffloadOnTopicLoad = false;
+ @Getter
+ @Setter
+ private String storageClassName;
@Getter
@Setter
private String shadowSourceName;
@@ -281,7 +285,7 @@ public ManagedLedgerConfig setPassword(String password) {
}
/**
- * should use {@link ConcurrentOpenLongPairRangeSet} to store unacked ranges.
+ * should use {@link OpenLongPairRangeSet} to store unacked ranges.
* @return
*/
public boolean isUnackedRangesOpenCacheSetEnabled() {
@@ -464,6 +468,17 @@ public void setAutoSkipNonRecoverableData(boolean skipNonRecoverableData) {
this.autoSkipNonRecoverableData = skipNonRecoverableData;
}
+ /**
+ * Skip managed ledger failure to recover managed ledger forcefully.
+ */
+ public boolean isLedgerForceRecovery() {
+ return ledgerForceRecovery;
+ }
+
+ public void setLedgerForceRecovery(boolean ledgerForceRecovery) {
+ this.ledgerForceRecovery = ledgerForceRecovery;
+ }
+
/**
* @return max unacked message ranges that will be persisted and recovered.
*
@@ -504,8 +519,10 @@ public int getMaxUnackedRangesToPersistInMetadataStore() {
return maxUnackedRangesToPersistInMetadataStore;
}
- public void setMaxUnackedRangesToPersistInMetadataStore(int maxUnackedRangesToPersistInMetadataStore) {
+ public ManagedLedgerConfig setMaxUnackedRangesToPersistInMetadataStore(
+ int maxUnackedRangesToPersistInMetadataStore) {
this.maxUnackedRangesToPersistInMetadataStore = maxUnackedRangesToPersistInMetadataStore;
+ return this;
}
/**
@@ -748,6 +765,22 @@ public void setMaxBacklogBetweenCursorsForCaching(int maxBacklogBetweenCursorsFo
this.maxBacklogBetweenCursorsForCaching = maxBacklogBetweenCursorsForCaching;
}
+ /**
+ * Trigger offload on topic load.
+ * @return
+ */
+ public boolean isTriggerOffloadOnTopicLoad() {
+ return triggerOffloadOnTopicLoad;
+ }
+
+ /**
+ * Set trigger offload on topic load.
+ * @param triggerOffloadOnTopicLoad
+ */
+ public void setTriggerOffloadOnTopicLoad(boolean triggerOffloadOnTopicLoad) {
+ this.triggerOffloadOnTopicLoad = triggerOffloadOnTopicLoad;
+ }
+
public String getShadowSource() {
return MapUtils.getString(properties, PROPERTY_SOURCE_TOPIC_KEY);
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java
index e09fd84ea55f2..d9c887fac468e 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactory.java
@@ -28,6 +28,8 @@
import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenReadOnlyCursorCallback;
import org.apache.bookkeeper.mledger.impl.cache.EntryCacheManager;
+import org.apache.pulsar.common.naming.TopicName;
+import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
/**
* A factory to open/create managed ledgers and delete them.
@@ -233,4 +235,14 @@ void asyncDelete(String name, CompletableFuture mlConfigFut
* @return properties of this managedLedger.
*/
CompletableFuture> getManagedLedgerPropertiesAsync(String name);
+
+ Map getManagedLedgers();
+
+ ManagedLedgerFactoryMXBean getCacheStats();
+
+
+ void estimateUnloadedTopicBacklog(PersistentOfflineTopicStats offlineTopicStats, TopicName topicName,
+ boolean accurate, Object ctx) throws Exception;
+
+ ManagedLedgerFactoryConfig getConfig();
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java
index 35c26c5dfdb89..43e8196daa9ae 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerFactoryMXBean.java
@@ -47,26 +47,51 @@ public interface ManagedLedgerFactoryMXBean {
*/
double getCacheHitsRate();
+ /**
+ * Cumulative number of cache hits.
+ */
+ long getCacheHitsTotal();
+
/**
* Get the number of cache misses per second.
*/
double getCacheMissesRate();
+ /**
+ * Cumulative number of cache misses.
+ */
+ long getCacheMissesTotal();
+
/**
* Get the amount of data is retrieved from the cache in byte/s.
*/
double getCacheHitsThroughput();
+ /**
+ * Cumulative amount of data retrieved from the cache in bytes.
+ */
+ long getCacheHitsBytesTotal();
+
/**
* Get the amount of data is retrieved from the bookkeeper in byte/s.
*/
double getCacheMissesThroughput();
+ /**
+ * Cumulative amount of data retrieved from the bookkeeper in bytes.
+ */
+ long getCacheMissesBytesTotal();
+
/**
* Get the number of cache evictions during the last minute.
*/
long getNumberOfCacheEvictions();
+ /**
+ * Cumulative number of cache evictions.
+ */
+ long getNumberOfCacheEvictionsTotal();
+
/**
* Cumulative number of entries inserted into the cache.
*/
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java
index cb6d3700afe3a..1d978e2378569 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ManagedLedgerMXBean.java
@@ -60,11 +60,21 @@ public interface ManagedLedgerMXBean {
*/
double getAddEntryBytesRate();
+ /**
+ * @return the total number of bytes written
+ */
+ long getAddEntryBytesTotal();
+
/**
* @return the bytes/s rate of messages added with replicas
*/
double getAddEntryWithReplicasBytesRate();
+ /**
+ * @return the total number of bytes written, including replicas
+ */
+ long getAddEntryWithReplicasBytesTotal();
+
/**
* @return the msg/s rate of messages read
*/
@@ -75,21 +85,41 @@ public interface ManagedLedgerMXBean {
*/
double getReadEntriesBytesRate();
+ /**
+ * @return the total number of bytes read
+ */
+ long getReadEntriesBytesTotal();
+
/**
* @return the rate of mark-delete ops/s
*/
double getMarkDeleteRate();
+ /**
+ * @return the number of mark-delete ops
+ */
+ long getMarkDeleteTotal();
+
/**
* @return the number of addEntry requests that succeeded
*/
long getAddEntrySucceed();
+ /**
+ * @return the total number of addEntry requests that succeeded
+ */
+ long getAddEntrySucceedTotal();
+
/**
* @return the number of addEntry requests that failed
*/
long getAddEntryErrors();
+ /**
+ * @return the total number of addEntry requests that failed
+ */
+ long getAddEntryErrorsTotal();
+
/**
* @return the number of entries read from the managed ledger (from cache or BK)
*/
@@ -100,16 +130,31 @@ public interface ManagedLedgerMXBean {
*/
long getReadEntriesSucceeded();
+ /**
+ * @return the total number of readEntries requests that succeeded
+ */
+ long getReadEntriesSucceededTotal();
+
/**
* @return the number of readEntries requests that failed
*/
long getReadEntriesErrors();
+ /**
+ * @return the total number of readEntries requests that failed
+ */
+ long getReadEntriesErrorsTotal();
+
/**
* @return the number of readEntries requests that cache miss Rate
*/
double getReadEntriesOpsCacheMissesRate();
+ /**
+ * @return the total number of readEntries requests that cache miss
+ */
+ long getReadEntriesOpsCacheMissesTotal();
+
// Entry size statistics
double getEntrySizeAverage();
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OpenTelemetryManagedLedgerCacheStats.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OpenTelemetryManagedLedgerCacheStats.java
new file mode 100644
index 0000000000000..13e7ed6ac6799
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/OpenTelemetryManagedLedgerCacheStats.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger;
+
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.metrics.BatchCallback;
+import io.opentelemetry.api.metrics.ObservableLongMeasurement;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl;
+import org.apache.bookkeeper.mledger.impl.cache.PooledByteBufAllocatorStats;
+import org.apache.bookkeeper.mledger.impl.cache.RangeEntryCacheImpl;
+import org.apache.pulsar.opentelemetry.Constants;
+import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.CacheEntryStatus;
+import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.CacheOperationStatus;
+import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.PoolArenaType;
+import org.apache.pulsar.opentelemetry.OpenTelemetryAttributes.PoolChunkAllocationType;
+
+public class OpenTelemetryManagedLedgerCacheStats implements AutoCloseable {
+
+ // Replaces pulsar_ml_count
+ public static final String MANAGED_LEDGER_COUNTER = "pulsar.broker.managed_ledger.count";
+ private final ObservableLongMeasurement managedLedgerCounter;
+
+ // Replaces pulsar_ml_cache_evictions
+ public static final String CACHE_EVICTION_OPERATION_COUNTER = "pulsar.broker.managed_ledger.cache.eviction.count";
+ private final ObservableLongMeasurement cacheEvictionOperationCounter;
+
+ // Replaces 'pulsar_ml_cache_entries',
+ // 'pulsar_ml_cache_inserted_entries_total',
+ // 'pulsar_ml_cache_evicted_entries_total'
+ public static final String CACHE_ENTRY_COUNTER = "pulsar.broker.managed_ledger.cache.entry.count";
+ private final ObservableLongMeasurement cacheEntryCounter;
+
+ // Replaces pulsar_ml_cache_used_size
+ public static final String CACHE_SIZE_COUNTER = "pulsar.broker.managed_ledger.cache.entry.size";
+ private final ObservableLongMeasurement cacheSizeCounter;
+
+ // Replaces pulsar_ml_cache_hits_rate, pulsar_ml_cache_misses_rate
+ public static final String CACHE_OPERATION_COUNTER = "pulsar.broker.managed_ledger.cache.operation.count";
+ private final ObservableLongMeasurement cacheOperationCounter;
+
+ // Replaces pulsar_ml_cache_hits_throughput, pulsar_ml_cache_misses_throughput
+ public static final String CACHE_OPERATION_BYTES_COUNTER = "pulsar.broker.managed_ledger.cache.operation.size";
+ private final ObservableLongMeasurement cacheOperationBytesCounter;
+
+ // Replaces 'pulsar_ml_cache_pool_active_allocations',
+ // 'pulsar_ml_cache_pool_active_allocations_huge',
+ // 'pulsar_ml_cache_pool_active_allocations_normal',
+ // 'pulsar_ml_cache_pool_active_allocations_small'
+ public static final String CACHE_POOL_ACTIVE_ALLOCATION_COUNTER =
+ "pulsar.broker.managed_ledger.cache.pool.allocation.active.count";
+ private final ObservableLongMeasurement cachePoolActiveAllocationCounter;
+
+ // Replaces ['pulsar_ml_cache_pool_allocated', 'pulsar_ml_cache_pool_used']
+ public static final String CACHE_POOL_ACTIVE_ALLOCATION_SIZE_COUNTER =
+ "pulsar.broker.managed_ledger.cache.pool.allocation.size";
+ private final ObservableLongMeasurement cachePoolActiveAllocationSizeCounter;
+
+ private final BatchCallback batchCallback;
+
+ public OpenTelemetryManagedLedgerCacheStats(OpenTelemetry openTelemetry, ManagedLedgerFactoryImpl factory) {
+ var meter = openTelemetry.getMeter(Constants.BROKER_INSTRUMENTATION_SCOPE_NAME);
+
+ managedLedgerCounter = meter
+ .upDownCounterBuilder(MANAGED_LEDGER_COUNTER)
+ .setUnit("{managed_ledger}")
+ .setDescription("The total number of managed ledgers.")
+ .buildObserver();
+
+ cacheEvictionOperationCounter = meter
+ .counterBuilder(CACHE_EVICTION_OPERATION_COUNTER)
+ .setUnit("{eviction}")
+ .setDescription("The total number of cache eviction operations.")
+ .buildObserver();
+
+ cacheEntryCounter = meter
+ .upDownCounterBuilder(CACHE_ENTRY_COUNTER)
+ .setUnit("{entry}")
+ .setDescription("The number of entries in the entry cache.")
+ .buildObserver();
+
+ cacheSizeCounter = meter
+ .upDownCounterBuilder(CACHE_SIZE_COUNTER)
+ .setUnit("{By}")
+ .setDescription("The byte amount of entries stored in the entry cache.")
+ .buildObserver();
+
+ cacheOperationCounter = meter
+ .counterBuilder(CACHE_OPERATION_COUNTER)
+ .setUnit("{entry}")
+ .setDescription("The number of cache operations.")
+ .buildObserver();
+
+ cacheOperationBytesCounter = meter
+ .counterBuilder(CACHE_OPERATION_BYTES_COUNTER)
+ .setUnit("{By}")
+ .setDescription("The byte amount of data retrieved from cache operations.")
+ .buildObserver();
+
+ cachePoolActiveAllocationCounter = meter
+ .upDownCounterBuilder(CACHE_POOL_ACTIVE_ALLOCATION_COUNTER)
+ .setUnit("{allocation}")
+ .setDescription("The number of currently active allocations in the direct arena.")
+ .buildObserver();
+
+ cachePoolActiveAllocationSizeCounter = meter
+ .upDownCounterBuilder(CACHE_POOL_ACTIVE_ALLOCATION_SIZE_COUNTER)
+ .setUnit("{By}")
+ .setDescription("The memory allocated in the direct arena.")
+ .buildObserver();
+
+
+ batchCallback = meter.batchCallback(() -> recordMetrics(factory),
+ managedLedgerCounter,
+ cacheEvictionOperationCounter,
+ cacheEntryCounter,
+ cacheSizeCounter,
+ cacheOperationCounter,
+ cacheOperationBytesCounter,
+ cachePoolActiveAllocationCounter,
+ cachePoolActiveAllocationSizeCounter);
+ }
+
+ @Override
+ public void close() {
+ batchCallback.close();
+ }
+
+ private void recordMetrics(ManagedLedgerFactoryImpl factory) {
+ var stats = factory.getCacheStats();
+
+ managedLedgerCounter.record(stats.getNumberOfManagedLedgers());
+ cacheEvictionOperationCounter.record(stats.getNumberOfCacheEvictionsTotal());
+
+ var entriesOut = stats.getCacheEvictedEntriesCount();
+ var entriesIn = stats.getCacheInsertedEntriesCount();
+ var entriesActive = entriesIn - entriesOut;
+ cacheEntryCounter.record(entriesActive, CacheEntryStatus.ACTIVE.attributes);
+ cacheEntryCounter.record(entriesIn, CacheEntryStatus.INSERTED.attributes);
+ cacheEntryCounter.record(entriesOut, CacheEntryStatus.EVICTED.attributes);
+ cacheSizeCounter.record(stats.getCacheUsedSize());
+
+ cacheOperationCounter.record(stats.getCacheHitsTotal(), CacheOperationStatus.HIT.attributes);
+ cacheOperationBytesCounter.record(stats.getCacheHitsBytesTotal(), CacheOperationStatus.HIT.attributes);
+ cacheOperationCounter.record(stats.getCacheMissesTotal(), CacheOperationStatus.MISS.attributes);
+ cacheOperationBytesCounter.record(stats.getCacheMissesBytesTotal(), CacheOperationStatus.MISS.attributes);
+
+ var allocatorStats = new PooledByteBufAllocatorStats(RangeEntryCacheImpl.ALLOCATOR);
+ cachePoolActiveAllocationCounter.record(allocatorStats.activeAllocationsSmall, PoolArenaType.SMALL.attributes);
+ cachePoolActiveAllocationCounter.record(allocatorStats.activeAllocationsNormal,
+ PoolArenaType.NORMAL.attributes);
+ cachePoolActiveAllocationCounter.record(allocatorStats.activeAllocationsHuge, PoolArenaType.HUGE.attributes);
+ cachePoolActiveAllocationSizeCounter.record(allocatorStats.totalAllocated,
+ PoolChunkAllocationType.ALLOCATED.attributes);
+ cachePoolActiveAllocationSizeCounter.record(allocatorStats.totalUsed, PoolChunkAllocationType.USED.attributes);
+ }
+}
\ No newline at end of file
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/Position.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/Position.java
index ac5810bbf01e7..d0d6d865c9558 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/Position.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/Position.java
@@ -18,6 +18,7 @@
*/
package org.apache.bookkeeper.mledger;
+import java.util.Optional;
import org.apache.bookkeeper.common.annotation.InterfaceAudience;
import org.apache.bookkeeper.common.annotation.InterfaceStability;
@@ -26,16 +27,108 @@
*/
@InterfaceAudience.LimitedPrivate
@InterfaceStability.Stable
-public interface Position {
+public interface Position extends Comparable {
+ /**
+ * Get the ledger id of the entry pointed by this position.
+ *
+ * @return the ledger id
+ */
+ long getLedgerId();
+
+ /**
+ * Get the entry id of the entry pointed by this position.
+ *
+ * @return the entry id
+ */
+ long getEntryId();
+
+ /**
+ * Compare this position with another position.
+ * The comparison is first based on the ledger id, and then on the entry id.
+ * This is implements the Comparable interface.
+ * @param that the other position to be compared.
+ * @return -1 if this position is less than the other, 0 if they are equal, 1 if this position is greater than
+ * the other.
+ */
+ default int compareTo(Position that) {
+ if (getLedgerId() != that.getLedgerId()) {
+ return Long.compare(getLedgerId(), that.getLedgerId());
+ }
+
+ return Long.compare(getEntryId(), that.getEntryId());
+ }
+
+ /**
+ * Compare this position with another position based on the ledger id and entry id.
+ * @param ledgerId the ledger id to compare
+ * @param entryId the entry id to compare
+ * @return -1 if this position is less than the other, 0 if they are equal, 1 if this position is greater than
+ * the other.
+ */
+ default int compareTo(long ledgerId, long entryId) {
+ if (getLedgerId() != ledgerId) {
+ return Long.compare(getLedgerId(), ledgerId);
+ }
+
+ return Long.compare(getEntryId(), entryId);
+ }
+
+ /**
+ * Calculate the hash code for the position based on ledgerId and entryId.
+ * This is used in Position implementations to implement the hashCode method.
+ * @return hash code
+ */
+ default int hashCodeForPosition() {
+ int result = Long.hashCode(getLedgerId());
+ result = 31 * result + Long.hashCode(getEntryId());
+ return result;
+ }
+
/**
* Get the position of the entry next to this one. The returned position might point to a non-existing, or not-yet
* existing entry
*
* @return the position of the next logical entry
*/
- Position getNext();
+ default Position getNext() {
+ if (getEntryId() < 0) {
+ return PositionFactory.create(getLedgerId(), 0);
+ } else {
+ return PositionFactory.create(getLedgerId(), getEntryId() + 1);
+ }
+ }
- long getLedgerId();
+ /**
+ * Position after moving entryNum messages,
+ * if entryNum < 1, then return the current position.
+ * */
+ default Position getPositionAfterEntries(int entryNum) {
+ if (entryNum < 1) {
+ return this;
+ }
+ if (getEntryId() < 0) {
+ return PositionFactory.create(getLedgerId(), entryNum - 1);
+ } else {
+ return PositionFactory.create(getLedgerId(), getEntryId() + entryNum);
+ }
+ }
- long getEntryId();
+ /**
+ * Check if the position implementation has an extension of the given class or interface.
+ *
+ * @param extensionClass the class of the extension
+ * @return true if the position has an extension of the given class, false otherwise
+ */
+ default boolean hasExtension(Class extensionClass) {
+ return getExtension(extensionClass).isPresent();
+ }
+
+ /**
+ * Get the extension instance of the given class or interface that is attached to this position.
+ * If the position does not have an extension of the given class, an empty optional is returned.
+ * @param extensionClass the class of the extension
+ */
+ default Optional getExtension(Class extensionClass) {
+ return Optional.empty();
+ }
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java
new file mode 100644
index 0000000000000..a7ab4a48a9b02
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionBound.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger;
+
+public enum PositionBound {
+ // define boundaries for position based seeks and searches
+ startIncluded, startExcluded
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionFactory.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionFactory.java
new file mode 100644
index 0000000000000..0b119844a6268
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/PositionFactory.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger;
+
+import org.apache.bookkeeper.mledger.impl.ImmutablePositionImpl;
+
+/**
+ * Factory for creating {@link Position} instances.
+ */
+public final class PositionFactory {
+ /**
+ * Earliest position.
+ */
+ public static final Position EARLIEST = create(-1, -1);
+ /**
+ * Latest position.
+ */
+ public static final Position LATEST = create(Long.MAX_VALUE, Long.MAX_VALUE);
+
+ private PositionFactory() {
+ }
+
+ /**
+ * Create a new position.
+ *
+ * @param ledgerId ledger id
+ * @param entryId entry id
+ * @return new position
+ */
+ public static Position create(long ledgerId, long entryId) {
+ return new ImmutablePositionImpl(ledgerId, entryId);
+ }
+
+ /**
+ * Create a new position.
+ *
+ * @param other other position
+ * @return new position
+ */
+ public static Position create(Position other) {
+ return new ImmutablePositionImpl(other);
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java
index 18d412f893152..016298cb108bb 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyCursor.java
@@ -24,7 +24,6 @@
import org.apache.bookkeeper.common.annotation.InterfaceAudience;
import org.apache.bookkeeper.common.annotation.InterfaceStability;
import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback;
-import org.apache.bookkeeper.mledger.impl.PositionImpl;
@InterfaceAudience.LimitedPrivate
@InterfaceStability.Stable
@@ -48,7 +47,7 @@ public interface ReadOnlyCursor {
* @see #readEntries(int)
*/
void asyncReadEntries(int numberOfEntriesToRead, ReadEntriesCallback callback,
- Object ctx, PositionImpl maxPosition);
+ Object ctx, Position maxPosition);
/**
* Asynchronously read entries from the ManagedLedger.
@@ -60,7 +59,7 @@ void asyncReadEntries(int numberOfEntriesToRead, ReadEntriesCallback callback,
* @param maxPosition max position can read
*/
void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback,
- Object ctx, PositionImpl maxPosition);
+ Object ctx, Position maxPosition);
/**
* Get the read position. This points to the next message to be read from the cursor.
@@ -116,7 +115,7 @@ Position findNewestMatching(ManagedCursor.FindPositionConstraint constraint, Pre
* @param range the range between two positions
* @return the number of entries in range
*/
- long getNumberOfEntries(Range range);
+ long getNumberOfEntries(Range range);
/**
* Close the cursor and releases the associated resources.
diff --git a/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/validators/MinNegativeOneValidator.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedger.java
similarity index 66%
rename from pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/validators/MinNegativeOneValidator.java
rename to managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedger.java
index 320e36812bfc2..91b8f92eb637e 100644
--- a/pulsar-cli-utils/src/main/java/org/apache/pulsar/cli/validators/MinNegativeOneValidator.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedger.java
@@ -16,15 +16,18 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pulsar.cli.validators;
+package org.apache.bookkeeper.mledger;
-import com.beust.jcommander.IValueValidator;
-import com.beust.jcommander.ParameterException;
-import org.apache.pulsar.cli.ValueValidationUtil;
+import java.util.Map;
+
+public interface ReadOnlyManagedLedger {
+
+ void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx);
+
+ long getNumberOfEntries();
+
+ ReadOnlyCursor createReadOnlyCursor(Position position);
+
+ Map getProperties();
-public class MinNegativeOneValidator implements IValueValidator {
- @Override
- public void validate(String name, Long value) throws ParameterException {
- ValueValidationUtil.minValueCheck(name, value, -1L);
- }
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedgerImplWrapper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedgerImplWrapper.java
new file mode 100644
index 0000000000000..5bc94c04beefd
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/ReadOnlyManagedLedgerImplWrapper.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger;
+
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.common.util.OrderedScheduler;
+import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl;
+import org.apache.bookkeeper.mledger.impl.MetaStore;
+import org.apache.bookkeeper.mledger.impl.ReadOnlyManagedLedgerImpl;
+
+public class ReadOnlyManagedLedgerImplWrapper implements ReadOnlyManagedLedger {
+
+ private final ReadOnlyManagedLedgerImpl readOnlyManagedLedger;
+
+ public ReadOnlyManagedLedgerImplWrapper(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, MetaStore store,
+ ManagedLedgerConfig config, OrderedScheduler scheduledExecutor,
+ String name) {
+ this.readOnlyManagedLedger =
+ new ReadOnlyManagedLedgerImpl(factory, bookKeeper, store, config, scheduledExecutor, name);
+ }
+
+ public CompletableFuture initialize() {
+ return readOnlyManagedLedger.initialize();
+ }
+
+ @Override
+ public void asyncReadEntry(Position position, AsyncCallbacks.ReadEntryCallback callback, Object ctx) {
+ readOnlyManagedLedger.asyncReadEntry(position, callback, ctx);
+ }
+
+ @Override
+ public long getNumberOfEntries() {
+ return readOnlyManagedLedger.getNumberOfEntries();
+ }
+
+ @Override
+ public ReadOnlyCursor createReadOnlyCursor(Position position) {
+ return readOnlyManagedLedger.createReadOnlyCursor(position);
+ }
+
+ @Override
+ public Map getProperties() {
+ return readOnlyManagedLedger.getProperties();
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetPositionImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetPositionImpl.java
new file mode 100644
index 0000000000000..22a99eb3607eb
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetPositionImpl.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import java.util.Optional;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
+
+/**
+ * Position implementation that includes the ack set.
+ * Use {@link AckSetStateUtil#createPositionWithAckSet(long, long, long[])} to create instances.
+ */
+public class AckSetPositionImpl implements Position, AckSetState {
+ private final Optional ackSetStateExtension = Optional.of(this);
+ protected final long ledgerId;
+ protected final long entryId;
+ protected volatile long[] ackSet;
+
+ public AckSetPositionImpl(long ledgerId, long entryId, long[] ackSet) {
+ this.ledgerId = ledgerId;
+ this.entryId = entryId;
+ this.ackSet = ackSet;
+ }
+
+ public long[] getAckSet() {
+ return ackSet;
+ }
+
+ public void setAckSet(long[] ackSet) {
+ this.ackSet = ackSet;
+ }
+
+ public long getLedgerId() {
+ return ledgerId;
+ }
+
+ public long getEntryId() {
+ return entryId;
+ }
+
+ @Override
+ public Position getNext() {
+ if (entryId < 0) {
+ return PositionFactory.create(ledgerId, 0);
+ } else {
+ return PositionFactory.create(ledgerId, entryId + 1);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return ledgerId + ":" + entryId + " (ackSet " + (ackSet == null ? "is null" :
+ "with long[] size of " + ackSet.length) + ")";
+ }
+
+ @Override
+ public int hashCode() {
+ return hashCodeForPosition();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof Position && compareTo((Position) obj) == 0;
+ }
+
+ @Override
+ public Optional getExtension(Class extensionClass) {
+ if (extensionClass == AckSetState.class) {
+ return (Optional) ackSetStateExtension;
+ }
+ return Position.super.getExtension(extensionClass);
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetState.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetState.java
new file mode 100644
index 0000000000000..363336e83113e
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetState.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+/**
+ * Interface to manage the ackSet state attached to a position.
+ * Helpers in {@link AckSetStateUtil} to create positions with
+ * ackSet state and to extract the state.
+ */
+public interface AckSetState {
+ /**
+ * Get the ackSet bitset information encoded as a long array.
+ * @return the ackSet
+ */
+ long[] getAckSet();
+
+ /**
+ * Set the ackSet bitset information as a long array.
+ * @param ackSet the ackSet
+ */
+ void setAckSet(long[] ackSet);
+
+ /**
+ * Check if the ackSet is set.
+ * @return true if the ackSet is set, false otherwise
+ */
+ default boolean hasAckSet() {
+ return getAckSet() != null;
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetStateUtil.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetStateUtil.java
new file mode 100644
index 0000000000000..11ab520b68e92
--- /dev/null
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/AckSetStateUtil.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.mledger.impl;
+
+import java.util.Optional;
+import lombok.experimental.UtilityClass;
+import org.apache.bookkeeper.mledger.Position;
+
+/**
+ * Utility class to manage the ackSet state attached to a position.
+ */
+@UtilityClass
+public class AckSetStateUtil {
+ /**
+ * Create a new position with the ackSet state.
+ *
+ * @param ledgerId ledger id
+ * @param entryId entry id
+ * @param ackSet ack set bitset information encoded as an array of longs
+ * @return new position
+ */
+ public static Position createPositionWithAckSet(long ledgerId, long entryId, long[] ackSet) {
+ return new AckSetPositionImpl(ledgerId, entryId, ackSet);
+ }
+
+ /**
+ * Get the AckSetState instance from the position if it exists.
+ * @param position position which possibly contains the AckSetState
+ */
+ public static Optional maybeGetAckSetState(Position position) {
+ return position.getExtension(AckSetState.class);
+ }
+
+ /**
+ * Get the ackSet bitset information encoded as a long array from the position if it exists.
+ * @param position position which possibly contains the AckSetState
+ * @return the ackSet or null if the position does not have the AckSetState, or it's not set
+ */
+ public static long[] getAckSetArrayOrNull(Position position) {
+ return maybeGetAckSetState(position).map(AckSetState::getAckSet).orElse(null);
+ }
+
+ /**
+ * Get the AckSetState instance from the position.
+ * @param position position which contains the AckSetState
+ * @return AckSetState instance
+ * @throws IllegalStateException if the position does not have AckSetState
+ */
+ public static AckSetState getAckSetState(Position position) {
+ return maybeGetAckSetState(position)
+ .orElseThrow(() ->
+ new IllegalStateException("Position does not have AckSetState. position=" + position));
+ }
+
+ /**
+ * Check if position contains the ackSet information and it is set.
+ * @param position position which possibly contains the AckSetState
+ * @return true if the ackSet is set, false otherwise
+ */
+ public static boolean hasAckSet(Position position) {
+ return maybeGetAckSetState(position).map(AckSetState::hasAckSet).orElse(false);
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java
index 803979313575a..e0e2b859794b5 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/EntryImpl.java
@@ -26,10 +26,13 @@
import io.netty.util.ReferenceCounted;
import org.apache.bookkeeper.client.api.LedgerEntry;
import org.apache.bookkeeper.mledger.Entry;
+import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
import org.apache.bookkeeper.mledger.util.AbstractCASReferenceCounted;
+import org.apache.bookkeeper.mledger.util.RangeCache;
public final class EntryImpl extends AbstractCASReferenceCounted implements Entry, Comparable,
- ReferenceCounted {
+ RangeCache.ValueWithKeyValidation {
private static final Recycler RECYCLER = new Recycler() {
@Override
@@ -42,7 +45,7 @@ protected EntryImpl newObject(Handle handle) {
private long timestamp;
private long ledgerId;
private long entryId;
- private PositionImpl position;
+ private Position position;
ByteBuf data;
private Runnable onDeallocate;
@@ -80,7 +83,7 @@ public static EntryImpl create(long ledgerId, long entryId, ByteBuf data) {
return entry;
}
- public static EntryImpl create(PositionImpl position, ByteBuf data) {
+ public static EntryImpl create(Position position, ByteBuf data) {
EntryImpl entry = RECYCLER.get();
entry.timestamp = System.nanoTime();
entry.ledgerId = position.getLedgerId();
@@ -151,9 +154,9 @@ public int getLength() {
}
@Override
- public PositionImpl getPosition() {
+ public Position getPosition() {
if (position == null) {
- position = PositionImpl.get(ledgerId, entryId);
+ position = PositionFactory.create(ledgerId, entryId);
}
return position;
}
@@ -205,4 +208,8 @@ protected void deallocate() {
recyclerHandle.recycle(this);
}
+ @Override
+ public boolean matchesKey(Position key) {
+ return key.compareTo(ledgerId, entryId) == 0;
+ }
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionImplRecyclable.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ImmutablePositionImpl.java
similarity index 50%
rename from managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionImplRecyclable.java
rename to managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ImmutablePositionImpl.java
index eb2b33e858d63..06245a6b5f33a 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/PositionImplRecyclable.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ImmutablePositionImpl.java
@@ -18,33 +18,45 @@
*/
package org.apache.bookkeeper.mledger.impl;
-import io.netty.util.Recycler;
-import io.netty.util.Recycler.Handle;
import org.apache.bookkeeper.mledger.Position;
-public class PositionImplRecyclable extends PositionImpl implements Position {
+public final class ImmutablePositionImpl implements Position {
+ private final long ledgerId;
+ private final long entryId;
- private final Handle recyclerHandle;
+ public ImmutablePositionImpl(long ledgerId, long entryId) {
+ this.ledgerId = ledgerId;
+ this.entryId = entryId;
+ }
- private static final Recycler RECYCLER = new Recycler() {
- @Override
- protected PositionImplRecyclable newObject(Recycler.Handle recyclerHandle) {
- return new PositionImplRecyclable(recyclerHandle);
- }
- };
+ public ImmutablePositionImpl(Position other) {
+ this.ledgerId = other.getLedgerId();
+ this.entryId = other.getEntryId();
+ }
- private PositionImplRecyclable(Handle recyclerHandle) {
- super(PositionImpl.EARLIEST);
- this.recyclerHandle = recyclerHandle;
+ public long getLedgerId() {
+ return ledgerId;
}
- public static PositionImplRecyclable create() {
- return RECYCLER.get();
+ public long getEntryId() {
+ return entryId;
}
- public void recycle() {
- ackSet = null;
- recyclerHandle.recycle(this);
+ /**
+ * String representation of virtual cursor - LedgerId:EntryId.
+ */
+ @Override
+ public String toString() {
+ return ledgerId + ":" + entryId;
}
-}
\ No newline at end of file
+ @Override
+ public int hashCode() {
+ return hashCodeForPosition();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof Position && compareTo((Position) obj) == 0;
+ }
+}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java
index 92f3d892b532d..ba901ece51c39 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainer.java
@@ -54,7 +54,7 @@ public class ManagedCursorContainer implements Iterable {
@Value
public static class CursorInfo {
ManagedCursor cursor;
- PositionImpl position;
+ Position position;
/**
* Cursor info's version.
@@ -67,10 +67,10 @@ public static class CursorInfo {
private static class Item {
final ManagedCursor cursor;
- PositionImpl position;
+ Position position;
int idx;
- Item(ManagedCursor cursor, PositionImpl position, int idx) {
+ Item(ManagedCursor cursor, Position position, int idx) {
this.cursor = cursor;
this.position = position;
this.idx = idx;
@@ -160,7 +160,7 @@ public ManagedCursorContainer() {}
public void add(ManagedCursor cursor, Position position) {
long stamp = rwLock.writeLock();
try {
- Item item = new Item(cursor, (PositionImpl) position, position != null ? heap.size() : -1);
+ Item item = new Item(cursor, position, position != null ? heap.size() : -1);
cursors.put(cursor.getName(), item);
if (position != null) {
heap.add(item);
@@ -229,7 +229,7 @@ public boolean removeCursor(String name) {
* @return a pair of positions, representing the previous slowest reader and the new slowest reader (after the
* update).
*/
- public Pair cursorUpdated(ManagedCursor cursor, Position newPosition) {
+ public Pair cursorUpdated(ManagedCursor cursor, Position newPosition) {
requireNonNull(cursor);
long stamp = rwLock.writeLock();
@@ -239,8 +239,8 @@ public Pair cursorUpdated(ManagedCursor cursor, Posi
return null;
}
- PositionImpl previousSlowestConsumer = heap.get(0).position;
- item.position = (PositionImpl) newPosition;
+ Position previousSlowestConsumer = heap.get(0).position;
+ item.position = newPosition;
version = DataVersion.getNextVersion(version);
if (heap.size() == 1) {
@@ -254,7 +254,7 @@ public Pair cursorUpdated(ManagedCursor cursor, Posi
} else {
siftUp(item);
}
- PositionImpl newSlowestConsumer = heap.get(0).position;
+ Position newSlowestConsumer = heap.get(0).position;
return Pair.of(previousSlowestConsumer, newSlowestConsumer);
} finally {
rwLock.unlockWrite(stamp);
@@ -266,7 +266,7 @@ public Pair cursorUpdated(ManagedCursor cursor, Posi
*
* @return the slowest reader position
*/
- public PositionImpl getSlowestReaderPosition() {
+ public Position getSlowestReaderPosition() {
long stamp = rwLock.readLock();
try {
return heap.isEmpty() ? null : heap.get(0).position;
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
index 8b13fc0f3424e..7c0d13108b1c4 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -59,6 +59,7 @@
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
import java.util.function.Predicate;
+import java.util.stream.Collectors;
import java.util.stream.LongStream;
import org.apache.bookkeeper.client.AsyncCallback.CloseCallback;
import org.apache.bookkeeper.client.AsyncCallback.OpenCallback;
@@ -77,6 +78,7 @@
import org.apache.bookkeeper.mledger.AsyncCallbacks.SkipEntriesCallback;
import org.apache.bookkeeper.mledger.Entry;
import org.apache.bookkeeper.mledger.ManagedCursor;
+import org.apache.bookkeeper.mledger.ManagedCursorAttributes;
import org.apache.bookkeeper.mledger.ManagedCursorMXBean;
import org.apache.bookkeeper.mledger.ManagedLedger;
import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
@@ -85,17 +87,23 @@
import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadException;
import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionBound;
+import org.apache.bookkeeper.mledger.PositionFactory;
import org.apache.bookkeeper.mledger.ScanOutcome;
-import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.PositionBound;
import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
import org.apache.bookkeeper.mledger.proto.MLDataFormats;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongListMap;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.MessageRange;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo;
+import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo.Builder;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.StringProperty;
+import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats;
+import org.apache.pulsar.common.util.DateFormatter;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.collections.BitSetRecyclable;
import org.apache.pulsar.common.util.collections.LongPairRangeSet;
@@ -119,30 +127,27 @@ public class ManagedCursorImpl implements ManagedCursor {
return 0;
};
protected final BookKeeper bookkeeper;
- protected final ManagedLedgerConfig config;
protected final ManagedLedgerImpl ledger;
private final String name;
- public static final String CURSOR_INTERNAL_PROPERTY_PREFIX = "#pulsar.internal.";
-
private volatile Map cursorProperties;
private final BookKeeper.DigestType digestType;
- protected volatile PositionImpl markDeletePosition;
+ protected volatile Position markDeletePosition;
// this position is have persistent mark delete position
- protected volatile PositionImpl persistentMarkDeletePosition;
- protected static final AtomicReferenceFieldUpdater
+ protected volatile Position persistentMarkDeletePosition;
+ protected static final AtomicReferenceFieldUpdater
INPROGRESS_MARKDELETE_PERSIST_POSITION_UPDATER =
- AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, PositionImpl.class,
+ AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, Position.class,
"inProgressMarkDeletePersistPosition");
- protected volatile PositionImpl inProgressMarkDeletePersistPosition;
+ protected volatile Position inProgressMarkDeletePersistPosition;
- protected static final AtomicReferenceFieldUpdater READ_POSITION_UPDATER =
- AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, PositionImpl.class, "readPosition");
- protected volatile PositionImpl readPosition;
+ protected static final AtomicReferenceFieldUpdater READ_POSITION_UPDATER =
+ AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, Position.class, "readPosition");
+ protected volatile Position readPosition;
// keeps sample of last read-position for validation and monitoring if read-position is not moving forward.
- protected volatile PositionImpl statsLastReadPosition;
+ protected volatile Position statsLastReadPosition;
protected static final AtomicReferenceFieldUpdater
LAST_MARK_DELETE_ENTRY_UPDATER = AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class,
@@ -177,6 +182,7 @@ public class ManagedCursorImpl implements ManagedCursor {
// Wether the current cursorLedger is read-only or writable
private boolean isCursorLedgerReadOnly = true;
+ private boolean ledgerForceRecovery;
// Stat of the cursor z-node
// NOTE: Don't update cursorLedgerStat alone,
@@ -184,23 +190,17 @@ public class ManagedCursorImpl implements ManagedCursor {
private volatile Stat cursorLedgerStat;
private volatile ManagedCursorInfo managedCursorInfo;
- private static final LongPairConsumer positionRangeConverter = PositionImpl::new;
+ private static final LongPairConsumer positionRangeConverter = PositionFactory::create;
- private static final RangeBoundConsumer positionRangeReverseConverter =
- (position) -> new LongPairRangeSet.LongPair(position.ledgerId, position.entryId);
+ private static final RangeBoundConsumer positionRangeReverseConverter =
+ (position) -> new LongPairRangeSet.LongPair(position.getLedgerId(), position.getEntryId());
- private static final LongPairConsumer recyclePositionRangeConverter = (key, value) -> {
- PositionImplRecyclable position = PositionImplRecyclable.create();
- position.ledgerId = key;
- position.entryId = value;
- position.ackSet = null;
- return position;
- };
- protected final RangeSetWrapper individualDeletedMessages;
+ private static final LongPairConsumer recyclePositionRangeConverter = PositionRecyclable::get;
+ protected final RangeSetWrapper individualDeletedMessages;
// Maintain the deletion status for batch messages
// (ledgerId, entryId) -> deletion indexes
- protected final ConcurrentSkipListMap batchDeletedIndexes;
+ protected final ConcurrentSkipListMap batchDeletedIndexes;
private final ReadWriteLock lock = new ReentrantReadWriteLock();
private RateLimiter markDeleteLimiter;
@@ -222,7 +222,7 @@ public class ManagedCursorImpl implements ManagedCursor {
private volatile boolean isActive = false;
class MarkDeleteEntry {
- final PositionImpl newPosition;
+ final Position newPosition;
final MarkDeleteCallback callback;
final Object ctx;
final Map properties;
@@ -232,7 +232,7 @@ class MarkDeleteEntry {
// group.
List callbackGroup;
- public MarkDeleteEntry(PositionImpl newPosition, Map properties,
+ public MarkDeleteEntry(Position newPosition, Map properties,
MarkDeleteCallback callback, Object ctx) {
this.newPosition = newPosition;
this.properties = properties;
@@ -292,6 +292,11 @@ public enum State {
protected final ManagedCursorMXBean mbean;
+ private volatile ManagedCursorAttributes managedCursorAttributes;
+ private static final AtomicReferenceFieldUpdater ATTRIBUTES_UPDATER =
+ AtomicReferenceFieldUpdater.newUpdater(ManagedCursorImpl.class, ManagedCursorAttributes.class,
+ "managedCursorAttributes");
+
@SuppressWarnings("checkstyle:javadoctype")
public interface VoidCallback {
void operationComplete();
@@ -299,36 +304,36 @@ public interface VoidCallback {
void operationFailed(ManagedLedgerException exception);
}
- ManagedCursorImpl(BookKeeper bookkeeper, ManagedLedgerConfig config, ManagedLedgerImpl ledger, String cursorName) {
+ ManagedCursorImpl(BookKeeper bookkeeper, ManagedLedgerImpl ledger, String cursorName) {
this.bookkeeper = bookkeeper;
this.cursorProperties = Collections.emptyMap();
- this.config = config;
this.ledger = ledger;
this.name = cursorName;
this.individualDeletedMessages = new RangeSetWrapper<>(positionRangeConverter,
positionRangeReverseConverter, this);
- if (config.isDeletionAtBatchIndexLevelEnabled()) {
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
this.batchDeletedIndexes = new ConcurrentSkipListMap<>();
} else {
this.batchDeletedIndexes = null;
}
- this.digestType = BookKeeper.DigestType.fromApiDigestType(config.getDigestType());
+ this.digestType = BookKeeper.DigestType.fromApiDigestType(getConfig().getDigestType());
STATE_UPDATER.set(this, State.Uninitialized);
PENDING_MARK_DELETED_SUBMITTED_COUNT_UPDATER.set(this, 0);
PENDING_READ_OPS_UPDATER.set(this, 0);
RESET_CURSOR_IN_PROGRESS_UPDATER.set(this, FALSE);
WAITING_READ_OP_UPDATER.set(this, null);
- this.clock = config.getClock();
+ this.clock = getConfig().getClock();
this.lastActive = this.clock.millis();
this.lastLedgerSwitchTimestamp = this.clock.millis();
- if (config.getThrottleMarkDelete() > 0.0) {
- markDeleteLimiter = RateLimiter.create(config.getThrottleMarkDelete());
+ if (getConfig().getThrottleMarkDelete() > 0.0) {
+ markDeleteLimiter = RateLimiter.create(getConfig().getThrottleMarkDelete());
} else {
// Disable mark-delete rate limiter
markDeleteLimiter = null;
}
this.mbean = new ManagedCursorMXBeanImpl(this);
+ this.ledgerForceRecovery = getConfig().isLedgerForceRecovery();
}
private void updateCursorLedgerStat(ManagedCursorInfo cursorInfo, Stat stat) {
@@ -343,7 +348,12 @@ public Map getProperties() {
@Override
public boolean isCursorDataFullyPersistable() {
- return individualDeletedMessages.size() <= config.getMaxUnackedRangesToPersist();
+ lock.readLock().lock();
+ try {
+ return individualDeletedMessages.size() <= getConfig().getMaxUnackedRangesToPersist();
+ } finally {
+ lock.readLock().unlock();
+ }
}
@Override
@@ -355,15 +365,19 @@ private CompletableFuture computeCursorProperties(
final Function, Map> updateFunction) {
CompletableFuture updateCursorPropertiesResult = new CompletableFuture<>();
- final Stat lastCursorLedgerStat = ManagedCursorImpl.this.cursorLedgerStat;
-
Map newProperties = updateFunction.apply(ManagedCursorImpl.this.cursorProperties);
+ if (!isDurable()) {
+ this.cursorProperties = Collections.unmodifiableMap(newProperties);
+ updateCursorPropertiesResult.complete(null);
+ return updateCursorPropertiesResult;
+ }
+
ManagedCursorInfo copy = ManagedCursorInfo
.newBuilder(ManagedCursorImpl.this.managedCursorInfo)
.clearCursorProperties()
.addAllCursorProperties(buildStringPropertiesMap(newProperties))
.build();
-
+ final Stat lastCursorLedgerStat = ManagedCursorImpl.this.cursorLedgerStat;
ledger.getStore().asyncUpdateCursorInfo(ledger.getName(),
name, copy, lastCursorLedgerStat, new MetaStoreCallback<>() {
@Override
@@ -473,9 +487,7 @@ void recover(final VoidCallback callback) {
ledger.getStore().asyncGetCursorInfo(ledger.getName(), name, new MetaStoreCallback() {
@Override
public void operationComplete(ManagedCursorInfo info, Stat stat) {
-
updateCursorLedgerStat(info, stat);
- lastActive = info.getLastActive() != 0 ? info.getLastActive() : lastActive;
if (log.isDebugEnabled()) {
log.debug("[{}] [{}] Recover cursor last active to [{}]", ledger.getName(), name, lastActive);
@@ -495,7 +507,7 @@ public void operationComplete(ManagedCursorInfo info, Stat stat) {
if (info.getCursorsLedgerId() == -1L) {
// There is no cursor ledger to read the last position from. It means the cursor has been properly
// closed and the last mark-delete position is stored in the ManagedCursorInfo itself.
- PositionImpl recoveredPosition = new PositionImpl(info.getMarkDeleteLedgerId(),
+ Position recoveredPosition = PositionFactory.create(info.getMarkDeleteLedgerId(),
info.getMarkDeleteEntryId());
if (info.getIndividualDeletedMessagesCount() > 0) {
recoverIndividualDeletedMessages(info.getIndividualDeletedMessagesList());
@@ -537,11 +549,11 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
if (log.isInfoEnabled()) {
log.info("[{}] Opened ledger {} for cursor {}. rc={}", ledger.getName(), ledgerId, name, rc);
}
- if (isBkErrorNotRecoverable(rc)) {
+ if (isBkErrorNotRecoverable(rc) || (rc != BKException.Code.OK && ledgerForceRecovery)) {
log.error("[{}] Error opening metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name,
BKException.getMessage(rc));
- // Rewind to oldest entry available
- initialize(getRollbackPosition(info), Collections.emptyMap(), Collections.emptyMap(), callback);
+ // Rewind to the oldest entry available
+ initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback);
return;
} else if (rc != BKException.Code.OK) {
log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", ledger.getName(), ledgerId, name,
@@ -565,10 +577,10 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
if (log.isDebugEnabled()) {
log.debug("[{}} readComplete rc={} entryId={}", ledger.getName(), rc1, lh1.getLastAddConfirmed());
}
- if (isBkErrorNotRecoverable(rc1)) {
+ if (isBkErrorNotRecoverable(rc1) || (rc1 != BKException.Code.OK && ledgerForceRecovery)) {
log.error("[{}] Error reading from metadata ledger {} for cursor {}: {}", ledger.getName(),
ledgerId, name, BKException.getMessage(rc1));
- // Rewind to oldest entry available
+ // Rewind to the oldest entry available
initialize(getRollbackPosition(info), Collections.emptyMap(), cursorProperties, callback);
return;
} else if (rc1 != BKException.Code.OK) {
@@ -599,11 +611,9 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
}
}
- PositionImpl position = new PositionImpl(positionInfo);
- if (positionInfo.getIndividualDeletedMessagesCount() > 0) {
- recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList());
- }
- if (config.isDeletionAtBatchIndexLevelEnabled()
+ Position position = PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId());
+ recoverIndividualDeletedMessages(positionInfo);
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled()
&& positionInfo.getBatchedEntryDeletionIndexInfoCount() > 0) {
recoverBatchDeletedIndexes(positionInfo.getBatchedEntryDeletionIndexInfoList());
}
@@ -612,7 +622,8 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
}, null);
};
try {
- bookkeeper.asyncOpenLedger(ledgerId, digestType, config.getPassword(), openCallback, null);
+ bookkeeper.asyncOpenLedger(ledgerId, digestType, getConfig().getPassword(), openCallback,
+ null);
} catch (Throwable t) {
log.error("[{}] Encountered error on opening cursor ledger {} for cursor {}",
ledger.getName(), ledgerId, name, t);
@@ -620,6 +631,45 @@ protected void recoverFromLedger(final ManagedCursorInfo info, final VoidCallbac
}
}
+ public void recoverIndividualDeletedMessages(PositionInfo positionInfo) {
+ if (positionInfo.getIndividualDeletedMessagesCount() > 0) {
+ recoverIndividualDeletedMessages(positionInfo.getIndividualDeletedMessagesList());
+ } else if (positionInfo.getIndividualDeletedMessageRangesCount() > 0) {
+ List rangeList = positionInfo.getIndividualDeletedMessageRangesList();
+ try {
+ Map rangeMap = rangeList.stream().collect(Collectors.toMap(LongListMap::getKey,
+ list -> list.getValuesList().stream().mapToLong(i -> i).toArray()));
+ individualDeletedMessages.build(rangeMap);
+ } catch (Exception e) {
+ log.warn("[{}]-{} Failed to recover individualDeletedMessages from serialized data", ledger.getName(),
+ name, e);
+ }
+ }
+ }
+
+ private List buildLongPropertiesMap(Map properties) {
+ if (properties.isEmpty()) {
+ return Collections.emptyList();
+ }
+ List longListMap = new ArrayList<>();
+ MutableInt serializedSize = new MutableInt();
+ properties.forEach((id, ranges) -> {
+ if (ranges == null || ranges.length <= 0) {
+ return;
+ }
+ org.apache.bookkeeper.mledger.proto.MLDataFormats.LongListMap.Builder lmBuilder = LongListMap.newBuilder()
+ .setKey(id);
+ for (long range : ranges) {
+ lmBuilder.addValues(range);
+ }
+ LongListMap lm = lmBuilder.build();
+ longListMap.add(lm);
+ serializedSize.add(lm.getSerializedSize());
+ });
+ individualDeletedMessagesSerializedSize = serializedSize.toInteger();
+ return longListMap;
+ }
+
private void recoverIndividualDeletedMessages(List individualDeletedMessagesList) {
lock.writeLock().lock();
try {
@@ -668,8 +718,10 @@ private void recoverBatchDeletedIndexes (
for (int i = 0; i < batchDeletedIndexInfo.getDeleteSetList().size(); i++) {
array[i] = batchDeletedIndexInfo.getDeleteSetList().get(i);
}
- this.batchDeletedIndexes.put(PositionImpl.get(batchDeletedIndexInfo.getPosition().getLedgerId(),
- batchDeletedIndexInfo.getPosition().getEntryId()), BitSetRecyclable.create().resetWords(array));
+ this.batchDeletedIndexes.put(
+ PositionFactory.create(batchDeletedIndexInfo.getPosition().getLedgerId(),
+ batchDeletedIndexInfo.getPosition().getEntryId()),
+ BitSetRecyclable.create().resetWords(array));
}
});
} finally {
@@ -677,7 +729,7 @@ private void recoverBatchDeletedIndexes (
}
}
- private void recoveredCursor(PositionImpl position, Map properties,
+ private void recoveredCursor(Position position, Map properties,
Map cursorProperties,
LedgerHandle recoveredFromCursorLedger) {
// if the position was at a ledger that didn't exist (since it will be deleted if it was previously empty),
@@ -688,7 +740,7 @@ private void recoveredCursor(PositionImpl position, Map properties
log.info("[{}] [{}] Couldn't find next next valid ledger for recovery {}", ledger.getName(), name,
position);
}
- position = nextExistingLedger != null ? PositionImpl.get(nextExistingLedger, -1) : position;
+ position = nextExistingLedger != null ? PositionFactory.create(nextExistingLedger, -1) : position;
}
if (position.compareTo(ledger.getLastPosition()) > 0) {
log.warn("[{}] [{}] Current position {} is ahead of last position {}", ledger.getName(), name, position,
@@ -710,7 +762,7 @@ private void recoveredCursor(PositionImpl position, Map properties
STATE_UPDATER.set(this, State.NoLedger);
}
- void initialize(PositionImpl position, Map properties, Map cursorProperties,
+ void initialize(Position position, Map properties, Map cursorProperties,
final VoidCallback callback) {
recoveredCursor(position, properties, cursorProperties, null);
if (log.isDebugEnabled()) {
@@ -756,7 +808,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
counter.countDown();
}
- }, null, PositionImpl.LATEST);
+ }, null, PositionFactory.LATEST);
counter.await();
@@ -769,19 +821,19 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
@Override
public void asyncReadEntries(final int numberOfEntriesToRead, final ReadEntriesCallback callback,
- final Object ctx, PositionImpl maxPosition) {
+ final Object ctx, Position maxPosition) {
asyncReadEntries(numberOfEntriesToRead, NO_MAX_SIZE_LIMIT, callback, ctx, maxPosition);
}
@Override
public void asyncReadEntries(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback,
- Object ctx, PositionImpl maxPosition) {
+ Object ctx, Position maxPosition) {
asyncReadEntriesWithSkip(numberOfEntriesToRead, maxSizeBytes, callback, ctx, maxPosition, null);
}
@Override
public void asyncReadEntriesWithSkip(int numberOfEntriesToRead, long maxSizeBytes, ReadEntriesCallback callback,
- Object ctx, PositionImpl maxPosition, Predicate skipCondition) {
+ Object ctx, Position maxPosition, Predicate skipCondition) {
checkArgument(numberOfEntriesToRead > 0);
if (isClosed()) {
callback.readEntriesFailed(new ManagedLedgerException
@@ -850,8 +902,8 @@ public void asyncGetNthEntry(int n, IndividualDeletedEntries deletedEntries, Rea
return;
}
- PositionImpl startPosition = ledger.getNextValidPosition(markDeletePosition);
- PositionImpl endPosition = ledger.getLastPosition();
+ Position startPosition = ledger.getNextValidPosition(markDeletePosition);
+ Position endPosition = ledger.getLastPosition();
if (startPosition.compareTo(endPosition) <= 0) {
long numOfEntries = getNumberOfEntries(Range.closed(startPosition, endPosition));
if (numOfEntries >= n) {
@@ -859,7 +911,7 @@ public void asyncGetNthEntry(int n, IndividualDeletedEntries deletedEntries, Rea
if (deletedEntries == IndividualDeletedEntries.Exclude) {
deletedMessages = getNumIndividualDeletedEntriesToSkip(n);
}
- PositionImpl positionAfterN = ledger.getPositionAfterN(markDeletePosition, n + deletedMessages,
+ Position positionAfterN = ledger.getPositionAfterN(markDeletePosition, n + deletedMessages,
PositionBound.startExcluded);
ledger.asyncReadEntry(positionAfterN, callback, ctx);
} else {
@@ -902,7 +954,7 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
counter.countDown();
}
- }, null, PositionImpl.LATEST);
+ }, null, PositionFactory.LATEST);
counter.await();
@@ -915,27 +967,27 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
@Override
public void asyncReadEntriesOrWait(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx,
- PositionImpl maxPosition) {
+ Position maxPosition) {
asyncReadEntriesOrWait(numberOfEntriesToRead, NO_MAX_SIZE_LIMIT, callback, ctx, maxPosition);
}
@Override
public void asyncReadEntriesOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallback callback, Object ctx,
- PositionImpl maxPosition) {
+ Position maxPosition) {
asyncReadEntriesWithSkipOrWait(maxEntries, maxSizeBytes, callback, ctx, maxPosition, null);
}
@Override
public void asyncReadEntriesWithSkipOrWait(int maxEntries, ReadEntriesCallback callback,
- Object ctx, PositionImpl maxPosition,
- Predicate skipCondition) {
+ Object ctx, Position maxPosition,
+ Predicate skipCondition) {
asyncReadEntriesWithSkipOrWait(maxEntries, NO_MAX_SIZE_LIMIT, callback, ctx, maxPosition, skipCondition);
}
@Override
public void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, ReadEntriesCallback callback,
- Object ctx, PositionImpl maxPosition,
- Predicate skipCondition) {
+ Object ctx, Position maxPosition,
+ Predicate skipCondition) {
checkArgument(maxEntries > 0);
if (isClosed()) {
callback.readEntriesFailed(new CursorAlreadyClosedException("Cursor was already closed"), ctx);
@@ -944,7 +996,7 @@ public void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, Re
int numberOfEntriesToRead = applyMaxSizeCap(maxEntries, maxSizeBytes);
- if (hasMoreEntries()) {
+ if (hasMoreEntries() && maxPosition.compareTo(readPosition) >= 0) {
// If we have available entries, we can read them immediately
if (log.isDebugEnabled()) {
log.debug("[{}] [{}] Read entries immediately", ledger.getName(), name);
@@ -969,10 +1021,10 @@ public void asyncReadEntriesWithSkipOrWait(int maxEntries, long maxSizeBytes, Re
// Check again for new entries after the configured time, then if still no entries are available register
// to be notified
- if (config.getNewEntriesCheckDelayInMillis() > 0) {
+ if (getConfig().getNewEntriesCheckDelayInMillis() > 0) {
ledger.getScheduledExecutor()
.schedule(() -> checkForNewEntries(op, callback, ctx),
- config.getNewEntriesCheckDelayInMillis(), TimeUnit.MILLISECONDS);
+ getConfig().getNewEntriesCheckDelayInMillis(), TimeUnit.MILLISECONDS);
} else {
// If there's no delay, check directly from the same thread
checkForNewEntries(op, callback, ctx);
@@ -986,13 +1038,18 @@ private void checkForNewEntries(OpReadEntry op, ReadEntriesCallback callback, Ob
log.debug("[{}] [{}] Re-trying the read at position {}", ledger.getName(), name, op.readPosition);
}
+ if (isClosed()) {
+ callback.readEntriesFailed(new CursorAlreadyClosedException("Cursor was already closed"), ctx);
+ return;
+ }
+
if (!hasMoreEntries()) {
if (log.isDebugEnabled()) {
log.debug("[{}] [{}] Still no entries available. Register for notification", ledger.getName(),
name);
}
// Let the managed ledger know we want to be notified whenever a new entry is published
- ledger.waitingCursors.add(this);
+ ledger.addWaitingCursor(this);
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] [{}] Skip notification registering since we do have entries available",
@@ -1060,7 +1117,7 @@ public boolean hasMoreEntries() {
// * Writer pointing to "invalid" entry -1 (meaning no entries in that ledger) --> Need to check if the reader
// is
// at the last entry in the previous ledger
- PositionImpl writerPosition = ledger.getLastPosition();
+ Position writerPosition = ledger.getLastPosition();
if (writerPosition.getEntryId() != -1) {
return readPosition.compareTo(writerPosition) <= 0;
} else {
@@ -1087,8 +1144,8 @@ public long getNumberOfEntries() {
public long getNumberOfEntriesSinceFirstNotAckedMessage() {
// sometimes for already caught up consumer: due to race condition markDeletePosition > readPosition. so,
// validate it before preparing range
- PositionImpl markDeletePosition = this.markDeletePosition;
- PositionImpl readPosition = this.readPosition;
+ Position markDeletePosition = this.markDeletePosition;
+ Position readPosition = this.readPosition;
return (markDeletePosition != null && readPosition != null && markDeletePosition.compareTo(readPosition) < 0)
? ledger.getNumberOfEntries(Range.openClosed(markDeletePosition, readPosition))
: 0;
@@ -1096,7 +1153,12 @@ public long getNumberOfEntriesSinceFirstNotAckedMessage() {
@Override
public int getTotalNonContiguousDeletedMessagesRange() {
- return individualDeletedMessages.size();
+ lock.readLock().lock();
+ try {
+ return individualDeletedMessages.size();
+ } finally {
+ lock.readLock().unlock();
+ }
}
@Override
@@ -1109,6 +1171,13 @@ public long getEstimatedSizeSinceMarkDeletePosition() {
return ledger.estimateBacklogFromPosition(markDeletePosition);
}
+ private long getNumberOfEntriesInBacklog() {
+ if (markDeletePosition.compareTo(ledger.getLastPosition()) >= 0) {
+ return 0;
+ }
+ return getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition()));
+ }
+
@Override
public long getNumberOfEntriesInBacklog(boolean isPrecise) {
if (log.isDebugEnabled()) {
@@ -1117,16 +1186,13 @@ public long getNumberOfEntriesInBacklog(boolean isPrecise) {
messagesConsumedCounter, markDeletePosition, readPosition);
}
if (isPrecise) {
- if (markDeletePosition.compareTo(ledger.getLastPosition()) >= 0) {
- return 0;
- }
- return getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition()));
+ return getNumberOfEntriesInBacklog();
}
long backlog = ManagedLedgerImpl.ENTRIES_ADDED_COUNTER_UPDATER.get(ledger) - messagesConsumedCounter;
if (backlog < 0) {
// In some case the counters get incorrect values, fall back to the precise backlog count
- backlog = getNumberOfEntries(Range.openClosed(markDeletePosition, ledger.getLastPosition()));
+ backlog = getNumberOfEntriesInBacklog();
}
return backlog;
@@ -1145,7 +1211,7 @@ public Position findNewestMatching(Predicate condition) throws Interrupte
public CompletableFuture scan(Optional position,
Predicate condition,
int batchSize, long maxEntries, long timeOutMs) {
- PositionImpl startPosition = (PositionImpl) position.orElseGet(
+ Position startPosition = position.orElseGet(
() -> ledger.getNextValidPosition(markDeletePosition));
CompletableFuture future = new CompletableFuture<>();
OpScan op = new OpScan(this, batchSize, startPosition, condition, new ScanCallback() {
@@ -1207,11 +1273,11 @@ public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate
public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate condition,
FindEntryCallback callback, Object ctx, boolean isFindFromLedger) {
OpFindNewest op;
- PositionImpl startPosition = null;
+ Position startPosition = null;
long max = 0;
switch (constraint) {
case SearchAllAvailableEntries:
- startPosition = (PositionImpl) getFirstPosition();
+ startPosition = getFirstPosition();
max = ledger.getNumberOfEntries() - 1;
break;
case SearchActiveEntries:
@@ -1265,15 +1331,15 @@ public void setAlwaysInactive() {
@Override
public Position getFirstPosition() {
Long firstLedgerId = ledger.getLedgersInfo().firstKey();
- return firstLedgerId == null ? null : new PositionImpl(firstLedgerId, 0);
+ return firstLedgerId == null ? null : PositionFactory.create(firstLedgerId, 0);
}
- protected void internalResetCursor(PositionImpl proposedReadPosition,
+ protected void internalResetCursor(Position proposedReadPosition,
AsyncCallbacks.ResetCursorCallback resetCursorCallback) {
- final PositionImpl newReadPosition;
- if (proposedReadPosition.equals(PositionImpl.EARLIEST)) {
+ final Position newReadPosition;
+ if (proposedReadPosition.equals(PositionFactory.EARLIEST)) {
newReadPosition = ledger.getFirstPosition();
- } else if (proposedReadPosition.equals(PositionImpl.LATEST)) {
+ } else if (proposedReadPosition.equals(PositionFactory.LATEST)) {
newReadPosition = ledger.getNextValidPosition(ledger.getLastPosition());
} else {
newReadPosition = proposedReadPosition;
@@ -1295,7 +1361,7 @@ protected void internalResetCursor(PositionImpl proposedReadPosition,
final AsyncCallbacks.ResetCursorCallback callback = resetCursorCallback;
- final PositionImpl newMarkDeletePosition = ledger.getPreviousPosition(newReadPosition);
+ final Position newMarkDeletePosition = ledger.getPreviousPosition(newReadPosition);
VoidCallback finalCallback = new VoidCallback() {
@Override
@@ -1315,17 +1381,19 @@ public void operationComplete() {
lastMarkDeleteEntry = new MarkDeleteEntry(newMarkDeletePosition, isCompactionCursor()
? getProperties() : Collections.emptyMap(), null, null);
individualDeletedMessages.clear();
- if (config.isDeletionAtBatchIndexLevelEnabled()) {
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
batchDeletedIndexes.values().forEach(BitSetRecyclable::recycle);
batchDeletedIndexes.clear();
- long[] resetWords = newReadPosition.ackSet;
- if (resetWords != null) {
- BitSetRecyclable ackSet = BitSetRecyclable.create().resetWords(resetWords);
- batchDeletedIndexes.put(newReadPosition, ackSet);
- }
+ AckSetStateUtil.maybeGetAckSetState(newReadPosition).ifPresent(ackSetState -> {
+ long[] resetWords = ackSetState.getAckSet();
+ if (resetWords != null) {
+ BitSetRecyclable ackSet = BitSetRecyclable.create().resetWords(resetWords);
+ batchDeletedIndexes.put(newReadPosition, ackSet);
+ }
+ });
}
- PositionImpl oldReadPosition = readPosition;
+ Position oldReadPosition = readPosition;
if (oldReadPosition.compareTo(newReadPosition) >= 0) {
log.info("[{}] reset readPosition to {} before current read readPosition {} on cursor {}",
ledger.getName(), newReadPosition, oldReadPosition, name);
@@ -1382,23 +1450,22 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
@Override
public void asyncResetCursor(Position newPos, boolean forceReset, AsyncCallbacks.ResetCursorCallback callback) {
- checkArgument(newPos instanceof PositionImpl);
- final PositionImpl newPosition = (PositionImpl) newPos;
+ final Position newPosition = newPos;
// order trim and reset operations on a ledger
ledger.getExecutor().execute(() -> {
- PositionImpl actualPosition = newPosition;
+ Position actualPosition = newPosition;
if (!ledger.isValidPosition(actualPosition)
- && !actualPosition.equals(PositionImpl.EARLIEST)
- && !actualPosition.equals(PositionImpl.LATEST)
+ && !actualPosition.equals(PositionFactory.EARLIEST)
+ && !actualPosition.equals(PositionFactory.LATEST)
&& !forceReset) {
actualPosition = ledger.getNextValidPosition(actualPosition);
if (actualPosition == null) {
// next valid position would only return null when newPos
// is larger than all available positions, then it's latest in effect.
- actualPosition = PositionImpl.LATEST;
+ actualPosition = PositionFactory.LATEST;
}
}
@@ -1552,18 +1619,18 @@ public String toString() {
positions.stream().filter(position -> !alreadyAcknowledgedPositions.contains(position))
.forEach(p ->{
- if (((PositionImpl) p).compareTo(this.readPosition) == 0) {
+ if (p.compareTo(this.readPosition) == 0) {
this.setReadPosition(this.readPosition.getNext());
log.warn("[{}][{}] replayPosition{} equals readPosition{}," + " need set next readPosition",
ledger.getName(), name, p, this.readPosition);
}
- ledger.asyncReadEntry((PositionImpl) p, cb, ctx);
+ ledger.asyncReadEntry(p, cb, ctx);
});
return alreadyAcknowledgedPositions;
}
- protected long getNumberOfEntries(Range range) {
+ protected long getNumberOfEntries(Range range) {
long allEntries = ledger.getNumberOfEntries(range);
if (log.isDebugEnabled()) {
@@ -1574,16 +1641,16 @@ protected long getNumberOfEntries(Range range) {
lock.readLock().lock();
try {
- if (config.isUnackedRangesOpenCacheSetEnabled()) {
+ if (getConfig().isUnackedRangesOpenCacheSetEnabled()) {
int cardinality = individualDeletedMessages.cardinality(
- range.lowerEndpoint().ledgerId, range.lowerEndpoint().entryId,
- range.upperEndpoint().ledgerId, range.upperEndpoint().entryId);
+ range.lowerEndpoint().getLedgerId(), range.lowerEndpoint().getEntryId(),
+ range.upperEndpoint().getLedgerId(), range.upperEndpoint().getEntryId());
deletedEntries.addAndGet(cardinality);
} else {
individualDeletedMessages.forEach((r) -> {
try {
if (r.isConnected(range)) {
- Range commonEntries = r.intersection(range);
+ Range commonEntries = r.intersection(range);
long commonCount = ledger.getNumberOfEntries(commonEntries);
if (log.isDebugEnabled()) {
log.debug("[{}] [{}] Discounting {} entries for already deleted range {}",
@@ -1593,9 +1660,9 @@ protected long getNumberOfEntries(Range range) {
}
return true;
} finally {
- if (r.lowerEndpoint() instanceof PositionImplRecyclable) {
- ((PositionImplRecyclable) r.lowerEndpoint()).recycle();
- ((PositionImplRecyclable) r.upperEndpoint()).recycle();
+ if (r.lowerEndpoint() instanceof PositionRecyclable) {
+ ((PositionRecyclable) r.lowerEndpoint()).recycle();
+ ((PositionRecyclable) r.upperEndpoint()).recycle();
}
}
}, recyclePositionRangeConverter);
@@ -1621,7 +1688,6 @@ public void markDelete(Position position) throws InterruptedException, ManagedLe
public void markDelete(Position position, Map properties)
throws InterruptedException, ManagedLedgerException {
requireNonNull(position);
- checkArgument(position instanceof PositionImpl);
class Result {
ManagedLedgerException exception = null;
@@ -1774,10 +1840,10 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
private static class InvidualDeletedMessagesHandlingState {
long totalEntriesToSkip = 0L;
long deletedMessages = 0L;
- PositionImpl startPosition;
- PositionImpl endPosition;
+ Position startPosition;
+ Position endPosition;
- InvidualDeletedMessagesHandlingState(PositionImpl startPosition) {
+ InvidualDeletedMessagesHandlingState(Position startPosition) {
this.startPosition = startPosition;
}
}
@@ -1790,7 +1856,7 @@ long getNumIndividualDeletedEntriesToSkip(long numEntries) {
try {
state.endPosition = r.lowerEndpoint();
if (state.startPosition.compareTo(state.endPosition) <= 0) {
- Range range = Range.openClosed(state.startPosition, state.endPosition);
+ Range range = Range.openClosed(state.startPosition, state.endPosition);
long entries = ledger.getNumberOfEntries(range);
if (state.totalEntriesToSkip + entries >= numEntries) {
// do not process further
@@ -1807,8 +1873,8 @@ long getNumIndividualDeletedEntriesToSkip(long numEntries) {
}
return true;
} finally {
- if (r.lowerEndpoint() instanceof PositionImplRecyclable) {
- ((PositionImplRecyclable) r.lowerEndpoint()).recycle();
+ if (r.lowerEndpoint() instanceof PositionRecyclable) {
+ ((PositionRecyclable) r.lowerEndpoint()).recycle();
}
}
}, recyclePositionRangeConverter);
@@ -1818,15 +1884,15 @@ long getNumIndividualDeletedEntriesToSkip(long numEntries) {
}
}
- boolean hasMoreEntries(PositionImpl position) {
- PositionImpl lastPositionInLedger = ledger.getLastPosition();
+ boolean hasMoreEntries(Position position) {
+ Position lastPositionInLedger = ledger.getLastPosition();
if (position.compareTo(lastPositionInLedger) <= 0) {
return getNumberOfEntries(Range.closed(position, lastPositionInLedger)) > 0;
}
return false;
}
- void initializeCursorPosition(Pair lastPositionCounter) {
+ void initializeCursorPosition(Pair lastPositionCounter) {
readPosition = ledger.getNextValidPosition(lastPositionCounter.getLeft());
ledger.onCursorReadPositionUpdated(this, readPosition);
markDeletePosition = lastPositionCounter.getLeft();
@@ -1845,14 +1911,14 @@ void initializeCursorPosition(Pair lastPositionCounter) {
* the new acknowledged position
* @return the previous acknowledged position
*/
- PositionImpl setAcknowledgedPosition(PositionImpl newMarkDeletePosition) {
+ Position setAcknowledgedPosition(Position newMarkDeletePosition) {
if (newMarkDeletePosition.compareTo(markDeletePosition) < 0) {
throw new MarkDeletingMarkedPosition(
"Mark deleting an already mark-deleted position. Current mark-delete: " + markDeletePosition
+ " -- attempted mark delete: " + newMarkDeletePosition);
}
- PositionImpl oldMarkDeletePosition = markDeletePosition;
+ Position oldMarkDeletePosition = markDeletePosition;
if (!newMarkDeletePosition.equals(oldMarkDeletePosition)) {
long skippedEntries = 0;
@@ -1865,14 +1931,14 @@ PositionImpl setAcknowledgedPosition(PositionImpl newMarkDeletePosition) {
skippedEntries = getNumberOfEntries(Range.openClosed(oldMarkDeletePosition, newMarkDeletePosition));
}
- PositionImpl positionAfterNewMarkDelete = ledger.getNextValidPosition(newMarkDeletePosition);
+ Position positionAfterNewMarkDelete = ledger.getNextValidPosition(newMarkDeletePosition);
// sometime ranges are connected but belongs to different ledgers so, they are placed sequentially
// eg: (2:10..3:15] can be returned as (2:10..2:15],[3:0..3:15]. So, try to iterate over connected range and
// found the last non-connected range which gives new markDeletePosition
while (positionAfterNewMarkDelete.compareTo(ledger.lastConfirmedEntry) <= 0) {
if (individualDeletedMessages.contains(positionAfterNewMarkDelete.getLedgerId(),
positionAfterNewMarkDelete.getEntryId())) {
- Range rangeToBeMarkDeleted = individualDeletedMessages.rangeContaining(
+ Range rangeToBeMarkDeleted = individualDeletedMessages.rangeContaining(
positionAfterNewMarkDelete.getLedgerId(), positionAfterNewMarkDelete.getEntryId());
newMarkDeletePosition = rangeToBeMarkDeleted.upperEndpoint();
positionAfterNewMarkDelete = ledger.getNextValidPosition(newMarkDeletePosition);
@@ -1898,7 +1964,7 @@ PositionImpl setAcknowledgedPosition(PositionImpl newMarkDeletePosition) {
// If the position that is mark-deleted is past the read position, it
// means that the client has skipped some entries. We need to move
// read position forward
- PositionImpl newReadPosition = ledger.getNextValidPosition(markDeletePosition);
+ Position newReadPosition = ledger.getNextValidPosition(markDeletePosition);
if (log.isDebugEnabled()) {
log.debug("[{}] Moved read position from: {} to: {}, and new mark-delete position {}",
ledger.getName(), currentReadPosition, newReadPosition, markDeletePosition);
@@ -1928,7 +1994,6 @@ public MarkDeletingMarkedPosition(String s) {
public void asyncMarkDelete(final Position position, Map properties,
final MarkDeleteCallback callback, final Object ctx) {
requireNonNull(position);
- checkArgument(position instanceof PositionImpl);
if (isClosed()) {
callback.markDeleteFailed(new ManagedLedgerException
@@ -1952,12 +2017,14 @@ public void asyncMarkDelete(final Position position, Map propertie
log.debug("[{}] Mark delete cursor {} up to position: {}", ledger.getName(), name, position);
}
- PositionImpl newPosition = (PositionImpl) position;
+ Position newPosition = position;
- if (config.isDeletionAtBatchIndexLevelEnabled()) {
- if (newPosition.ackSet != null) {
+ Optional ackSetStateOptional = AckSetStateUtil.maybeGetAckSetState(newPosition);
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
+ if (ackSetStateOptional.isPresent()) {
AtomicReference bitSetRecyclable = new AtomicReference<>();
- BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(newPosition.ackSet);
+ BitSetRecyclable givenBitSet =
+ BitSetRecyclable.create().resetWords(ackSetStateOptional.map(AckSetState::getAckSet).get());
// In order to prevent the batch index recorded in batchDeletedIndexes from rolling back,
// only update batchDeletedIndexes when the submitted batch index is greater
// than the recorded index.
@@ -1979,15 +2046,19 @@ public void asyncMarkDelete(final Position position, Map propertie
}
newPosition = ledger.getPreviousPosition(newPosition);
}
- Map subMap = batchDeletedIndexes.subMap(PositionImpl.EARLIEST, newPosition);
+ Map subMap = batchDeletedIndexes.subMap(PositionFactory.EARLIEST, newPosition);
subMap.values().forEach(BitSetRecyclable::recycle);
subMap.clear();
- } else if (newPosition.ackSet != null) {
- newPosition = ledger.getPreviousPosition(newPosition);
- newPosition.ackSet = null;
+ } else {
+ if (ackSetStateOptional.isPresent()) {
+ AckSetState ackSetState = ackSetStateOptional.get();
+ if (ackSetState.getAckSet() != null) {
+ newPosition = ledger.getPreviousPosition(newPosition);
+ }
+ }
}
- if (((PositionImpl) ledger.getLastConfirmedEntry()).compareTo(newPosition) < 0) {
+ if (ledger.getLastConfirmedEntry().compareTo(newPosition) < 0) {
boolean shouldCursorMoveForward = false;
try {
long ledgerEntries = ledger.getLedgerInfo(markDeletePosition.getLedgerId()).get().getEntries();
@@ -2032,7 +2103,7 @@ public void asyncMarkDelete(final Position position, Map propertie
internalAsyncMarkDelete(newPosition, properties, callback, ctx);
}
- protected void internalAsyncMarkDelete(final PositionImpl newPosition, Map properties,
+ protected void internalAsyncMarkDelete(final Position newPosition, Map properties,
final MarkDeleteCallback callback, final Object ctx) {
ledger.mbean.addMarkDeleteOp();
@@ -2087,7 +2158,7 @@ void internalMarkDelete(final MarkDeleteEntry mdEntry) {
return;
}
- PositionImpl inProgressLatest = INPROGRESS_MARKDELETE_PERSIST_POSITION_UPDATER.updateAndGet(this, current -> {
+ Position inProgressLatest = INPROGRESS_MARKDELETE_PERSIST_POSITION_UPDATER.updateAndGet(this, current -> {
if (current != null && current.compareTo(mdEntry.newPosition) > 0) {
return current;
} else {
@@ -2137,9 +2208,9 @@ public void operationComplete() {
try {
individualDeletedMessages.removeAtMost(mdEntry.newPosition.getLedgerId(),
mdEntry.newPosition.getEntryId());
- if (config.isDeletionAtBatchIndexLevelEnabled()) {
- Map subMap = batchDeletedIndexes.subMap(PositionImpl.EARLIEST,
- false, PositionImpl.get(mdEntry.newPosition.getLedgerId(),
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
+ Map subMap = batchDeletedIndexes.subMap(PositionFactory.EARLIEST,
+ false, PositionFactory.create(mdEntry.newPosition.getLedgerId(),
mdEntry.newPosition.getEntryId()), true);
subMap.values().forEach(BitSetRecyclable::recycle);
subMap.clear();
@@ -2178,8 +2249,7 @@ public void operationFailed(ManagedLedgerException exception) {
if (ledger.isNoMessagesAfterPos(mdEntry.newPosition)) {
persistPositionToMetaStore(mdEntry, cb);
} else {
- mdEntry.callback.markDeleteFailed(new ManagedLedgerException("Create new cursor ledger failed"),
- mdEntry.ctx);
+ cb.operationFailed(new ManagedLedgerException("Switch new cursor ledger failed"));
}
} else {
persistPositionToLedger(cursorLedger, mdEntry, cb);
@@ -2253,7 +2323,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
return;
}
- PositionImpl newMarkDeletePosition = null;
+ Position newMarkDeletePosition = null;
lock.writeLock().lock();
@@ -2264,8 +2334,8 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
}
for (Position pos : positions) {
- PositionImpl position = (PositionImpl) requireNonNull(pos);
- if (((PositionImpl) ledger.getLastConfirmedEntry()).compareTo(position) < 0) {
+ Position position = requireNonNull(pos);
+ if (ledger.getLastConfirmedEntry().compareTo(position) < 0) {
if (log.isDebugEnabled()) {
log.debug(
"[{}] Failed mark delete due to invalid markDelete {} is ahead of last-confirmed-entry {} "
@@ -2276,7 +2346,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
}
if (isMessageDeleted(position)) {
- if (config.isDeletionAtBatchIndexLevelEnabled()) {
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position);
if (bitSetRecyclable != null) {
bitSetRecyclable.recycle();
@@ -2287,8 +2357,9 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
}
continue;
}
- if (position.ackSet == null) {
- if (config.isDeletionAtBatchIndexLevelEnabled()) {
+ long[] ackSet = AckSetStateUtil.getAckSetArrayOrNull(position);
+ if (ackSet == null) {
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position);
if (bitSetRecyclable != null) {
bitSetRecyclable.recycle();
@@ -2296,7 +2367,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
}
// Add a range (prev, pos] to the set. Adding the previous entry as an open limit to the range will
// make the RangeSet recognize the "continuity" between adjacent Positions.
- PositionImpl previousPosition = ledger.getPreviousPosition(position);
+ Position previousPosition = ledger.getPreviousPosition(position);
individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(),
previousPosition.getEntryId(), position.getLedgerId(), position.getEntryId());
MSG_CONSUMED_COUNTER_UPDATER.incrementAndGet(this);
@@ -2305,15 +2376,15 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
log.debug("[{}] [{}] Individually deleted messages: {}", ledger.getName(), name,
individualDeletedMessages);
}
- } else if (config.isDeletionAtBatchIndexLevelEnabled()) {
- BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(position.ackSet);
+ } else if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
+ BitSetRecyclable givenBitSet = BitSetRecyclable.create().resetWords(ackSet);
BitSetRecyclable bitSet = batchDeletedIndexes.computeIfAbsent(position, (v) -> givenBitSet);
if (givenBitSet != bitSet) {
bitSet.and(givenBitSet);
givenBitSet.recycle();
}
if (bitSet.isEmpty()) {
- PositionImpl previousPosition = ledger.getPreviousPosition(position);
+ Position previousPosition = ledger.getPreviousPosition(position);
individualDeletedMessages.addOpenClosed(previousPosition.getLedgerId(),
previousPosition.getEntryId(),
position.getLedgerId(), position.getEntryId());
@@ -2333,7 +2404,7 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
// If the lower bound of the range set is the current mark delete position, then we can trigger a new
// mark-delete to the upper bound of the first range segment
- Range range = individualDeletedMessages.firstRange();
+ Range range = individualDeletedMessages.firstRange();
// If the upper bound is before the mark-delete position, we need to move ahead as these
// individualDeletedMessages are now irrelevant
@@ -2371,8 +2442,9 @@ public void asyncDelete(Iterable positions, AsyncCallbacks.DeleteCallb
callback.deleteFailed(getManagedLedgerException(e), ctx);
return;
} finally {
+ boolean empty = individualDeletedMessages.isEmpty();
lock.writeLock().unlock();
- if (individualDeletedMessages.isEmpty()) {
+ if (empty) {
callback.deleteComplete(ctx);
}
}
@@ -2413,7 +2485,7 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
}
// update lastMarkDeleteEntry field if newPosition is later than the current lastMarkDeleteEntry.newPosition
- private void updateLastMarkDeleteEntryToLatest(final PositionImpl newPosition,
+ private void updateLastMarkDeleteEntryToLatest(final Position newPosition,
final Map properties) {
LAST_MARK_DELETE_ENTRY_UPDATER.updateAndGet(this, last -> {
if (last != null && last.newPosition.compareTo(newPosition) > 0) {
@@ -2438,13 +2510,13 @@ private void updateLastMarkDeleteEntryToLatest(final PositionImpl newPosition,
List filterReadEntries(List entries) {
lock.readLock().lock();
try {
- Range entriesRange = Range.closed((PositionImpl) entries.get(0).getPosition(),
- (PositionImpl) entries.get(entries.size() - 1).getPosition());
+ Range entriesRange = Range.closed(entries.get(0).getPosition(),
+ entries.get(entries.size() - 1).getPosition());
if (log.isDebugEnabled()) {
log.debug("[{}] [{}] Filtering entries {} - alreadyDeleted: {}", ledger.getName(), name, entriesRange,
individualDeletedMessages);
}
- Range span = individualDeletedMessages.isEmpty() ? null : individualDeletedMessages.span();
+ Range span = individualDeletedMessages.isEmpty() ? null : individualDeletedMessages.span();
if (span == null || !entriesRange.isConnected(span)) {
// There are no individually deleted messages in this entry list, no need to perform filtering
if (log.isDebugEnabled()) {
@@ -2525,9 +2597,9 @@ public void rewind() {
public void rewind(boolean readCompacted) {
lock.writeLock().lock();
try {
- PositionImpl newReadPosition =
+ Position newReadPosition =
readCompacted ? markDeletePosition.getNext() : ledger.getNextValidPosition(markDeletePosition);
- PositionImpl oldReadPosition = readPosition;
+ Position oldReadPosition = readPosition;
log.info("[{}-{}] Rewind from {} to {}", ledger.getName(), name, oldReadPosition, newReadPosition);
@@ -2540,8 +2612,7 @@ public void rewind(boolean readCompacted) {
@Override
public void seek(Position newReadPositionInt, boolean force) {
- checkArgument(newReadPositionInt instanceof PositionImpl);
- PositionImpl newReadPosition = (PositionImpl) newReadPositionInt;
+ Position newReadPosition = newReadPositionInt;
lock.writeLock().lock();
try {
@@ -2609,7 +2680,7 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) {
* @param callback
* @param ctx
*/
- void persistPositionWhenClosing(PositionImpl position, Map properties,
+ void persistPositionWhenClosing(Position position, Map properties,
final AsyncCallbacks.CloseCallback callback, final Object ctx) {
if (shouldPersistUnackRangesToLedger()) {
@@ -2650,13 +2721,18 @@ public void operationFailed(MetaStoreException e) {
}
private boolean shouldPersistUnackRangesToLedger() {
- return cursorLedger != null
- && !isCursorLedgerReadOnly
- && config.getMaxUnackedRangesToPersist() > 0
- && individualDeletedMessages.size() > config.getMaxUnackedRangesToPersistInMetadataStore();
+ lock.readLock().lock();
+ try {
+ return cursorLedger != null
+ && !isCursorLedgerReadOnly
+ && getConfig().getMaxUnackedRangesToPersist() > 0
+ && individualDeletedMessages.size() > getConfig().getMaxUnackedRangesToPersistInMetadataStore();
+ } finally {
+ lock.readLock().unlock();
+ }
}
- private void persistPositionMetaStore(long cursorsLedgerId, PositionImpl position, Map properties,
+ private void persistPositionMetaStore(long cursorsLedgerId, Position position, Map properties,
MetaStoreCallback callback, boolean persistIndividualDeletedMessageRanges) {
if (state == State.Closed) {
ledger.getExecutor().execute(() -> callback.operationFailed(new MetaStoreException(
@@ -2678,7 +2754,7 @@ private void persistPositionMetaStore(long cursorsLedgerId, PositionImpl positio
info.addAllCursorProperties(buildStringPropertiesMap(cursorProperties));
if (persistIndividualDeletedMessageRanges) {
info.addAllIndividualDeletedMessages(buildIndividualDeletedMessageRanges());
- if (config.isDeletionAtBatchIndexLevelEnabled()) {
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
info.addAllBatchedEntryDeletionIndexInfo(buildBatchEntryDeletionIndexInfoList());
}
}
@@ -2774,10 +2850,9 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) {
* @param newReadPositionInt
*/
void setReadPosition(Position newReadPositionInt) {
- checkArgument(newReadPositionInt instanceof PositionImpl);
if (this.markDeletePosition == null
- || ((PositionImpl) newReadPositionInt).compareTo(this.markDeletePosition) > 0) {
- this.readPosition = (PositionImpl) newReadPositionInt;
+ || newReadPositionInt.compareTo(this.markDeletePosition) > 0) {
+ this.readPosition = newReadPositionInt;
ledger.onCursorReadPositionUpdated(this, newReadPositionInt);
}
}
@@ -2799,7 +2874,7 @@ public void skipNonRecoverableLedger(final long ledgerId){
log.warn("[{}] [{}] Since the ledger [{}] is lost and the autoSkipNonRecoverableData is true, this ledger will"
+ " be auto acknowledge in subscription", ledger.getName(), name, ledgerId);
asyncDelete(() -> LongStream.range(0, ledgerInfo.getEntries())
- .mapToObj(i -> (Position) PositionImpl.get(ledgerId, i)).iterator(),
+ .mapToObj(i -> (Position) PositionFactory.create(ledgerId, i)).iterator(),
new AsyncCallbacks.DeleteCallback() {
@Override
public void deleteComplete(Object ctx) {
@@ -2852,9 +2927,19 @@ public void operationFailed(ManagedLedgerException exception) {
synchronized (pendingMarkDeleteOps) {
// At this point we don't have a ledger ready
STATE_UPDATER.set(ManagedCursorImpl.this, State.NoLedger);
- // Note: if the stat is NoLedger, will persist the mark deleted position to metadata store.
- // Before giving up, try to persist the position in the metadata store.
- flushPendingMarkDeletes();
+ // There are two case may cause switch ledger fails.
+ // 1. No enough BKs; BKs are in read-only mode...
+ // 2. Write ZK fails.
+ // Regarding the case "No enough BKs", try to persist the position in the metadata store before
+ // giving up.
+ if (!(exception instanceof MetaStoreException)) {
+ flushPendingMarkDeletes();
+ } else {
+ while (!pendingMarkDeleteOps.isEmpty()) {
+ MarkDeleteEntry entry = pendingMarkDeleteOps.poll();
+ entry.callback.markDeleteFailed(exception, entry.ctx);
+ }
+ }
}
}
});
@@ -2933,7 +3018,7 @@ public void operationFailed(ManagedLedgerException exception) {
private CompletableFuture doCreateNewMetadataLedger() {
CompletableFuture future = new CompletableFuture<>();
- ledger.asyncCreateLedger(bookkeeper, config, digestType, (rc, lh, ctx) -> {
+ ledger.asyncCreateLedger(bookkeeper, getConfig(), digestType, (rc, lh, ctx) -> {
if (ledger.checkAndCompleteLedgerOpTask(rc, lh, ctx)) {
future.complete(null);
@@ -3003,7 +3088,7 @@ private static List buildStringPropertiesMap(Map
}
private List buildIndividualDeletedMessageRanges() {
- lock.readLock().lock();
+ lock.writeLock().lock();
try {
if (individualDeletedMessages.isEmpty()) {
this.individualDeletedMessagesSerializedSize = 0;
@@ -3038,21 +3123,21 @@ private List buildIndividualDeletedMessageRanges() {
acksSerializedSize.addAndGet(messageRange.getSerializedSize());
rangeList.add(messageRange);
- return rangeList.size() <= config.getMaxUnackedRangesToPersist();
+ return rangeList.size() <= getConfig().getMaxUnackedRangesToPersist();
});
this.individualDeletedMessagesSerializedSize = acksSerializedSize.get();
individualDeletedMessages.resetDirtyKeys();
return rangeList;
} finally {
- lock.readLock().unlock();
+ lock.writeLock().unlock();
}
}
private List buildBatchEntryDeletionIndexInfoList() {
lock.readLock().lock();
try {
- if (!config.isDeletionAtBatchIndexLevelEnabled() || batchDeletedIndexes.isEmpty()) {
+ if (!getConfig().isDeletionAtBatchIndexLevelEnabled() || batchDeletedIndexes.isEmpty()) {
return Collections.emptyList();
}
MLDataFormats.NestedPositionInfo.Builder nestedPositionBuilder = MLDataFormats.NestedPositionInfo
@@ -3060,9 +3145,9 @@ private List buildBatchEntryDeletio
MLDataFormats.BatchedEntryDeletionIndexInfo.Builder batchDeletedIndexInfoBuilder = MLDataFormats
.BatchedEntryDeletionIndexInfo.newBuilder();
List result = new ArrayList<>();
- Iterator> iterator = batchDeletedIndexes.entrySet().iterator();
- while (iterator.hasNext() && result.size() < config.getMaxBatchDeletedIndexToPersist()) {
- Map.Entry entry = iterator.next();
+ Iterator> iterator = batchDeletedIndexes.entrySet().iterator();
+ while (iterator.hasNext() && result.size() < getConfig().getMaxBatchDeletedIndexToPersist()) {
+ Map.Entry entry = iterator.next();
nestedPositionBuilder.setLedgerId(entry.getKey().getLedgerId());
nestedPositionBuilder.setEntryId(entry.getKey().getEntryId());
batchDeletedIndexInfoBuilder.setPosition(nestedPositionBuilder.build());
@@ -3082,13 +3167,24 @@ private List buildBatchEntryDeletio
}
void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, final VoidCallback callback) {
- PositionImpl position = mdEntry.newPosition;
- PositionInfo pi = PositionInfo.newBuilder().setLedgerId(position.getLedgerId())
+ Position position = mdEntry.newPosition;
+ Builder piBuilder = PositionInfo.newBuilder().setLedgerId(position.getLedgerId())
.setEntryId(position.getEntryId())
- .addAllIndividualDeletedMessages(buildIndividualDeletedMessageRanges())
.addAllBatchedEntryDeletionIndexInfo(buildBatchEntryDeletionIndexInfoList())
- .addAllProperties(buildPropertiesMap(mdEntry.properties)).build();
+ .addAllProperties(buildPropertiesMap(mdEntry.properties));
+ Map internalRanges = null;
+ try {
+ internalRanges = individualDeletedMessages.toRanges(getConfig().getMaxUnackedRangesToPersist());
+ } catch (Exception e) {
+ log.warn("[{}]-{} Failed to serialize individualDeletedMessages", ledger.getName(), name, e);
+ }
+ if (internalRanges != null && !internalRanges.isEmpty()) {
+ piBuilder.addAllIndividualDeletedMessageRanges(buildLongPropertiesMap(internalRanges));
+ } else {
+ piBuilder.addAllIndividualDeletedMessages(buildIndividualDeletedMessageRanges());
+ }
+ PositionInfo pi = piBuilder.build();
if (log.isDebugEnabled()) {
log.debug("[{}] Cursor {} Appending to ledger={} position={}", ledger.getName(), name, lh.getId(),
@@ -3104,12 +3200,7 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin
lh1.getId());
}
- if (shouldCloseLedger(lh1)) {
- if (log.isDebugEnabled()) {
- log.debug("[{}] Need to create new metadata ledger for cursor {}", ledger.getName(), name);
- }
- startCreatingNewMetadataLedger();
- }
+ rolloverLedgerIfNeeded(lh1);
mbean.persistToLedger(true);
mbean.addWriteCursorLedgerSize(data.length);
@@ -3127,8 +3218,37 @@ void persistPositionToLedger(final LedgerHandle lh, MarkDeleteEntry mdEntry, fin
}, null);
}
+ public boolean periodicRollover() {
+ LedgerHandle lh = cursorLedger;
+ if (State.Open.equals(STATE_UPDATER.get(this))
+ && lh != null && lh.getLength() > 0) {
+ boolean triggered = rolloverLedgerIfNeeded(lh);
+ if (triggered) {
+ log.info("[{}] Periodic rollover triggered for cursor {} (length={} bytes)",
+ ledger.getName(), name, lh.getLength());
+ } else {
+ log.debug("[{}] Periodic rollover skipped for cursor {} (length={} bytes)",
+ ledger.getName(), name, lh.getLength());
+
+ }
+ return triggered;
+ }
+ return false;
+ }
+
+ boolean rolloverLedgerIfNeeded(LedgerHandle lh1) {
+ if (shouldCloseLedger(lh1)) {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Need to create new metadata ledger for cursor {}", ledger.getName(), name);
+ }
+ startCreatingNewMetadataLedger();
+ return true;
+ }
+ return false;
+ }
+
void persistPositionToMetaStore(MarkDeleteEntry mdEntry, final VoidCallback callback) {
- final PositionImpl newPosition = mdEntry.newPosition;
+ final Position newPosition = mdEntry.newPosition;
STATE_UPDATER.compareAndSet(ManagedCursorImpl.this, State.Open, State.NoLedger);
mbean.persistToLedger(false);
// Before giving up, try to persist the position in the metadata store
@@ -3157,8 +3277,8 @@ public void operationFailed(MetaStoreException e) {
boolean shouldCloseLedger(LedgerHandle lh) {
long now = clock.millis();
if (ledger.getFactory().isMetadataServiceAvailable()
- && (lh.getLastAddConfirmed() >= config.getMetadataMaxEntriesPerLedger()
- || lastLedgerSwitchTimestamp < (now - config.getLedgerRolloverTimeout() * 1000))
+ && (lh.getLastAddConfirmed() >= getConfig().getMetadataMaxEntriesPerLedger()
+ || lastLedgerSwitchTimestamp < (now - getConfig().getLedgerRolloverTimeout() * 1000))
&& (STATE_UPDATER.get(this) != State.Closed && STATE_UPDATER.get(this) != State.Closing)) {
// It's safe to modify the timestamp since this method will be only called from a callback, implying that
// calls will be serialized on one single thread
@@ -3218,7 +3338,7 @@ void notifyEntriesAvailable() {
}
PENDING_READ_OPS_UPDATER.incrementAndGet(this);
- opReadEntry.readPosition = (PositionImpl) getReadPosition();
+ opReadEntry.readPosition = getReadPosition();
ledger.asyncReadEntries(opReadEntry);
} else {
// No one is waiting to be notified. Ignore
@@ -3358,9 +3478,10 @@ public static boolean isBkErrorNotRecoverable(int rc) {
*
* @param info
*/
- private PositionImpl getRollbackPosition(ManagedCursorInfo info) {
- PositionImpl firstPosition = ledger.getFirstPosition();
- PositionImpl snapshottedPosition = new PositionImpl(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId());
+ private Position getRollbackPosition(ManagedCursorInfo info) {
+ Position firstPosition = ledger.getFirstPosition();
+ Position snapshottedPosition =
+ PositionFactory.create(info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId());
if (firstPosition == null) {
// There are no ledgers in the ML, any position is good
return snapshottedPosition;
@@ -3401,22 +3522,37 @@ public String getIndividuallyDeletedMessages() {
}
@VisibleForTesting
- public LongPairRangeSet getIndividuallyDeletedMessagesSet() {
+ public LongPairRangeSet getIndividuallyDeletedMessagesSet() {
return individualDeletedMessages;
}
+ public Position processIndividuallyDeletedMessagesAndGetMarkDeletedPosition(
+ LongPairRangeSet.RangeProcessor processor) {
+ final Position mdp;
+ lock.readLock().lock();
+ try {
+ mdp = markDeletePosition;
+ individualDeletedMessages.forEach(processor);
+ } finally {
+ lock.readLock().unlock();
+ }
+ return mdp;
+ }
+
+ @Override
public boolean isMessageDeleted(Position position) {
- checkArgument(position instanceof PositionImpl);
- return ((PositionImpl) position).compareTo(markDeletePosition) <= 0
- || individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId());
+ lock.readLock().lock();
+ try {
+ return position.compareTo(markDeletePosition) <= 0
+ || individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId());
+ } finally {
+ lock.readLock().unlock();
+ }
}
//this method will return a copy of the position's ack set
+ @Override
public long[] getBatchPositionAckSet(Position position) {
- if (!(position instanceof PositionImpl)) {
- return null;
- }
-
if (batchDeletedIndexes != null) {
BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.get(position);
if (bitSetRecyclable == null) {
@@ -3436,19 +3572,25 @@ public long[] getBatchPositionAckSet(Position position) {
* @param position
* @return next available position
*/
- public PositionImpl getNextAvailablePosition(PositionImpl position) {
- Range range = individualDeletedMessages.rangeContaining(position.getLedgerId(),
- position.getEntryId());
- if (range != null) {
- PositionImpl nextPosition = range.upperEndpoint().getNext();
- return (nextPosition != null && nextPosition.compareTo(position) > 0) ? nextPosition : position.getNext();
+ public Position getNextAvailablePosition(Position position) {
+ lock.readLock().lock();
+ try {
+ Range range = individualDeletedMessages.rangeContaining(position.getLedgerId(),
+ position.getEntryId());
+ if (range != null) {
+ Position nextPosition = range.upperEndpoint().getNext();
+ return (nextPosition != null && nextPosition.compareTo(position) > 0)
+ ? nextPosition : position.getNext();
+ }
+ return position.getNext();
+ } finally {
+ lock.readLock().unlock();
}
- return position.getNext();
}
public Position getNextLedgerPosition(long currentLedgerId) {
Long nextExistingLedger = ledger.getNextValidLedger(currentLedgerId);
- return nextExistingLedger != null ? PositionImpl.get(nextExistingLedger, 0) : null;
+ return nextExistingLedger != null ? PositionFactory.create(nextExistingLedger, 0) : null;
}
public boolean isIndividuallyDeletedEntriesEmpty() {
@@ -3493,8 +3635,13 @@ public ManagedLedger getManagedLedger() {
}
@Override
- public Range getLastIndividualDeletedRange() {
- return individualDeletedMessages.lastRange();
+ public Range getLastIndividualDeletedRange() {
+ lock.readLock().lock();
+ try {
+ return individualDeletedMessages.lastRange();
+ } finally {
+ lock.readLock().unlock();
+ }
}
@Override
@@ -3513,8 +3660,8 @@ private ManagedCursorImpl cursorImpl() {
}
@Override
- public long[] getDeletedBatchIndexesAsLongArray(PositionImpl position) {
- if (config.isDeletionAtBatchIndexLevelEnabled()) {
+ public long[] getDeletedBatchIndexesAsLongArray(Position position) {
+ if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
BitSetRecyclable bitSet = batchDeletedIndexes.get(position);
return bitSet == null ? null : bitSet.toLongArray();
} else {
@@ -3527,6 +3674,7 @@ public ManagedCursorMXBean getStats() {
return this.mbean;
}
+ @Override
public void updateReadStats(int readEntriesCount, long readEntriesSize) {
this.entriesReadCount += readEntriesCount;
this.entriesReadSize += readEntriesSize;
@@ -3559,6 +3707,7 @@ public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
}, null);
}
+ @Override
public int applyMaxSizeCap(int maxEntries, long maxSizeBytes) {
if (maxSizeBytes == NO_MAX_SIZE_LIMIT) {
return maxEntries;
@@ -3587,7 +3736,7 @@ public int applyMaxSizeCap(int maxEntries, long maxSizeBytes) {
@Override
public boolean checkAndUpdateReadPositionChanged() {
- PositionImpl lastEntry = ledger.lastConfirmedEntry;
+ Position lastEntry = ledger.lastConfirmedEntry;
boolean isReadPositionOnTail = lastEntry == null || readPosition == null
|| (lastEntry.compareTo(readPosition) <= 0);
boolean isReadPositionChanged = readPosition != null && !readPosition.equals(statsLastReadPosition);
@@ -3615,31 +3764,65 @@ public boolean isCacheReadEntry() {
private static final Logger log = LoggerFactory.getLogger(ManagedCursorImpl.class);
public ManagedLedgerConfig getConfig() {
- return config;
+ return getManagedLedger().getConfig();
}
/***
* Create a non-durable cursor and copy the ack stats.
*/
+ @Override
public ManagedCursor duplicateNonDurableCursor(String nonDurableCursorName) throws ManagedLedgerException {
NonDurableCursorImpl newNonDurableCursor =
(NonDurableCursorImpl) ledger.newNonDurableCursor(getMarkDeletedPosition(), nonDurableCursorName);
- if (individualDeletedMessages != null) {
- this.individualDeletedMessages.forEach(range -> {
- newNonDurableCursor.individualDeletedMessages.addOpenClosed(
- range.lowerEndpoint().getLedgerId(),
- range.lowerEndpoint().getEntryId(),
- range.upperEndpoint().getLedgerId(),
- range.upperEndpoint().getEntryId());
- return true;
- });
+ lock.readLock().lock();
+ try {
+ if (individualDeletedMessages != null) {
+ this.individualDeletedMessages.forEach(range -> {
+ newNonDurableCursor.individualDeletedMessages.addOpenClosed(
+ range.lowerEndpoint().getLedgerId(),
+ range.lowerEndpoint().getEntryId(),
+ range.upperEndpoint().getLedgerId(),
+ range.upperEndpoint().getEntryId());
+ return true;
+ });
+ }
+ } finally {
+ lock.readLock().unlock();
}
if (batchDeletedIndexes != null) {
- for (Map.Entry entry : this.batchDeletedIndexes.entrySet()) {
+ for (Map.Entry entry : this.batchDeletedIndexes.entrySet()) {
BitSetRecyclable copiedBitSet = BitSetRecyclable.valueOf(entry.getValue());
newNonDurableCursor.batchDeletedIndexes.put(entry.getKey(), copiedBitSet);
}
}
return newNonDurableCursor;
}
+
+ @Override
+ public ManagedCursorAttributes getManagedCursorAttributes() {
+ if (managedCursorAttributes != null) {
+ return managedCursorAttributes;
+ }
+ return ATTRIBUTES_UPDATER.updateAndGet(this, old -> old != null ? old : new ManagedCursorAttributes(this));
+ }
+
+ @Override
+ public ManagedLedgerInternalStats.CursorStats getCursorStats() {
+ ManagedLedgerInternalStats.CursorStats cs = new ManagedLedgerInternalStats.CursorStats();
+ cs.markDeletePosition = getMarkDeletedPosition().toString();
+ cs.readPosition = getReadPosition().toString();
+ cs.waitingReadOp = hasPendingReadRequest();
+ cs.pendingReadOps = getPendingReadOpsCount();
+ cs.messagesConsumedCounter = getMessagesConsumedCounter();
+ cs.cursorLedger = getCursorLedger();
+ cs.cursorLedgerLastEntry = getCursorLedgerLastEntry();
+ cs.individuallyDeletedMessages = getIndividuallyDeletedMessages();
+ cs.lastLedgerSwitchTimestamp = DateFormatter.format(getLastLedgerSwitchTimestamp());
+ cs.state = getState();
+ cs.active = isActive();
+ cs.numberOfEntriesSinceFirstNotAckedMessage = getNumberOfEntriesSinceFirstNotAckedMessage();
+ cs.totalNonContiguousDeletedMessagesRange = getTotalNonContiguousDeletedMessagesRange();
+ cs.properties = getProperties();
+ return cs;
+ }
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java
index 48465e6294b0e..a183c0d61ce16 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorMXBeanImpl.java
@@ -90,7 +90,8 @@ public long getPersistZookeeperErrors() {
@Override
public void addWriteCursorLedgerSize(final long size) {
- writeCursorLedgerSize.add(size * ((ManagedCursorImpl) managedCursor).config.getWriteQuorumSize());
+ writeCursorLedgerSize.add(
+ size * managedCursor.getManagedLedger().getConfig().getWriteQuorumSize());
writeCursorLedgerLogicalSize.add(size);
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
index 51c5c91234f21..f546a487f84be 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryImpl.java
@@ -18,17 +18,23 @@
*/
package org.apache.bookkeeper.mledger.impl;
-import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.bookkeeper.mledger.ManagedLedgerException.getManagedLedgerException;
+import static org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.NULL_OFFLOAD_PROMISE;
import static org.apache.pulsar.common.util.Runnables.catchingAndLoggingThrowables;
import com.google.common.base.Predicates;
+import com.google.common.collect.BoundType;
import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
+import com.google.protobuf.InvalidProtocolBufferException;
import io.netty.util.concurrent.DefaultThreadFactory;
+import io.opentelemetry.api.OpenTelemetry;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@@ -44,8 +50,12 @@
import java.util.function.Supplier;
import java.util.stream.Collectors;
import lombok.Getter;
+import org.apache.bookkeeper.client.AsyncCallback;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.client.LedgerEntry;
+import org.apache.bookkeeper.client.LedgerHandle;
import org.apache.bookkeeper.common.util.OrderedScheduler;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
@@ -67,8 +77,12 @@
import org.apache.bookkeeper.mledger.ManagedLedgerInfo.MessageRangeInfo;
import org.apache.bookkeeper.mledger.ManagedLedgerInfo.PositionInfo;
import org.apache.bookkeeper.mledger.MetadataCompressionConfig;
+import org.apache.bookkeeper.mledger.OpenTelemetryManagedLedgerCacheStats;
import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionFactory;
import org.apache.bookkeeper.mledger.ReadOnlyCursor;
+import org.apache.bookkeeper.mledger.ReadOnlyManagedLedger;
+import org.apache.bookkeeper.mledger.ReadOnlyManagedLedgerImplWrapper;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.ManagedLedgerInitializeLedgerCallback;
import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl.State;
import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
@@ -79,11 +93,14 @@
import org.apache.bookkeeper.mledger.proto.MLDataFormats.LongProperty;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.MessageRange;
+import org.apache.bookkeeper.mledger.util.Errors;
import org.apache.bookkeeper.mledger.util.Futures;
import org.apache.bookkeeper.stats.NullStatsLogger;
import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.policies.data.EnsemblePlacementPolicyConfig;
+import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
import org.apache.pulsar.common.util.DateFormatter;
import org.apache.pulsar.common.util.FutureUtil;
import org.apache.pulsar.common.util.Runnables;
@@ -118,6 +135,10 @@ public class ManagedLedgerFactoryImpl implements ManagedLedgerFactory {
private volatile long cacheEvictionTimeThresholdNanos;
private final MetadataStore metadataStore;
+ private final OpenTelemetryManagedLedgerCacheStats openTelemetryCacheStats;
+ private final OpenTelemetryManagedLedgerStats openTelemetryManagedLedgerStats;
+ private final OpenTelemetryManagedCursorStats openTelemetryManagedCursorStats;
+
//indicate whether shutdown() is called.
private volatile boolean closed;
@@ -149,7 +170,7 @@ public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, ClientConfi
ManagedLedgerFactoryConfig config)
throws Exception {
this(metadataStore, new DefaultBkFactory(bkClientConfiguration),
- true /* isBookkeeperManaged */, config, NullStatsLogger.INSTANCE);
+ true /* isBookkeeperManaged */, config, NullStatsLogger.INSTANCE, OpenTelemetry.noop());
}
public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, BookKeeper bookKeeper)
@@ -160,7 +181,7 @@ public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, BookKeeper
public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore, BookKeeper bookKeeper,
ManagedLedgerFactoryConfig config)
throws Exception {
- this(metadataStore, (policyConfig) -> bookKeeper, config);
+ this(metadataStore, (policyConfig) -> CompletableFuture.completedFuture(bookKeeper), config);
}
public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
@@ -168,21 +189,24 @@ public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
ManagedLedgerFactoryConfig config)
throws Exception {
this(metadataStore, bookKeeperGroupFactory, false /* isBookkeeperManaged */,
- config, NullStatsLogger.INSTANCE);
+ config, NullStatsLogger.INSTANCE, OpenTelemetry.noop());
}
public ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
BookkeeperFactoryForCustomEnsemblePlacementPolicy bookKeeperGroupFactory,
- ManagedLedgerFactoryConfig config, StatsLogger statsLogger)
+ ManagedLedgerFactoryConfig config, StatsLogger statsLogger,
+ OpenTelemetry openTelemetry)
throws Exception {
this(metadataStore, bookKeeperGroupFactory, false /* isBookkeeperManaged */,
- config, statsLogger);
+ config, statsLogger, openTelemetry);
}
private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
BookkeeperFactoryForCustomEnsemblePlacementPolicy bookKeeperGroupFactory,
boolean isBookkeeperManaged,
- ManagedLedgerFactoryConfig config, StatsLogger statsLogger) throws Exception {
+ ManagedLedgerFactoryConfig config,
+ StatsLogger statsLogger,
+ OpenTelemetry openTelemetry) throws Exception {
MetadataCompressionConfig compressionConfigForManagedLedgerInfo =
config.getCompressionConfigForManagedLedgerInfo();
MetadataCompressionConfig compressionConfigForManagedCursorInfo =
@@ -204,7 +228,7 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
compressionConfigForManagedCursorInfo);
this.config = config;
this.mbean = new ManagedLedgerFactoryMBeanImpl(this);
- this.entryCacheManager = new RangeEntryCacheManagerImpl(this);
+ this.entryCacheManager = new RangeEntryCacheManagerImpl(this, openTelemetry);
this.statsTask = scheduledExecutor.scheduleWithFixedDelay(catchingAndLoggingThrowables(this::refreshStats),
0, config.getStatsPeriodSeconds(), TimeUnit.SECONDS);
this.flushCursorsTask = scheduledExecutor.scheduleAtFixedRate(catchingAndLoggingThrowables(this::flushCursors),
@@ -220,6 +244,10 @@ private ManagedLedgerFactoryImpl(MetadataStoreExtended metadataStore,
closed = false;
metadataStore.registerSessionListener(this::handleMetadataStoreNotification);
+
+ openTelemetryCacheStats = new OpenTelemetryManagedLedgerCacheStats(openTelemetry, this);
+ openTelemetryManagedLedgerStats = new OpenTelemetryManagedLedgerStats(openTelemetry, this);
+ openTelemetryManagedCursorStats = new OpenTelemetryManagedCursorStats(openTelemetry, this);
}
static class DefaultBkFactory implements BookkeeperFactoryForCustomEnsemblePlacementPolicy {
@@ -232,8 +260,8 @@ public DefaultBkFactory(ClientConfiguration bkClientConfiguration)
}
@Override
- public BookKeeper get(EnsemblePlacementPolicyConfig policy) {
- return bkClient;
+ public CompletableFuture get(EnsemblePlacementPolicyConfig policy) {
+ return CompletableFuture.completedFuture(bkClient);
}
}
@@ -288,7 +316,8 @@ private synchronized void doCacheEviction() {
*
* @return
*/
- public Map getManagedLedgers() {
+ @Override
+ public Map getManagedLedgers() {
// Return a view of already created ledger by filtering futures not yet completed
return Maps.filterValues(Maps.transformValues(ledgers, future -> future.getNow(null)), Predicates.notNull());
}
@@ -377,55 +406,68 @@ public void asyncOpen(final String name, final ManagedLedgerConfig config, final
ledgers.computeIfAbsent(name, (mlName) -> {
// Create the managed ledger
CompletableFuture future = new CompletableFuture<>();
- BookKeeper bk = bookkeeperFactory.get(
- new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(),
- config.getBookKeeperEnsemblePlacementPolicyProperties()));
- final ManagedLedgerImpl newledger = config.getShadowSource() == null
- ? new ManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name, mlOwnershipChecker)
- : new ShadowManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name,
- mlOwnershipChecker);
- PendingInitializeManagedLedger pendingLedger = new PendingInitializeManagedLedger(newledger);
- pendingInitializeLedgers.put(name, pendingLedger);
- newledger.initialize(new ManagedLedgerInitializeLedgerCallback() {
- @Override
- public void initializeComplete() {
- log.info("[{}] Successfully initialize managed ledger", name);
- pendingInitializeLedgers.remove(name, pendingLedger);
- future.complete(newledger);
+ bookkeeperFactory.get(
+ new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(),
+ config.getBookKeeperEnsemblePlacementPolicyProperties()))
+ .thenAccept(bk -> {
+ final ManagedLedgerImpl newledger = config.getShadowSource() == null
+ ? new ManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name,
+ mlOwnershipChecker)
+ : new ShadowManagedLedgerImpl(this, bk, store, config, scheduledExecutor, name,
+ mlOwnershipChecker);
+ PendingInitializeManagedLedger pendingLedger = new PendingInitializeManagedLedger(newledger);
+ pendingInitializeLedgers.put(name, pendingLedger);
+ newledger.initialize(new ManagedLedgerInitializeLedgerCallback() {
+ @Override
+ public void initializeComplete() {
+ log.info("[{}] Successfully initialize managed ledger", name);
+ pendingInitializeLedgers.remove(name, pendingLedger);
+ future.complete(newledger);
+
+ // May need to update the cursor position
+ newledger.maybeUpdateCursorBeforeTrimmingConsumedLedger();
+ // May need to trigger offloading
+ if (config.isTriggerOffloadOnTopicLoad()) {
+ newledger.maybeOffloadInBackground(NULL_OFFLOAD_PROMISE);
+ }
+ }
- // May need to update the cursor position
- newledger.maybeUpdateCursorBeforeTrimmingConsumedLedger();
- }
+ @Override
+ public void initializeFailed(ManagedLedgerException e) {
+ if (config.isCreateIfMissing()) {
+ log.error("[{}] Failed to initialize managed ledger: {}", name, e.getMessage());
+ }
- @Override
- public void initializeFailed(ManagedLedgerException e) {
- if (config.isCreateIfMissing()) {
- log.error("[{}] Failed to initialize managed ledger: {}", name, e.getMessage());
- }
+ // Clean the map if initialization fails
+ ledgers.remove(name, future);
+ entryCacheManager.removeEntryCache(name);
- // Clean the map if initialization fails
- ledgers.remove(name, future);
+ if (pendingInitializeLedgers.remove(name, pendingLedger)) {
+ pendingLedger.ledger.asyncClose(new CloseCallback() {
+ @Override
+ public void closeComplete(Object ctx) {
+ // no-op
+ }
- if (pendingInitializeLedgers.remove(name, pendingLedger)) {
- pendingLedger.ledger.asyncClose(new CloseCallback() {
- @Override
- public void closeComplete(Object ctx) {
- // no-op
- }
+ @Override
+ public void closeFailed(ManagedLedgerException exception, Object ctx) {
+ log.warn("[{}] Failed to a pending initialization managed ledger", name,
+ exception);
+ }
+ }, null);
+ }
- @Override
- public void closeFailed(ManagedLedgerException exception, Object ctx) {
- log.warn("[{}] Failed to a pending initialization managed ledger", name, exception);
+ future.completeExceptionally(e);
}
}, null);
- }
-
- future.completeExceptionally(e);
- }
- }, null);
+ }).exceptionally(ex -> {
+ future.completeExceptionally(ex);
+ return null;
+ });
return future;
}).thenAccept(ml -> callback.openLedgerComplete(ml, ctx)).exceptionally(exception -> {
- callback.openLedgerFailed((ManagedLedgerException) exception.getCause(), ctx);
+ callback.openLedgerFailed(ManagedLedgerException
+ .getManagedLedgerException(FutureUtil.unwrapCompletionException(exception)), ctx);
return null;
});
}
@@ -438,20 +480,23 @@ public void asyncOpenReadOnlyManagedLedger(String managedLedgerName,
callback.openReadOnlyManagedLedgerFailed(
new ManagedLedgerException.ManagedLedgerFactoryClosedException(), ctx);
}
- ReadOnlyManagedLedgerImpl roManagedLedger = new ReadOnlyManagedLedgerImpl(this,
- bookkeeperFactory
- .get(new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(),
- config.getBookKeeperEnsemblePlacementPolicyProperties())),
- store, config, scheduledExecutor, managedLedgerName);
- roManagedLedger.initialize().thenRun(() -> {
- log.info("[{}] Successfully initialize Read-only managed ledger", managedLedgerName);
- callback.openReadOnlyManagedLedgerComplete(roManagedLedger, ctx);
-
- }).exceptionally(e -> {
- log.error("[{}] Failed to initialize Read-only managed ledger", managedLedgerName, e);
- callback.openReadOnlyManagedLedgerFailed((ManagedLedgerException) e.getCause(), ctx);
- return null;
- });
+
+ bookkeeperFactory
+ .get(new EnsemblePlacementPolicyConfig(config.getBookKeeperEnsemblePlacementPolicyClassName(),
+ config.getBookKeeperEnsemblePlacementPolicyProperties()))
+ .thenCompose(bk -> {
+ ReadOnlyManagedLedgerImplWrapper roManagedLedger = new ReadOnlyManagedLedgerImplWrapper(this, bk,
+ store, config, scheduledExecutor, managedLedgerName);
+ return roManagedLedger.initialize().thenApply(v -> roManagedLedger);
+ }).thenAccept(roManagedLedger -> {
+ log.info("[{}] Successfully initialize Read-only managed ledger", managedLedgerName);
+ callback.openReadOnlyManagedLedgerComplete(roManagedLedger, ctx);
+ }).exceptionally(e -> {
+ log.error("[{}] Failed to initialize Read-only managed ledger", managedLedgerName, e);
+ callback.openReadOnlyManagedLedgerFailed(ManagedLedgerException
+ .getManagedLedgerException(FutureUtil.unwrapCompletionException(e)), ctx);
+ return null;
+ });
}
@Override
@@ -493,13 +538,12 @@ public void asyncOpenReadOnlyCursor(String managedLedgerName, Position startPosi
callback.openReadOnlyCursorFailed(new ManagedLedgerException.ManagedLedgerFactoryClosedException(), ctx);
return;
}
- checkArgument(startPosition instanceof PositionImpl);
AsyncCallbacks.OpenReadOnlyManagedLedgerCallback openReadOnlyManagedLedgerCallback =
new AsyncCallbacks.OpenReadOnlyManagedLedgerCallback() {
@Override
- public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedgerImpl readOnlyManagedLedger, Object ctx) {
+ public void openReadOnlyManagedLedgerComplete(ReadOnlyManagedLedger readOnlyManagedLedger, Object ctx) {
callback.openReadOnlyCursorComplete(readOnlyManagedLedger.
- createReadOnlyCursor((PositionImpl) startPosition), ctx);
+ createReadOnlyCursor(startPosition), ctx);
}
@Override
@@ -573,49 +617,38 @@ public void closeFailed(ManagedLedgerException exception, Object ctx) {
ledgerFuture.completeExceptionally(new ManagedLedgerException.ManagedLedgerFactoryClosedException());
}
}
- CompletableFuture bookkeeperFuture = new CompletableFuture<>();
- futures.add(bookkeeperFuture);
- futures.add(CompletableFuture.runAsync(() -> {
- if (isBookkeeperManaged) {
- try {
- BookKeeper bookkeeper = bookkeeperFactory.get();
- if (bookkeeper != null) {
- bookkeeper.close();
- }
- bookkeeperFuture.complete(null);
- } catch (Throwable throwable) {
- bookkeeperFuture.completeExceptionally(throwable);
- }
- } else {
- bookkeeperFuture.complete(null);
- }
- if (!ledgers.isEmpty()) {
- log.info("Force closing {} ledgers.", ledgers.size());
- //make sure all callbacks is called.
- ledgers.forEach(((ledgerName, ledgerFuture) -> {
- if (!ledgerFuture.isDone()) {
- ledgerFuture.completeExceptionally(
- new ManagedLedgerException.ManagedLedgerFactoryClosedException());
- } else {
- ManagedLedgerImpl managedLedger = ledgerFuture.getNow(null);
- if (managedLedger == null) {
- return;
- }
- try {
- managedLedger.close();
- } catch (Throwable throwable) {
- log.warn("[{}] Got exception when closing managed ledger: {}", managedLedger.getName(),
- throwable);
+ CompletableFuture bookkeeperFuture = isBookkeeperManaged
+ ? bookkeeperFactory.get()
+ : CompletableFuture.completedFuture(null);
+ return bookkeeperFuture
+ .thenRun(() -> {
+ log.info("Closing {} ledgers.", ledgers.size());
+ //make sure all callbacks is called.
+ ledgers.forEach(((ledgerName, ledgerFuture) -> {
+ if (!ledgerFuture.isDone()) {
+ ledgerFuture.completeExceptionally(
+ new ManagedLedgerException.ManagedLedgerFactoryClosedException());
+ } else {
+ ManagedLedgerImpl managedLedger = ledgerFuture.getNow(null);
+ if (managedLedger == null) {
+ return;
+ }
+ try {
+ managedLedger.close();
+ } catch (Throwable throwable) {
+ log.warn("[{}] Got exception when closing managed ledger: {}", managedLedger.getName(),
+ throwable);
+ }
}
- }
- }));
- }
- }));
- return FutureUtil.waitForAll(futures).thenAcceptAsync(__ -> {
- //wait for tasks in scheduledExecutor executed.
- scheduledExecutor.shutdownNow();
- entryCacheManager.clear();
- });
+ }));
+ }).thenAcceptAsync(__ -> {
+ //wait for tasks in scheduledExecutor executed.
+ openTelemetryManagedCursorStats.close();
+ openTelemetryManagedLedgerStats.close();
+ openTelemetryCacheStats.close();
+ scheduledExecutor.shutdownNow();
+ entryCacheManager.clear();
+ });
}
@Override
@@ -694,6 +727,7 @@ public void operationComplete(MLDataFormats.ManagedLedgerInfo pbInfo, Stat stat)
ledgerInfo.ledgerId = pbLedgerInfo.getLedgerId();
ledgerInfo.entries = pbLedgerInfo.hasEntries() ? pbLedgerInfo.getEntries() : null;
ledgerInfo.size = pbLedgerInfo.hasSize() ? pbLedgerInfo.getSize() : null;
+ ledgerInfo.timestamp = pbLedgerInfo.hasTimestamp() ? pbLedgerInfo.getTimestamp() : null;
ledgerInfo.isOffloaded = pbLedgerInfo.hasOffloadContext();
if (pbLedgerInfo.hasOffloadContext()) {
MLDataFormats.OffloadContext offloadContext = pbLedgerInfo.getOffloadContext();
@@ -855,14 +889,14 @@ void deleteManagedLedger(String managedLedgerName, CompletableFuture> futures = info.cursors.entrySet().stream()
- .map(e -> deleteCursor(bkc, managedLedgerName, e.getKey(), e.getValue()))
- .collect(Collectors.toList());
- Futures.waitForAll(futures).thenRun(() -> {
- deleteManagedLedgerData(bkc, managedLedgerName, info, mlConfigFuture, callback, ctx);
+ getBookKeeper().thenCompose(bk -> {
+ // First delete all cursors resources
+ List> futures = info.cursors.entrySet().stream()
+ .map(e -> deleteCursor(bk, managedLedgerName, e.getKey(), e.getValue()))
+ .collect(Collectors.toList());
+ return Futures.waitForAll(futures).thenApply(v -> bk);
+ }).thenAccept(bk -> {
+ deleteManagedLedgerData(bk, managedLedgerName, info, mlConfigFuture, callback, ctx);
}).exceptionally(ex -> {
callback.deleteLedgerFailed(new ManagedLedgerException(ex), ctx);
return null;
@@ -1024,6 +1058,7 @@ public MetaStore getMetaStore() {
return store;
}
+ @Override
public ManagedLedgerFactoryConfig getConfig() {
return config;
}
@@ -1043,20 +1078,422 @@ public long getCacheEvictionTimeThreshold(){
return cacheEvictionTimeThresholdNanos;
}
+ @Override
public ManagedLedgerFactoryMXBean getCacheStats() {
return this.mbean;
}
- public BookKeeper getBookKeeper() {
+ public CompletableFuture getBookKeeper() {
return bookkeeperFactory.get();
}
+ @Override
+ public void estimateUnloadedTopicBacklog(PersistentOfflineTopicStats offlineTopicStats,
+ TopicName topicName, boolean accurate, Object ctx)
+ throws Exception {
+ String managedLedgerName = topicName.getPersistenceNamingEncoding();
+ long numberOfEntries = 0;
+ long totalSize = 0;
+ BookKeeper.DigestType digestType = (BookKeeper.DigestType) ((List) ctx).get(0);
+ byte[] password = (byte[]) ((List) ctx).get(1);
+ NavigableMap ledgers =
+ getManagedLedgersInfo(topicName, accurate, digestType, password);
+ for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers.values()) {
+ numberOfEntries += ls.getEntries();
+ totalSize += ls.getSize();
+ if (accurate) {
+ offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId());
+ }
+ }
+ offlineTopicStats.totalMessages = numberOfEntries;
+ offlineTopicStats.storageSize = totalSize;
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Total number of entries - {} and size - {}", managedLedgerName, numberOfEntries, totalSize);
+ }
+
+ // calculate per cursor message backlog
+ calculateCursorBacklogs(topicName, ledgers, offlineTopicStats, accurate, digestType, password);
+ offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis());
+ }
+
+ private NavigableMap getManagedLedgersInfo(
+ final TopicName topicName, boolean accurate, BookKeeper.DigestType digestType, byte[] password)
+ throws Exception {
+ final NavigableMap ledgers = new ConcurrentSkipListMap<>();
+
+ String managedLedgerName = topicName.getPersistenceNamingEncoding();
+ MetaStore store = getMetaStore();
+
+ final CountDownLatch mlMetaCounter = new CountDownLatch(1);
+ store.getManagedLedgerInfo(managedLedgerName, false /* createIfMissing */,
+ new MetaStore.MetaStoreCallback() {
+ @Override
+ public void operationComplete(MLDataFormats.ManagedLedgerInfo mlInfo, Stat stat) {
+ for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : mlInfo.getLedgerInfoList()) {
+ ledgers.put(ls.getLedgerId(), ls);
+ }
+
+ // find no of entries in last ledger
+ if (!ledgers.isEmpty()) {
+ final long id = ledgers.lastKey();
+ AsyncCallback.OpenCallback opencb = (rc, lh, ctx1) -> {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Opened ledger {}: {}", managedLedgerName, id,
+ BKException.getMessage(rc));
+ }
+ if (rc == BKException.Code.OK) {
+ MLDataFormats.ManagedLedgerInfo.LedgerInfo info =
+ MLDataFormats.ManagedLedgerInfo.LedgerInfo
+ .newBuilder().setLedgerId(id)
+ .setEntries(lh.getLastAddConfirmed() + 1)
+ .setSize(lh.getLength()).setTimestamp(System.currentTimeMillis())
+ .build();
+ ledgers.put(id, info);
+ mlMetaCounter.countDown();
+ } else if (Errors.isNoSuchLedgerExistsException(rc)) {
+ log.warn("[{}] Ledger not found: {}", managedLedgerName, ledgers.lastKey());
+ ledgers.remove(ledgers.lastKey());
+ mlMetaCounter.countDown();
+ } else {
+ log.error("[{}] Failed to open ledger {}: {}", managedLedgerName, id,
+ BKException.getMessage(rc));
+ mlMetaCounter.countDown();
+ }
+ };
+
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Opening ledger {}", managedLedgerName, id);
+ }
+ getBookKeeper()
+ .thenAccept(bk -> {
+ bk.asyncOpenLedgerNoRecovery(id, digestType, password, opencb, null);
+ }).exceptionally(ex -> {
+ log.warn("[{}] Failed to open ledger {}: {}", managedLedgerName, id, ex);
+ opencb.openComplete(-1, null, null);
+ mlMetaCounter.countDown();
+ return null;
+ });
+ } else {
+ log.warn("[{}] Ledger list empty", managedLedgerName);
+ mlMetaCounter.countDown();
+ }
+ }
+
+ @Override
+ public void operationFailed(ManagedLedgerException.MetaStoreException e) {
+ log.warn("[{}] Unable to obtain managed ledger metadata - {}", managedLedgerName, e);
+ mlMetaCounter.countDown();
+ }
+ });
+
+ if (accurate) {
+ // block until however long it takes for operation to complete
+ mlMetaCounter.await();
+ } else {
+ mlMetaCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+ return ledgers;
+ }
+
+ public void calculateCursorBacklogs(final TopicName topicName,
+ final NavigableMap ledgers,
+ final PersistentOfflineTopicStats offlineTopicStats, boolean accurate,
+ BookKeeper.DigestType digestType, byte[] password) throws Exception {
+ if (ledgers.isEmpty()) {
+ return;
+ }
+ String managedLedgerName = topicName.getPersistenceNamingEncoding();
+ MetaStore store = getMetaStore();
+ BookKeeper bk = getBookKeeper().get();
+ final CountDownLatch allCursorsCounter = new CountDownLatch(1);
+ final long errorInReadingCursor = -1;
+ final var ledgerRetryMap = new ConcurrentHashMap();
+
+ final MLDataFormats.ManagedLedgerInfo.LedgerInfo ledgerInfo = ledgers.lastEntry().getValue();
+ final Position lastLedgerPosition =
+ PositionFactory.create(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1);
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Last ledger position {}", managedLedgerName, lastLedgerPosition);
+ }
+
+ store.getCursors(managedLedgerName, new MetaStore.MetaStoreCallback>() {
+ @Override
+ public void operationComplete(List cursors, Stat v) {
+ // Load existing cursors
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Found {} cursors", managedLedgerName, cursors.size());
+ }
+
+ if (cursors.isEmpty()) {
+ allCursorsCounter.countDown();
+ return;
+ }
+
+ final CountDownLatch cursorCounter = new CountDownLatch(cursors.size());
+
+ for (final String cursorName : cursors) {
+ // determine subscription position from cursor ledger
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Loading cursor {}", managedLedgerName, cursorName);
+ }
+
+ AsyncCallback.OpenCallback cursorLedgerOpenCb = (rc, lh, ctx1) -> {
+ long ledgerId = lh.getId();
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Opened cursor ledger {} for cursor {}. rc={}", managedLedgerName, ledgerId,
+ cursorName, rc);
+ }
+ if (rc != BKException.Code.OK) {
+ log.warn("[{}] Error opening metadata ledger {} for cursor {}: {}", managedLedgerName,
+ ledgerId, cursorName, BKException.getMessage(rc));
+ cursorCounter.countDown();
+ return;
+ }
+ long lac = lh.getLastAddConfirmed();
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac,
+ ledgerId);
+ }
+
+ if (lac == LedgerHandle.INVALID_ENTRY_ID) {
+ // save the ledger id and cursor to retry outside of this call back
+ // since we are trying to read the same cursor ledger, we will block until
+ // this current callback completes, since an attempt to read the entry
+ // will block behind this current operation to complete
+ ledgerRetryMap.put(cursorName, ledgerId);
+ log.info("[{}] Cursor {} LAC {} read from ledger {}", managedLedgerName, cursorName, lac,
+ ledgerId);
+ cursorCounter.countDown();
+ return;
+ }
+ final long entryId = lac;
+ // read last acked message position for subscription
+ lh.asyncReadEntries(entryId, entryId, new AsyncCallback.ReadCallback() {
+ @Override
+ public void readComplete(int rc, LedgerHandle lh, Enumeration seq,
+ Object ctx) {
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug("readComplete rc={} entryId={}", rc, entryId);
+ }
+ if (rc != BKException.Code.OK) {
+ log.warn("[{}] Error reading from metadata ledger {} for cursor {}: {}",
+ managedLedgerName, ledgerId, cursorName, BKException.getMessage(rc));
+ // indicate that this cursor should be excluded
+ offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor,
+ lh.getId());
+ } else {
+ LedgerEntry entry = seq.nextElement();
+ MLDataFormats.PositionInfo positionInfo;
+ try {
+ positionInfo = MLDataFormats.PositionInfo.parseFrom(entry.getEntry());
+ } catch (InvalidProtocolBufferException e) {
+ log.warn(
+ "[{}] Error reading position from metadata ledger {} for cursor "
+ + "{}: {}", managedLedgerName, ledgerId, cursorName, e);
+ offlineTopicStats.addCursorDetails(cursorName, errorInReadingCursor,
+ lh.getId());
+ return;
+ }
+ final Position lastAckedMessagePosition =
+ PositionFactory.create(positionInfo.getLedgerId(),
+ positionInfo.getEntryId());
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Cursor {} MD {} read last ledger position {}",
+ managedLedgerName, cursorName, lastAckedMessagePosition,
+ lastLedgerPosition);
+ }
+ // calculate cursor backlog
+ Range range = Range.openClosed(lastAckedMessagePosition,
+ lastLedgerPosition);
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Calculating backlog for cursor {} using range {}",
+ managedLedgerName, cursorName, range);
+ }
+ long cursorBacklog = getNumberOfEntries(range, ledgers);
+ offlineTopicStats.messageBacklog += cursorBacklog;
+ offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, lh.getId());
+ }
+ } finally {
+ cursorCounter.countDown();
+ }
+ }
+ }, null);
+
+ }; // end of cursor meta read callback
+
+ store.asyncGetCursorInfo(managedLedgerName, cursorName,
+ new MetaStore.MetaStoreCallback() {
+ @Override
+ public void operationComplete(MLDataFormats.ManagedCursorInfo info,
+ Stat stat) {
+ long cursorLedgerId = info.getCursorsLedgerId();
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Cursor {} meta-data read ledger id {}", managedLedgerName,
+ cursorName, cursorLedgerId);
+ }
+ if (cursorLedgerId != -1) {
+ bk.asyncOpenLedgerNoRecovery(cursorLedgerId, digestType, password,
+ cursorLedgerOpenCb, null);
+ } else {
+ Position lastAckedMessagePosition = PositionFactory.create(
+ info.getMarkDeleteLedgerId(), info.getMarkDeleteEntryId());
+ Range range = Range.openClosed(lastAckedMessagePosition,
+ lastLedgerPosition);
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Calculating backlog for cursor {} using range {}",
+ managedLedgerName, cursorName, range);
+ }
+ long cursorBacklog = getNumberOfEntries(range, ledgers);
+ offlineTopicStats.messageBacklog += cursorBacklog;
+ offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, cursorLedgerId);
+ cursorCounter.countDown();
+ }
+
+ }
+
+ @Override
+ public void operationFailed(ManagedLedgerException.MetaStoreException e) {
+ log.warn("[{}] Unable to obtain cursor ledger for cursor {}: {}", managedLedgerName,
+ cursorName, e);
+ cursorCounter.countDown();
+ }
+ });
+ } // for every cursor find backlog
+ try {
+ if (accurate) {
+ cursorCounter.await();
+ } else {
+ cursorCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+ } catch (Exception e) {
+ log.warn("[{}] Error reading subscription positions{}", managedLedgerName, e);
+ } finally {
+ allCursorsCounter.countDown();
+ }
+ }
+
+ @Override
+ public void operationFailed(ManagedLedgerException.MetaStoreException e) {
+ log.warn("[{}] Failed to get the cursors list", managedLedgerName, e);
+ allCursorsCounter.countDown();
+ }
+ });
+ if (accurate) {
+ allCursorsCounter.await();
+ } else {
+ allCursorsCounter.await(META_READ_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ }
+
+ // go through ledgers where LAC was -1
+ if (accurate && ledgerRetryMap.size() > 0) {
+ ledgerRetryMap.forEach((cursorName, ledgerId) -> {
+ if (log.isDebugEnabled()) {
+ log.debug("Cursor {} Ledger {} Trying to obtain MD from BkAdmin", cursorName, ledgerId);
+ }
+ Position lastAckedMessagePosition = tryGetMDPosition(bk, ledgerId, cursorName);
+ if (lastAckedMessagePosition == null) {
+ log.warn("[{}] Cursor {} read from ledger {}. Unable to determine cursor position",
+ managedLedgerName, cursorName, ledgerId);
+ } else {
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Cursor {} read from ledger using bk admin {}. position {}", managedLedgerName,
+ cursorName, ledgerId, lastAckedMessagePosition);
+ }
+ // calculate cursor backlog
+ Range range = Range.openClosed(lastAckedMessagePosition, lastLedgerPosition);
+ if (log.isDebugEnabled()) {
+ log.debug("[{}] Calculating backlog for cursor {} using range {}", managedLedgerName,
+ cursorName, range);
+ }
+ long cursorBacklog = getNumberOfEntries(range, ledgers);
+ offlineTopicStats.messageBacklog += cursorBacklog;
+ offlineTopicStats.addCursorDetails(cursorName, cursorBacklog, ledgerId);
+ }
+ });
+ }
+ }
+
+ // need a better way than to duplicate the functionality below from ML
+ private long getNumberOfEntries(Range range,
+ NavigableMap ledgers) {
+ Position fromPosition = range.lowerEndpoint();
+ boolean fromIncluded = range.lowerBoundType() == BoundType.CLOSED;
+ Position toPosition = range.upperEndpoint();
+ boolean toIncluded = range.upperBoundType() == BoundType.CLOSED;
+
+ if (fromPosition.getLedgerId() == toPosition.getLedgerId()) {
+ // If the 2 positions are in the same ledger
+ long count = toPosition.getEntryId() - fromPosition.getEntryId() - 1;
+ count += fromIncluded ? 1 : 0;
+ count += toIncluded ? 1 : 0;
+ return count;
+ } else {
+ long count = 0;
+ // If the from & to are pointing to different ledgers, then we need to :
+ // 1. Add the entries in the ledger pointed by toPosition
+ count += toPosition.getEntryId();
+ count += toIncluded ? 1 : 0;
+
+ // 2. Add the entries in the ledger pointed by fromPosition
+ MLDataFormats.ManagedLedgerInfo.LedgerInfo li = ledgers.get(fromPosition.getLedgerId());
+ if (li != null) {
+ count += li.getEntries() - (fromPosition.getEntryId() + 1);
+ count += fromIncluded ? 1 : 0;
+ }
+
+ // 3. Add the whole ledgers entries in between
+ for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers
+ .subMap(fromPosition.getLedgerId(), false, toPosition.getLedgerId(), false).values()) {
+ count += ls.getEntries();
+ }
+
+ return count;
+ }
+ }
+
+
+ private Position tryGetMDPosition(BookKeeper bookKeeper, long ledgerId, String cursorName) {
+ BookKeeperAdmin bookKeeperAdmin = null;
+ long lastEntry = LedgerHandle.INVALID_ENTRY_ID;
+ Position lastAckedMessagePosition = null;
+ try {
+ bookKeeperAdmin = new BookKeeperAdmin(bookKeeper);
+ for (LedgerEntry ledgerEntry : bookKeeperAdmin.readEntries(ledgerId, 0, lastEntry)) {
+ lastEntry = ledgerEntry.getEntryId();
+ if (log.isDebugEnabled()) {
+ log.debug(" Read entry {} from ledger {} for cursor {}", lastEntry, ledgerId, cursorName);
+ }
+ MLDataFormats.PositionInfo positionInfo = MLDataFormats.PositionInfo.parseFrom(ledgerEntry.getEntry());
+ lastAckedMessagePosition =
+ PositionFactory.create(positionInfo.getLedgerId(), positionInfo.getEntryId());
+ if (log.isDebugEnabled()) {
+ log.debug("Cursor {} read position {}", cursorName, lastAckedMessagePosition);
+ }
+ }
+ } catch (Exception e) {
+ log.warn("Unable to determine LAC for ledgerId {} for cursor {}: {}", ledgerId, cursorName, e);
+ } finally {
+ if (bookKeeperAdmin != null) {
+ try {
+ bookKeeperAdmin.close();
+ } catch (Exception e) {
+ log.warn("Unable to close bk admin for ledgerId {} for cursor {}", ledgerId, cursorName, e);
+ }
+ }
+
+ }
+ return lastAckedMessagePosition;
+ }
+
+ private static final int META_READ_TIMEOUT_SECONDS = 60;
+
/**
* Factory to create Bookkeeper-client for a given ensemblePlacementPolicy.
*
*/
public interface BookkeeperFactoryForCustomEnsemblePlacementPolicy {
- default BookKeeper get() {
+ default CompletableFuture get() {
return get(null);
}
@@ -1067,7 +1504,7 @@ default BookKeeper get() {
* @param ensemblePlacementPolicyMetadata
* @return
*/
- BookKeeper get(EnsemblePlacementPolicyConfig ensemblePlacementPolicyMetadata);
+ CompletableFuture get(EnsemblePlacementPolicyConfig ensemblePlacementPolicyMetadata);
}
private static final Logger log = LoggerFactory.getLogger(ManagedLedgerFactoryImpl.class);
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java
index cf3d7142d617e..a3038a0e7ff76 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryMBeanImpl.java
@@ -99,26 +99,51 @@ public double getCacheHitsRate() {
return cacheHits.getRate();
}
+ @Override
+ public long getCacheHitsTotal() {
+ return cacheHits.getTotalCount();
+ }
+
@Override
public double getCacheMissesRate() {
return cacheMisses.getRate();
}
+ @Override
+ public long getCacheMissesTotal() {
+ return cacheMisses.getTotalCount();
+ }
+
@Override
public double getCacheHitsThroughput() {
return cacheHits.getValueRate();
}
+ @Override
+ public long getCacheHitsBytesTotal() {
+ return cacheHits.getTotalValue();
+ }
+
@Override
public double getCacheMissesThroughput() {
return cacheMisses.getValueRate();
}
+ @Override
+ public long getCacheMissesBytesTotal() {
+ return cacheMisses.getTotalValue();
+ }
+
@Override
public long getNumberOfCacheEvictions() {
return cacheEvictions.getCount();
}
+ @Override
+ public long getNumberOfCacheEvictionsTotal() {
+ return cacheEvictions.getTotalCount();
+ }
+
public long getCacheInsertedEntriesCount() {
return insertedEntryCount.sum();
}
diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
index 1c0a0465507a1..c1081761b601f 100644
--- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
+++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -34,6 +34,7 @@
import java.io.IOException;
import java.time.Clock;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -59,6 +60,7 @@
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLongFieldUpdater;
import java.util.concurrent.atomic.AtomicReference;
@@ -76,10 +78,13 @@
import org.apache.bookkeeper.client.BookKeeper;
import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.client.api.LedgerEntry;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.common.util.Backoff;
import org.apache.bookkeeper.common.util.OrderedScheduler;
import org.apache.bookkeeper.common.util.Retries;
+import org.apache.bookkeeper.discover.RegistrationClient;
import org.apache.bookkeeper.mledger.AsyncCallbacks;
import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback;
@@ -93,8 +98,10 @@
import org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback;
import org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback;
import org.apache.bookkeeper.mledger.Entry;
+import org.apache.bookkeeper.mledger.LedgerOffloader;
import org.apache.bookkeeper.mledger.ManagedCursor;
import org.apache.bookkeeper.mledger.ManagedLedger;
+import org.apache.bookkeeper.mledger.ManagedLedgerAttributes;
import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
import org.apache.bookkeeper.mledger.ManagedLedgerException;
import org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException;
@@ -112,6 +119,8 @@
import org.apache.bookkeeper.mledger.ManagedLedgerException.TooManyRequestsException;
import org.apache.bookkeeper.mledger.ManagedLedgerMXBean;
import org.apache.bookkeeper.mledger.Position;
+import org.apache.bookkeeper.mledger.PositionBound;
+import org.apache.bookkeeper.mledger.PositionFactory;
import org.apache.bookkeeper.mledger.WaitingEntryCallBack;
import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback;
import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
@@ -125,6 +134,7 @@
import org.apache.bookkeeper.mledger.proto.MLDataFormats.OffloadContext;
import org.apache.bookkeeper.mledger.util.CallbackMutex;
import org.apache.bookkeeper.mledger.util.Futures;
+import org.apache.bookkeeper.mledger.util.ManagedLedgerImplUtils;
import org.apache.bookkeeper.net.BookieId;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition;
@@ -212,8 +222,8 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
private final CallbackMutex trimmerMutex = new CallbackMutex();
private final CallbackMutex offloadMutex = new CallbackMutex();
- private static final CompletableFuture NULL_OFFLOAD_PROMISE = CompletableFuture
- .completedFuture(PositionImpl.LATEST);
+ public static final CompletableFuture NULL_OFFLOAD_PROMISE = CompletableFuture
+ .completedFuture(PositionFactory.LATEST);
protected volatile LedgerHandle currentLedger;
protected volatile long currentLedgerEntries = 0;
protected volatile long currentLedgerSize = 0;
@@ -233,13 +243,16 @@ public class ManagedLedgerImpl implements ManagedLedger, CreateCallback {
private long maximumRolloverTimeMs;
protected final Supplier> mlOwnershipChecker;
- volatile PositionImpl lastConfirmedEntry;
+ volatile Position lastConfirmedEntry;
protected ManagedLedgerInterceptor managedLedgerInterceptor;
protected volatile long lastAddEntryTimeMs = 0;
private long inactiveLedgerRollOverTimeMs = 0;
+ /** A signal that may trigger all the subsequent OpAddEntry of current ledger to be failed due to timeout. **/
+ protected volatile AtomicBoolean currentLedgerTimeoutTriggered;
+
protected static final int DEFAULT_LEDGER_DELETE_RETRIES = 3;
protected static final int DEFAULT_LEDGER_DELETE_BACKOFF_TIME_SEC = 60;
private static final String MIGRATION_STATE_PROPERTY = "migrated";
@@ -279,10 +292,7 @@ public boolean isFenced() {
}
}
- // define boundaries for position based seeks and searches
- public enum PositionBound {
- startIncluded, startExcluded
- }
+
protected static final AtomicReferenceFieldUpdater STATE_UPDATER =
AtomicReferenceFieldUpdater.newUpdater(ManagedLedgerImpl.class, State.class, "state");
@@ -321,6 +331,9 @@ public enum PositionBound {
*/
final ConcurrentLinkedQueue pendingAddEntries = new ConcurrentLinkedQueue<>();
+ @Getter
+ private final ManagedLedgerAttributes managedLedgerAttributes;
+
/**
* This variable is used for testing the tests.
* ManagedLedgerTest#testManagedLedgerWithPlacementPolicyInCustomMetadata()
@@ -333,6 +346,7 @@ public ManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper
final String name) {
this(factory, bookKeeper, store, config, scheduledExecutor, name, null);
}
+
public ManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper, MetaStore store,
ManagedLedgerConfig config, OrderedScheduler scheduledExecutor,
final String name, final Supplier> mlOwnershipChecker) {
@@ -365,12 +379,10 @@ public ManagedLedgerImpl(ManagedLedgerFactoryImpl factory, BookKeeper bookKeeper
this.mlOwnershipChecker = mlOwnershipChecker;
this.propertiesMap = new ConcurrentHashMap<>();
this.inactiveLedgerRollOverTimeMs = config.getInactiveLedgerRollOverTimeMs();
- if (config.getManagedLedgerInterceptor() != null) {
- this.managedLedgerInterceptor = config.getManagedLedgerInterceptor();
- }
this.minBacklogCursorsForCaching = config.getMinimumBacklogCursorsForCaching();
this.minBacklogEntriesForCaching = config.getMinimumBacklogEntriesForCaching();
this.maxBacklogBetweenCursorsForCaching = config.getMaxBacklogBetweenCursorsForCaching();
+ this.managedLedgerAttributes = new ManagedLedgerAttributes(this);
}
synchronized void initialize(final ManagedLedgerInitializeLedgerCallback callback, final Object ctx) {
@@ -384,7 +396,9 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
ledgersStat = stat;
if (mlInfo.hasTerminatedPosition()) {
state = State.Terminated;
- lastConfirmedEntry = new PositionImpl(mlInfo.getTerminatedPosition());
+ NestedPositionInfo terminatedPosition = mlInfo.getTerminatedPosition();
+ lastConfirmedEntry =
+ PositionFactory.create(terminatedPosition.getLedgerId(), terminatedPosition.getEntryId());
log.info("[{}] Recovering managed ledger terminated at {}", name, lastConfirmedEntry);
}
for (LedgerInfo ls : mlInfo.getLedgerInfoList()) {
@@ -418,13 +432,14 @@ public void operationComplete(ManagedLedgerInfo mlInfo, Stat stat) {
.setTimestamp(clock.millis()).build();
ledgers.put(id, info);
if (managedLedgerInterceptor != null) {
- managedLedgerInterceptor.onManagedLedgerLastLedgerInitialize(name, lh)
- .thenRun(() -> initializeBookKeeper(callback))
- .exceptionally(ex -> {
- callback.initializeFailed(
- new ManagedLedgerInterceptException(ex.getCause()));
- return null;
- });
+ managedLedgerInterceptor
+ .onManagedLedgerLastLedgerInitialize(name, createLastEntryHandle(lh))
+ .thenRun(() -> initializeBookKeeper(callback))
+ .exceptionally(ex -> {
+ callback.initializeFailed(
+ new ManagedLedgerInterceptException(ex.getCause()));
+ return null;
+ });
} else {
initializeBookKeeper(callback);
}
@@ -464,6 +479,42 @@ public void operationFailed(MetaStoreException e) {
scheduleTimeoutTask();
}
+ protected ManagedLedgerInterceptor.LastEntryHandle createLastEntryHandle(LedgerHandle lh) {
+ return () -> {
+ CompletableFuture> promise = new CompletableFuture<>();
+ if (lh.getLastAddConfirmed() >= 0) {
+ lh.readAsync(lh.getLastAddConfirmed(), lh.getLastAddConfirmed())
+ .whenComplete((entries, ex) -> {
+ if (ex != null) {
+ promise.completeExceptionally(ex);
+ } else {
+ if (entries != null) {
+ try {
+ LedgerEntry ledgerEntry =
+ entries.getEntry(lh.getLastAddConfirmed());
+ if (ledgerEntry != null) {
+ promise.complete(
+ Optional.of(EntryImpl.create(ledgerEntry)));
+ } else {
+ promise.complete(Optional.empty());
+ }
+ entries.close();
+ } catch (Exception e) {
+ entries.close();
+ promise.completeExceptionally(e);
+ }
+ } else {
+ promise.complete(Optional.empty());
+ }
+ }
+ });
+ } else {
+ promise.complete(Optional.empty());
+ }
+ return promise;
+ };
+ }
+
protected synchronized void initializeBookKeeper(final ManagedLedgerInitializeLedgerCallback callback) {
if (log.isDebugEnabled()) {
log.debug("[{}] initializing bookkeeper; ledgers {}", name, ledgers);
@@ -532,14 +583,16 @@ public void operationFailed(MetaStoreException e) {
STATE_UPDATER.set(this, State.LedgerOpened);
updateLastLedgerCreatedTimeAndScheduleRolloverTask();
currentLedger = lh;
+ currentLedgerTimeoutTriggered = new AtomicBoolean();
- lastConfirmedEntry = new PositionImpl(lh.getId(), -1);
+ lastConfirmedEntry = PositionFactory.create(lh.getId(), -1);
// bypass empty ledgers, find last ledger with Message if possible.
while (lastConfirmedEntry.getEntryId() == -1) {
Map.Entry formerLedger = ledgers.lowerEntry(lastConfirmedEntry.getLedgerId());
if (formerLedger != null) {
LedgerInfo ledgerInfo = formerLedger.getValue();
- lastConfirmedEntry = PositionImpl.get(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1);
+ lastConfirmedEntry =
+ PositionFactory.create(ledgerInfo.getLedgerId(), ledgerInfo.getEntries() - 1);
} else {
break;
}
@@ -578,7 +631,7 @@ public void operationComplete(List consumers, Stat s) {
for (final String cursorName : consumers) {
log.info("[{}] Loading cursor {}", name, cursorName);
final ManagedCursorImpl cursor;
- cursor = new ManagedCursorImpl(bookKeeper, config, ManagedLedgerImpl.this, cursorName);
+ cursor = new ManagedCursorImpl(bookKeeper, ManagedLedgerImpl.this, cursorName);
cursor.recover(new VoidCallback() {
@Override
@@ -609,7 +662,7 @@ public void operationFailed(ManagedLedgerException exception) {
log.debug("[{}] Recovering cursor {} lazily", name, cursorName);
}
final ManagedCursorImpl cursor;
- cursor = new ManagedCursorImpl(bookKeeper, config, ManagedLedgerImpl.this, cursorName);
+ cursor = new ManagedCursorImpl(bookKeeper, ManagedLedgerImpl.this, cursorName);
CompletableFuture cursorRecoveryFuture = new CompletableFuture<>();
uninitializedCursors.put(cursorName, cursorRecoveryFuture);
@@ -652,7 +705,7 @@ private void addCursor(ManagedCursorImpl cursor) {
if (cursor.isDurable()) {
positionForOrdering = cursor.getMarkDeletedPosition();
if (positionForOrdering == null) {
- positionForOrdering = PositionImpl.EARLIEST;
+ positionForOrdering = PositionFactory.EARLIEST;
}
}
cursors.add(cursor, positionForOrdering);
@@ -675,37 +728,7 @@ public Position addEntry(byte[] data, int numberOfMessages) throws InterruptedEx
@Override
public Position addEntry(byte[] data, int offset, int length) throws InterruptedException, ManagedLedgerException {
- final CountDownLatch counter = new CountDownLatch(1);
- // Result list will contain the status exception and the resulting
- // position
- class Result {
- ManagedLedgerException status = null;
- Position position = null;
- }
- final Result result = new Result();
-
- asyncAddEntry(data, offset, length, new AddEntryCallback() {
- @Override
- public void addComplete(Position position, ByteBuf entryData, Object ctx) {
- result.position = position;
- counter.countDown();
- }
-
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- result.status = exception;
- counter.countDown();
- }
- }, null);
-
- counter.await();
-
- if (result.status != null) {
- log.error("[{}] Error adding entry", name, result.status);
- throw result.status;
- }
-
- return result.position;
+ return addEntry(data, 1, offset, length);
}
@Override
@@ -765,18 +788,7 @@ public void asyncAddEntry(final byte[] data, int numberOfMessages, int offset, i
@Override
public void asyncAddEntry(ByteBuf buffer, AddEntryCallback callback, Object ctx) {
- if (log.isDebugEnabled()) {
- log.debug("[{}] asyncAddEntry size={} state={}", name, buffer.readableBytes(), state);
- }
-
- // retain buffer in this thread
- buffer.retain();
-
- // Jump to specific thread to avoid contention from writers writing from different threads
- executor.execute(() -> {
- OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, callback, ctx);
- internalAsyncAddEntry(addOperation);
- });
+ asyncAddEntry(buffer, 1, callback, ctx);
}
@Override
@@ -790,7 +802,8 @@ public void asyncAddEntry(ByteBuf buffer, int numberOfMessages, AddEntryCallback
// Jump to specific thread to avoid contention from writers writing from different threads
executor.execute(() -> {
- OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, numberOfMessages, callback, ctx);
+ OpAddEntry addOperation = OpAddEntry.createNoRetainBuffer(this, buffer, numberOfMessages, callback, ctx,
+ currentLedgerTimeoutTriggered);
internalAsyncAddEntry(addOperation);
});
}
@@ -842,6 +855,7 @@ protected synchronized void internalAsyncAddEntry(OpAddEntry addOperation) {
// Write into lastLedger
addOperation.setLedger(currentLedger);
+ addOperation.setTimeoutTriggered(currentLedgerTimeoutTriggered);
++currentLedgerEntries;
currentLedgerSize += addOperation.data.readableBytes();
@@ -973,7 +987,8 @@ public synchronized void asyncOpenCursor(final String cursorName, final InitialP
if (uninitializedCursors.containsKey(cursorName)) {
uninitializedCursors.get(cursorName).thenAccept(cursor -> callback.openCursorComplete(cursor, ctx))
.exceptionally(ex -> {
- callback.openCursorFailed((ManagedLedgerException) ex, ctx);
+ callback.openCursorFailed(ManagedLedgerException
+ .getManagedLedgerException(FutureUtil.unwrapCompletionException(ex)), ctx);
return null;
});
return;
@@ -991,10 +1006,10 @@ public synchronized void asyncOpenCursor(final String cursorName, final InitialP
if (log.isDebugEnabled()) {
log.debug("[{}] Creating new cursor: {}", name, cursorName);
}
- final ManagedCursorImpl cursor = new ManagedCursorImpl(bookKeeper, config, this, cursorName);
+ final ManagedCursorImpl cursor = new ManagedCursorImpl(bookKeeper, this, cursorName);
CompletableFuture cursorFuture = new CompletableFuture<>();
uninitializedCursors.put(cursorName, cursorFuture);
- PositionImpl position = InitialPosition.Earliest == initialPosition ? getFirstPosition() : getLastPosition();
+ Position position = InitialPosition.Earliest == initialPosition ? getFirstPosition() : getLastPosition();
cursor.initialize(position, properties, cursorProperties, new VoidCallback() {
@Override
public void operationComplete() {
@@ -1032,6 +1047,7 @@ public synchronized void asyncDeleteCursor(final String consumerName, final Dele
+ consumerName), ctx);
return;
} else if (!cursor.isDurable()) {
+ cursor.setState(ManagedCursorImpl.State.Closed);
cursors.removeCursor(consumerName);
deactivateCursorByName(consumerName);
callback.deleteCursorComplete(ctx);
@@ -1123,8 +1139,8 @@ public ManagedCursor newNonDurableCursor(Position startCursorPosition, String cu
return cachedCursor;
}
- NonDurableCursorImpl cursor = new NonDurableCursorImpl(bookKeeper, config, this, cursorName,
- (PositionImpl) startCursorPosition, initialPosition, isReadCompacted);
+ NonDurableCursorImpl cursor = new NonDurableCursorImpl(bookKeeper, this, cursorName,
+ startCursorPosition, initialPosition, isReadCompacted);
cursor.setActive();
log.info("[{}] Opened new cursor: {}", name, cursor);
@@ -1163,7 +1179,7 @@ public long getNumberOfEntries() {
@Override
public long getNumberOfActiveEntries() {
long totalEntries = getNumberOfEntries();
- PositionImpl pos = cursors.getSlowestReaderPosition();
+ Position pos = cursors.getSlowestReaderPosition();
if (pos == null) {
// If there are no consumers, there are no active entries
return 0;
@@ -1182,7 +1198,7 @@ public long getTotalSize() {
@Override
public long getEstimatedBacklogSize() {
- PositionImpl pos = getMarkDeletePositionOfSlowestConsumer();
+ Position pos = getMarkDeletePositionOfSlowestConsumer();
while (true) {
if (pos == null) {
@@ -1226,18 +1242,18 @@ public long getEstimatedBacklogSize() {
@Override
public CompletableFuture getEarliestMessagePublishTimeInBacklog() {
- PositionImpl pos = getMarkDeletePositionOfSlowestConsumer();
+ Position pos = getMarkDeletePositionOfSlowestConsumer();
return getEarliestMessagePublishTimeOfPos(pos);
}
- public CompletableFuture getEarliestMessagePublishTimeOfPos(PositionImpl pos) {
+ private CompletableFuture getEarliestMessagePublishTimeOfPos(Position pos) {
CompletableFuture future = new CompletableFuture<>();
if (pos == null) {
future.complete(0L);
return future;
}
- PositionImpl nextPos = getNextValidPosition(pos);
+ Position nextPos = getNextValidPosition(pos);
if (nextPos.compareTo(lastConfirmedEntry) > 0) {
return CompletableFuture.completedFuture(-1L);
@@ -1276,14 +1292,14 @@ public String toString() {
/**
* Get estimated backlog size from a specific position.
*/
- public long getEstimatedBacklogSize(PositionImpl pos) {
+ public long getEstimatedBacklogSize(Position pos) {
if (pos == null) {
return 0;
}
return estimateBacklogFromPosition(pos);
}
- long estimateBacklogFromPosition(PositionImpl pos) {
+ long estimateBacklogFromPosition(Position pos) {
synchronized (this) {
long sizeBeforePosLedger = ledgers.headMap(pos.getLedgerId()).values()
.stream().mapToLong(LedgerInfo::getSize).sum();
@@ -1362,7 +1378,7 @@ public synchronized void asyncTerminate(TerminateCallback callback, Object ctx)
if (rc != BKException.Code.OK) {
callback.terminateFailed(createManagedLedgerException(rc), ctx);
} else {
- lastConfirmedEntry = new PositionImpl(lh.getId(), lh.getLastAddConfirmed());
+ lastConfirmedEntry = PositionFactory.create(lh.getId(), lh.getLastAddConfirmed());
// Store the new state in metadata
store.asyncUpdateLedgerIds(name, getManagedLedgerInfo(), ledgersStat, new MetaStoreCallback() {
@Override
@@ -1584,6 +1600,7 @@ public void operationComplete(Void v, Stat stat) {
LedgerHandle originalCurrentLedger = currentLedger;
ledgers.put(lh.getId(), newLedger);
currentLedger = lh;
+ currentLedgerTimeoutTriggered = new AtomicBoolean();
currentLedgerEntries = 0;
currentLedgerSize = 0;
updateLedgersIdsComplete(originalCurrentLedger);
@@ -1667,9 +1684,12 @@ void createNewOpAddEntryForNewLedger() {
if (existsOp != null) {
// If op is used by another ledger handle, we need to close it and create a new one
if (existsOp.ledger != null) {
- existsOp.close();
- existsOp = OpAddEntry.createNoRetainBuffer(existsOp.ml, existsOp.data,
- existsOp.getNumberOfMessages(), existsOp.callback, existsOp.ctx);
+ existsOp = existsOp.duplicateAndClose(currentLedgerTimeoutTriggered);
+ } else {
+ // It may happen when the following operations execute at the same time, so it is expected.
+ // - Adding entry.
+ // - Switching ledger.
+ existsOp.setTimeoutTriggered(currentLedgerTimeoutTriggered);
}
existsOp.setLedger(currentLedger);
pendingAddEntries.add(existsOp);
@@ -1819,7 +1839,7 @@ public CompletableFuture asyncFindPosition(Predicate predicate)
CompletableFuture future = new CompletableFuture<>();
Long firstLedgerId = ledgers.firstKey();
- final PositionImpl startPosition = firstLedgerId == null ? null : new PositionImpl(firstLedgerId, 0);
+ final Position startPosition = firstLedgerId == null ? null : PositionFactory.create(firstLedgerId, 0);
if (startPosition == null) {
future.complete(null);
return future;
@@ -1833,7 +1853,7 @@ public void findEntryComplete(Position position, Object ctx) {
log.info("[{}] Unable to find position for predicate {}. Use the first position {} instead.", name,
predicate, startPosition);
} else {
- finalPosition = getNextValidPosition((PositionImpl) position);
+ finalPosition = getNextValidPosition(position);
}
future.complete(finalPosition);
}
@@ -1884,7 +1904,7 @@ void asyncReadEntries(OpReadEntry opReadEntry) {
if (ledgerInfo == null || ledgerInfo.getEntries() == 0) {
// Cursor is pointing to an empty ledger, there's no need to try opening it. Skip this ledger and
// move to the next one
- opReadEntry.updateReadPosition(new PositionImpl(opReadEntry.readPosition.getLedgerId() + 1, 0));
+ opReadEntry.updateReadPosition(PositionFactory.create(opReadEntry.readPosition.getLedgerId() + 1, 0));
opReadEntry.checkReadCompletion();
return;
}
@@ -1901,12 +1921,12 @@ void asyncReadEntries(OpReadEntry opReadEntry) {
}
}
- public CompletableFuture getLedgerMetadata(long ledgerId) {
+ public CompletableFuture getLedgerMetadata(long ledgerId) {
LedgerHandle currentLedger = this.currentLedger;
if (currentLedger != null && ledgerId == currentLedger.getId()) {
- return CompletableFuture.completedFuture(currentLedger.getLedgerMetadata().toSafeString());
+ return CompletableFuture.completedFuture(currentLedger.getLedgerMetadata());
} else {
- return getLedgerHandle(ledgerId).thenApply(rh -> rh.getLedgerMetadata().toSafeString());
+ return getLedgerHandle(ledgerId).thenApply(rh -> rh.getLedgerMetadata());
}
}
@@ -2012,7 +2032,8 @@ public void invalidateLedgerHandle(ReadHandle ledgerHandle) {
}
}
- public void asyncReadEntry(PositionImpl position, ReadEntryCallback callback, Object ctx) {
+ @Override
+ public void asyncReadEntry(Position position, ReadEntryCallback callback, Object ctx) {
LedgerHandle currentLedger = this.currentLedger;
if (log.isDebugEnabled()) {
log.debug("[{}] Reading entry ledger {}: {}", name, position.getLedgerId(), position.getEntryId());
@@ -2045,7 +2066,7 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry)
long firstEntry = opReadEntry.readPosition.getEntryId();
long lastEntryInLedger;
- PositionImpl lastPosition = lastConfirmedEntry;
+ Position lastPosition = lastConfirmedEntry;
if (ledger.getId() == lastPosition.getLedgerId()) {
// For the current ledger, we only give read visibility to the last entry we have received a confirmation in
@@ -2072,9 +2093,9 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry)
// beginning of the next ledger
Long nextLedgerId = ledgers.ceilingKey(ledger.getId() + 1);
if (nextLedgerId != null) {
- opReadEntry.updateReadPosition(new PositionImpl(nextLedgerId, 0));
+ opReadEntry.updateReadPosition(PositionFactory.create(nextLedgerId, 0));
} else {
- opReadEntry.updateReadPosition(new PositionImpl(ledger.getId() + 1, 0));
+ opReadEntry.updateReadPosition(PositionFactory.create(ledger.getId() + 1, 0));
}
} else {
opReadEntry.updateReadPosition(opReadEntry.readPosition);
@@ -2092,7 +2113,7 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry)
long lastValidEntry = -1L;
long entryId = firstEntry;
for (; entryId <= lastEntry; entryId++) {
- if (opReadEntry.skipCondition.test(PositionImpl.get(ledger.getId(), entryId))) {
+ if (opReadEntry.skipCondition.test(PositionFactory.create(ledger.getId(), entryId))) {
if (firstValidEntry != -1L) {
break;
}
@@ -2109,7 +2130,7 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry)
// then manual call internalReadEntriesComplete to advance read position.
if (firstValidEntry == -1L) {
opReadEntry.internalReadEntriesComplete(Collections.emptyList(), opReadEntry.ctx,
- PositionImpl.get(ledger.getId(), lastEntry));
+ PositionFactory.create(ledger.getId(), lastEntry));
return;
}
@@ -2124,7 +2145,7 @@ private void internalReadFromLedger(ReadHandle ledger, OpReadEntry opReadEntry)
asyncReadEntry(ledger, firstEntry, lastEntry, opReadEntry, opReadEntry.ctx);
}
- protected void asyncReadEntry(ReadHandle ledger, PositionImpl position, ReadEntryCallback callback, Object ctx) {
+ protected void asyncReadEntry(ReadHandle ledger, Position position, ReadEntryCallback callback, Object ctx) {
mbean.addEntriesRead(1);
if (config.getReadEntryTimeoutSeconds() > 0) {
// set readOpCount to uniquely validate if ReadEntryCallbackWrapper is already recycled
@@ -2208,13 +2229,11 @@ public void readEntryComplete(Entry entry, Object ctx) {
Object cbCtx = this.cntx;
if (recycle(reOpCount)) {
callback.readEntryComplete(entry, cbCtx);
- return;
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] read entry already completed for {}-{}", name, ledgerId, entryId);
}
entry.release();
- return;
}
}
@@ -2225,7 +2244,6 @@ public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
Object cbCtx = this.cntx;
if (recycle(reOpCount)) {
callback.readEntryFailed(exception, cbCtx);
- return;
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] read entry already completed for {}-{}", name, ledgerId, entryId);
@@ -2240,13 +2258,11 @@ public void readEntriesComplete(List returnedEntries, Object ctx) {
Object cbCtx = this.cntx;
if (recycle(reOpCount)) {
callback.readEntriesComplete(returnedEntries, cbCtx);
- return;
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] read entry already completed for {}-{}", name, ledgerId, entryId);
}
returnedEntries.forEach(Entry::release);
- return;
}
}
@@ -2257,12 +2273,10 @@ public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
Object cbCtx = this.cntx;
if (recycle(reOpCount)) {
callback.readEntriesFailed(exception, cbCtx);
- return;
} else {
if (log.isDebugEnabled()) {
log.debug("[{}] read entry already completed for {}-{}", name, ledgerId, entryId);
}
- return;
}
}
@@ -2309,8 +2323,8 @@ public ManagedLedgerMXBean getStats() {
return mbean;
}
- public boolean hasMoreEntries(PositionImpl position) {
- PositionImpl lastPos = lastConfirmedEntry;
+ public boolean hasMoreEntries(Position position) {
+ Position lastPos = lastConfirmedEntry;
boolean result = position.compareTo(lastPos) <= 0;
if (log.isDebugEnabled()) {
log.debug("[{}] hasMoreEntries: pos={} lastPos={} res={}", name, position, lastPos, result);
@@ -2331,7 +2345,7 @@ private void invalidateEntriesUpToSlowestReaderPosition() {
return;
}
if (!activeCursors.isEmpty()) {
- PositionImpl evictionPos = activeCursors.getSlowestReaderPosition();
+ Position evictionPos = activeCursors.getSlowestReaderPosition();
if (evictionPos != null) {
entryCache.invalidateEntries(evictionPos);
}
@@ -2340,7 +2354,7 @@ private void invalidateEntriesUpToSlowestReaderPosition() {
}
}
- void onCursorMarkDeletePositionUpdated(ManagedCursorImpl cursor, PositionImpl newPosition) {
+ void onCursorMarkDeletePositionUpdated(ManagedCursorImpl cursor, Position newPosition) {
if (config.isCacheEvictionByMarkDeletedPosition()) {
updateActiveCursor(cursor, newPosition);
}
@@ -2348,15 +2362,15 @@ void onCursorMarkDeletePositionUpdated(ManagedCursorImpl cursor, PositionImpl ne
// non-durable cursors aren't tracked for trimming
return;
}
- Pair pair = cursors.cursorUpdated(cursor, newPosition);
+ Pair pair = cursors.cursorUpdated(cursor, newPosition);
if (pair == null) {
// Cursor has been removed in the meantime
trimConsumedLedgersInBackground();
return;
}
- PositionImpl previousSlowestReader = pair.getLeft();
- PositionImpl currentSlowestReader = pair.getRight();
+ Position previousSlowestReader = pair.getLeft();
+ Position currentSlowestReader = pair.getRight();
if (previousSlowestReader.compareTo(currentSlowestReader) == 0) {
// The slowest consumer has not changed position. Nothing to do right now
@@ -2370,7 +2384,7 @@ void onCursorMarkDeletePositionUpdated(ManagedCursorImpl cursor, PositionImpl ne
}
private void updateActiveCursor(ManagedCursorImpl cursor, Position newPosition) {
- Pair slowestPositions = activeCursors.cursorUpdated(cursor, newPosition);
+ Pair slowestPositions = activeCursors.cursorUpdated(cursor, newPosition);
if (slowestPositions != null
&& !slowestPositions.getLeft().equals(slowestPositions.getRight())) {
invalidateEntriesUpToSlowestReaderPosition();
@@ -2383,12 +2397,12 @@ public void onCursorReadPositionUpdated(ManagedCursorImpl cursor, Position newRe
}
}
- PositionImpl startReadOperationOnLedger(PositionImpl position) {
+ Position startReadOperationOnLedger(Position position) {
Long ledgerId = ledgers.ceilingKey(position.getLedgerId());
if (ledgerId != null && ledgerId != position.getLedgerId()) {
// The ledger pointed by this position does not exist anymore. It was deleted because it was empty. We need
// to skip on the next available ledger
- position = new PositionImpl(ledgerId, 0);
+ position = PositionFactory.create(ledgerId, 0);
}
return position;
@@ -2422,7 +2436,7 @@ public void addWaitingEntryCallBack(WaitingEntryCallBack cb) {
public void maybeUpdateCursorBeforeTrimmingConsumedLedger() {
for (ManagedCursor cursor : cursors) {
- PositionImpl lastAckedPosition = (PositionImpl) cursor.getMarkDeletedPosition();
+ Position lastAckedPosition = cursor.getMarkDeletedPosition();
LedgerInfo currPointedLedger = ledgers.get(lastAckedPosition.getLedgerId());
LedgerInfo nextPointedLedger = Optional.ofNullable(ledgers.higherEntry(lastAckedPosition.getLedgerId()))
.map(Map.Entry::getValue).orElse(null);
@@ -2431,7 +2445,7 @@ public void maybeUpdateCursorBeforeTrimmingConsumedLedger() {
if (nextPointedLedger != null) {
if (lastAckedPosition.getEntryId() != -1
&& lastAckedPosition.getEntryId() + 1 >= currPointedLedger.getEntries()) {
- lastAckedPosition = new PositionImpl(nextPointedLedger.getLedgerId(), -1);
+ lastAckedPosition = PositionFactory.create(nextPointedLedger.getLedgerId(), -1);
}
} else {
log.debug("No need to reset cursor: {}, current ledger is the last ledger.", cursor);
@@ -2471,9 +2485,8 @@ private void scheduleDeferredTrimming(boolean isTruncate, CompletableFuture p
100, TimeUnit.MILLISECONDS);
}
- private void maybeOffloadInBackground(CompletableFuture promise) {
- if (config.getLedgerOffloader() == null || config.getLedgerOffloader() == NullLedgerOffloader.INSTANCE
- || config.getLedgerOffloader().getOffloadPolicies() == null) {
+ public void maybeOffloadInBackground(CompletableFuture promise) {
+ if (getOffloadPoliciesIfAppendable().isEmpty()) {
return;
}
@@ -2488,9 +2501,8 @@ private void maybeOffloadInBackground(CompletableFuture promise) {
}
private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInSeconds,
- CompletableFuture finalPromise) {
- if (config.getLedgerOffloader() == null || config.getLedgerOffloader() == NullLedgerOffloader.INSTANCE
- || config.getLedgerOffloader().getOffloadPolicies() == null) {
+ CompletableFuture finalPromise) {
+ if (getOffloadPoliciesIfAppendable().isEmpty()) {
String msg = String.format("[%s] Nothing to offload due to offloader or offloadPolicies is NULL", name);
finalPromise.completeExceptionally(new IllegalArgumentException(msg));
return;
@@ -2509,7 +2521,7 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS
return;
}
- CompletableFuture unlockingPromise = new CompletableFuture<>();
+ CompletableFuture unlockingPromise = new CompletableFuture<>();
unlockingPromise.whenComplete((res, ex) -> {
offloadMutex.unlock();
if (ex != null) {
@@ -2555,7 +2567,7 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS
+ ", total size = {}, already offloaded = {}, to offload = {}",
name, toOffload.stream().map(LedgerInfo::getLedgerId).collect(Collectors.toList()),
sizeSummed, alreadyOffloadedSize, toOffloadSize);
- offloadLoop(unlockingPromise, toOffload, PositionImpl.LATEST, Optional.empty());
+ offloadLoop(unlockingPromise, toOffload, PositionFactory.LATEST, Optional.empty());
} else {
// offloadLoop will complete immediately with an empty list to offload
log.debug("[{}] Nothing to offload, total size = {}, already offloaded = {}, "
@@ -2563,7 +2575,7 @@ private void maybeOffload(long offloadThresholdInBytes, long offloadThresholdInS
+ "managedLedgerOffloadThresholdInSeconds:{}]",
name, sizeSummed, alreadyOffloadedSize, offloadThresholdInBytes,
TimeUnit.MILLISECONDS.toSeconds(offloadTimeThresholdMillis));
- unlockingPromise.complete(PositionImpl.LATEST);
+ unlockingPromise.complete(PositionFactory.LATEST);
}
}
@@ -2580,6 +2592,7 @@ boolean isOffloadedNeedsDelete(OffloadContext offload, Optional
long elapsedMs = clock.millis() - offload.getTimestamp();
return offloadPolicies.filter(policies -> offload.getComplete() && !offload.getBookkeeperDeleted()
&& policies.getManagedLedgerOffloadDeletionLagInMillis() != null
+ && policies.getManagedLedgerOffloadDeletionLagInMillis() >= 0
&& elapsedMs > policies.getManagedLedgerOffloadDeletionLagInMillis()).isPresent();
}
@@ -2592,6 +2605,16 @@ void internalTrimConsumedLedgers(CompletableFuture promise) {
internalTrimLedgers(false, promise);
}
+ private Optional getOffloadPoliciesIfAppendable() {
+ LedgerOffloader ledgerOffloader = config.getLedgerOffloader();
+ if (ledgerOffloader == null
+ || !ledgerOffloader.isAppendable()
+ || ledgerOffloader.getOffloadPolicies() == null) {
+ return Optional.empty();
+ }
+ return Optional.ofNullable(ledgerOffloader.getOffloadPolicies());
+ }
+
void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) {
if (!factory.isMetadataServiceAvailable()) {
// Defer trimming of ledger if we cannot connect to metadata service
@@ -2607,10 +2630,7 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) {
List ledgersToDelete = new ArrayList<>();
List offloadedLedgersToDelete = new ArrayList<>();
- Optional optionalOffloadPolicies = Optional.ofNullable(config.getLedgerOffloader() != null
- && config.getLedgerOffloader() != NullLedgerOffloader.INSTANCE
- ? config.getLedgerOffloader().getOffloadPolicies()
- : null);
+ Optional optionalOffloadPolicies = getOffloadPoliciesIfAppendable();
synchronized (this) {
if (log.isDebugEnabled()) {
log.debug("[{}] Start TrimConsumedLedgers. ledgers={} totalSize={}", name, ledgers.keySet(),
@@ -2643,7 +2663,7 @@ void internalTrimLedgers(boolean isTruncate, CompletableFuture promise) {
// include lastLedger in the trimming.
slowestReaderLedgerId = currentLedger.getId() + 1;
} else {
- PositionImpl slowestReaderPosition = cursors.getSlowestReaderPosition();
+ Position slowestReaderPosition = cursors.getSlowestReaderPosition();
if (slowestReaderPosition != null) {
// The slowest reader position is the mark delete position.
// If the slowest reader position point the last entry in the ledger x,
@@ -2808,6 +2828,15 @@ public void operationFailed(MetaStoreException e) {
}
}
+ @Override
+ public void rolloverCursorsInBackground() {
+ if (cursors.hasDurableCursors()) {
+ executor.execute(() -> {
+ cursors.forEach(ManagedCursor::periodicRollover);
+ });
+ }
+ }
+
/**
* @param ledgerId the ledger handle which maybe will be released.
* @return if the ledger handle was released.
@@ -2824,7 +2853,7 @@ private boolean releaseReadHandleIfNoLongerRead(long ledgerId, long slowestNonDu
}
protected void doDeleteLedgers(List ledgersToDelete) {
- PositionImpl currentLastConfirmedEntry = lastConfirmedEntry;
+ Position currentLastConfirmedEntry = lastConfirmedEntry;
// Update metadata
for (LedgerInfo ls : ledgersToDelete) {
if (currentLastConfirmedEntry != null && ls.getLedgerId() == currentLastConfirmedEntry.getLedgerId()) {
@@ -2856,7 +2885,7 @@ void advanceCursorsIfNecessary(List