Skip to content

Commit

Permalink
Merge pull request #135 from confluentinc/squah-cache-topicids-topic-…
Browse files Browse the repository at this point in the history
…resolution

Updates for #17285
  • Loading branch information
squah-confluent authored Sep 30, 2024
2 parents 7281eb2 + f5d98e0 commit 7e718fb
Show file tree
Hide file tree
Showing 49 changed files with 1,164 additions and 444 deletions.
4 changes: 2 additions & 2 deletions LICENSE-binary
Original file line number Diff line number Diff line change
Expand Up @@ -260,11 +260,11 @@ rocksdbjni-7.9.2
scala-collection-compat_2.12-2.10.0
scala-collection-compat_2.13-2.10.0
scala-library-2.12.19
scala-library-2.13.14
scala-library-2.13.15
scala-logging_2.12-3.9.5
scala-logging_2.13-3.9.5
scala-reflect-2.12.19
scala-reflect-2.13.14
scala-reflect-2.13.15
scala-java8-compat_2.12-1.0.2
scala-java8-compat_2.13-1.0.2
snappy-java-1.1.10.5
Expand Down
2 changes: 1 addition & 1 deletion bin/kafka-run-class.sh
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ should_include_file() {
base_dir=$(dirname $0)/..

if [ -z "$SCALA_VERSION" ]; then
SCALA_VERSION=2.13.14
SCALA_VERSION=2.13.15
if [[ -f "$base_dir/gradle.properties" ]]; then
SCALA_VERSION=`grep "^scalaVersion=" "$base_dir/gradle.properties" | cut -d= -f 2`
fi
Expand Down
2 changes: 1 addition & 1 deletion bin/windows/kafka-run-class.bat
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ set BASE_DIR=%CD%
popd

IF ["%SCALA_VERSION%"] EQU [""] (
set SCALA_VERSION=2.13.14
set SCALA_VERSION=2.13.15
)

IF ["%SCALA_BINARY_VERSION%"] EQU [""] (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -492,7 +492,7 @@ ShareFetchCollector<K, V> build(
public Set<String> subscription() {
acquireAndEnsureOpen();
try {
return subscriptions.subscription();
return Collections.unmodifiableSet(subscriptions.subscription());
} finally {
release();
}
Expand Down Expand Up @@ -594,7 +594,6 @@ public synchronized ConsumerRecords<K, V> poll(final Duration timeout) {
return ConsumerRecords.empty();
} finally {
kafkaShareConsumerMetrics.recordPollEnd(timer.currentTimeMs());
wakeupTrigger.clearTask();
release();
}
}
Expand All @@ -612,13 +611,16 @@ private ShareFetch<K, V> pollForFetches(final Timer timer) {

// Wait a bit - this is where we will fetch records
Timer pollTimer = time.timer(pollTimeout);
wakeupTrigger.setShareFetchAction(fetchBuffer);

try {
fetchBuffer.awaitNotEmpty(pollTimer);
} catch (InterruptException e) {
log.trace("Timeout during fetch", e);
throw e;
} finally {
timer.update(pollTimer.currentTimeMs());
wakeupTrigger.clearTask();
}

return collect(Collections.emptyMap());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,18 +53,21 @@ public void wakeup() {
FetchAction fetchAction = (FetchAction) task;
fetchAction.fetchBuffer().wakeup();
return new WakeupFuture();
} else if (task instanceof ShareFetchAction) {
ShareFetchAction shareFetchAction = (ShareFetchAction) task;
shareFetchAction.fetchBuffer().wakeup();
return new WakeupFuture();
} else {
return task;
}
});
}

/**
* If there is no pending task, set the pending task active.
* If wakeup was called before setting an active task, the current task will complete exceptionally with
* WakeupException right
* away.
* if there is an active task, throw exception.
* If there is no pending task, set the pending task active.
* If wakeup was called before setting an active task, the current task will complete exceptionally with
* WakeupException right away.
* If there is an active task, throw exception.
* @param currentTask
* @param <T>
* @return
Expand Down Expand Up @@ -105,6 +108,25 @@ public void setFetchAction(final FetchBuffer fetchBuffer) {
}
}

public void setShareFetchAction(final ShareFetchBuffer fetchBuffer) {
final AtomicBoolean throwWakeupException = new AtomicBoolean(false);
pendingTask.getAndUpdate(task -> {
if (task == null) {
return new ShareFetchAction(fetchBuffer);
} else if (task instanceof WakeupFuture) {
throwWakeupException.set(true);
return null;
} else if (task instanceof DisabledWakeups) {
return task;
}
// last active state is still active
throw new IllegalStateException("Last active task is still active");
});
if (throwWakeupException.get()) {
throw new WakeupException();
}
}

public void disableWakeups() {
pendingTask.set(new DisabledWakeups());
}
Expand All @@ -113,7 +135,7 @@ public void clearTask() {
pendingTask.getAndUpdate(task -> {
if (task == null) {
return null;
} else if (task instanceof ActiveFuture || task instanceof FetchAction) {
} else if (task instanceof ActiveFuture || task instanceof FetchAction || task instanceof ShareFetchAction) {
return null;
}
return task;
Expand Down Expand Up @@ -172,4 +194,17 @@ public FetchBuffer fetchBuffer() {
return fetchBuffer;
}
}

static class ShareFetchAction implements Wakeupable {

private final ShareFetchBuffer fetchBuffer;

public ShareFetchAction(ShareFetchBuffer fetchBuffer) {
this.fetchBuffer = fetchBuffer;
}

public ShareFetchBuffer fetchBuffer() {
return fetchBuffer;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.common.internals;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.security.PrivilegedAction;
import java.util.Objects;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;

import javax.security.auth.Subject;

/**
* This strategy combines the functionality of the {@link LegacyStrategy}, {@link ModernStrategy}, and
* {@link UnsupportedStrategy} strategies to provide the legacy APIs as long as they are present and not degraded.
* If the legacy APIs are missing or degraded, this falls back to the modern APIs.
*/
class CompositeStrategy implements SecurityManagerCompatibility {

private static final Logger log = LoggerFactory.getLogger(CompositeStrategy.class);
static final CompositeStrategy INSTANCE = new CompositeStrategy(ReflectiveStrategy.Loader.forName());

private final SecurityManagerCompatibility fallbackStrategy;
private final AtomicReference<SecurityManagerCompatibility> activeStrategy;

// Visible for testing
CompositeStrategy(ReflectiveStrategy.Loader loader) {
SecurityManagerCompatibility initial;
SecurityManagerCompatibility fallback = null;
try {
initial = new LegacyStrategy(loader);
try {
fallback = new ModernStrategy(loader);
// This is expected for JRE 18+
log.debug("Loaded legacy SecurityManager methods, will fall back to modern methods after UnsupportedOperationException");
} catch (NoSuchMethodException | ClassNotFoundException ex) {
// This is expected for JRE <= 17
log.debug("Unable to load modern Subject methods, relying only on legacy methods", ex);
}
} catch (ClassNotFoundException | NoSuchMethodException e) {
try {
initial = new ModernStrategy(loader);
// This is expected for JREs after the removal takes place.
log.debug("Unable to load legacy SecurityManager methods, relying only on modern methods", e);
} catch (NoSuchMethodException | ClassNotFoundException ex) {
initial = new UnsupportedStrategy(e, ex);
// This is not expected in normal use, only in test environments.
log.error("Unable to load legacy SecurityManager methods", e);
log.error("Unable to load modern Subject methods", ex);
}
}
Objects.requireNonNull(initial, "initial strategy must be defined");
activeStrategy = new AtomicReference<>(initial);
fallbackStrategy = fallback;
}

private <T> T performAction(Function<SecurityManagerCompatibility, T> action) {
SecurityManagerCompatibility active = activeStrategy.get();
try {
return action.apply(active);
} catch (UnsupportedOperationException e) {
// If we chose a fallback strategy during loading, switch to it and retry this operation.
if (active != fallbackStrategy && fallbackStrategy != null) {
if (activeStrategy.compareAndSet(active, fallbackStrategy)) {
log.debug("Using fallback strategy after encountering degraded legacy method", e);
}
return action.apply(fallbackStrategy);
}
// If we're already using the fallback strategy, then there's nothing to do to handle these exceptions.
throw e;
}
}

@Override
public <T> T doPrivileged(PrivilegedAction<T> action) {
return performAction(compatibility -> compatibility.doPrivileged(action));
}

@Override
public Subject current() {
return performAction(SecurityManagerCompatibility::current);
}

@Override
public <T> T callAs(Subject subject, Callable<T> action) throws CompletionException {
return performAction(compatibility -> compatibility.callAs(subject, action));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.common.internals;

import java.lang.reflect.Method;
import java.security.PrivilegedAction;
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionException;

import javax.security.auth.Subject;

/**
* This class implements reflective access to the deprecated-for-removal methods of AccessController and Subject.
* <p>Instantiating this class may fail if any of the required classes or methods are not found.
* Method invocations for this class may fail with {@link UnsupportedOperationException} if all methods are found,
* but the operation is not permitted to be invoked.
* <p>This class is expected to be instantiable in JRE >=8 until the removal finally takes place.
*/
@SuppressWarnings("unchecked")
class LegacyStrategy implements SecurityManagerCompatibility {

private final Method doPrivileged;
private final Method getContext;
private final Method getSubject;
private final Method doAs;

// Visible for testing
LegacyStrategy(ReflectiveStrategy.Loader loader) throws ClassNotFoundException, NoSuchMethodException {
Class<?> accessController = loader.loadClass("java.security.AccessController");
doPrivileged = accessController.getDeclaredMethod("doPrivileged", PrivilegedAction.class);
getContext = accessController.getDeclaredMethod("getContext");
Class<?> accessControlContext = loader.loadClass("java.security.AccessControlContext");
Class<?> subject = loader.loadClass(Subject.class.getName());
getSubject = subject.getDeclaredMethod("getSubject", accessControlContext);
// Note that the Subject class isn't deprecated or removed, so reference it as an argument type.
// This allows for mocking out the method implementation while still accepting Subject instances as arguments.
doAs = subject.getDeclaredMethod("doAs", Subject.class, PrivilegedExceptionAction.class);
}

@Override
public <T> T doPrivileged(PrivilegedAction<T> action) {
return (T) ReflectiveStrategy.invoke(doPrivileged, null, action);
}

/**
* @return the result of AccessController.getContext(), of type AccessControlContext
*/
private Object getContext() {
return ReflectiveStrategy.invoke(getContext, null);
}

/**
* @param context The current AccessControlContext
* @return The result of Subject.getSubject(AccessControlContext)
*/
private Subject getSubject(Object context) {
return (Subject) ReflectiveStrategy.invoke(getSubject, null, context);
}

@Override
public Subject current() {
return getSubject(getContext());
}

/**
* @return The result of Subject.doAs(Subject, PrivilegedExceptionAction)
*/
private <T> T doAs(Subject subject, PrivilegedExceptionAction<T> action) throws PrivilegedActionException {
return (T) ReflectiveStrategy.invokeChecked(doAs, PrivilegedActionException.class, null, subject, action);
}

@Override
public <T> T callAs(Subject subject, Callable<T> callable) throws CompletionException {
try {
return doAs(subject, callable::call);
} catch (PrivilegedActionException e) {
throw new CompletionException(e.getCause());
}
}
}
Loading

0 comments on commit 7e718fb

Please sign in to comment.