Merge branch 'thelastpickle:master' into fail-repair-after-range-move…

…ment

.github/scripts/run-tests.sh

@@ -217,33 +217,33 @@ case "${TEST_TYPE}" in
         sudo apt-get update
         sudo apt-get install jq -y
         mvn -B package -DskipTests
-        docker-compose -f ./src/packaging/docker-build/docker-compose.yml build
-        docker-compose -f ./src/packaging/docker-build/docker-compose.yml run build
+        docker compose -f ./src/packaging/docker-build/docker-compose.yml build
+        docker compose -f ./src/packaging/docker-build/docker-compose.yml run build
         VERSION=$(printf 'VER\t${project.version}' | mvn help:evaluate | grep '^VER' | cut -f2)
         docker build --build-arg SHADED_JAR=src/server/target/cassandra-reaper-${VERSION}.jar -f src/server/src/main/docker/Dockerfile -t cassandra-reaper:latest .
         docker images
 
         # Clear out Cassandra data before starting a new cluster
         sudo rm -vfr ./src/packaging/data/
 
-        docker-compose -f ./src/packaging/docker-compose.yml up -d cassandra
-        sleep 30 && docker-compose -f ./src/packaging/docker-compose.yml run cqlsh-initialize-reaper_db
-        sleep 10 && docker-compose -f ./src/packaging/docker-compose.yml up -d reaper
+        docker compose -f ./src/packaging/docker-compose.yml up -d cassandra
+        sleep 30 && docker compose -f ./src/packaging/docker-compose.yml run cqlsh-initialize-reaper_db
+        sleep 10 && docker compose -f ./src/packaging/docker-compose.yml up -d reaper
         docker ps -a
 
         # requests python package is needed to use spreaper
         pip install requests
         mkdir -p ~/.reaper
         echo "admin" > ~/.reaper/credentials
         sleep 30 && src/packaging/bin/spreaper login admin
-        src/packaging/bin/spreaper add-cluster $(docker-compose -f ./src/packaging/docker-compose.yml run nodetool status | grep UN | tr -s ' ' | cut -d' ' -f2) 7199 > cluster.json
+        src/packaging/bin/spreaper add-cluster $(docker compose -f ./src/packaging/docker-compose.yml run nodetool status | grep UN | tr -s ' ' | cut -d' ' -f2) 7199 > cluster.json
         cat cluster.json
         cluster_name=$(cat cluster.json|grep -v "#" | jq -r '.name')
         if [[ "$cluster_name" != "reaper-cluster" ]]; then
             echo "Failed registering cluster in Reaper running in Docker"
             exit 1
         fi
-        sleep 5 && docker-compose -f ./src/packaging/docker-compose.yml down
+        sleep 5 && docker compose -f ./src/packaging/docker-compose.yml down
         ;;
     *)
         echo "Skipping, no actions for TEST_TYPE=${TEST_TYPE}."

src/docs/content/docs/backends/memory.md

@@ -11,6 +11,9 @@ To use in memory storage as the storage type for Reaper, the `storageType` setti
 
 ```yaml
 storageType: memory
+persistenceStoragePath: /var/lib/cassandra-reaper/storage
 ```
 
 In-memory storage is volatile and as such all registered cluster, column families and repair information will be lost upon service restart. This storage setting is intended for testing purposes only.
+
+Starting from 3.6.0, persistenceStoragePath is required for memory storage type. This enable lightweight deployments of Reaper, without requiring the use of a Cassandra database. It will store the data locally and reload them consistently upon startup.

src/server/pom.xml

@@ -33,7 +33,7 @@
         <jersey.version>2.35</jersey.version>
         <logback.version>1.3.14</logback.version>
         <cxf.version>3.4.5</cxf.version>
-        <shiro.version>1.12.0</shiro.version>
+        <shiro.version>1.13.0</shiro.version>
         <prometheus.version>0.12.0</prometheus.version>
         <docker.directory>src/main/docker</docker.directory>
         <timestamp>${maven.build.timestamp}</timestamp>
@@ -169,12 +169,12 @@
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-handler</artifactId>
-            <version>4.1.70.Final</version>
+            <version>4.1.94.Final</version>
         </dependency>
         <dependency>
             <groupId>com.datastax.cassandra</groupId>
             <artifactId>cassandra-driver-core</artifactId>
-            <version>3.11.0</version>
+            <version>3.11.5</version>
             <exclusions>
 	            <exclusion>
 		            <groupId>com.fasterxml.jackson.core</groupId>

src/server/src/main/docker/Dockerfile

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM amazoncorretto:11.0.22-alpine
+FROM amazoncorretto:11.0.25-alpine
 
 ARG SHADED_JAR
 
@@ -84,6 +84,7 @@ ENV REAPER_SEGMENT_COUNT_PER_NODE=64 \
     REAPER_HTTP_MANAGEMENT_ENABLE="false" \
     REAPER_HTTP_MANAGEMENT_KEYSTORE_PATH="" \
     REAPER_HTTP_MANAGEMENT_TRUSTSTORE_PATH="" \
+    REAPER_HTTP_MANAGEMENT_TRUSTSTORES_DIR="" \
     REAPER_TMP_DIRECTORY="/var/tmp/cassandra-reaper" \
     REAPER_MEMORY_STORAGE_DIRECTORY="/var/lib/cassandra-reaper/storage"
 

src/server/src/main/docker/cassandra-reaper.yml

@@ -88,4 +88,5 @@ httpManagement:
   mgmtApiMetricsPort: ${REAPER_MGMT_API_METRICS_PORT}
   keystore: ${REAPER_HTTP_MANAGEMENT_KEYSTORE_PATH}
   truststore: ${REAPER_HTTP_MANAGEMENT_TRUSTSTORE_PATH}
+  truststoresDir: ${REAPER_HTTP_MANAGEMENT_TRUSTSTORES_DIR}
 

src/server/src/main/java/io/cassandrareaper/ReaperApplication.java

@@ -50,6 +50,8 @@
 import io.cassandrareaper.storage.IDistributedStorage;
 import io.cassandrareaper.storage.InitializeStorage;
 
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.EnumSet;
 import java.util.Optional;
 import java.util.concurrent.ScheduledExecutorService;
@@ -341,8 +343,9 @@ private void maybeInitializeSidecarMode(ClusterResource addClusterResource) thro
 
   private boolean selfRegisterClusterForSidecar(ClusterResource addClusterResource, String seedHost)
       throws ReaperException {
-    final Optional<Cluster> cluster = addClusterResource.findClusterWithSeedHost(seedHost, Optional.empty(),
-        Optional.empty());
+    final Optional<Cluster> cluster = addClusterResource.findClusterWithSeedHost(
+        seedHost, Optional.empty(),Optional.empty()
+    );
     if (!cluster.isPresent()) {
       return false;
     }
@@ -427,6 +430,19 @@ private void checkConfiguration(ReaperApplicationConfiguration config) {
     LOG.debug("repairParallelism: {}", config.getRepairParallelism());
     LOG.debug("hangingRepairTimeoutMins: {}", config.getHangingRepairTimeoutMins());
     LOG.debug("jmxPorts: {}", config.getJmxPorts());
+
+    if (config.getHttpManagement() != null) {
+      if (config.getHttpManagement().isEnabled()) {
+        if (config.getHttpManagement().getTruststoresDir() != null) {
+          if (!Files.exists(Paths.get(config.getHttpManagement().getTruststoresDir()))) {
+            throw new RuntimeException(String.format(
+                "HttpManagement truststores directory is configured as %s but it does not exist",
+                config.getHttpManagement().getTruststoresDir()
+            ));
+          }
+        }
+      }
+    }
   }
 
   private void tryInitializeStorage(ReaperApplicationConfiguration config, Environment environment)

src/server/src/main/java/io/cassandrareaper/ReaperApplicationConfiguration.java

@@ -775,6 +775,9 @@ public static final class HttpManagement {
     @JsonProperty
     private String truststore;
 
+    @JsonProperty
+    private String truststoresDir;
+
     @JsonProperty
     private Integer mgmtApiMetricsPort;
 
@@ -794,6 +797,10 @@ public String getTruststore() {
       return truststore;
     }
 
+    public String getTruststoresDir() {
+      return truststoresDir;
+    }
+
     @VisibleForTesting
     public void setEnabled(Boolean enabled) {
       this.enabled = enabled;
@@ -809,6 +816,11 @@ public void setTruststore(String truststore) {
       this.truststore = truststore;
     }
 
+    @VisibleForTesting
+    public void setTruststoresDir(String truststoresDir) {
+      this.truststoresDir = truststoresDir;
+    }
+
     public int getMgmtApiMetricsPort() {
       return mgmtApiMetricsPort == null ? DEFAULT_MGMT_API_METRICS_PORT : mgmtApiMetricsPort;
     }

src/server/src/main/java/io/cassandrareaper/management/ClusterFacade.java

@@ -893,12 +893,10 @@ public ICassandraManagementProxy connect(Node node, Collection<String> endpoints
   private ICassandraManagementProxy connectImpl(Cluster cluster, Collection<String> endpoints)
           throws ReaperException {
     try {
-      ICassandraManagementProxy proxy = context.managementConnectionFactory
-              .connectAny(
-          endpoints
-              .stream()
-              .map(host -> Node.builder().withCluster(cluster).withHostname(host).build())
-              .collect(Collectors.toList()));
+      ICassandraManagementProxy proxy = context.managementConnectionFactory.connectAny(endpoints.stream()
+          .map(host -> Node.builder().withCluster(cluster).withHostname(host).build())
+          .collect(Collectors.toList())
+      );
 
       Async.markClusterActive(cluster, context);
       return proxy;

src/server/src/main/java/io/cassandrareaper/management/http/HttpManagementConnectionFactory.java

@@ -77,6 +77,11 @@
 
 public class HttpManagementConnectionFactory implements IManagementConnectionFactory {
   private static final char[] KEYSTORE_PASSWORD = "changeit".toCharArray();
+
+  private static final String KEYSTORE_COMPONENT_NAME = "keystore.jks";
+
+  private static final String TRUSTSTORE_COMPONENT_NAME = "truststore.jks";
+
   private static final Logger LOG = LoggerFactory.getLogger(HttpManagementConnectionFactory.class);
   private static final ConcurrentMap<String, HttpCassandraManagementProxy> HTTP_CONNECTIONS = Maps.newConcurrentMap();
   private final MetricRegistry metricRegistry;
@@ -95,13 +100,22 @@ public HttpManagementConnectionFactory(AppContext context, ScheduledExecutorServ
     this.config = context.config;
     registerConnectionsGauge();
     this.jobStatusPollerExecutor = jobStatusPollerExecutor;
-    if (context.config.getHttpManagement().getKeystore() != null && !context.config.getHttpManagement().getKeystore()
-        .isEmpty()) {
-      try {
-        createSslWatcher();
-      } catch (IOException e) {
-        throw new RuntimeException(e);
+
+    String ts = context.config.getHttpManagement().getTruststore();
+    boolean watchTruststore = ts != null && !ts.isEmpty();
+    String ks = context.config.getHttpManagement().getKeystore();
+    boolean watchKeystore = ks != null && !ks.isEmpty();
+    String tsd = context.config.getHttpManagement().getTruststoresDir();
+    boolean watchTruststoreDir = tsd != null && !tsd.isEmpty() && Files.isDirectory(Paths.get(tsd));
+
+    try {
+      if (watchKeystore || watchTruststore || watchTruststoreDir) {
+        createSslWatcher(watchTruststore, watchKeystore, watchTruststoreDir);
+      } else {
+        LOG.debug("Not setting up any SSL watchers");
       }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
     }
   }
 
@@ -170,19 +184,26 @@ private HttpCassandraManagementProxy connectImpl(Node node)
       @Override
       public HttpCassandraManagementProxy apply(@Nullable String hostName) {
         ReaperApplicationConfiguration.HttpManagement httpConfig = config.getHttpManagement();
-        boolean useMtls = httpConfig.getKeystore() != null && !httpConfig.getKeystore().isEmpty();
+
+        boolean useMtls = (httpConfig.getKeystore() != null && !httpConfig.getKeystore().isEmpty())
+            || (httpConfig.getTruststoresDir() != null && !httpConfig.getTruststoresDir().isEmpty());
 
         OkHttpClient.Builder clientBuilder = new OkHttpClient().newBuilder();
 
         String protocol = "http";
+
         if (useMtls) {
+
+          Path truststoreName = getTruststoreComponentPath(node, TRUSTSTORE_COMPONENT_NAME);
+          Path keystoreName = getTruststoreComponentPath(node, KEYSTORE_COMPONENT_NAME);
+
           LOG.debug("Using TLS connection to " + node.getHostname());
           // We have to split TrustManagers to its own function to please OkHttpClient
           TrustManager[] trustManagers;
           SSLContext sslContext;
           try {
-            trustManagers = getTrustManagers();
-            sslContext = createSslContext(trustManagers);
+            trustManagers = getTrustManagers(truststoreName);
+            sslContext = createSslContext(trustManagers, keystoreName);
           } catch (ReaperException e) {
             LOG.error("Failed to create SSLContext: " + e.getLocalizedMessage(), e);
             throw new RuntimeException(e);
@@ -218,8 +239,7 @@ public HttpCassandraManagementProxy apply(@Nullable String hostName) {
   }
 
   @VisibleForTesting
-  SSLContext createSslContext(TrustManager[] tms) throws ReaperException {
-    Path keyStorePath = Paths.get(config.getHttpManagement().getKeystore());
+  SSLContext createSslContext(TrustManager[] tms, Path keyStorePath) throws ReaperException {
 
     try (InputStream ksIs = Files.newInputStream(keyStorePath, StandardOpenOption.READ)) {
 
@@ -238,8 +258,11 @@ SSLContext createSslContext(TrustManager[] tms) throws ReaperException {
     }
   }
 
-  private TrustManager[] getTrustManagers() throws ReaperException {
-    Path trustStorePath = Paths.get(config.getHttpManagement().getTruststore());
+  @VisibleForTesting
+  TrustManager[] getTrustManagers(Path trustStorePath) throws ReaperException {
+
+    LOG.trace(String.format("Calling getSingleTrustManager with %s", trustStorePath));
+
     try (InputStream tsIs = Files.newInputStream(trustStorePath, StandardOpenOption.READ)) {
       KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
       trustStore.load(tsIs, KEYSTORE_PASSWORD);
@@ -249,30 +272,67 @@ private TrustManager[] getTrustManagers() throws ReaperException {
 
       return tmf.getTrustManagers();
     } catch (IOException | NoSuchAlgorithmException | KeyStoreException | CertificateException e) {
-      throw new ReaperException(e);
+      throw new ReaperException("Error loading trust managers");
+    }
+  }
+
+  @VisibleForTesting
+  Path getTruststoreComponentPath(Node node, String truststoreComponentName) {
+    Path trustStorePath;
+
+    String clusterName = node.getClusterName();
+    // the cluster name is not available, or we don't have the per-cluster truststores
+    // we fall back to the global trust stores
+    if (clusterName.equals("") || config.getHttpManagement().getTruststoresDir() == null) {
+
+      trustStorePath = truststoreComponentName.equals(TRUSTSTORE_COMPONENT_NAME)
+          ? Paths.get(config.getHttpManagement().getTruststore()).toAbsolutePath()
+          : Paths.get(config.getHttpManagement().getKeystore()).toAbsolutePath();
+    } else {
+      // load a cluster-specific trust store otherwise
+      Path storesRootPath = Paths.get(config.getHttpManagement().getTruststoresDir());
+      trustStorePath = storesRootPath
+          .resolve(String.format("%s-%s", clusterName, truststoreComponentName))
+          .toAbsolutePath();
     }
+
+    return trustStorePath;
   }
 
   @VisibleForTesting
-  void createSslWatcher() throws IOException {
+  void createSslWatcher(boolean watchTruststore, boolean watchKeystore, boolean watchTruststoreDir) throws IOException {
+
     WatchService watchService = FileSystems.getDefault().newWatchService();
-    Path trustStorePath = Paths.get(config.getHttpManagement().getTruststore());
-    Path keyStorePath = Paths.get(config.getHttpManagement().getKeystore());
-    Path keystoreParent = trustStorePath.getParent();
-    Path trustStoreParent = keyStorePath.getParent();
-
-    keystoreParent.register(
-        watchService,
-        StandardWatchEventKinds.ENTRY_CREATE,
-        StandardWatchEventKinds.ENTRY_DELETE,
-        StandardWatchEventKinds.ENTRY_MODIFY);
-
-    if (!keystoreParent.equals(trustStoreParent)) {
-      trustStoreParent.register(
+
+    Path trustStorePath = watchTruststore ? Paths.get(config.getHttpManagement().getTruststore()) : null;
+    Path keyStorePath = watchKeystore ? Paths.get(config.getHttpManagement().getKeystore()) : null ;
+    Path truststoreDirPath = watchTruststoreDir ? Paths.get(config.getHttpManagement().getTruststoresDir()) : null ;
+
+    if (watchKeystore) {
+      keyStorePath.getParent().register(
+          watchService,
+          StandardWatchEventKinds.ENTRY_CREATE,
+          StandardWatchEventKinds.ENTRY_DELETE,
+          StandardWatchEventKinds.ENTRY_MODIFY
+      );
+    }
+    if (watchTruststore && watchKeystore) {
+      if (!trustStorePath.getParent().equals(keyStorePath.getParent())) {
+        trustStorePath.getParent().register(
+            watchService,
+            StandardWatchEventKinds.ENTRY_CREATE,
+            StandardWatchEventKinds.ENTRY_DELETE,
+            StandardWatchEventKinds.ENTRY_MODIFY
+        );
+      }
+    }
+    if (watchTruststoreDir) {
+      truststoreDirPath.register(
           watchService,
           StandardWatchEventKinds.ENTRY_CREATE,
           StandardWatchEventKinds.ENTRY_DELETE,
-          StandardWatchEventKinds.ENTRY_MODIFY);
+          StandardWatchEventKinds.ENTRY_MODIFY
+      );
     }
 
     ExecutorService executorService = Executors.newSingleThreadExecutor();
@@ -290,11 +350,23 @@ void createSslWatcher() throws IOException {
                 WatchEvent<java.nio.file.Path> ev = (WatchEvent<Path>) event;
                 Path eventFilename = ev.context();
 
-                if (keystoreParent.resolve(eventFilename).equals(keyStorePath)
-                    || trustStoreParent.resolve(eventFilename).equals(trustStorePath)) {
-                  // Something in the TLS has been modified.. recreate HTTP connections
-                  reloadNeeded = true;
+                if (watchKeystore) {
+                  if (keyStorePath.getParent().resolve(eventFilename).equals(keyStorePath)) {
+                    reloadNeeded = true;
+                  }
+                }
+                if (watchTruststore) {
+                  if (trustStorePath.getParent().resolve(eventFilename).equals(trustStorePath)) {
+                    // Something in the TLS has been modified.. recreate HTTP connections
+                    reloadNeeded = true;
+                  }
                 }
+                if (watchTruststoreDir) {
+                  if (eventFilename.toString().endsWith(".jks")) {
+                    reloadNeeded = true;
+                  }
+                }
+
               }
               if (!key.reset()) {
                 // The watched directories have disappeared..