Bloodhound / Opensearch

changelog.d/0-release-notes/elasticsearch-to-opensearch

@@ -0,0 +1,10 @@
+Switch from ElasticSearch 6.8 to OpenSearch 1.3 (we're testing with OpenSearch
+1.3.19.) Please note that the APIs of ElasticSearch 6.8 and OpenSearch 1.3 are
+not compatible. I.e., either a migration is required
+(https://opensearch.org/docs/latest/upgrade-to/) or the indexes need to be
+rebuilt from scratch. The latter is done automatically when the
+`elasticsearch-index-create` job of the `elasticsearch-index` Helm chart faces
+an OpenSearch instance where Wire's indexes are missing. Alternatively, this can
+be started manually; see
+https://docs.wire.com/developer/reference/elastic-search.html#refill-es-documents-from-cassandra
+. Depending on the amount of users in the database, this may take long.

charts/integration/values.yaml

@@ -113,7 +113,7 @@ config:
     replicationFactor: 1
 
   elasticsearch:
-    host: elasticsearch-ephemeral
+    host: opensearch-cluster-master
 
   sqsEndpointUrl: http://fake-aws-sqs:4568
   sesEndpointUrl: http://fake-aws-ses:4569

deploy/dockerephemeral/docker-compose.yaml

@@ -204,15 +204,9 @@ services:
       redis:
         ipv4_address: 172.20.0.36
 
-  elasticsearch:
-    container_name: demo_wire_elasticsearch
-    build:
-      context: .
-      dockerfile_inline: |
-        FROM quay.io/wire/elasticsearch:0.0.9-amd64
-        RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack -b
-        # this seems to be necessary to run X-Pack on Alpine (https://discuss.elastic.co/t/elasticsearch-failing-to-start-due-to-x-pack/85125/7)
-        RUN rm -rf /usr/share/elasticsearch/plugins/x-pack/platform/linux-x86_64
+  opensearch:
+    container_name: opensearch
+    image: opensearchproject/opensearch:1.3.19
     ulimits:
       nofile:
         soft: 65536
@@ -221,17 +215,37 @@ services:
       - "127.0.0.1:9200:9200"
       - "127.0.0.1:9300:9300"
     environment:
-      - "xpack.ml.enabled=false"
-      - "xpack.security.enabled=true"
-      - "xpack.security.http.ssl.enabled=true"
-      - "xpack.ssl.certificate=certs/elasticsearch-cert.pem"
-      - "xpack.ssl.key=certs/elasticsearch-key.pem"
       - "bootstrap.system_call_filter=false"
       - "JVM_OPTIONS_ES=-Xmx512m -Xms512m"
       - "discovery.type=single-node"
+
+      - "DISABLE_INSTALL_DEMO_CONFIG=true"
+      - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=Ch4ng3m3Secr3t!"
+    volumes:
+      - ./docker/elasticsearch-cert.pem:/usr/share/opensearch/config/certs/tls.crt
+      - ./docker/elasticsearch-key.pem:/usr/share/opensearch/config/certs/tls.key
+      - ./docker/elasticsearch-ca.pem:/usr/share/opensearch/config/certs/ca.crt
+      - ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
+      - ./docker/opensearch/opensearch-security/config.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/config.yml
+      - ./docker/opensearch/opensearch-security/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml
+      - ./docker/opensearch/opensearch-security/roles_mapping.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml
+      - ./docker/opensearch/opensearch-security/allowlist.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/allowlist.yml
+      - ./docker/opensearch/opensearch-security/roles.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles.yml
+      - ./docker/opensearch/opensearch-security/nodes_dn.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/nodes_dn.yml
+      - ./docker/opensearch/opensearch-security/action_groups.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/action_groups.yml
+      - ./docker/opensearch/opensearch-security/tenants.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml
+    networks:
+      - demo_wire
+
+  opensearch-dashboard:
+    image: opensearchproject/opensearch-dashboards:1
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601
+    expose:
+      - "5601"
     volumes:
-      - ./docker/elasticsearch-cert.pem:/usr/share/elasticsearch/config/certs/elasticsearch-cert.pem
-      - ./docker/elasticsearch-key.pem:/usr/share/elasticsearch/config/certs/elasticsearch-key.pem
+      - ./docker/opensearch/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
     networks:
       - demo_wire
 

deploy/dockerephemeral/docker/opensearch/opensearch-security/action_groups.yml

@@ -0,0 +1,3 @@
+_meta:
+  type: "actiongroups"
+  config_version: 2

deploy/dockerephemeral/docker/opensearch/opensearch-security/allowlist.yml

@@ -0,0 +1,6 @@
+_meta:
+  type: "allowlist"
+  config_version: 2
+
+config:
+  enabled: false

deploy/dockerephemeral/docker/opensearch/opensearch-security/config.yml

@@ -0,0 +1,17 @@
+_meta:
+  type: "config"
+  config_version: 2
+
+config:
+  dynamic:
+    authc:
+      basic_internal_auth_domain:
+        description: "Authenticate using HTTP basic against the internal users database"
+        http_enabled: true
+        transport_enabled: true
+        order: 1
+        http_authenticator:
+          type: basic
+          challenge: true
+        authentication_backend:
+          type: internal

deploy/dockerephemeral/docker/opensearch/opensearch-security/internal_users.yml

@@ -0,0 +1,10 @@
+_meta:
+  type: "internalusers"
+  config_version: 2
+
+elastic:
+  hash: "$2y$12$GRc68jkEX1m4uQpTVbwURu79xHxZ7vsbyEctOAADQwPjlhYS4LJVa"
+  reserved: true
+  description: "Wire User"
+  backend_roles:
+    - index_manager

deploy/dockerephemeral/docker/opensearch/opensearch-security/nodes_dn.yml

@@ -0,0 +1,3 @@
+_meta:
+  type: "nodesdn"
+  config_version: 2

deploy/dockerephemeral/docker/opensearch/opensearch-security/roles.yml

@@ -0,0 +1,3 @@
+_meta:
+  type: "roles"
+  config_version: 2

deploy/dockerephemeral/docker/opensearch/opensearch-security/roles_mapping.yml

@@ -0,0 +1,9 @@
+_meta:
+  type: "rolesmapping"
+  config_version: 2
+
+all_access:
+  reserved: false
+  backend_roles:
+    - index_manager
+  description: "Map index_manager to full_access"

deploy/dockerephemeral/docker/opensearch/opensearch-security/tenants.yml

@@ -0,0 +1,3 @@
+_meta:
+  type: "tenants"
+  config_version: 2

deploy/dockerephemeral/docker/opensearch/opensearch.yml

@@ -0,0 +1,45 @@
+cluster.name: opensearch-cluster
+
+# Bind to all interfaces because we don't know what IP address Docker will assign to us.
+network.host: 0.0.0.0
+
+# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again.
+discovery.type: single-node
+
+path.data: /usr/share/opensearch/data
+
+# WARNING: This is not a production-ready config! (Good enough for testing,
+# though.)
+plugins:
+  security:
+    ssl:
+      transport:
+          pemcert_filepath: certs/tls.crt
+          pemkey_filepath: certs/tls.key
+          pemtrustedcas_filepath: certs/ca.crt
+          enforce_hostname_verification: false
+      http:
+        enabled: true
+        pemcert_filepath: certs/tls.crt
+        pemkey_filepath: certs/tls.key
+        pemtrustedcas_filepath: certs/ca.crt
+    allow_unsafe_democertificates: true
+    allow_default_init_securityindex: true
+    audit.type: internal_opensearch
+    restapi:
+      roles_enabled: ["all_access", "security_rest_api_access"]
+    system_indices:
+      enabled: true
+      indices:
+        [
+          ".opendistro-alerting-config",
+          ".opendistro-alerting-alert*",
+          ".opendistro-anomaly-results*",
+          ".opendistro-anomaly-detector*",
+          ".opendistro-anomaly-checkpoints",
+          ".opendistro-anomaly-detection-state",
+          ".opendistro-reports-*",
+          ".opendistro-notifications-*",
+          ".opendistro-notebooks",
+          ".opendistro-asynchronous-search-response*",
+        ]

deploy/dockerephemeral/docker/opensearch/opensearch_dashboards.yml

@@ -0,0 +1,8 @@
+opensearch.hosts: [https://opensearch:9200]
+opensearch.ssl.verificationMode: none
+opensearch.username: elastic
+opensearch.password: changeme
+
+# Use this setting if you are running opensearch-dashboards without https
+opensearch_security.cookie.secure: false
+server.host: '0.0.0.0'

hack/bin/integration-setup-federation.sh

@@ -53,6 +53,7 @@ echo "Installing charts..."
 set +e
 # This exists because we need to run `helmfile` with `--skip-deps`, without that it doesn't work.
 helm repo add bedag https://bedag.github.io/helm-charts/
+helm repo add opensearch https://opensearch-project.github.io/helm-charts/
 
 helmfile --environment "$HELMFILE_ENV" --file "${TOP_LEVEL}/hack/helmfile.yaml" sync --skip-deps --concurrency 0
 EXIT_CODE=$?