diff --git a/migration/dump-db b/migration/dump-db deleted file mode 100755 index d564f3912..000000000 --- a/migration/dump-db +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -e - -SELFDIR=$(dirname "$(readlink -f "$0")") -SCRIPT=${SCRIPT:=./run} - -SERVICES="\ - mender-inventory \ - mender-useradm \ - mender-device-auth \ - mender-deployments \ - mender-create-artifact-worker \ - mender-workflows-server \ - mender-workflows-worker \ - mender-deviceconnect \ - mender-deviceconfig \ -" -DB_SERVICES="\ - mender-mongo -" - -# stop service that are using DBs -# shellcheck disable=SC2086 -${SCRIPT} stop ${SERVICES} - -# make sure that DB services are up -# shellcheck disable=SC2086 -${SCRIPT} start ${DB_SERVICES} - -# dump DB -# shellcheck disable=SC2016 -DUMP_PATH=$PWD ${SCRIPT} -f "${SELFDIR}/migration-helper.yml" \ - run \ - --rm \ - -e DB_SERVICES="${DB_SERVICES}" \ - mongo-helper \ - sh -c 'for s in ${DB_SERVICES}; do mongodump -h "$s" --out "/srv/db-dump/$s"; done' diff --git a/migration/migration-helper.yml b/migration/migration-helper.yml deleted file mode 100644 index 642f76969..000000000 --- a/migration/migration-helper.yml +++ /dev/null @@ -1,11 +0,0 @@ -services: - - # - # mongo migration helper service - # - mongo-helper: - image: mongo:8.0 - networks: - - mender - volumes: - - ${DUMP_PATH}:/srv diff --git a/migration/restore-db b/migration/restore-db deleted file mode 100755 index c0aea16f9..000000000 --- a/migration/restore-db +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -e - -SELFDIR=$(dirname "$(readlink -f "$0")") -SCRIPT=${SCRIPT:=./run} - -SERVICES="\ - mender-inventory \ - mender-useradm \ - mender-device-auth \ - mender-deployments \ - mender-create-artifact-worker \ - mender-workflows-server \ - mender-workflows-worker \ - mender-deviceconnect \ - mender-deviceconfig \ -" - -DB_SERVICES="\ - mender-mongo -" - -# stop service that are using DBs -# shellcheck disable=SC2086 -${SCRIPT} stop ${SERVICES} - -# make sure that DB services are up -# shellcheck disable=SC2086 -${SCRIPT} start ${DB_SERVICES} - -# dump DB -# shellcheck disable=SC2016 -DUMP_PATH=$PWD ${SCRIPT} -f "${SELFDIR}/migration-helper.yml" \ - run \ - --rm \ - -e DB_SERVICES="${DB_SERVICES}" \ - mongo-helper \ - sh -c 'for s in ${DB_SERVICES}; do mongorestore -h "$s" --drop "/srv/db-dump/$s"; done' diff --git a/production/config/enterprise.yml.template b/production/config/enterprise.yml.template deleted file mode 100644 index 1f7a645ec..000000000 --- a/production/config/enterprise.yml.template +++ /dev/null @@ -1,24 +0,0 @@ -# This is a template file for running Mender Enterprise. It builds on the -# prod.yml template file, so everything described there applies to this template -# as well. -# -# Notes: -# - integration/docker-compose.enterprise.yml file is assumed to be included - -version: '2.3' -services: - - mender-device-auth: - environment: - # Set this to a tenant token that should be used for clients that - # don't supply a tenant token. If empty, clients without a tenant - # token will be rejected. - DEVICEAUTH_DEFAULT_TENANT_TOKEN: '' - - mender-deviceconnect: - environment: - DEVICECONNECT_ENABLE_AUDIT: "1" - - mender-useradm: - volumes: - - ./production/keys-generated/keys/useradm/private.key:/etc/useradm-enterprise/rsa/private.pem:ro diff --git a/production/config/prod.yml.template b/production/config/prod.yml.template deleted file mode 100644 index 1ac0e2c21..000000000 --- a/production/config/prod.yml.template +++ /dev/null @@ -1,146 +0,0 @@ -# this is a template file for production setup, consult -# https://docs.docker.com/compose/compose-file/ for details on syntax and usage -# -# Notes: -# - integration/docker-compose.yml file is assumed to be included -# - integration/docker-compose.storage.minio.yml is assumed to be included -# - all services are part of `mender` network (service names are unchanged) -# - keys and certificates are generated using keygen utility from integration -# repository, keys and certificates are stored in ./keys-generated directory -# - certificates and key are mounted into containers using volumes -# - minio artifacts are stored in a named volume `mender-artifacts`; volume -# needs to be created manually using `docker volume create mender-artifacts` - -# related compose bugs: -# - https://github.com/docker/compose/issues/3874 -# - https://github.com/docker/compose/issues/3568 -# - https://github.com/docker/compose/issues/3219 - -version: '2.3' -services: - - mender-iot-manager: - command: server --automigrate - - mender-workflows-server: - command: server --automigrate - - mender-workflows-worker: - command: worker --automigrate --excluded-workflows generate_artifact - - mender-create-artifact-worker: - command: --automigrate - - mender-useradm: - command: server --automigrate - volumes: - - ./production/keys-generated/keys/useradm/private.key:/etc/useradm/rsa/private.pem:ro - logging: - options: - max-file: "10" - max-size: "50m" - - mender-device-auth: - command: server --automigrate - volumes: - - ./production/keys-generated/keys/deviceauth/private.key:/etc/deviceauth/rsa/private.pem:ro - environment: - DEVICEAUTH_SERVER_PRIV_KEY_PATH: /etc/deviceauth/rsa/private.pem - logging: - options: - max-file: "10" - max-size: "50m" - - mender-inventory: - command: server --automigrate - logging: - options: - max-file: "10" - max-size: "50m" - - mender-api-gateway: - ports: - # list of ports API gateway is made available on - - "443:443" - networks: - mender: - aliases: - # mender-api-gateway is a proxy to storage - # and has to use exactly the same name as devices - # and the deployments service will; - # - # if devices and deployments will access storage - # using https://s3.acme.org, then - # set this to s3.acme.org - - set-my-alias-here.com - command: - - --accesslog=true - - --entrypoints.http.address=:80 - - --entrypoints.http.http.redirections.entryPoint.scheme=https - - --entrypoints.http.http.redirections.entryPoint.to=https - - --entrypoints.https.address=:443 - - --entryPoints.https.transport.respondingTimeouts.idleTimeout=7200 - - --entryPoints.https.transport.respondingTimeouts.readTimeout=7200 - - --entryPoints.https.transport.respondingTimeouts.writeTimeout=7200 - - --providers.file.directory=/etc/traefik/config - volumes: - - ./config/traefik/traefik.yaml:/etc/traefik/config/traefik.yaml:ro - - ./config/traefik/traefik.middlewares.yaml:/etc/traefik/config/traefik.middlewares.yaml:ro - - ./config/traefik/traefik.tls.yaml:/etc/traefik/config/traefik.tls.yaml:ro - - ./production/keys-generated/cert/cert.crt:/etc/traefik/certs/cert.crt:ro - - ./production/keys-generated/cert/private.key:/etc/traefik/certs/private.key:ro - logging: - options: - max-file: "10" - max-size: "50m" - environment: - # ALLOWED_HOSTS is a comma-separated list of allowed hostnames - ALLOWED_HOSTS: "my-gateway-dns-name" - - mender-deployments: - command: server --automigrate - volumes: - - ./production/keys-generated/cert/cert.crt:/etc/ssl/certs/docker.mender.io.crt:ro - environment: - STORAGE_BACKEND_CERT: /etc/ssl/certs/docker.mender.io.crt - # access key, the same value as MINIO_ACCESS_KEY - DEPLOYMENTS_AWS_AUTH_KEY: - # secret, the same valie as MINIO_SECRET_KEY - DEPLOYMENTS_AWS_AUTH_SECRET: - - # deployments service uses signed URLs, hence it needs to access - # storage-proxy using exactly the same name as devices will; if - # devices will access storage using https://s3.acme.org, then - # set this to https://s3.acme.org - DEPLOYMENTS_AWS_EXTERNAL_URI: https://set-my-alias-here.com - DEPLOYMENTS_AWS_URI: http://minio:9000 - logging: - options: - max-file: "10" - max-size: "50m" - - minio: - environment: - # access key - MINIO_ACCESS_KEY: - # secret - MINIO_SECRET_KEY: - volumes: - # mounts a docker volume named `mender-artifacts` as /export directory - - mender-artifacts:/export:rw - - mender-mongo: - volumes: - - mender-db:/data/db:rw - -volumes: - # mender artifacts storage - mender-artifacts: - external: - # use external volume created manually - name: mender-artifacts - # mongo service database - mender-db: - external: - # use external volume created manually - name: mender-db diff --git a/production/run b/production/run deleted file mode 100755 index 329da4609..000000000 --- a/production/run +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -set -e - -../verify-docker-versions - -# Detect docker-compose command -DOCKER_COMPOSE_COMMAND=$(docker compose version &>/dev/null && echo 'docker compose' || echo 'docker-compose') -# Pass this value on to the GUI container as an env variable -export INTEGRATION_VERSION=$(git describe --tags --abbrev=0) -# Parse the Mender-Artifact version used from the other-components.yml file's image tag -export MENDER_ARTIFACT_VERSION=$(awk -F':' '/mendersoftware\/mender-artifact/ {print $3}' ../other-components.yml) -# Parse the mender version from docker-compose.yml mender image's tag -export MENDER_VERSION=$(awk -F':' '/mendersoftware\/mender-client/ {print $3}' ../docker-compose.client.yml) -export MENDER_DEB_PACKAGE_VERSION=$MENDER_VERSION - -if [[ "$OSTYPE" == "darwin"* ]]; then - export GATEWAY_IP=$(ifconfig | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -1) -else - export GATEWAY_IP=$(ip route get 1 | awk '{print $7;exit}') -fi - -ENTERPRISE_DOCKER_COMPOSE= -ENTERPRISE_PROD= -if [ -f ./config/enterprise.yml ]; then - ENTERPRISE_DOCKER_COMPOSE="-f ../docker-compose.enterprise.yml" - ENTERPRISE_PROD="-f ./config/enterprise.yml" -fi - -exec ${DOCKER_COMPOSE_COMMAND} \ - -p menderproduction \ - -f ../docker-compose.yml \ - -f ../docker-compose.storage.minio.yml \ - $ENTERPRISE_DOCKER_COMPOSE \ - -f ./config/prod.yml \ - $ENTERPRISE_PROD \ - "$@" diff --git a/tests/production_test_env.py b/tests/production_test_env.py deleted file mode 100755 index bd5a275ac..000000000 --- a/tests/production_test_env.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 Northern.tech AS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import subprocess -import argparse - -parser = argparse.ArgumentParser(description="Helper script to bring up production env") - -parser.add_argument( - "--start", dest="start", action="store_true", help="start production environment" -) - -parser.add_argument( - "--kill", dest="kill", action="store_true", help="destroy production environment" -) - -parser.add_argument( - "--docker-compose-instance", - required=True, - help="The docker-compose instance to use (project name)", -) - -if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - -args = parser.parse_args() - -docker_compose_project = args.docker_compose_instance - - -def fill_production_template(): - - # copy production environment yml file - subprocess.check_output( - ["cp", "production/config/prod.yml.template", "production-testing-env.yml"], - cwd="../", - ) - subprocess.check_output( - "sed -i 's,/production/,/,g' ../production-testing-env.yml", shell=True - ) - subprocess.check_output( - "sed -i 's/ALLOWED_HOSTS: my-gateway-dns-name/ALLOWED_HOSTS: ~./' ../production-testing-env.yml", - shell=True, - ) - subprocess.check_output( - "sed -i '0,/set-my-alias-here.com/s/set-my-alias-here.com/localhost/' ../production-testing-env.yml", - shell=True, - ) - subprocess.check_output( - "sed -i 's|DEPLOYMENTS_AWS_URI:.*|DEPLOYMENTS_AWS_URI: https://mender-api-gateway|' ../production-testing-env.yml", - shell=True, - ) - subprocess.check_output( - "sed -i 's/MINIO_ACCESS_KEY:.*/MINIO_ACCESS_KEY: Q3AM3UQ867SPQQA43P2F/' ../production-testing-env.yml", - shell=True, - ) - subprocess.check_output( - "sed -i 's/MINIO_SECRET_KEY:.*/MINIO_SECRET_KEY: abcssadasdssado798dsfjhkksd/' ../production-testing-env.yml", - shell=True, - ) - subprocess.check_output( - "sed -i 's/DEPLOYMENTS_AWS_AUTH_KEY:.*/DEPLOYMENTS_AWS_AUTH_KEY: Q3AM3UQ867SPQQA43P2F/' ../production-testing-env.yml", - shell=True, - ) - subprocess.check_output( - "sed -i 's/DEPLOYMENTS_AWS_AUTH_SECRET:.*/DEPLOYMENTS_AWS_AUTH_SECRET: abcssadasdssado798dsfjhkksd/' ../production-testing-env.yml", - shell=True, - ) - - -def setup_docker_volumes(): - docker_volumes = [ - "mender-artifacts", - "mender-db", - ] - - for volume in docker_volumes: - ret = subprocess.call(["docker", "volume", "create", "--name=%s" % volume]) - assert ret == 0, "failed to create docker volumes" - - -if args.start: - # create volumes required for production environment - setup_docker_volumes() - - # add keys for production environment - if not os.path.exists("../keys-generated"): - ret = subprocess.call( - ["./keygen"], - env={ - "CERT_CN": "localhost", - "CERT_SAN": "DNS:localhost,DNS:mender-api-gateway", - }, - cwd="../", - ) - assert ret == 0, "failed to generate keys" - fill_production_template() - - # start docker-compose - ret = subprocess.call( - [ - "docker-compose", - "-p", - docker_compose_project, - "-f", - "docker-compose.yml", - "-f", - "docker-compose.storage.minio.yml", - "-f", - "./production-testing-env.yml", - "up", - "-d", - ], - cwd="../", - ) - - assert ret == 0, "failed to start docker-compose" - -if args.kill: - subprocess.call( - [ - "docker-compose", - "-p", - docker_compose_project, - "down", - "-v", - "--remove-orphans", - ] - ) diff --git a/tests/tests/test_security.py b/tests/tests/test_security.py index 9a23e4acc..d01c8c714 100644 --- a/tests/tests/test_security.py +++ b/tests/tests/test_security.py @@ -53,62 +53,6 @@ def do_test_token_token_expiration(self, env, valid_image_with_mender_conf): class TestSecurityOpenSource(BaseTestSecurity): - def test_ssl_only(self, running_custom_production_setup): - """ make sure we are not exposing any non-ssl connections in production environment """ - done = False - sleep_time = 2 - # start production environment - subprocess.call( - [ - "./production_test_env.py", - "--start", - "--docker-compose-instance", - running_custom_production_setup.name, - ] - ) - - try: - - # get all exposed ports from docker - - for _ in range(3): - exposed_hosts = subprocess.check_output( - "docker ps | grep %s | grep -o -E '0.0.0.0:[0-9]*' | cat" - % running_custom_production_setup.name, - shell=True, - ).decode() - - try: - for host in exposed_hosts.split(): - with contextlib.closing( - ssl.SSLContext().wrap_socket(socket.socket()) - ) as sock: - logger.info("%s: connect to host with TLS" % host) - host, port = host.split(":") - sock.connect((host, int(port))) - done = True - except: - sleep_time *= 2 - time.sleep(sleep_time) - continue - - if done: - break - - if not done: - pytest.fail("failed to connect to production env. using SSL") - - finally: - # tear down production env - subprocess.call( - [ - "./production_test_env.py", - "--kill", - "--docker-compose-instance", - running_custom_production_setup.name, - ] - ) - def test_token_token_expiration( self, standard_setup_with_short_lived_token, valid_image_with_mender_conf ):