Modify packaging to make consoleme runnable out-of-the-box (#9258)

.github/workflows/build-and-package.yml

@@ -0,0 +1,89 @@
+name: Build & Package
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Install Terraform
+        uses: hashicorp/setup-terraform@v1
+      - name: Install dependencies
+        run: |
+          curl -sL https://deb.nodesource.com/setup_14.x | sudo bash
+          curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
+          echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+          sudo apt-get update
+          sudo apt-get install \
+            libssl-dev \
+            libcurl4-openssl-dev \
+            python3-dev \
+            build-essential \
+            libxml2-dev \
+            libxmlsec1-dev \
+            libxmlsec1-openssl \
+            musl-dev \
+            yarn \
+            nodejs
+          pip install \
+            -r requirements.txt \
+            -r requirements-test.txt \
+            -r requirements-docs.txt
+          pip install .
+          yarn --cwd ui install
+          yarn --cwd ui build:prod
+          terraform -chdir=terraform/central-account/ init
+          terraform -chdir=terraform/spoke-accounts/ init
+          pre-commit install
+      - name: Run pre-commit
+        run: |
+          pre-commit run -a
+  package:
+    runs-on: ubuntu-latest
+    needs: build
+    # Always run on the master branch.
+    # Builds on non-tag refs will be uploaded to PyPI as prerelease versions.
+    if: ${{ github.ref == 'refs/heads/master' && github.repository == 'Netflix/consoleme' }}
+    steps:
+      - uses: actions/checkout@v2
+      - run: git fetch --prune --unshallow --tags
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Build UI assets
+        run: |
+          curl -sL https://deb.nodesource.com/setup_14.x | sudo bash
+          curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
+          echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+          sudo apt-get update
+          sudo apt-get install \
+            yarn \
+            nodejs
+          yarn --cwd ui install
+          yarn --cwd ui build:prod
+      - name: Install build dependencies
+        run: pip install build wheel setupmeta
+      - name: Build Python package
+        run: python -m build --sdist --wheel --outdir dist/ .
+      - name: Publish package to PyPI Test
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+          repository_url: https://test.pypi.org/legacy/
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

.isort.cfg

@@ -1,6 +1,6 @@
 [settings]
-known_first_party=consoleme,consoleme_default_plugins,consoleme_ecs_cdk
-known_third_party = aiozipkin,asgiref,atlassian,aws_cdk,bcrypt,billiard,bleach,boto3,botocore,celery,cfnresponse,click,click_log,cloudaux,cryptography,dateutil,deepdiff,ed25519,elasticsearch,email_validator,furl,git,google,googleapiclient,jsonschema,jwt,logmatic,marshmallow,mock,mockredis,moto,nested_stacks,okta_jwt,onelogin,pandas,parliament,password_strength,pip,pkg_resources,policy_sentry,policyuniverse,pydantic,pytest,pytz,questionary,redis,redislite,requests,retrying,ruamel,sentry_sdk,setuptools,simplejson,tenacity,tornado,ujson,uvloop,validate_email,yaml
+known_first_party=consoleme,consoleme_ecs_cdk
+known_third_party = aiozipkin,asgiref,atlassian,aws_cdk,bcrypt,billiard,bleach,boto3,botocore,celery,cfnresponse,click,click_log,cloudaux,consoleme_default,cryptography,dateutil,deepdiff,ed25519,elasticsearch,email_validator,furl,git,google,googleapiclient,jsonschema,jwt,logmatic,marshmallow,mock,mockredis,moto,nested_stacks,okta_jwt,onelogin,pandas,parliament,password_strength,pkg_resources,policy_sentry,policyuniverse,pydantic,pytest,pytz,questionary,redis,redislite,requests,retrying,ruamel,sentry_sdk,setuptools,simplejson,tenacity,tornado,ujson,uvloop,validate_email,yaml
 multi_line_output=3
 include_trailing_comma=True
 balanced_wrapping=True

CONTRIBUTING.md

@@ -69,6 +69,6 @@ Some of the key files/folders for you as a new developer on `consoleme`:
 - `terraform/` - Terraform config if you choose to [deploy consoleme using Terraform](https://hawkins.gitbook.io/consoleme/deployment-strategies)
 - `packer/` - Packer config if you choose to [deploy consoleme using a EC2 instance](https://hawkins.gitbook.io/consoleme/deployment-strategies)
 - `docs/gitbook/` - our documentation site, written in markdown, built using gitbook
-- `default_plugins/` - examples of plugins used to extend and customize ConsoleMe
+- `consoleme/default_plugins/` - examples of plugins used to extend and customize ConsoleMe
 - `.github/workflows` - our CI config (using GitHub Actions)
 - `.run/` - configurations to run consoleme in JetBrains IDEs (i.e. PyCharm)

Dockerfile

@@ -1,6 +1,6 @@
 # Dockerfile should instantiate AWS Project with configurable plugins
 FROM python:3.8
-MAINTAINER Curtis Castrapel
+MAINTAINER Netflix Security
 WORKDIR /apps/consoleme
 # NODE_OPTIONS meeded to increase memory size of Node for the `yarn build` step. The Monaco Editor
 # appears to be the culprit requiring this.
@@ -9,33 +9,30 @@ ENV SETUPTOOLS_USE_DISTUTILS=stdlib
 
 # Install OS dependencies
 RUN curl -sL https://deb.nodesource.com/setup_14.x | bash
-RUN apt-get clean
-RUN apt-get update
-RUN apt-get install build-essential libxml2-dev libxmlsec1-dev libxmlsec1-openssl musl-dev libcurl4-nss-dev python3-dev nodejs -y
-
-# Install python dependencies
-COPY requirements.txt requirements-test.txt setup.py /apps/consoleme/
-COPY default_plugins /apps/consoleme/default_plugins/
-
-RUN pip install -U setuptools pip cython
-RUN pip install --no-cache-dir -r requirements.txt
-RUN pip install --no-cache-dir -r requirements-test.txt
-RUN pip install -e .
-RUN pip install -e default_plugins
+RUN apt clean
+RUN apt update
+RUN apt install -y \
+    build-essential \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libxmlsec1-openssl \
+    musl-dev \
+    libcurl4-nss-dev \
+    python3-dev \
+    nodejs
+RUN npm install yarn -g
 # Install watchdog. Used to automatically restart ConsoleMe in Docker, for development.
 RUN pip install watchdog argh
 
 # Run ConsoleMe tornado server using configuration
-RUN mkdir /apps/consoleme/consoleme
-COPY consoleme /apps/consoleme/consoleme/
-COPY example_config /apps/consoleme/example_config/
-COPY scripts /apps/consoleme/scripts/
-COPY ui /apps/consoleme/ui/
+COPY . /apps/consoleme
+RUN pip install -U setuptools pip
+RUN pip install --no-cache-dir -r requirements.txt -r requirements-test.txt
+RUN pip install -e .
 
 # Install SPA frontend
-RUN npm install yarn -g
 RUN yarn --cwd ui
 RUN yarn --cwd ui build:prod
 
-CMD python scripts/retrieve_or_decode_configuration.py ; python /apps/consoleme/consoleme/__main__.py
+CMD python scripts/retrieve_or_decode_configuration.py; consoleme
 EXPOSE 8081

Makefile

@@ -16,9 +16,7 @@ CONSOLEME_CONFIG_ENTRYPOINT := $(or ${CONSOLEME_CONFIG_ENTRYPOINT},${CONSOLEME_C
 .PHONY: env_install
 env_install:
 	pip install wheel
-	pip install -e default_plugins ;\
-	pip install -r requirements.txt ;\
-	pip install -r requirements-test.txt ;\
+	pip install -r requirements.txt -r requirements-test.txt
 	pip install -e .
 
 .PHONY: install
@@ -138,7 +136,7 @@ endif
 	packer build --debug -var 'app_archive=consoleme.tar.gz' packer/create_consoleme_ami.json
 
 .PHONY: packer_ubuntu_oss
-packer_ubuntu_oss: ubuntu_redis env_install default_plugins
+packer_ubuntu_oss: ubuntu_redis env_install
 
 .PHONY: ubuntu_redis
 ubuntu_redis:
@@ -153,7 +151,3 @@ endif
 	sudo apt-get install -y redis-server
 	sudo systemctl enable redis-server.service
 	sudo systemctl restart redis-server.service
-
-.PHONY: default_plugins
-default_plugins:
-	pip install -e default_plugins

consoleme/__main__.py

@@ -36,26 +36,26 @@ def main():
 
 
 def init():
-    if __name__ == "__main__":
-        port = config.get("tornado.port")
-        stats.count("start")
+    port = config.get("tornado.port")
+    stats.count("start")
 
-        server = tornado.httpserver.HTTPServer(app)
+    server = tornado.httpserver.HTTPServer(app)
 
-        if port:
-            server.bind(port, address=config.get("tornado.address"))
+    if port:
+        server.bind(port, address=config.get("tornado.address"))
 
-        server.start()  # forks one process per cpu
+    server.start()  # forks one process per cpu
 
-        if config.get("tornado.debug", False):
-            for directory, _, files in os.walk("consoleme/templates"):
-                [
-                    tornado.autoreload.watch(directory + "/" + f)
-                    for f in files
-                    if not f.startswith(".")
-                ]
-        log.debug({"message": "Server started"})
-        asyncio.get_event_loop().run_forever()
+    if config.get("tornado.debug", False):
+        for directory, _, files in os.walk("consoleme/templates"):
+            [
+                tornado.autoreload.watch(directory + "/" + f)
+                for f in files
+                if not f.startswith(".")
+            ]
+    log.debug({"message": "Server started"})
+    asyncio.get_event_loop().run_forever()
 
 
-init()
+if __name__ == "__main__":
+    init()

default_plugins/consoleme_default_plugins/plugins/celery_tasks/celery_tasks.py → consoleme/default_plugins/plugins/celery_tasks/celery_tasks.py

@@ -86,12 +86,12 @@ def task_1():
 
 internal_schedule = {
     "task1": {
-        "task": "consoleme_default_plugins.plugins.celery_tasks.celery_tasks.task_1",
+        "task": "consoleme.default_plugins.plugins.celery_tasks.celery_tasks.task_1",
         "options": {"expires": 4000},
         "schedule": schedule,
     },
     "cache_application_information": {
-        "task": "consoleme_default_plugins.plugins.celery_tasks.celery_tasks.cache_application_information",
+        "task": "consoleme.default_plugins.plugins.celery_tasks.celery_tasks.cache_application_information",
         "options": {"expires": 4000},
         "schedule": schedule,
     },

default_plugins/consoleme_default_plugins/plugins/config/config.py → consoleme/default_plugins/plugins/config/config.py

@@ -25,16 +25,16 @@ def get_config_location():
                 client = boto3.client("s3")
                 bucket, key = split_s3_path(config_location)
                 obj = client.get_object(Bucket=bucket, Key=key)
-                s3_object_content = obj["Body"].read()
+                config_data = obj["Body"].read()
                 with open(default_save_location, "w") as f:
-                    f.write(s3_object_content.decode())
+                    f.write(config_data.decode())
             elif config_location.startswith("AWS_SECRETS_MANAGER:"):
                 secret_name = "".join(config_location.split("AWS_SECRETS_MANAGER:")[1:])
-                aws_secret_content = get_aws_secret(
+                config_data = get_aws_secret(
                     secret_name, os.environ.get("EC2_REGION", "us-east-1")
                 )
                 with open(default_save_location, "w") as f:
-                    f.write(aws_secret_content)
+                    f.write(config_data)
             else:
                 return config_location
         config_locations: List[str] = [
@@ -78,7 +78,7 @@ def get_employee_photo_url(user):
         # Fall back to Gravatar
         gravatar_url = (
             "https://www.gravatar.com/avatar/"
-            + hashlib.md5(user.lower().encode("utf-8")).hexdigest()
+            + hashlib.md5(user.lower().encode("utf-8")).hexdigest()  # nosec
             + "?"
         )
         gravatar_url += urllib.parse.urlencode({"d": "mp"})

default_plugins/consoleme_default_plugins/plugins/internal_routes/internal_routes.py → consoleme/default_plugins/plugins/internal_routes/internal_routes.py

@@ -1,4 +1,4 @@
-from consoleme_default_plugins.plugins.internal_routes.handlers.internal_demo_route import (
+from consoleme.default_plugins.plugins.internal_routes.handlers.internal_demo_route import (
     InternalDemoRouteHandler,
 )
 

default_plugins/consoleme_default_plugins/plugins/metrics/cloudwatch/__init__.py → consoleme/default_plugins/plugins/metrics/cloudwatch/__init__.py

@@ -6,7 +6,7 @@
 import sentry_sdk
 
 from consoleme.config import config
-from consoleme_default_plugins.plugins.metrics.base_metric import Metric
+from consoleme.default_plugins.plugins.metrics.base_metric import Metric
 
 cloudwatch = boto3.client(
     "cloudwatch", region_name=config.region, **config.get("boto3.client_kwargs", {})

default_plugins/consoleme_default_plugins/plugins/metrics/default_metrics.py → consoleme/default_plugins/plugins/metrics/default_metrics.py

@@ -1,6 +1,6 @@
 from typing import Dict, Optional, Union
 
-from consoleme_default_plugins.plugins.metrics.base_metric import Metric
+from consoleme.default_plugins.plugins.metrics.base_metric import Metric
 
 
 class DefaultMetric(Metric):

default_plugins/consoleme_default_plugins/plugins/metrics/metrics.py → consoleme/default_plugins/plugins/metrics/metrics.py

@@ -3,7 +3,7 @@
 
 desired_metric_plugin = config.get(
     "metrics.metrics_plugin",
-    "consoleme_default_plugins.plugins.metrics.default_metrics.DefaultMetric",
+    "consoleme.default_plugins.plugins.metrics.default_metrics.DefaultMetric",
 )
 
 try: