Merge pull request #384 from bugout-dev/setup-requirements

Setup instead of requirements.txt

.github/workflows/lint.backend.yml

@@ -16,12 +16,14 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
+      - name: Upgrade pip and setuptools
+        run: pip install --upgrade pip setuptools
       - name: Install test requirements
         working-directory: ./backend
-        run: pip install -r requirements.txt
+        run: pip install -e .[dev]
       - name: Mypy type check
         working-directory: ./backend
-        run: mypy moonstream/
+        run: mypy moonstreamapi/
       - name: Black syntax check
         working-directory: ./backend
-        run: black --check moonstream/
+        run: black --check moonstreamapi/

.github/workflows/lint.crawlers.yml

@@ -16,6 +16,8 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
+      - name: Upgrade pip and setuptools
+        run: pip install --upgrade pip setuptools
       - name: Install test requirements
         working-directory: ./crawlers/mooncrawl
         run: pip install -e .[dev]

.github/workflows/lint.db.yml

@@ -16,6 +16,8 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
+      - name: Upgrade pip and setuptools
+        run: pip install --upgrade pip setuptools
       - name: Install test requirements
         working-directory: ./db
         run: pip install -e .[dev]

.github/workflows/release.mooncrawl.yml

@@ -0,0 +1,30 @@
+name: Release mooncrawl package
+
+on:
+  push:
+    tags:
+      - 'mooncrawl/v*'
+
+defaults:
+  run:
+    working-directory: crawlers/mooncrawl
+
+jobs:
+  publish:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[distribute]
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*

.github/workflows/release.moonstreamapi.yml

@@ -0,0 +1,30 @@
+name: Release moonstreamapi package
+
+on:
+  push:
+    tags:
+      - 'moonstreamapi/v*'
+
+defaults:
+  run:
+    working-directory: backend
+
+jobs:
+  publish:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[distribute]
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*

.github/workflows/release.moonstreamdb.yml

@@ -0,0 +1,30 @@
+name: Release moonstreamdb package
+
+on:
+  push:
+    tags:
+      - 'moonstreamdb/v*'
+
+defaults:
+  run:
+    working-directory: db
+
+jobs:
+  publish:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[distribute]
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*

backend/deploy/deploy.bash

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Deployment script - intended to run on Moonstream servers
+# Deployment script - intended to run on Moonstream API server
 
 # Colors
 C_RESET='\033[0m'
@@ -21,24 +21,29 @@ PYTHON_ENV_DIR="${PYTHON_ENV_DIR:-/home/ubuntu/moonstream-env}"
 PYTHON="${PYTHON_ENV_DIR}/bin/python"
 PIP="${PYTHON_ENV_DIR}/bin/pip"
 SCRIPT_DIR="$(realpath $(dirname $0))"
-PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
-PARAMETERS_BASH_SCRIPT="${SCRIPT_DIR}/parameters.bash"
 SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
 PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
 AWS_SSM_PARAMETER_PATH="${AWS_SSM_PARAMETER_PATH:-/moonstream/prod}"
-SERVICE_FILE="${SCRIPT_DIR}/moonstream.service"
+
+# Parameters scripts
+PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.py"
+CHECKENV_PARAMETERS_SCRIPT="${SCRIPT_DIR}/parameters.bash"
+CHECKENV_NODES_CONNECTIONS_SCRIPT="${SCRIPT_DIR}/nodes-connections.bash"
+
+# API server service file
+SERVICE_FILE="${SCRIPT_DIR}/moonstreamapi.service"
 
 set -eu
 
 echo
 echo
-echo -e "${PREFIX_INFO} Updating pip and setuptools"
-"${PIP}" install -U pip setuptools
+echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
+"${PIP}" install --upgrade pip setuptools
 
 echo
 echo
-echo -e "${PREFIX_INFO} Updating Python dependencies"
-"${PIP}" install -r "${APP_BACKEND_DIR}/requirements.txt"
+echo -e "${PREFIX_INFO} Installing Python dependencies"
+"${PIP}" install -e "${APP_BACKEND_DIR}/"
 
 echo
 echo
@@ -49,13 +54,18 @@ AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" "${PYTHON}" "${PARAMETERS_SCRIPT}" "$
 echo
 echo
 echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
-bash "${PARAMETERS_BASH_SCRIPT}" -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
+bash "${CHECKENV_PARAMETERS_SCRIPT}" -v -p "moonstream" -o "${PARAMETERS_ENV_PATH}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Updating nodes connection parameters"
+bash "${CHECKENV_NODES_CONNECTIONS_SCRIPT}" -v -f "${PARAMETERS_ENV_PATH}"
 
 echo
 echo
-echo -e "${PREFIX_INFO} Replacing existing Moonstream service definition with ${SERVICE_FILE}"
+echo -e "${PREFIX_INFO} Replacing existing Moonstream API service definition with ${SERVICE_FILE}"
 chmod 644 "${SERVICE_FILE}"
-cp "${SERVICE_FILE}" /etc/systemd/system/moonstream.service
+cp "${SERVICE_FILE}" /etc/systemd/system/moonstreamapi.service
 systemctl daemon-reload
-systemctl restart moonstream.service
-systemctl status moonstream.service
+systemctl restart moonstreamapi.service
+systemctl status moonstreamapi.service

backend/deploy/moonstream.service → backend/deploy/moonstreamapi.service

@@ -1,14 +1,14 @@
 [Unit]
-Description=moonstream-service
+Description=moonstreamapi-service
 After=network.target
 
 [Service]
 User=ubuntu
 Group=www-data
 WorkingDirectory=/home/ubuntu/moonstream/backend
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstream.api:app
-SyslogIdentifier=moonstream
+ExecStart=/home/ubuntu/moonstream-env/bin/uvicorn --host 0.0.0.0 --port 7481 --workers 8 moonstreamapi.api:app
+SyslogIdentifier=moonstreamapi
 
 [Install]
 WantedBy=multi-user.target

backend/deploy/nodes-connections.bash

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+#
+# Update nodes connection address environment variables 
+# from AWS Route53 internal hosted zone
+
+VERSION='0.0.1'
+
+# Colors
+C_RESET='\033[0m'
+C_RED='\033[1;31m'
+C_GREEN='\033[1;32m'
+C_YELLOW='\033[1;33m'
+
+# Logs
+PREFIX_INFO="${C_GREEN}[INFO]${C_RESET} [$(date +%d-%m\ %T)]"
+PREFIX_WARN="${C_YELLOW}[WARN]${C_RESET} [$(date +%d-%m\ %T)]"
+PREFIX_CRIT="${C_RED}[CRIT]${C_RESET} [$(date +%d-%m\ %T)]"
+
+# Print help message
+function usage {
+  echo "Usage: $0 [-h] -p PRODUCT -f FILEPATH"  
+  echo
+  echo "CLI to update nodes connection address environment 
+  variables from AWS Route53 internal hosted zone"
+  echo
+  echo "Optional arguments:"
+  echo "  -h  Show this help message and exit"
+  echo "  -f  File path where environment variables update at"
+}
+
+file_flag=""
+verbose_flag="false"
+
+while getopts 'f:v' flag; do
+  case "${flag}" in
+    f) file_flag="${OPTARG}" ;;
+    h) usage
+      exit 1 ;;
+    v) verbose_flag="true" ;;
+    *) usage
+      exit 1 ;;
+  esac
+done
+
+# Log messages
+function verbose {
+  if [ "${verbose_flag}" == "true" ]; then
+    echo -e "$1"
+  fi
+}
+
+# File flag should be specified
+if [ -z "${file_flag}" ]; then
+  verbose "${PREFIX_CRIT} Please specify file path"
+  usage
+  exit 1
+fi
+
+if [ ! -f "${file_flag}" ]; then
+  verbose "${PREFIX_CRIT} Provided file does not exist"
+  usage
+  exit 1
+fi
+
+verbose "${PREFIX_INFO} Script version: v${VERSION}"
+
+verbose "${PREFIX_INFO} Source environment variables"
+. ${file_flag}
+
+verbose "${PREFIX_INFO} Retrieving Ethereum node address"
+RETRIEVED_NODE_ETHEREUM_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
+if [ "$RETRIEVED_NODE_ETHEREUM_IPC_ADDR" == "null" ]; then
+  verbose "${PREFIX_CRIT} Ethereum node internal DNS record address is null"
+  exit 1
+fi
+
+verbose "${PREFIX_INFO} Retrieving Polygon node address"
+RETRIEVED_NODE_POLYGON_IPC_ADDR=$(aws route53 list-resource-record-sets --hosted-zone-id "${MOONSTREAM_INTERNAL_HOSTED_ZONE_ID}" --query "ResourceRecordSets[?Name == '${MOONSTREAM_POLYGON_WEB3_PROVIDER_URI}.'].ResourceRecords[].Value" | jq -r .[0])
+if [ "$RETRIEVED_NODE_POLYGON_IPC_ADDR" == "null" ]; then
+  verbose "${PREFIX_CRIT} Polygon node internal DNS record address is null"
+  exit 1
+fi
+
+# TODO(kompotkot): Modify regexp to work with export prefix
+verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_ETHEREUM_IPC_ADDR with ${RETRIEVED_NODE_ETHEREUM_IPC_ADDR}"
+sed -i "s|^MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=.*|MOONSTREAM_NODE_ETHEREUM_IPC_ADDR=\"$RETRIEVED_NODE_ETHEREUM_IPC_ADDR\"|" ${file_flag}
+
+verbose "${PREFIX_INFO} Updating MOONSTREAM_NODE_POLYGON_IPC_ADDR with ${RETRIEVED_NODE_POLYGON_IPC_ADDR}"
+sed -i "s|^MOONSTREAM_NODE_POLYGON_IPC_ADDR=.*|MOONSTREAM_NODE_POLYGON_IPC_ADDR=\"$RETRIEVED_NODE_POLYGON_IPC_ADDR\"|" ${file_flag}

backend/deploy/parameters.bash

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
 #
-# Collect secrets from AWS SSM Parameter Store and output as environment variable exports.
+# Collect secrets from AWS SSM Parameter Store and 
+# opt out as environment variable exports.
+
+VERSION='0.0.1'
 
 # Colors
 C_RESET='\033[0m'
@@ -26,6 +29,7 @@ and output as environment variable exports"
   echo "  -o  Output file name environment variables export to"
 }
 
+# TODO(kompotkot): Flag for export prefix
 product_flag=""
 output_flag=""
 verbose_flag="false"
@@ -57,6 +61,8 @@ if [ -z "${product_flag}" ]; then
   exit 1
 fi
 
+verbose "${PREFIX_INFO} Script version: v${VERSION}"
+
 verbose "${PREFIX_INFO} Retrieving deployment parameters with tag ${C_GREEN}Product:${product_flag}${C_RESET}"
 ENV_PARAMETERS=$(aws ssm describe-parameters \
   --parameter-filters Key=tag:Product,Values=${product_flag} \

backend/dev.sh

@@ -6,4 +6,4 @@ set -e
 MOONSTREAM_HOST="${MOONSTREAM_HOST:-0.0.0.0}"
 MOONSTREAM_PORT="${MOONSTREAM_PORT:-7481}"
 
-uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstream.api:app --reload
+uvicorn --port "$MOONSTREAM_PORT" --host "$MOONSTREAM_HOST" moonstreamapi.api:app --reload

backend/moonstreamapi/__init__.py

@@ -0,0 +1,11 @@
+try:
+    from .reporter import reporter
+    from .version import MOONSTREAMAPI_VERSION
+
+    # Reporting
+    reporter.tags.append(f"version:{MOONSTREAMAPI_VERSION}")
+    reporter.system_report(publish=True)
+    reporter.setup_excepthook(publish=True)
+except:
+    # Pass it to be able import MOONSTREAMAPI_VERSION in setup.py with pip
+    pass

backend/moonstream/api.py → backend/moonstreamapi/api.py

@@ -19,7 +19,7 @@
 from .routes.whales import router as whales_router
 from .middleware import BroodAuthMiddleware, MoonstreamHTTPException
 from .settings import DOCS_TARGET_PATH, ORIGINS
-from .version import MOONSTREAM_VERSION
+from .version import MOONSTREAMAPI_VERSION
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -44,7 +44,7 @@
 app = FastAPI(
     title=f"Moonstream API",
     description="Moonstream API endpoints.",
-    version=MOONSTREAM_VERSION,
+    version=MOONSTREAMAPI_VERSION,
     openapi_tags=tags_metadata,
     openapi_url="/openapi.json",
     docs_url=None,
@@ -90,7 +90,7 @@ async def version_handler() -> data.VersionResponse:
     """
     Get server version.
     """
-    return data.VersionResponse(version=MOONSTREAM_VERSION)
+    return data.VersionResponse(version=MOONSTREAMAPI_VERSION)
 
 
 @app.get("/now", tags=["time"])

backend/moonstream/reporter.py → backend/moonstreamapi/reporter.py

@@ -9,7 +9,7 @@
 client_id = "moonstream-backend"
 
 reporter = HumbugReporter(
-    name="moonstream",
+    name="moonstreamapi",
     consent=HumbugConsent(True),
     client_id=client_id,
     session_id=session_id,