diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..1cf54843 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,147 @@ +name: Deploy Docker + +# Trigger the workflow on both pushes and pull requests +on: [ push, pull_request ] + +# Use bash as the default shell for run commands +defaults: + run: + shell: bash + +env: + DOCKER_CLI_EXPERIMENTAL: enabled # Enable Docker experimental features (eg. `docker manifest ...`) + +jobs: + + deploy-linux: + name: Deploy (linux) + runs-on: ubuntu-latest + steps: + + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Login to Docker Registry + if: github.event_name != 'pull_request' + run: |- + # Login to GitHub Package Registry + docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} + + # Login to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ ! -z "${{ secrets.DOCKER_HUB_TOKEN }}" ]; then + docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} -p ${{ secrets.DOCKER_HUB_TOKEN }} + fi + + - name: Build Docker image + run: |- + # Parse the tag + TAG=${GITHUB_REF##*/} + TAG=${TAG:-$GITHUB_SHA} + + # Build the image + docker build --rm -f docker/Dockerfile -t corpbot.py:${TAG}-linux-amd64 . + + - name: Publish to Docker Registry + if: github.event_name != 'pull_request' + run: |- + # Parse the tag + TAG=${GITHUB_REF##*/} + TAG=${TAG:-$GITHUB_SHA} + + # Publish to GitHub Package Registry + docker tag corpbot.py:${TAG}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${TAG}-linux-amd64 + docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${TAG}-linux-amd64 + + # Publish to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then + docker tag corpbot.py:${TAG}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-linux-amd64 + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-linux-amd64 + fi + + deploy-windows: + name: Deploy (windows) + runs-on: windows-latest + steps: + + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Login to Docker Registry + if: github.event_name != 'pull_request' + run: |- + # Login to GitHub Package Registry + docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} + + # Login to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ ! -z "${{ secrets.DOCKER_HUB_TOKEN }}" ]; then + docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} -p ${{ secrets.DOCKER_HUB_TOKEN }} + fi + + - name: Build Docker image + run: |- + # Parse the tag + TAG=${GITHUB_REF##*/} + TAG=${TAG:-$GITHUB_SHA} + + # Build the image + docker build --rm -f docker/Dockerfile.win -t corpbot.py:${TAG}-windows-amd64 . + + - name: Publish to Docker Registry + if: github.event_name != 'pull_request' + run: |- + # Parse the tag + TAG=${GITHUB_REF##*/} + TAG=${TAG:-$GITHUB_SHA} + + ## FIXME: GitHub Package Registry doesn't support Windows images (yet) + # Publish to GitHub Package Registry + # docker tag corpbot.py:${TAG}-windows-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${TAG}-windows-amd64 + # docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${TAG}-windows-amd64 + + # Publish to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then + docker tag corpbot.py:${TAG}-windows-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-windows-amd64 + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-windows-amd64 + fi + + deploy-manifest: + name: Deploy (manifest) + runs-on: ubuntu-latest + needs: [ deploy-linux, deploy-windows ] + steps: + + - name: Checkout Repository + uses: actions/checkout@v1 + + - name: Login to Docker Registry + if: github.event_name != 'pull_request' + run: |- + # Login to GitHub Package Registry + docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} + + # Login to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ ! -z "${{ secrets.DOCKER_HUB_TOKEN }}" ]; then + docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} -p ${{ secrets.DOCKER_HUB_TOKEN }} + fi + + - name: Publish to Docker Registry + if: github.event_name != 'pull_request' + run: |- + # Parse the tag + TAG=${GITHUB_REF##*/} + TAG=${TAG:-$GITHUB_SHA} + + ## FIXME: GitHub Package Registry doesn't support Windows images (yet) + # Create and push a combined image manifest to GitHub Package Registry + # docker manifest create docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG}-windows-amd64 + # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG}-linux-amd64 --os linux --arch amd64 + # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG}-windows-amd64 --os windows --arch amd64 + # docker manifest push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG} --purge + + # Create and push a combined image manifest to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then + docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-windows-amd64 + docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-linux-amd64 --os linux --arch amd64 + docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-windows-amd64 --os windows --arch amd64 + docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG} --purge + fi diff --git a/.gitignore b/.gitignore index 2811a5f0..e250d837 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # Anythings starting with . .* +# Don't ignore .github +!.github + # Ignore redis stuff Redis* diff --git a/Cogs/Settings.py b/Cogs/Settings.py index 06729604..c892351b 100755 --- a/Cogs/Settings.py +++ b/Cogs/Settings.py @@ -138,10 +138,10 @@ class Settings(commands.Cog): def __init__(self, bot, prefix = "$", file : str = None): if file == None: # We weren't given a file, default to ./Settings.json - file = "Settings.json" + file = bot.settings_dict.get("settings_path","Settings.json") self.file = file - self.backupDir = "Settings-Backup" + self.backupDir = bot.settings_dict.get("settings_backup_path","Settings-Backup") self.backupMax = 100 self.backupTime = 7200 # runs every 2 hours self.backupWait = 10 # initial wait time before first backup @@ -309,14 +309,14 @@ def __init__(self, bot, prefix = "$", file : str = None): def load_json(self, file): - if os.path.exists(file): + if os.path.exists(file) and os.path.getsize(file): print("Since no mongoDB instance was running, I'm reverting back to the Settings.json") self.serverDict = json.load(open(file)) else: self.serverDict = {} def migrate(self, _file): - if os.path.exists(_file): + if os.path.exists(_file) and os.path.getsize(_file): try: settings_json = json.load(open(_file)) if "mongodb_migrated" not in settings_json: @@ -478,7 +478,7 @@ async def backup(self): os.makedirs(self.backupDir) # Flush backup timeStamp = datetime.today().strftime("%Y-%m-%d %H.%M") - self.flushSettings("./{}/Backup-{}.json".format(self.backupDir, timeStamp), True) + self.flushSettings("{}/Backup-{}.json".format(self.backupDir, timeStamp), True) # Get curr dir and change curr dir retval = os.getcwd() diff --git a/Cogs/TempRole.py b/Cogs/TempRole.py index 6ba595be..1845f13f 100644 --- a/Cogs/TempRole.py +++ b/Cogs/TempRole.py @@ -86,7 +86,7 @@ def check_temp(self): # Bail if we're not the current instance return temp_roles = self.settings.getUserStat(member, server, "TempRoles") - if len(temp_roles): + if temp_roles is not None and len(temp_roles): # We have a list remove_temps = [] for temp_role in temp_roles: diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..ebba2f28 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,66 @@ +# Base the image on Python 3 running on Alpine +# FROM python:3.9.4-alpine3.13 +FROM python:3.8.9-alpine3.13 + +# Install base dependencies +RUN apk add --update --no-cache \ + bash \ + git \ + openssh \ + ca-certificates \ + libffi \ + libxml2 \ + zlib \ + libxslt \ + libjpeg + +# Install build dependencies +RUN apk add --update --no-cache --virtual build-dependencies \ + alpine-sdk \ + linux-headers \ + python3-dev \ + libffi-dev \ + libxml2-dev \ + zlib-dev \ + libxslt-dev \ + jpeg-dev + +# Disable host key checking for git +RUN mkdir -p ~/.ssh && \ + echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config + +# Switch to the image provided work directory +WORKDIR /usr/src/app + +# Set default git merge strategy +RUN git config --global pull.rebase false + +# Copy the installation script and install dependencies +COPY Install.py ./Install.py +RUN yes '' | python ./Install.py + +# Remove build dependencies +RUN apk del build-dependencies + +# Copy the rest of the application over +COPY . . + +# Create the data directory +RUN mkdir -p /data + +# Define default environment variables +ENV SETTINGS_DICT_PREFIX "" +ENV SETTINGS_DICT_TOKEN "" +ENV SETTINGS_DICT_WEATHER "" +ENV SETTINGS_DICT_CURRENCY "" +ENV SETTINGS_DICT_SETTINGS_PATH "/data/Settings.json" +ENV SETTINGS_DICT_SETTINGS_BACKUP_PATH "/data" +ENV SETTINGS_DICT_LAVALINK_HOST "lavalink" +ENV SETTINGS_DICT_LAVALINK_REGION "us_central" +ENV SETTINGS_DICT_LAVALINK_PASSWORD "youshallnotpass" + +# Expose volumes +VOLUME [ "/data", "/usr/local/lib/python3.8/site-packages" ] + +# Set the startup Python script +CMD [ "docker/start.sh" ] diff --git a/docker/Dockerfile.lavalink b/docker/Dockerfile.lavalink new file mode 100644 index 00000000..76102e2d --- /dev/null +++ b/docker/Dockerfile.lavalink @@ -0,0 +1,3 @@ +FROM 1conan/lavalink:latest + +COPY lavalink.yaml /config/application.yml diff --git a/docker/Dockerfile.win b/docker/Dockerfile.win new file mode 100755 index 00000000..32a7e372 --- /dev/null +++ b/docker/Dockerfile.win @@ -0,0 +1,48 @@ +# Base the image on Python 3 running on Windows Server Core +FROM python:3.8.2-windowsservercore + +# Install git +ENV GIT_VERSION 2.26.0 +ENV GIT_PATCH_VERSION 1 +RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ;\ + Invoke-WebRequest $('https://github.com/git-for-windows/git/releases/download/v{0}.windows.{1}/MinGit-{0}-busybox-64-bit.zip' -f $env:GIT_VERSION, $env:GIT_PATCH_VERSION) -OutFile 'mingit.zip' -UseBasicParsing ;\ + Expand-Archive mingit.zip -DestinationPath c:\mingit ;\ + Remove-Item mingit.zip -Force ;\ + setx /M PATH $('c:\mingit\cmd;{0}' -f $env:PATH) + +# Update certificates +RUN certutil -generateSSTFromWU roots.sst ;\ + certutil -addstore -f root roots.sst ;\ + del roots.sst + +# Create the app directory +RUN mkdir C:/app + +# Set the working directory +WORKDIR C:/app + +# Copy the application +ADD ./ C:/app + +# Install dependencies +RUN cmd /c "break|C:/app/Install.bat" + +## TODO: Install Lavalink and use "StartBot.bat" +## https://github.com/Frederikam/Lavalink/releases/tag/3.3.1 + +# Define default environment variables +ENV SETTINGS_DICT_PREFIX "" +ENV SETTINGS_DICT_TOKEN "" +ENV SETTINGS_DICT_WEATHER "" +ENV SETTINGS_DICT_CURRENCY "" +ENV SETTINGS_DICT_SETTINGS_PATH "C:/app/docker/data/Settings.json" +ENV SETTINGS_DICT_SETTINGS_BACKUP_PATH "C:/app/docker/data/Settings-Backup" +ENV SETTINGS_DICT_LAVALINK_HOST "lavalink" +ENV SETTINGS_DICT_LAVALINK_REGION "us_central" +ENV SETTINGS_DICT_LAVALINK_PASSWORD "youshallnotpass" + +# Expose volumes +VOLUME [ "C:/app/docker/data" ] + +# Set the startup command +CMD [ "Start.bat" ] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..e0eded52 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,43 @@ +# Docker support for CorpBot.py + +This is a short document detailing the current status of the Docker images, including how to set them up. + +## Features + +#### Implemented + +- [x] Bot running smoothly inside Docker (tested and works well) +- [x] Full data persistence (data is stored under `/data`, see `docker-compose.yml`) +- [x] Lavalink support both as a Docker container and an external service (playback tested and works well) +- [x] Docker image for Linux (tested and working) +- [x] Docker image for Windows (largely untested) + +#### Planned + +- [ ] Fix/update CI workflow to use new GH registry URL +- [ ] Fix/update CI workflow to support Windows (if/when available) +- [ ] ARM image support (CorpBot itself runs fine on ARM) +- [ ] Redis support (redis branch) +- [ ] MongoDB support (same deal) +- [ ] Optimized, minimal image sizes (Alpine, multi-staged builds, Windows image needs special attention) + +## Running + +Create a `.env` file at the root of this project with the following environment variables): +``` +SETTINGS_DICT_PREFIX="" +SETTINGS_DICT_TOKEN="" +SETTINGS_DICT_WEATHER="" +SETTINGS_DICT_CURRENCY="" +SETTINGS_DICT_SETTINGS_PATH=" (optional) +SETTINGS_DICT_SETTINGS_BACKUP_PATH="" (optional) +SETTINGS_DICT_LAVALINK_HOST="" (optional) +SETTINGS_DICT_LAVALINK_REGION="" (optional) +SETTINGS_DICT_LAVALINK_PASSWORD="" (optional) +``` + +Build and run with Docker Compose while inside the `./docker/` directory: +> docker-compose up --build + +To bring it down with Docker Compose: +> docker-compose down diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..55fb0f2e --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,51 @@ +version: "2.1" # We use v2.x for resource limit support with Docker Compose + +services: + + corpbot: + image: corpnewt/corpbot:latest + build: + context: ../ + dockerfile: docker/Dockerfile + # ## This can be used for building a Windows image instead + # #dockerfile: docker/Dockerfile.win + env_file: ../.env ## Use a .env file at the root + ## Alternatively you can comment out the + ## `environment:` object below and use that instead of the .env file# + # environment: + # SETTINGS_DICT_PREFIX: "bot_prefix" + # SETTINGS_DICT_TOKEN: "discord_token" + # SETTINGS_DICT_WEATHER: "weather_api_key" + # SETTINGS_DICT_CURRENCY: "currency_api_key" + depends_on: + - lavalink + networks: + - corpbot_network + volumes: + - corpbot_data:/data + - corpbot_packages:/usr/local/lib/python3.8/site-packages + # The following can be used to limit resource usage + # cpus: 0.5 + # mem_limit: 1024m + # mem_reservation: 512m + + # NOTE: There is no publicly available Lavalink images for Windows, + # so it might be necessary to run it separately instead + lavalink: + image: 1conan/lavalink:latest + build: + context: . + dockerfile: Dockerfile.lavalink + networks: + - corpbot_network + # The following can be used to limit resource usage + # cpus: 0.5 + # mem_limit: 1024m + # mem_reservation: 512m + +networks: + corpbot_network: + +volumes: + corpbot_data: + corpbot_packages: diff --git a/docker/lavalink.yaml b/docker/lavalink.yaml new file mode 100644 index 00000000..c58ab1b6 --- /dev/null +++ b/docker/lavalink.yaml @@ -0,0 +1,38 @@ +server: # REST and WS server + port: 2333 + address: 0.0.0.0 +spring: + main: + banner-mode: log +lavalink: + server: + password: "youshallnotpass" + sources: + youtube: true + bandcamp: true + soundcloud: true + twitch: true + vimeo: true + mixer: true + http: true + local: false + bufferDurationMs: 400 # How much to buffer (default 400ms) + youtubePlaylistLoadLimit: 6 # Number of pages at 100 each + youtubeSearchEnabled: true + soundcloudSearchEnabled: true + gc-warnings: true + +metrics: + prometheus: + enabled: false + endpoint: /metrics + +logging: + file: + max-history: 30 + max-size: 10MB + path: ./logs/ + + level: + root: INFO + lavalink: INFO diff --git a/docker/start.sh b/docker/start.sh new file mode 100755 index 00000000..b3ce5e4f --- /dev/null +++ b/docker/start.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Enable error handling +set -e +set -o pipefail + +# Enable debugging +#set -x + +# Switch to the application directory +cd /usr/src/app + +# Parse all environment variables that start with "SETTINGS_DICT_", +# then create a matching JSON object string and store it in "settings_dict.json" +JSON="{" # Starts a new JSON object +for setting in "${!SETTINGS_DICT_@}"; do + KEY=${setting#"SETTINGS_DICT_"} # Gets the key name without the prefix + KEY="$(echo "$KEY" | tr -d '"')" # Removes double quotes from the key + VALUE="${!setting}" # Gets the value + VALUE="$(echo "$VALUE" | tr -d '"')" # Removes double quotes from the value + JSON="$JSON\n \"${KEY,,}\": \"$VALUE\"," # Appends the key-value entry to the JSON object +done +JSON="${JSON::-1}\n}" # Removes the last character (comma) and finishes the JSON object +JSON="$(echo -e $JSON)" # Applies the new lines +echo ${JSON} > settings_dict.json # Writes the JSON string to the file + +# Replace git SSH urls with HTTPS to get around WatchDog issues +sed -i "s/git@github.com:/https:\/\/github.com\//g" ./.git/config + +# Start the application +python ./WatchDog.py