From 24e8db7c649a6a7b62e1c9b5e609978db0ee3643 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 23 Mar 2020 13:28:22 +0200 Subject: [PATCH 01/30] Initial work on Docker support Bot now runs inside a small(ish) Alpine linux image, with support for Lavalink as the music player. All features have not been tested, and the audio quality/experience may require further testing/tweaking. Support for Windows containers is also planned, but no ETA yet. --- docker/Dockerfile | 50 ++++++++++++++++++++++++++++++++++++++ docker/README.md | 27 ++++++++++++++++++++ docker/docker-compose.yaml | 37 ++++++++++++++++++++++++++++ docker/lavalink.yaml | 38 +++++++++++++++++++++++++++++ docker/start.sh | 35 ++++++++++++++++++++++++++ 5 files changed, 187 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100644 docker/docker-compose.yaml create mode 100644 docker/lavalink.yaml create mode 100755 docker/start.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..b081c212 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,50 @@ +## TODO: Use build stages to create separate containers for building and running + +# Base the image on Python 3 running on Alpine +FROM python:3.7.7-alpine + +# Install base dependencies +RUN apk add --update --no-cache \ + bash \ + libffi \ + libxml2 \ + zlib \ + libxslt \ + libjpeg + +# Install build dependencies +RUN apk add --update --no-cache --virtual build-dependencies \ + alpine-sdk \ + linux-headers \ + python-dev \ + libffi-dev \ + libxml2-dev \ + zlib-dev \ + libxslt-dev \ + jpeg-dev + +# Switch to the image provided work directory +WORKDIR /usr/src/app + +# Copy the installation script and install dependencies +COPY Install.py ./Install.py +RUN yes '' | python ./Install.py + +# Remove build dependencies +RUN apk del build-dependencies + +# Copy the rest of the application over +COPY . . + +# Define default environment variables +ENV SETTINGS_DICT_PREFIX "" +ENV SETTINGS_DICT_TOKEN "" +ENV SETTINGS_DICT_WEATHER "" +ENV SETTINGS_DICT_CURRENCY "" +ENV LAVALINK_HOST "lavalink" +ENV LAVALINK_REGION "us_central" + +# Set the startup Python script +#CMD [ "python", "./WatchDog.py" ] +#CMD [ "python", "./Main.py" ] +CMD [ "docker/start.sh" ] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..d9c8778c --- /dev/null +++ b/docker/README.md @@ -0,0 +1,27 @@ +# Docker support for CorpBot.py + +*NOTE:* This is still in progress and may not be fully stable or bug free! + +## Features + +- [x] Bot and its basic features running inside a container +- [x] Working music playback through Lavalink +- [x] Linux support (Alpine) +- [ ] Minimal image size (Alpine, multi-staged builds) +- [ ] Windows support (coming soon, maybe) + +## Running + +Create a `.env` file at the root of this project with the following environment variables: +``` +SETTINGS_DICT_PREFIX="" +SETTINGS_DICT_TOKEN="" +SETTINGS_DICT_WEATHER="" +SETTINGS_DICT_CURRENCY="" +``` + +Build and run with Docker Compose while inside the `./docker/` directory: +> docker-compose up --build + +To bring it down with Docker Compose: +> docker-compose down diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 00000000..ff5d88e2 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,37 @@ +version: "2.4" # Use v2.x for resource limit support on Docker Compose + +services: + + corpbot: + image: corpnewt/corpbot:latest + build: + context: ../ + dockerfile: docker/Dockerfile + env_file: ../.env ## Use a .env file at the root + ## Alternatively you can comment out the + ## `environment:` object below and use that instead + # environment: + # SETTINGS_DICT_PREFIX: "bot_prefix" + # SETTINGS_DICT_TOKEN: "discord_token" + # SETTINGS_DICT_WEATHER: "weather_api_key" + # SETTINGS_DICT_CURRENCY: "currency_api_key" + depends_on: + - lavalink + networks: + - bot_network + cpus: 0.5 + mem_limit: 2048m + mem_reservation: 1024m + + lavalink: + image: 1conan/lavalink:latest + volumes: + - ${PWD}/lavalink.yaml:/config/application.yml + networks: + - bot_network + cpus: 0.5 + mem_limit: 1024m + mem_reservation: 512m + +networks: + bot_network: diff --git a/docker/lavalink.yaml b/docker/lavalink.yaml new file mode 100644 index 00000000..c58ab1b6 --- /dev/null +++ b/docker/lavalink.yaml @@ -0,0 +1,38 @@ +server: # REST and WS server + port: 2333 + address: 0.0.0.0 +spring: + main: + banner-mode: log +lavalink: + server: + password: "youshallnotpass" + sources: + youtube: true + bandcamp: true + soundcloud: true + twitch: true + vimeo: true + mixer: true + http: true + local: false + bufferDurationMs: 400 # How much to buffer (default 400ms) + youtubePlaylistLoadLimit: 6 # Number of pages at 100 each + youtubeSearchEnabled: true + soundcloudSearchEnabled: true + gc-warnings: true + +metrics: + prometheus: + enabled: false + endpoint: /metrics + +logging: + file: + max-history: 30 + max-size: 10MB + path: ./logs/ + + level: + root: INFO + lavalink: INFO diff --git a/docker/start.sh b/docker/start.sh new file mode 100755 index 00000000..83eb6516 --- /dev/null +++ b/docker/start.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Enable error handling +set -e +set -o pipefail + +# Enable debugging +#set -x + +# Switch to the application directory +cd /usr/src/app + +# Parse all environment variables that start with "SETTINGS_DICT_", +# then create a matching JSON object string and store it in "settings_dict.json" +JSON="{" # Starts a new JSON object +for setting in "${!SETTINGS_DICT_@}"; do + KEY=${setting#"SETTINGS_DICT_"} # Gets the key name without the prefix + KEY="$(echo "$KEY" | tr -d '"')" # Removes double quotes from the key + VALUE="${!setting}" # Gets the value + VALUE="$(echo "$VALUE" | tr -d '"')" # Removes double quotes from the value + JSON="$JSON\n \"${KEY,,}\": \"$VALUE\"," # Appends the key-value entry to the JSON object +done +JSON="${JSON::-1}\n}" # Removes the last character (comma) and finishes the JSON object +JSON="$(echo -e $JSON)" # Applies the new lines +echo ${JSON} > settings_dict.json # Writes the JSON string to the file + +# Modify the lavalink host in Music.py to use the configured lavalink host instead +sed -i "s/127.0.0.1/$LAVALINK_HOST/g" ./Cogs/Music.py + +# Modify the lavalink region in Music.py to use the configured lavalink region instead +sed -i "s/us_central/$LAVALINK_REGION/g" ./Cogs/Music.py + +# Start the application +#python ./WatchDog.py +python ./Main.py From 47106ca708dead8078546e0aa80d47d6ab302bd6 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 23 Mar 2020 15:53:59 +0200 Subject: [PATCH 02/30] Settings dict values for lavalink Added optional settings dict values for configuring Lavalink --- Cogs/Music.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cogs/Music.py b/Cogs/Music.py index ce3b3018..447c3ca7 100644 --- a/Cogs/Music.py +++ b/Cogs/Music.py @@ -40,12 +40,12 @@ def __init__(self, bot, settings): async def start_nodes(self): node = self.bot.wavelink.get_best_node() if not node: - node = await self.bot.wavelink.initiate_node(host='127.0.0.1', + node = await self.bot.wavelink.initiate_node(host=self.bot.settings_dict.get("lavalink_host","127.0.0.1"), port=2333, - rest_uri='http://127.0.0.1:2333', - password='youshallnotpass', + rest_uri='http://'+self.bot.settings_dict.get("lavalink_host","127.0.0.1")+':2333', + password=self.bot.settings_dict.get("lavalink_password","youshallnotpass"), identifier='TEST', - region='us_central') + region=self.bot.settings_dict.get("lavalink_region","us_central")) node.set_hook(self.on_event_hook) def skip_pop(self, ctx): From 685a583268835b7eec3f06c18ffb90c86912c9bc Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 23 Mar 2020 15:54:18 +0200 Subject: [PATCH 03/30] Check for file size when checking if file exists --- Cogs/Settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cogs/Settings.py b/Cogs/Settings.py index d9d22287..e61ee2b1 100755 --- a/Cogs/Settings.py +++ b/Cogs/Settings.py @@ -309,14 +309,14 @@ def __init__(self, bot, prefix = "$", file : str = None): def load_json(self, file): - if os.path.exists(file): + if os.path.exists(file) and os.path.getsize(file): print("Since no mongoDB instance was running, I'm reverting back to the Settings.json") self.serverDict = json.load(open(file)) else: self.serverDict = {} def migrate(self, _file): - if os.path.exists(_file): + if os.path.exists(_file) and os.path.getsize(_file): try: settings_json = json.load(open(_file)) if "mongodb_migrated" not in settings_json: From 80bb43d20f4b1f606bbb383670d15f2207abc559 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 23 Mar 2020 15:54:55 +0200 Subject: [PATCH 04/30] More containerization work Mounting Settings.json is the only confirmed not working thing right now, everything else seems to work --- docker/Dockerfile | 30 ++++++++++++++++++++---------- docker/README.md | 14 ++++++++++++-- docker/docker-compose.yaml | 7 +++++++ docker/start.sh | 10 +++------- 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b081c212..36835af7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,11 +1,12 @@ -## TODO: Use build stages to create separate containers for building and running - # Base the image on Python 3 running on Alpine FROM python:3.7.7-alpine # Install base dependencies RUN apk add --update --no-cache \ bash \ + git \ + openssh \ + ca-certificates \ libffi \ libxml2 \ zlib \ @@ -23,6 +24,10 @@ RUN apk add --update --no-cache --virtual build-dependencies \ libxslt-dev \ jpeg-dev +# Disable host key checking for git +RUN mkdir -p ~/.ssh && \ + echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config + # Switch to the image provided work directory WORKDIR /usr/src/app @@ -36,15 +41,20 @@ RUN apk del build-dependencies # Copy the rest of the application over COPY . . +# Make sure Settings.json exists +RUN touch /usr/src/app/Settings.json && chown 755 /usr/src/app/Settings.json + # Define default environment variables -ENV SETTINGS_DICT_PREFIX "" -ENV SETTINGS_DICT_TOKEN "" -ENV SETTINGS_DICT_WEATHER "" -ENV SETTINGS_DICT_CURRENCY "" -ENV LAVALINK_HOST "lavalink" -ENV LAVALINK_REGION "us_central" +ENV SETTINGS_DICT_PREFIX "" +ENV SETTINGS_DICT_TOKEN "" +ENV SETTINGS_DICT_WEATHER "" +ENV SETTINGS_DICT_CURRENCY "" +ENV SETTINGS_DICT_LAVALINK_HOST "lavalink" +ENV SETTINGS_DICT_LAVALINK_REGION "us_central" +ENV SETTINGS_DICT_LAVALINK_PASSWORD "youshallnotpass" + +# Expose volumes +VOLUME [ "/usr/src/app/Settings-Backup" ] # Set the startup Python script -#CMD [ "python", "./WatchDog.py" ] -#CMD [ "python", "./Main.py" ] CMD [ "docker/start.sh" ] diff --git a/docker/README.md b/docker/README.md index d9c8778c..9f5854d9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,25 +1,35 @@ # Docker support for CorpBot.py -*NOTE:* This is still in progress and may not be fully stable or bug free! +**NOTE:** This is still a work in progress and may not be fully stable or bug free! ## Features - [x] Bot and its basic features running inside a container - [x] Working music playback through Lavalink - [x] Linux support (Alpine) +- [x] Patch `Music.py` to read Lavalink settings from the config file instead +- [x] Expose persistable data (`Settings.json` and `Settings-Backup/`) +- [ ] Fix `Settings.json` mounting (it should ideally be in a folder that we can mount) +- [ ] Redis support (redis branch) +- [ ] MongoDB support (?) - [ ] Minimal image size (Alpine, multi-staged builds) - [ ] Windows support (coming soon, maybe) ## Running -Create a `.env` file at the root of this project with the following environment variables: +Create a `.env` file at the root of this project with the following environment variables): ``` SETTINGS_DICT_PREFIX="" SETTINGS_DICT_TOKEN="" SETTINGS_DICT_WEATHER="" SETTINGS_DICT_CURRENCY="" +SETTINGS_DICT_LAVALINK_HOST="" (optional) +SETTINGS_DICT_LAVALINK_REGION="" (optional) +SETTINGS_DICT_LAVALINK_PASSWORD="" (optional) ``` +Create an empty `Settings.json` file at the root of this project, which will be mounted by Docker Compose. + Build and run with Docker Compose while inside the `./docker/` directory: > docker-compose up --build diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ff5d88e2..67253726 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -19,6 +19,10 @@ services: - lavalink networks: - bot_network + volumes: + ## FIXME: This doesn't work properly, bot can't seem to write to it (we can though) + #- ${PWD}/../Settings.json:/usr/src/app/Settings.json:rw + - bot_settings_backup:/usr/src/app/Settings-Backup cpus: 0.5 mem_limit: 2048m mem_reservation: 1024m @@ -35,3 +39,6 @@ services: networks: bot_network: + +volumes: + bot_settings_backup: diff --git a/docker/start.sh b/docker/start.sh index 83eb6516..b3ce5e4f 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -24,12 +24,8 @@ JSON="${JSON::-1}\n}" # Removes the last character (comma) and finishes the JSON JSON="$(echo -e $JSON)" # Applies the new lines echo ${JSON} > settings_dict.json # Writes the JSON string to the file -# Modify the lavalink host in Music.py to use the configured lavalink host instead -sed -i "s/127.0.0.1/$LAVALINK_HOST/g" ./Cogs/Music.py - -# Modify the lavalink region in Music.py to use the configured lavalink region instead -sed -i "s/us_central/$LAVALINK_REGION/g" ./Cogs/Music.py +# Replace git SSH urls with HTTPS to get around WatchDog issues +sed -i "s/git@github.com:/https:\/\/github.com\//g" ./.git/config # Start the application -#python ./WatchDog.py -python ./Main.py +python ./WatchDog.py From 7804905874b108cd9e506291facb11ca09c1b5bb Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 23 Mar 2020 20:45:21 +0200 Subject: [PATCH 05/30] Attempting to use a separate data directory --- docker/Dockerfile | 7 +++++-- docker/docker-compose.yaml | 5 +++-- docker/start.sh | 13 +++++++++---- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 36835af7..42ac2a52 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -41,8 +41,11 @@ RUN apk del build-dependencies # Copy the rest of the application over COPY . . +# Create the data directory +RUN mkdir -p /data + # Make sure Settings.json exists -RUN touch /usr/src/app/Settings.json && chown 755 /usr/src/app/Settings.json +#RUN touch /usr/src/app/Settings.json && chown 755 /usr/src/app/Settings.json # Define default environment variables ENV SETTINGS_DICT_PREFIX "" @@ -54,7 +57,7 @@ ENV SETTINGS_DICT_LAVALINK_REGION "us_central" ENV SETTINGS_DICT_LAVALINK_PASSWORD "youshallnotpass" # Expose volumes -VOLUME [ "/usr/src/app/Settings-Backup" ] +VOLUME [ "/data", "/usr/src/app/Settings-Backup" ] # Set the startup Python script CMD [ "docker/start.sh" ] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 67253726..91a07b59 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -22,10 +22,11 @@ services: volumes: ## FIXME: This doesn't work properly, bot can't seem to write to it (we can though) #- ${PWD}/../Settings.json:/usr/src/app/Settings.json:rw + - ./data:/data - bot_settings_backup:/usr/src/app/Settings-Backup cpus: 0.5 - mem_limit: 2048m - mem_reservation: 1024m + mem_limit: 1024m + mem_reservation: 512m lavalink: image: 1conan/lavalink:latest diff --git a/docker/start.sh b/docker/start.sh index b3ce5e4f..e5b8132d 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -10,6 +10,12 @@ set -o pipefail # Switch to the application directory cd /usr/src/app +# Replace git SSH urls with HTTPS to get around WatchDog issues +sed -i "s/git@github.com:/https:\/\/github.com\//g" ./.git/config + +# Switch to the data directory +cd /data + # Parse all environment variables that start with "SETTINGS_DICT_", # then create a matching JSON object string and store it in "settings_dict.json" JSON="{" # Starts a new JSON object @@ -24,8 +30,7 @@ JSON="${JSON::-1}\n}" # Removes the last character (comma) and finishes the JSON JSON="$(echo -e $JSON)" # Applies the new lines echo ${JSON} > settings_dict.json # Writes the JSON string to the file -# Replace git SSH urls with HTTPS to get around WatchDog issues -sed -i "s/git@github.com:/https:\/\/github.com\//g" ./.git/config - # Start the application -python ./WatchDog.py +#python ./WatchDog.py +#python /usr/src/app/WatchDog.py +python /usr/src/app/Main.py From 63e71787450bfa346af1989730972a2634a6b6d5 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Tue, 24 Mar 2020 10:39:19 +0200 Subject: [PATCH 06/30] Settings.py now supports override paths Added optional settings_dict overrides for both Settings.json and Settings-Backup. This allows the important/persisted data to be separate, but also allows the Docker integration to work better with how the bot stores files etc. --- Cogs/Settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cogs/Settings.py b/Cogs/Settings.py index e61ee2b1..72fdb618 100755 --- a/Cogs/Settings.py +++ b/Cogs/Settings.py @@ -138,10 +138,10 @@ class Settings(commands.Cog): def __init__(self, bot, prefix = "$", file : str = None): if file == None: # We weren't given a file, default to ./Settings.json - file = "Settings.json" + file = bot.settings_dict.get("settings_path","Settings.json") self.file = file - self.backupDir = "Settings-Backup" + self.backupDir = bot.settings_dict.get("settings_backup_path","Settings-Backup") self.backupMax = 100 self.backupTime = 7200 # runs every 2 hours self.backupWait = 10 # initial wait time before first backup From 8d941e01431053d8baad9173ba757d96820fcf44 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Tue, 24 Mar 2020 10:41:17 +0200 Subject: [PATCH 07/30] Updated Docker stuff with new settings changes Docker will now default to storing bot data under /data (inside the container), which can easily be mounted in any environment. --- docker/Dockerfile | 21 ++++++++++----------- docker/README.md | 10 ++++++---- docker/docker-compose.yaml | 2 +- docker/start.sh | 13 ++++--------- 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 42ac2a52..592b23be 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,20 +44,19 @@ COPY . . # Create the data directory RUN mkdir -p /data -# Make sure Settings.json exists -#RUN touch /usr/src/app/Settings.json && chown 755 /usr/src/app/Settings.json - # Define default environment variables -ENV SETTINGS_DICT_PREFIX "" -ENV SETTINGS_DICT_TOKEN "" -ENV SETTINGS_DICT_WEATHER "" -ENV SETTINGS_DICT_CURRENCY "" -ENV SETTINGS_DICT_LAVALINK_HOST "lavalink" -ENV SETTINGS_DICT_LAVALINK_REGION "us_central" -ENV SETTINGS_DICT_LAVALINK_PASSWORD "youshallnotpass" +ENV SETTINGS_DICT_PREFIX "" +ENV SETTINGS_DICT_TOKEN "" +ENV SETTINGS_DICT_WEATHER "" +ENV SETTINGS_DICT_CURRENCY "" +ENV SETTINGS_DICT_SETTINGS_PATH "/data/Settings.json" +ENV SETTINGS_DICT_SETTINGS_BACKUP_PATH "/data/Settings-Backup" +ENV SETTINGS_DICT_LAVALINK_HOST "lavalink" +ENV SETTINGS_DICT_LAVALINK_REGION "us_central" +ENV SETTINGS_DICT_LAVALINK_PASSWORD "youshallnotpass" # Expose volumes -VOLUME [ "/data", "/usr/src/app/Settings-Backup" ] +VOLUME [ "/data" ] # Set the startup Python script CMD [ "docker/start.sh" ] diff --git a/docker/README.md b/docker/README.md index 9f5854d9..a008465e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -9,7 +9,7 @@ - [x] Linux support (Alpine) - [x] Patch `Music.py` to read Lavalink settings from the config file instead - [x] Expose persistable data (`Settings.json` and `Settings-Backup/`) -- [ ] Fix `Settings.json` mounting (it should ideally be in a folder that we can mount) +- [x] Fix `Settings.json` mounting (it should ideally be in a folder that we can mount) - [ ] Redis support (redis branch) - [ ] MongoDB support (?) - [ ] Minimal image size (Alpine, multi-staged builds) @@ -23,9 +23,11 @@ SETTINGS_DICT_PREFIX="" SETTINGS_DICT_TOKEN="" SETTINGS_DICT_WEATHER="" SETTINGS_DICT_CURRENCY="" -SETTINGS_DICT_LAVALINK_HOST="" (optional) -SETTINGS_DICT_LAVALINK_REGION="" (optional) -SETTINGS_DICT_LAVALINK_PASSWORD="" (optional) +SETTINGS_DICT_SETTINGS_PATH=" (optional) +SETTINGS_DICT_SETTINGS_BACKUP_PATH="" (optional) +SETTINGS_DICT_LAVALINK_HOST="" (optional) +SETTINGS_DICT_LAVALINK_REGION="" (optional) +SETTINGS_DICT_LAVALINK_PASSWORD="" (optional) ``` Create an empty `Settings.json` file at the root of this project, which will be mounted by Docker Compose. diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 91a07b59..ed7048fa 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -23,7 +23,7 @@ services: ## FIXME: This doesn't work properly, bot can't seem to write to it (we can though) #- ${PWD}/../Settings.json:/usr/src/app/Settings.json:rw - ./data:/data - - bot_settings_backup:/usr/src/app/Settings-Backup + #- bot_settings_backup:/usr/src/app/Settings-Backup cpus: 0.5 mem_limit: 1024m mem_reservation: 512m diff --git a/docker/start.sh b/docker/start.sh index e5b8132d..b3ce5e4f 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -10,12 +10,6 @@ set -o pipefail # Switch to the application directory cd /usr/src/app -# Replace git SSH urls with HTTPS to get around WatchDog issues -sed -i "s/git@github.com:/https:\/\/github.com\//g" ./.git/config - -# Switch to the data directory -cd /data - # Parse all environment variables that start with "SETTINGS_DICT_", # then create a matching JSON object string and store it in "settings_dict.json" JSON="{" # Starts a new JSON object @@ -30,7 +24,8 @@ JSON="${JSON::-1}\n}" # Removes the last character (comma) and finishes the JSON JSON="$(echo -e $JSON)" # Applies the new lines echo ${JSON} > settings_dict.json # Writes the JSON string to the file +# Replace git SSH urls with HTTPS to get around WatchDog issues +sed -i "s/git@github.com:/https:\/\/github.com\//g" ./.git/config + # Start the application -#python ./WatchDog.py -#python /usr/src/app/WatchDog.py -python /usr/src/app/Main.py +python ./WatchDog.py From 7c29d4c5a4aa409dc6ed619b6dfcfe9aae75e7cd Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Tue, 24 Mar 2020 10:44:12 +0200 Subject: [PATCH 08/30] Docker cleanup --- docker/README.md | 2 -- docker/docker-compose.yaml | 17 +++++++---------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/docker/README.md b/docker/README.md index a008465e..302e3021 100644 --- a/docker/README.md +++ b/docker/README.md @@ -30,8 +30,6 @@ SETTINGS_DICT_LAVALINK_REGION="" (optional) SETTINGS_DICT_LAVALINK_PASSWORD="" (optional) ``` -Create an empty `Settings.json` file at the root of this project, which will be mounted by Docker Compose. - Build and run with Docker Compose while inside the `./docker/` directory: > docker-compose up --build diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ed7048fa..f1ace8c7 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,4 +1,4 @@ -version: "2.4" # Use v2.x for resource limit support on Docker Compose +version: "2.4" # We use v2.x for resource limit support with Docker Compose services: @@ -9,7 +9,7 @@ services: dockerfile: docker/Dockerfile env_file: ../.env ## Use a .env file at the root ## Alternatively you can comment out the - ## `environment:` object below and use that instead + ## `environment:` object below and use that instead of the .env file# # environment: # SETTINGS_DICT_PREFIX: "bot_prefix" # SETTINGS_DICT_TOKEN: "discord_token" @@ -18,12 +18,9 @@ services: depends_on: - lavalink networks: - - bot_network + - corpbot_network volumes: - ## FIXME: This doesn't work properly, bot can't seem to write to it (we can though) - #- ${PWD}/../Settings.json:/usr/src/app/Settings.json:rw - - ./data:/data - #- bot_settings_backup:/usr/src/app/Settings-Backup + - corpbot_data:/data cpus: 0.5 mem_limit: 1024m mem_reservation: 512m @@ -33,13 +30,13 @@ services: volumes: - ${PWD}/lavalink.yaml:/config/application.yml networks: - - bot_network + - corpbot_network cpus: 0.5 mem_limit: 1024m mem_reservation: 512m networks: - bot_network: + corpbot: volumes: - bot_settings_backup: + corpbot_data: From 7ec7bb962bcd9907ded3979492cb194e7c954f3c Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Fri, 27 Mar 2020 09:15:32 +0200 Subject: [PATCH 09/30] Initial work on Dockerfile for Windows --- docker/Dockerfile.win | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docker/Dockerfile.win diff --git a/docker/Dockerfile.win b/docker/Dockerfile.win new file mode 100644 index 00000000..a7ffdcfc --- /dev/null +++ b/docker/Dockerfile.win @@ -0,0 +1,22 @@ +# Base the image on Python 3 running on Windows Server Core +FROM python:3.7.7-windowsservercore + +# Create the app directory +RUN mkdir C:/app + +# Set the working directory +WORKDIR C:/app + +# Copy the application +ADD ./ C:/app + +# Install dependencies +RUN Install.bat +# RUN python Install.py + +## TODO: Install Lavalink and use "StartBot.bat" +## https://github.com/Frederikam/Lavalink/releases/tag/3.3.1 + +# Set the startup command +CMD [ "Start.bat" ] +#CMD [ "python", "Start.py" ] From 5e84c0e396f740d3fede8668e4fa6168dd8a583c Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Fri, 27 Mar 2020 15:49:05 +0200 Subject: [PATCH 10/30] Dockerfile for Windows now compiles and runs (kinda) --- docker/Dockerfile.win | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) mode change 100644 => 100755 docker/Dockerfile.win diff --git a/docker/Dockerfile.win b/docker/Dockerfile.win old mode 100644 new mode 100755 index a7ffdcfc..f688335c --- a/docker/Dockerfile.win +++ b/docker/Dockerfile.win @@ -1,5 +1,20 @@ # Base the image on Python 3 running on Windows Server Core -FROM python:3.7.7-windowsservercore +FROM python:3.8.2-windowsservercore + +# Install git +ENV GIT_VERSION 2.26.0 +ENV GIT_PATCH_VERSION 1 +#RUN powershell -Command $ErrorActionPreference='Stop' ;\ +RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ;\ + Invoke-WebRequest $('https://github.com/git-for-windows/git/releases/download/v{0}.windows.{1}/MinGit-{0}-busybox-64-bit.zip' -f $env:GIT_VERSION, $env:GIT_PATCH_VERSION) -OutFile 'mingit.zip' -UseBasicParsing ;\ + Expand-Archive mingit.zip -DestinationPath c:\mingit ;\ + Remove-Item mingit.zip -Force ;\ + setx /M PATH $('c:\mingit\cmd;{0}' -f $env:PATH) + +# Update certificates +RUN certutil -generateSSTFromWU roots.sst ;\ + certutil -addstore -f root roots.sst ;\ + del roots.sst # Create the app directory RUN mkdir C:/app @@ -11,8 +26,8 @@ WORKDIR C:/app ADD ./ C:/app # Install dependencies -RUN Install.bat -# RUN python Install.py +RUN cmd /c "break|C:/app/Install.bat" +# RUN python C:/app/Install.py ## TODO: Install Lavalink and use "StartBot.bat" ## https://github.com/Frederikam/Lavalink/releases/tag/3.3.1 From e672dfd2f09dd6366d9f0b2c2a23f7126a0b9e3d Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 20 Apr 2020 10:23:40 +0300 Subject: [PATCH 11/30] Fixed docker-compose.yaml --- docker/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index f1ace8c7..04d10b73 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -36,7 +36,7 @@ services: mem_reservation: 512m networks: - corpbot: + corpbot_network: volumes: corpbot_data: From a46f3b505c050c250ee05931d3d470ed29a768a8 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 20 Apr 2020 10:32:27 +0300 Subject: [PATCH 12/30] First pass on Docker CI workflow --- .github/workflows/docker.yml | 41 ++++++++++++++++++++++++++++++++++++ .gitignore | 3 +++ 2 files changed, 44 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..29b48170 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,41 @@ +name: Deploy Docker + +jobs: + + deploy-linux: + name: Deploy (linux) + runs-on: ubuntu-latest + steps: + + - name: Checkout Repository + uses: actions/checkout@v1 + + - name: Build + run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:linux-amd64 . + + - name: Publish (GitHub Package Registry) + uses: elgohr/Publish-Docker-Github-Action@master + with: + name: $GITHUB_ACTOR/corpbot.py/corpbot.py:linux-amd64 + username: $GITHUB_ACTOR + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com + + deploy-windows: + name: Deploy (windows) + runs-on: windows-latest + steps: + + - name: Checkout Repository + uses: actions/checkout@v1 + + - name: Build + run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:linux-amd64 . + + - name: Publish (GitHub Package Registry) + uses: elgohr/Publish-Docker-Github-Action@master + with: + name: $GITHUB_ACTOR/corpbot.py/corpbot.py:windows-amd64 + username: $GITHUB_ACTOR + password: ${{ secrets.GITHUB_TOKEN }} + registry: docker.pkg.github.com diff --git a/.gitignore b/.gitignore index 2811a5f0..e250d837 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # Anythings starting with . .* +# Don't ignore .github +!.github + # Ignore redis stuff Redis* From 90fcfc9da38d8e2615e9f8f59c9dac162fd82a8a Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 20 Apr 2020 10:36:05 +0300 Subject: [PATCH 13/30] Fixing Docker CI workflow --- .github/workflows/docker.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 29b48170..bc187d33 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,5 +1,12 @@ name: Deploy Docker +on: + push: + # Only build when pushing to the "rewrite" branch + branches: + - rewrite + - docker/feature ## TODO: Remove when done + jobs: deploy-linux: @@ -11,12 +18,12 @@ jobs: uses: actions/checkout@v1 - name: Build - run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:linux-amd64 . + run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-linux-amd64 . - name: Publish (GitHub Package Registry) uses: elgohr/Publish-Docker-Github-Action@master with: - name: $GITHUB_ACTOR/corpbot.py/corpbot.py:linux-amd64 + name: $GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-linux-amd64 username: $GITHUB_ACTOR password: ${{ secrets.GITHUB_TOKEN }} registry: docker.pkg.github.com @@ -30,12 +37,12 @@ jobs: uses: actions/checkout@v1 - name: Build - run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:linux-amd64 . + run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-windows-amd64 . - name: Publish (GitHub Package Registry) uses: elgohr/Publish-Docker-Github-Action@master with: - name: $GITHUB_ACTOR/corpbot.py/corpbot.py:windows-amd64 + name: $GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-windows-amd64 username: $GITHUB_ACTOR password: ${{ secrets.GITHUB_TOKEN }} registry: docker.pkg.github.com From 93b92cf9c837f0921119ac2a7735333a18f4fdbd Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 20 Apr 2020 10:36:05 +0300 Subject: [PATCH 14/30] Fixing Docker CI workflow --- .github/workflows/docker.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 29b48170..669ef8bc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,5 +1,12 @@ name: Deploy Docker +on: + push: + # Only build when pushing to the "rewrite" branch + branches: + - rewrite + - feature/docker ## TODO: Remove when done + jobs: deploy-linux: @@ -11,12 +18,12 @@ jobs: uses: actions/checkout@v1 - name: Build - run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:linux-amd64 . + run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-linux-amd64 . - name: Publish (GitHub Package Registry) uses: elgohr/Publish-Docker-Github-Action@master with: - name: $GITHUB_ACTOR/corpbot.py/corpbot.py:linux-amd64 + name: $GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-linux-amd64 username: $GITHUB_ACTOR password: ${{ secrets.GITHUB_TOKEN }} registry: docker.pkg.github.com @@ -30,12 +37,12 @@ jobs: uses: actions/checkout@v1 - name: Build - run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:linux-amd64 . + run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-windows-amd64 . - name: Publish (GitHub Package Registry) uses: elgohr/Publish-Docker-Github-Action@master with: - name: $GITHUB_ACTOR/corpbot.py/corpbot.py:windows-amd64 + name: $GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-windows-amd64 username: $GITHUB_ACTOR password: ${{ secrets.GITHUB_TOKEN }} registry: docker.pkg.github.com From 6bbd3cea651c086e824ee2891e754c4db0800d65 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 20 Apr 2020 10:38:12 +0300 Subject: [PATCH 15/30] Fixing Docker CI workflow --- .github/workflows/docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 669ef8bc..84fa6645 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,3 +1,7 @@ +## Builds a Docker image for both Linux and Windows, +## then deploys these to GitHub Package Registry, +## where they are publicly available for everyone + name: Deploy Docker on: From 56f87f60cafa2c26a07c633a325251b34ed6918d Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 20 Apr 2020 10:39:46 +0300 Subject: [PATCH 16/30] Fixing Docker CI workflow --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 84fa6645..9826548f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,7 +41,7 @@ jobs: uses: actions/checkout@v1 - name: Build - run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-windows-amd64 . + run: docker build --rm -f docker/Dockerfile.win -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-windows-amd64 . - name: Publish (GitHub Package Registry) uses: elgohr/Publish-Docker-Github-Action@master From 14370ce5645913bb146167e1afa22a4c601cf434 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 20 Apr 2020 10:40:50 +0300 Subject: [PATCH 17/30] Fixing Docker CI workflow --- .github/workflows/docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9826548f..ae5813c9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,12 +22,12 @@ jobs: uses: actions/checkout@v1 - name: Build - run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-linux-amd64 . + run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 . - name: Publish (GitHub Package Registry) uses: elgohr/Publish-Docker-Github-Action@master with: - name: $GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-linux-amd64 + name: ${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 username: $GITHUB_ACTOR password: ${{ secrets.GITHUB_TOKEN }} registry: docker.pkg.github.com @@ -41,12 +41,12 @@ jobs: uses: actions/checkout@v1 - name: Build - run: docker build --rm -f docker/Dockerfile.win -t docker.pkg.github.com/$GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-windows-amd64 . + run: docker build --rm -f docker/Dockerfile.win -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 . - name: Publish (GitHub Package Registry) uses: elgohr/Publish-Docker-Github-Action@master with: - name: $GITHUB_ACTOR/corpbot.py/corpbot.py:rewrite-windows-amd64 + name: ${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 username: $GITHUB_ACTOR password: ${{ secrets.GITHUB_TOKEN }} registry: docker.pkg.github.com From 494afa5f67dd2f21b560a04d96489a4606326fdb Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 20 Apr 2020 11:19:58 +0300 Subject: [PATCH 18/30] Fixing Docker CI workflow --- .github/workflows/docker.yml | 64 +++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ae5813c9..74aeae26 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,32 +21,42 @@ jobs: - name: Checkout Repository uses: actions/checkout@v1 - - name: Build - run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 . - - - name: Publish (GitHub Package Registry) - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: ${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 - username: $GITHUB_ACTOR - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com - - deploy-windows: - name: Deploy (windows) - runs-on: windows-latest - steps: + - name: Login to GitHub Package Registry + run: docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} - - name: Checkout Repository - uses: actions/checkout@v1 - - - name: Build - run: docker build --rm -f docker/Dockerfile.win -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 . + - name: Build Docker image + run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 . - - name: Publish (GitHub Package Registry) - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: ${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 - username: $GITHUB_ACTOR - password: ${{ secrets.GITHUB_TOKEN }} - registry: docker.pkg.github.com + - name: Publish to GitHub Package Registry + run: docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 + + # - name: Publish (GitHub Package Registry) + # uses: elgohr/Publish-Docker-Github-Action@master + # with: + # name: ${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 + # username: $GITHUB_ACTOR + # password: ${{ secrets.GITHUB_TOKEN }} + # registry: docker.pkg.github.com + + # deploy-windows: + # name: Deploy (windows) + # runs-on: windows-latest + # steps: + + # - name: Checkout Repository + # uses: actions/checkout@v1 + + # ## FIXME: Bash variables don't work in Windows, duh + # - name: Build + # run: docker build --rm -f docker/Dockerfile.win -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 . + + # - name: Publish (GitHub Package Registry) + # uses: elgohr/Publish-Docker-Github-Action@master + # with: + # name: ${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 + # username: $GITHUB_ACTOR + # password: ${{ secrets.GITHUB_TOKEN }} + # registry: docker.pkg.github.com + +## TODO: Use the manifest tool to combine the platforms into one image, +## and use "depends_on" to ensure both are built before the manifest job From 31503d362aa9c8a0e984fe9040641b17b0fe0c69 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 20 Apr 2020 11:51:58 +0300 Subject: [PATCH 19/30] Fixing Docker CI workflow for Windows --- .github/workflows/docker.yml | 46 +++++++++++++++--------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 74aeae26..22a05ace 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,33 +30,25 @@ jobs: - name: Publish to GitHub Package Registry run: docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 - # - name: Publish (GitHub Package Registry) - # uses: elgohr/Publish-Docker-Github-Action@master - # with: - # name: ${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 - # username: $GITHUB_ACTOR - # password: ${{ secrets.GITHUB_TOKEN }} - # registry: docker.pkg.github.com - - # deploy-windows: - # name: Deploy (windows) - # runs-on: windows-latest - # steps: - - # - name: Checkout Repository - # uses: actions/checkout@v1 - - # ## FIXME: Bash variables don't work in Windows, duh - # - name: Build - # run: docker build --rm -f docker/Dockerfile.win -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 . - - # - name: Publish (GitHub Package Registry) - # uses: elgohr/Publish-Docker-Github-Action@master - # with: - # name: ${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 - # username: $GITHUB_ACTOR - # password: ${{ secrets.GITHUB_TOKEN }} - # registry: docker.pkg.github.com + deploy-windows: + name: Deploy (windows) + runs-on: windows-latest + steps: + + - name: Checkout Repository + uses: actions/checkout@v1 + + - name: Login to GitHub Package Registry + shell: bash + run: docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} + + - name: Build Docker image + shell: bash + run: docker build --rm -f docker/Dockerfile.win -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 . + + - name: Publish to GitHub Package Registry + shell: bash + run: docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 ## TODO: Use the manifest tool to combine the platforms into one image, ## and use "depends_on" to ensure both are built before the manifest job From be53a3ac6bfdf837ad6d6d0bd0872ad65fe46dc3 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Mon, 20 Apr 2020 12:28:00 +0300 Subject: [PATCH 20/30] Finishing touches on Docker support --- .github/workflows/docker.yml | 36 ++++++++++++++++-------------------- docker/Dockerfile.win | 17 ++++++++++++++--- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 22a05ace..ec89b2cc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,7 +1,3 @@ -## Builds a Docker image for both Linux and Windows, -## then deploys these to GitHub Package Registry, -## where they are publicly available for everyone - name: Deploy Docker on: @@ -9,7 +5,6 @@ on: # Only build when pushing to the "rewrite" branch branches: - rewrite - - feature/docker ## TODO: Remove when done jobs: @@ -30,25 +25,26 @@ jobs: - name: Publish to GitHub Package Registry run: docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 - deploy-windows: - name: Deploy (windows) - runs-on: windows-latest - steps: + ## TODO: GitHub Package Registry doesn't support Windows images (yet) + # deploy-windows: + # name: Deploy (windows) + # runs-on: windows-latest + # steps: - - name: Checkout Repository - uses: actions/checkout@v1 + # - name: Checkout Repository + # uses: actions/checkout@v1 - - name: Login to GitHub Package Registry - shell: bash - run: docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} + # - name: Login to GitHub Package Registry + # shell: bash + # run: docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} - - name: Build Docker image - shell: bash - run: docker build --rm -f docker/Dockerfile.win -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 . + # - name: Build Docker image + # shell: bash + # run: docker build --rm -f docker/Dockerfile.win -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 . - - name: Publish to GitHub Package Registry - shell: bash - run: docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 + # - name: Publish to GitHub Package Registry + # shell: bash + # run: docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 ## TODO: Use the manifest tool to combine the platforms into one image, ## and use "depends_on" to ensure both are built before the manifest job diff --git a/docker/Dockerfile.win b/docker/Dockerfile.win index f688335c..32a7e372 100755 --- a/docker/Dockerfile.win +++ b/docker/Dockerfile.win @@ -4,7 +4,6 @@ FROM python:3.8.2-windowsservercore # Install git ENV GIT_VERSION 2.26.0 ENV GIT_PATCH_VERSION 1 -#RUN powershell -Command $ErrorActionPreference='Stop' ;\ RUN [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 ;\ Invoke-WebRequest $('https://github.com/git-for-windows/git/releases/download/v{0}.windows.{1}/MinGit-{0}-busybox-64-bit.zip' -f $env:GIT_VERSION, $env:GIT_PATCH_VERSION) -OutFile 'mingit.zip' -UseBasicParsing ;\ Expand-Archive mingit.zip -DestinationPath c:\mingit ;\ @@ -27,11 +26,23 @@ ADD ./ C:/app # Install dependencies RUN cmd /c "break|C:/app/Install.bat" -# RUN python C:/app/Install.py ## TODO: Install Lavalink and use "StartBot.bat" ## https://github.com/Frederikam/Lavalink/releases/tag/3.3.1 +# Define default environment variables +ENV SETTINGS_DICT_PREFIX "" +ENV SETTINGS_DICT_TOKEN "" +ENV SETTINGS_DICT_WEATHER "" +ENV SETTINGS_DICT_CURRENCY "" +ENV SETTINGS_DICT_SETTINGS_PATH "C:/app/docker/data/Settings.json" +ENV SETTINGS_DICT_SETTINGS_BACKUP_PATH "C:/app/docker/data/Settings-Backup" +ENV SETTINGS_DICT_LAVALINK_HOST "lavalink" +ENV SETTINGS_DICT_LAVALINK_REGION "us_central" +ENV SETTINGS_DICT_LAVALINK_PASSWORD "youshallnotpass" + +# Expose volumes +VOLUME [ "C:/app/docker/data" ] + # Set the startup command CMD [ "Start.bat" ] -#CMD [ "python", "Start.py" ] From 3feb10ac3e1c940b28579bbd58ea90482d93432e Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Thu, 23 Apr 2020 09:21:28 +0300 Subject: [PATCH 21/30] Cleanup and optional Docker Hub support for CI --- .github/workflows/docker.yml | 118 ++++++++++++++---- docker/README.md | 24 ++-- ...docker-compose.yaml => docker-compose.yml} | 4 + 3 files changed, 113 insertions(+), 33 deletions(-) rename docker/{docker-compose.yaml => docker-compose.yml} (80%) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ec89b2cc..395fce91 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -6,6 +6,10 @@ on: branches: - rewrite +env: + # Enable Docker experimental features (eg. `docker manifest ...`) + DOCKER_CLI_EXPERIMENTAL: enabled + jobs: deploy-linux: @@ -16,35 +20,103 @@ jobs: - name: Checkout Repository uses: actions/checkout@v1 - - name: Login to GitHub Package Registry - run: docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Registry + run: |- + # Login to GitHub Package Registry + docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} + + # Login to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ ! -z "${{ secrets.DOCKER_HUB_TOKEN }}" ]; then + docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} -p ${{ secrets.DOCKER_HUB_TOKEN }} + fi + + - name: Build Docker image + run: |- + docker build --rm -f docker/Dockerfile -t corpbot.py:rewrite-linux-amd64 . + + - name: Publish to Docker Registry + run: |- + # Publish to GitHub Package Registry + docker tag corpbot.py:rewrite-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 + docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 + + # Publish to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then + docker tag corpbot.py:rewrite-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-linux-amd64 + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-linux-amd64 + fi + + deploy-windows: + name: Deploy (windows) + runs-on: windows-latest + steps: + + - name: Checkout Repository + uses: actions/checkout@v1 + + - name: Login to Docker Registry + shell: bash + run: |- + # Login to GitHub Package Registry + docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} + + # Login to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ ! -z "${{ secrets.DOCKER_HUB_TOKEN }}" ]; then + docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} -p ${{ secrets.DOCKER_HUB_TOKEN }} + fi - name: Build Docker image - run: docker build --rm -f docker/Dockerfile -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 . + shell: bash + run: |- + docker build --rm -f docker/Dockerfile -t corpbot.py:rewrite-windows-amd64 . - - name: Publish to GitHub Package Registry - run: docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 + - name: Publish to Docker Registry + shell: bash + run: |- + ## FIXME: GitHub Package Registry doesn't support Windows images (yet) + # Publish to GitHub Package Registry + # docker tag corpbot.py:rewrite-windows-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 + # docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 - ## TODO: GitHub Package Registry doesn't support Windows images (yet) - # deploy-windows: - # name: Deploy (windows) - # runs-on: windows-latest - # steps: + # Publish to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then + docker tag corpbot.py:rewrite-windows-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-windows-amd64 + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-windows-amd64 + fi - # - name: Checkout Repository - # uses: actions/checkout@v1 + deploy-manifest: + name: Deploy (manifest) + runs-on: ubuntu-latest + needs: [ deploy-linux, deploy-windows ] + steps: + + - name: Checkout Repository + uses: actions/checkout@v1 - # - name: Login to GitHub Package Registry - # shell: bash - # run: docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Registry + run: |- + # Login to GitHub Package Registry + docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} - # - name: Build Docker image - # shell: bash - # run: docker build --rm -f docker/Dockerfile.win -t docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 . + # Login to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ] && [ ! -z "${{ secrets.DOCKER_HUB_TOKEN }}" ]; then + docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} -p ${{ secrets.DOCKER_HUB_TOKEN }} + fi - # - name: Publish to GitHub Package Registry - # shell: bash - # run: docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 + - name: Publish to Docker Registry + shell: bash + run: |- + ## FIXME: GitHub Package Registry doesn't support Windows images (yet) + # Create and push a combined image manifest to GitHub Package Registry + # docker manifest create docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite-windows-amd64 + # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite-linux-amd64 --os linux --arch amd64 + # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite-windows-amd64 --os windows --arch amd64 + # docker manifest push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite --purge -## TODO: Use the manifest tool to combine the platforms into one image, -## and use "depends_on" to ensure both are built before the manifest job + # Create and push a combined image manifest to Docker Hub + if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then + docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-windows-amd64 + docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-linux-amd64 --os linux --arch amd64 + docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-windows-amd64 --os windows --arch amd64 + docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite --purge + fi diff --git a/docker/README.md b/docker/README.md index 302e3021..6874c31b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,19 +1,23 @@ # Docker support for CorpBot.py -**NOTE:** This is still a work in progress and may not be fully stable or bug free! +This is a short document detailing the current status of the Docker images, including how to set them up. ## Features -- [x] Bot and its basic features running inside a container -- [x] Working music playback through Lavalink -- [x] Linux support (Alpine) -- [x] Patch `Music.py` to read Lavalink settings from the config file instead -- [x] Expose persistable data (`Settings.json` and `Settings-Backup/`) -- [x] Fix `Settings.json` mounting (it should ideally be in a folder that we can mount) +#### Implemented + +- [x] Bot running smoothly inside Docker (tested and works well) +- [x] Full data persistence (data is stored under `/data`, see `docker-compose.yml`) +- [x] Lavalink support both as a Docker container and an external service (playback tested and works well) +- [x] Docker image for Linux (tested and working) +- [x] Docker image for Windows (largely untested) + +#### Planned + - [ ] Redis support (redis branch) -- [ ] MongoDB support (?) -- [ ] Minimal image size (Alpine, multi-staged builds) -- [ ] Windows support (coming soon, maybe) +- [ ] MongoDB support (same deal) +- [ ] Optimized, minimal image sizes (Alpine, multi-staged builds) +- [ ] A single image for multiple platforms (needs Docker Hub support) ## Running diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yml similarity index 80% rename from docker/docker-compose.yaml rename to docker/docker-compose.yml index 04d10b73..a74d09be 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yml @@ -7,6 +7,8 @@ services: build: context: ../ dockerfile: docker/Dockerfile + ## This can be used for building a Windows image instead + #dockerfile: docker/Dockerfile.win env_file: ../.env ## Use a .env file at the root ## Alternatively you can comment out the ## `environment:` object below and use that instead of the .env file# @@ -25,6 +27,8 @@ services: mem_limit: 1024m mem_reservation: 512m + ## NOTE: There is no publicly available Lavalink images for Windows, + ## so it might be necessary to run it separately instead lavalink: image: 1conan/lavalink:latest volumes: From 0ab73065d28c58d3dc3dee086e843e5b2dfc6fb4 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Thu, 23 Apr 2020 09:24:26 +0300 Subject: [PATCH 22/30] Added feature/docker branch back to CI build rules --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 395fce91..d46c2ab5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,6 +5,7 @@ on: # Only build when pushing to the "rewrite" branch branches: - rewrite + - feature/docker ## TODO: Remove this when this branch is no longer used env: # Enable Docker experimental features (eg. `docker manifest ...`) From 4a6aad47ff0b89fd04dd47f7272cbebaab1841fc Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Thu, 23 Apr 2020 09:25:53 +0300 Subject: [PATCH 23/30] Fixed typo in CI workflow --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d46c2ab5..6fa9cc36 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -69,7 +69,7 @@ jobs: - name: Build Docker image shell: bash run: |- - docker build --rm -f docker/Dockerfile -t corpbot.py:rewrite-windows-amd64 . + docker build --rm -f docker/Dockerfile.win -t corpbot.py:rewrite-windows-amd64 . - name: Publish to Docker Registry shell: bash From 378fbdd4476c3cf5f12bc38beba31b943e1d8a39 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Thu, 23 Apr 2020 09:56:23 +0300 Subject: [PATCH 24/30] Minor documentation update --- docker/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index 6874c31b..3d91046a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -16,8 +16,7 @@ This is a short document detailing the current status of the Docker images, incl - [ ] Redis support (redis branch) - [ ] MongoDB support (same deal) -- [ ] Optimized, minimal image sizes (Alpine, multi-staged builds) -- [ ] A single image for multiple platforms (needs Docker Hub support) +- [ ] Optimized, minimal image sizes (Alpine, multi-staged builds, Windows image needs special attention) ## Running From 50e4a13380b8298975eb111f9af7910ffce29b59 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Thu, 23 Apr 2020 13:41:22 +0300 Subject: [PATCH 25/30] Updated the CI workflow to be more dynamic - CI will now build both pushes (commits) and pull requests - CI will only deploy images for pushes (commits) - CI will attempt to use the current branch name for the tag, falling back to the commit hash --- .github/workflows/docker.yml | 63 +++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6fa9cc36..65efd660 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,15 +1,16 @@ name: Deploy Docker -on: - push: - # Only build when pushing to the "rewrite" branch - branches: - - rewrite - - feature/docker ## TODO: Remove this when this branch is no longer used +# Trigger the workflow on both pushes and pull requests +on: [ push, pull_request ] + +# Use bash as the default shell for run commands +defaults: + run: + shell: bash env: - # Enable Docker experimental features (eg. `docker manifest ...`) - DOCKER_CLI_EXPERIMENTAL: enabled + DOCKER_CLI_EXPERIMENTAL: enabled # Enable Docker experimental features (eg. `docker manifest ...`) + GITHUB_BRANCH: ${GITHUB_REF##*/} jobs: @@ -22,6 +23,7 @@ jobs: uses: actions/checkout@v1 - name: Login to Docker Registry + if: github.event_name != 'pull_request' run: |- # Login to GitHub Package Registry docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} @@ -33,18 +35,19 @@ jobs: - name: Build Docker image run: |- - docker build --rm -f docker/Dockerfile -t corpbot.py:rewrite-linux-amd64 . + docker build --rm -f docker/Dockerfile -t corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 . - name: Publish to Docker Registry + if: github.event_name != 'pull_request' run: |- # Publish to GitHub Package Registry - docker tag corpbot.py:rewrite-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 - docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-linux-amd64 + docker tag corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 + docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 # Publish to Docker Hub if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then - docker tag corpbot.py:rewrite-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-linux-amd64 - docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-linux-amd64 + docker tag corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 fi deploy-windows: @@ -56,7 +59,7 @@ jobs: uses: actions/checkout@v1 - name: Login to Docker Registry - shell: bash + if: github.event_name != 'pull_request' run: |- # Login to GitHub Package Registry docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} @@ -67,22 +70,21 @@ jobs: fi - name: Build Docker image - shell: bash run: |- - docker build --rm -f docker/Dockerfile.win -t corpbot.py:rewrite-windows-amd64 . + docker build --rm -f docker/Dockerfile.win -t corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 . - name: Publish to Docker Registry - shell: bash + if: github.event_name != 'pull_request' run: |- ## FIXME: GitHub Package Registry doesn't support Windows images (yet) # Publish to GitHub Package Registry - # docker tag corpbot.py:rewrite-windows-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 - # docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:rewrite-windows-amd64 + # docker tag corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 + # docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 # Publish to Docker Hub if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then - docker tag corpbot.py:rewrite-windows-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-windows-amd64 - docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-windows-amd64 + docker tag corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 fi deploy-manifest: @@ -95,6 +97,7 @@ jobs: uses: actions/checkout@v1 - name: Login to Docker Registry + if: github.event_name != 'pull_request' run: |- # Login to GitHub Package Registry docker login docker.pkg.github.com -u $GITHUB_ACTOR -p ${{ secrets.GITHUB_TOKEN }} @@ -105,19 +108,19 @@ jobs: fi - name: Publish to Docker Registry - shell: bash + if: github.event_name != 'pull_request' run: |- ## FIXME: GitHub Package Registry doesn't support Windows images (yet) # Create and push a combined image manifest to GitHub Package Registry - # docker manifest create docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite-windows-amd64 - # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite-linux-amd64 --os linux --arch amd64 - # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite-windows-amd64 --os windows --arch amd64 - # docker manifest push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:rewrite --purge + # docker manifest create docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 + # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 --os linux --arch amd64 + # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 --os windows --arch amd64 + # docker manifest push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} --purge # Create and push a combined image manifest to Docker Hub if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then - docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-windows-amd64 - docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-linux-amd64 --os linux --arch amd64 - docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite-windows-amd64 --os windows --arch amd64 - docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:rewrite --purge + docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 + docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 --os linux --arch amd64 + docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 --os windows --arch amd64 + docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} --purge fi From e39b5e81c177c2cfbc3d9c42327902cd9e3ca55f Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Thu, 23 Apr 2020 13:46:10 +0300 Subject: [PATCH 26/30] Fixing Docker CI workflow --- .github/workflows/docker.yml | 38 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 65efd660..b63fb226 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,7 +10,7 @@ defaults: env: DOCKER_CLI_EXPERIMENTAL: enabled # Enable Docker experimental features (eg. `docker manifest ...`) - GITHUB_BRANCH: ${GITHUB_REF##*/} + #GITHUB_BRANCH: ${GITHUB_REF##*/} jobs: @@ -35,19 +35,19 @@ jobs: - name: Build Docker image run: |- - docker build --rm -f docker/Dockerfile -t corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 . + docker build --rm -f docker/Dockerfile -t corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 . - name: Publish to Docker Registry if: github.event_name != 'pull_request' run: |- # Publish to GitHub Package Registry - docker tag corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 - docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 + docker tag corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 + docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 # Publish to Docker Hub if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then - docker tag corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 - docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 + docker tag corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 fi deploy-windows: @@ -71,20 +71,20 @@ jobs: - name: Build Docker image run: |- - docker build --rm -f docker/Dockerfile.win -t corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 . + docker build --rm -f docker/Dockerfile.win -t corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 . - name: Publish to Docker Registry if: github.event_name != 'pull_request' run: |- ## FIXME: GitHub Package Registry doesn't support Windows images (yet) # Publish to GitHub Package Registry - # docker tag corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 - # docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 + # docker tag corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 + # docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 # Publish to Docker Hub if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then - docker tag corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 - docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 + docker tag corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 fi deploy-manifest: @@ -112,15 +112,15 @@ jobs: run: |- ## FIXME: GitHub Package Registry doesn't support Windows images (yet) # Create and push a combined image manifest to GitHub Package Registry - # docker manifest create docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 - # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 --os linux --arch amd64 - # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 --os windows --arch amd64 - # docker manifest push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} --purge + # docker manifest create docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 + # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 --os linux --arch amd64 + # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 --os windows --arch amd64 + # docker manifest push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} --purge # Create and push a combined image manifest to Docker Hub if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then - docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 - docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-linux-amd64 --os linux --arch amd64 - docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA}-windows-amd64 --os windows --arch amd64 - docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_BRANCH:-$GITHUB_SHA} --purge + docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 + docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 --os linux --arch amd64 + docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 --os windows --arch amd64 + docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} --purge fi From 0d00c39afdd7b9471e490e7b3d20f38b381bb8b4 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Thu, 23 Apr 2020 13:49:44 +0300 Subject: [PATCH 27/30] Fixing Docker CI workflow --- .github/workflows/docker.yml | 59 ++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b63fb226..e101ab96 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,7 +10,6 @@ defaults: env: DOCKER_CLI_EXPERIMENTAL: enabled # Enable Docker experimental features (eg. `docker manifest ...`) - #GITHUB_BRANCH: ${GITHUB_REF##*/} jobs: @@ -35,19 +34,28 @@ jobs: - name: Build Docker image run: |- - docker build --rm -f docker/Dockerfile -t corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 . + # Parse the tag + TAG=${GITHUB_REF##*/} + TAG=${TAG:-$GITHUB_SHA} + + # Build the image + docker build --rm -f docker/Dockerfile -t corpbot.py:${TAG}-linux-amd64 . - name: Publish to Docker Registry if: github.event_name != 'pull_request' run: |- + # Parse the tag + TAG=${GITHUB_REF##*/} + TAG=${TAG:-$GITHUB_SHA} + # Publish to GitHub Package Registry - docker tag corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 - docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 + docker tag corpbot.py:${TAG}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${TAG}-linux-amd64 + docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${TAG}-linux-amd64 # Publish to Docker Hub if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then - docker tag corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 - docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 + docker tag corpbot.py:${TAG}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-linux-amd64 + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-linux-amd64 fi deploy-windows: @@ -71,20 +79,29 @@ jobs: - name: Build Docker image run: |- - docker build --rm -f docker/Dockerfile.win -t corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 . + # Parse the tag + TAG=${GITHUB_REF##*/} + TAG=${TAG:-$GITHUB_SHA} + + # Build the image + docker build --rm -f docker/Dockerfile.win -t corpbot.py:${TAG}-windows-amd64 . - name: Publish to Docker Registry if: github.event_name != 'pull_request' run: |- + # Parse the tag + TAG=${GITHUB_REF##*/} + TAG=${TAG:-$GITHUB_SHA} + ## FIXME: GitHub Package Registry doesn't support Windows images (yet) # Publish to GitHub Package Registry - # docker tag corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 - # docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 + # docker tag corpbot.py:${TAG}-windows-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${TAG}-windows-amd64 + # docker push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py/corpbot.py:${TAG}-windows-amd64 # Publish to Docker Hub if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then - docker tag corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 - docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 + docker tag corpbot.py:${TAG}-windows-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-windows-amd64 + docker push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-windows-amd64 fi deploy-manifest: @@ -110,17 +127,21 @@ jobs: - name: Publish to Docker Registry if: github.event_name != 'pull_request' run: |- + # Parse the tag + TAG=${GITHUB_REF##*/} + TAG=${TAG:-$GITHUB_SHA} + ## FIXME: GitHub Package Registry doesn't support Windows images (yet) # Create and push a combined image manifest to GitHub Package Registry - # docker manifest create docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 - # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 --os linux --arch amd64 - # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 --os windows --arch amd64 - # docker manifest push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} --purge + # docker manifest create docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG}-linux-amd64 docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG}-windows-amd64 + # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG}-linux-amd64 --os linux --arch amd64 + # docker manifest annotate docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG} docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG}-windows-amd64 --os windows --arch amd64 + # docker manifest push docker.pkg.github.com/${GITHUB_ACTOR,,}/corpbot.py:${TAG} --purge # Create and push a combined image manifest to Docker Hub if [ ! -z "${{ secrets.DOCKER_HUB_USERNAME }}" ]; then - docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 - docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-linux-amd64 --os linux --arch amd64 - docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA}-windows-amd64 --os windows --arch amd64 - docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${GITHUB_REF##*/:-$GITHUB_SHA} --purge + docker manifest create ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-linux-amd64 ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-windows-amd64 + docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-linux-amd64 --os linux --arch amd64 + docker manifest annotate ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG} ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG}-windows-amd64 --os windows --arch amd64 + docker manifest push ${{ secrets.DOCKER_HUB_USERNAME }}/corpbot.py:${TAG} --purge fi From 69bdd15aa7b3c35c4a1cc4ff13d938dbc325ffb6 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Tue, 20 Apr 2021 15:41:28 +0300 Subject: [PATCH 28/30] Update Settings.py Allow absolute paths --- Cogs/Settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cogs/Settings.py b/Cogs/Settings.py index 3cf00251..c892351b 100755 --- a/Cogs/Settings.py +++ b/Cogs/Settings.py @@ -478,7 +478,7 @@ async def backup(self): os.makedirs(self.backupDir) # Flush backup timeStamp = datetime.today().strftime("%Y-%m-%d %H.%M") - self.flushSettings("./{}/Backup-{}.json".format(self.backupDir, timeStamp), True) + self.flushSettings("{}/Backup-{}.json".format(self.backupDir, timeStamp), True) # Get curr dir and change curr dir retval = os.getcwd() From dbe956e869b21f50860d63307343851da5e48ab2 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Tue, 20 Apr 2021 15:41:46 +0300 Subject: [PATCH 29/30] Update Docker Updates and fixes left and right --- .github/workflows/docker.yml | 4 ++-- docker/Dockerfile | 7 ++++--- docker/Dockerfile.lavalink | 3 +++ docker/README.md | 7 +++++-- docker/docker-compose.yml | 29 ++++++++++++++++------------- 5 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 docker/Dockerfile.lavalink diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e101ab96..1cf54843 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Login to Docker Registry if: github.event_name != 'pull_request' @@ -64,7 +64,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Login to Docker Registry if: github.event_name != 'pull_request' diff --git a/docker/Dockerfile b/docker/Dockerfile index 592b23be..e906f213 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,6 @@ # Base the image on Python 3 running on Alpine -FROM python:3.7.7-alpine +# FROM python:3.9.4-alpine3.13 +FROM python:3.8.9-alpine3.13 # Install base dependencies RUN apk add --update --no-cache \ @@ -17,7 +18,7 @@ RUN apk add --update --no-cache \ RUN apk add --update --no-cache --virtual build-dependencies \ alpine-sdk \ linux-headers \ - python-dev \ + python3-dev \ libffi-dev \ libxml2-dev \ zlib-dev \ @@ -50,7 +51,7 @@ ENV SETTINGS_DICT_TOKEN "" ENV SETTINGS_DICT_WEATHER "" ENV SETTINGS_DICT_CURRENCY "" ENV SETTINGS_DICT_SETTINGS_PATH "/data/Settings.json" -ENV SETTINGS_DICT_SETTINGS_BACKUP_PATH "/data/Settings-Backup" +ENV SETTINGS_DICT_SETTINGS_BACKUP_PATH "/data" ENV SETTINGS_DICT_LAVALINK_HOST "lavalink" ENV SETTINGS_DICT_LAVALINK_REGION "us_central" ENV SETTINGS_DICT_LAVALINK_PASSWORD "youshallnotpass" diff --git a/docker/Dockerfile.lavalink b/docker/Dockerfile.lavalink new file mode 100644 index 00000000..76102e2d --- /dev/null +++ b/docker/Dockerfile.lavalink @@ -0,0 +1,3 @@ +FROM 1conan/lavalink:latest + +COPY lavalink.yaml /config/application.yml diff --git a/docker/README.md b/docker/README.md index 3d91046a..e0eded52 100644 --- a/docker/README.md +++ b/docker/README.md @@ -14,6 +14,9 @@ This is a short document detailing the current status of the Docker images, incl #### Planned +- [ ] Fix/update CI workflow to use new GH registry URL +- [ ] Fix/update CI workflow to support Windows (if/when available) +- [ ] ARM image support (CorpBot itself runs fine on ARM) - [ ] Redis support (redis branch) - [ ] MongoDB support (same deal) - [ ] Optimized, minimal image sizes (Alpine, multi-staged builds, Windows image needs special attention) @@ -24,8 +27,8 @@ Create a `.env` file at the root of this project with the following environment ``` SETTINGS_DICT_PREFIX="" SETTINGS_DICT_TOKEN="" -SETTINGS_DICT_WEATHER="" -SETTINGS_DICT_CURRENCY="" +SETTINGS_DICT_WEATHER="" +SETTINGS_DICT_CURRENCY="" SETTINGS_DICT_SETTINGS_PATH=" (optional) SETTINGS_DICT_SETTINGS_BACKUP_PATH="" (optional) SETTINGS_DICT_LAVALINK_HOST="" (optional) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a74d09be..7153ae3f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,4 +1,4 @@ -version: "2.4" # We use v2.x for resource limit support with Docker Compose +version: "2.1" # We use v2.x for resource limit support with Docker Compose services: @@ -7,8 +7,8 @@ services: build: context: ../ dockerfile: docker/Dockerfile - ## This can be used for building a Windows image instead - #dockerfile: docker/Dockerfile.win + # ## This can be used for building a Windows image instead + # #dockerfile: docker/Dockerfile.win env_file: ../.env ## Use a .env file at the root ## Alternatively you can comment out the ## `environment:` object below and use that instead of the .env file# @@ -23,21 +23,24 @@ services: - corpbot_network volumes: - corpbot_data:/data - cpus: 0.5 - mem_limit: 1024m - mem_reservation: 512m + # The following can be used to limit resource usage + # cpus: 0.5 + # mem_limit: 1024m + # mem_reservation: 512m - ## NOTE: There is no publicly available Lavalink images for Windows, - ## so it might be necessary to run it separately instead + # NOTE: There is no publicly available Lavalink images for Windows, + # so it might be necessary to run it separately instead lavalink: image: 1conan/lavalink:latest - volumes: - - ${PWD}/lavalink.yaml:/config/application.yml + build: + context: . + dockerfile: Dockerfile.lavalink networks: - corpbot_network - cpus: 0.5 - mem_limit: 1024m - mem_reservation: 512m + # The following can be used to limit resource usage + # cpus: 0.5 + # mem_limit: 1024m + # mem_reservation: 512m networks: corpbot_network: From 85ab3e87e219c43c2bd6dce37a119290f4239626 Mon Sep 17 00:00:00 2001 From: Pauli Jokela Date: Wed, 21 Apr 2021 10:45:39 +0300 Subject: [PATCH 30/30] Better data persisting, handling TempRole errors --- Cogs/TempRole.py | 2 +- docker/Dockerfile | 5 ++++- docker/docker-compose.yml | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Cogs/TempRole.py b/Cogs/TempRole.py index 6ba595be..1845f13f 100644 --- a/Cogs/TempRole.py +++ b/Cogs/TempRole.py @@ -86,7 +86,7 @@ def check_temp(self): # Bail if we're not the current instance return temp_roles = self.settings.getUserStat(member, server, "TempRoles") - if len(temp_roles): + if temp_roles is not None and len(temp_roles): # We have a list remove_temps = [] for temp_role in temp_roles: diff --git a/docker/Dockerfile b/docker/Dockerfile index e906f213..ebba2f28 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -32,6 +32,9 @@ RUN mkdir -p ~/.ssh && \ # Switch to the image provided work directory WORKDIR /usr/src/app +# Set default git merge strategy +RUN git config --global pull.rebase false + # Copy the installation script and install dependencies COPY Install.py ./Install.py RUN yes '' | python ./Install.py @@ -57,7 +60,7 @@ ENV SETTINGS_DICT_LAVALINK_REGION "us_central" ENV SETTINGS_DICT_LAVALINK_PASSWORD "youshallnotpass" # Expose volumes -VOLUME [ "/data" ] +VOLUME [ "/data", "/usr/local/lib/python3.8/site-packages" ] # Set the startup Python script CMD [ "docker/start.sh" ] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7153ae3f..55fb0f2e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -23,6 +23,7 @@ services: - corpbot_network volumes: - corpbot_data:/data + - corpbot_packages:/usr/local/lib/python3.8/site-packages # The following can be used to limit resource usage # cpus: 0.5 # mem_limit: 1024m @@ -47,3 +48,4 @@ networks: volumes: corpbot_data: + corpbot_packages: