From 6adbbaece5fbbd8aaade835375ad7b502f967f3a Mon Sep 17 00:00:00 2001 From: Eduardo Teles <edu.teles@pm.me> Date: Sat, 19 Aug 2023 20:51:58 -0300 Subject: [PATCH 1/2] add backup and restore with pg_dumpall --- Dockerfile | 1 + README.md | 4 +++- docker-compose.yaml | 3 ++- src/backup.sh | 26 ++++++++++++++++++-------- src/env.sh | 2 +- src/restore.sh | 11 ++++++++--- 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 16915d7..c2fddc0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ ENV S3_S3V4 'no' ENV SCHEDULE '' ENV PASSPHRASE '' ENV BACKUP_KEEP_DAYS '' +ENV BACKUP_ALL 'false' ADD src/run.sh run.sh ADD src/env.sh env.sh diff --git a/README.md b/README.md index 0164acf..a516de6 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,14 @@ services: SCHEDULE: '@weekly' # optional BACKUP_KEEP_DAYS: 7 # optional PASSPHRASE: passphrase # optional + BACKUP_ALL: 'false' # optional S3_REGION: region S3_ACCESS_KEY_ID: key S3_SECRET_ACCESS_KEY: secret S3_BUCKET: my-bucket S3_PREFIX: backup POSTGRES_HOST: postgres - POSTGRES_DATABASE: dbname + POSTGRES_DATABASE: dbname # optional if BACKUP_ALL is 'true' POSTGRES_USER: user POSTGRES_PASSWORD: password ``` @@ -31,6 +32,7 @@ services: - Images are tagged by the major PostgreSQL version supported: `11`, `12`, `13`, `14`, or `15`. - The `SCHEDULE` variable determines backup frequency. See go-cron schedules documentation [here](http://godoc.org/github.com/robfig/cron#hdr-Predefined_schedules). Omit to run the backup immediately and then exit. - If `PASSPHRASE` is provided, the backup will be encrypted using GPG. +- If `BACKUP_ALL` is `'true'`, all databases will be backed up. Otherwise, only `POSTGRES_DATABASE` will be backed up. - Run `docker exec <container name> sh backup.sh` to trigger a backup ad-hoc. - If `BACKUP_KEEP_DAYS` is set, backups older than this many days will be deleted from S3. - Set `S3_ENDPOINT` if you're using a non-AWS S3-compatible storage provider. diff --git a/docker-compose.yaml b/docker-compose.yaml index 16ac481..4ea3864 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -17,12 +17,13 @@ services: SCHEDULE: '@weekly' # optional BACKUP_KEEP_DAYS: 7 # optional PASSPHRASE: passphrase # optional + BACKUP_ALL: 'true' # optional S3_REGION: S3_ACCESS_KEY_ID: S3_SECRET_ACCESS_KEY: S3_BUCKET: S3_PREFIX: backup POSTGRES_HOST: postgres - POSTGRES_DATABASE: postgres + POSTGRES_DATABASE: postgres # optional if BACKUP_ALL is true POSTGRES_USER: user POSTGRES_PASSWORD: password diff --git a/src/backup.sh b/src/backup.sh index ff1bf15..5aeb6d6 100644 --- a/src/backup.sh +++ b/src/backup.sh @@ -5,14 +5,24 @@ set -o pipefail source ./env.sh -echo "Creating backup of $POSTGRES_DATABASE database..." -pg_dump --format=custom \ - -h $POSTGRES_HOST \ - -p $POSTGRES_PORT \ - -U $POSTGRES_USER \ - -d $POSTGRES_DATABASE \ - $PGDUMP_EXTRA_OPTS \ - > db.dump +if [ "$BACKUP_ALL" = "true" ]; then + echo "Creating backup of all databases..." + pg_dumpall --clean \ + -h $POSTGRES_HOST \ + -p $POSTGRES_PORT \ + -U $POSTGRES_USER \ + $PGDUMP_EXTRA_OPTS \ + > db.dump +else + echo "Creating backup of $POSTGRES_DATABASE database..." + pg_dump --format=custom \ + -h $POSTGRES_HOST \ + -p $POSTGRES_PORT \ + -U $POSTGRES_USER \ + -d $POSTGRES_DATABASE \ + $PGDUMP_EXTRA_OPTS \ + > db.dump +fi timestamp=$(date +"%Y-%m-%dT%H:%M:%S") s3_uri_base="s3://${S3_BUCKET}/${S3_PREFIX}/${POSTGRES_DATABASE}_${timestamp}.dump" diff --git a/src/env.sh b/src/env.sh index af495e4..7a1ab9e 100644 --- a/src/env.sh +++ b/src/env.sh @@ -3,7 +3,7 @@ if [ -z "$S3_BUCKET" ]; then exit 1 fi -if [ -z "$POSTGRES_DATABASE" ]; then +if [ "$BACKUP_ALL" != "true" ] && [ -z "$POSTGRES_DATABASE" ]; then echo "You need to set the POSTGRES_DATABASE environment variable." exit 1 fi diff --git a/src/restore.sh b/src/restore.sh index 3040146..4113b2d 100644 --- a/src/restore.sh +++ b/src/restore.sh @@ -35,10 +35,15 @@ if [ -n "$PASSPHRASE" ]; then rm db.dump.gpg fi -conn_opts="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DATABASE" +conn_opts="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER" + +if [ "$BACKUP_ALL" = "true" ]; then + psql -f db.dump $conn_opts +else + echo "Restoring from backup..." + pg_restore $conn_opts -d $POSTGRES_DATABASE --clean --if-exists db.dump +fi -echo "Restoring from backup..." -pg_restore $conn_opts --clean --if-exists db.dump rm db.dump echo "Restore complete." From 53c6148132cf18be0ad4fa7e242c6025118b4f8f Mon Sep 17 00:00:00 2001 From: eduardoteles17 <edu.teles@pm.me> Date: Tue, 5 Sep 2023 14:07:07 -0300 Subject: [PATCH 2/2] add gzip compression on BACKUP_ALL --- src/backup.sh | 15 +++++++++------ src/restore.sh | 16 ++++++++++++---- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/backup.sh b/src/backup.sh index 5aeb6d6..b7cbff8 100644 --- a/src/backup.sh +++ b/src/backup.sh @@ -7,14 +7,17 @@ source ./env.sh if [ "$BACKUP_ALL" = "true" ]; then echo "Creating backup of all databases..." + file_ext="dump.gz" pg_dumpall --clean \ -h $POSTGRES_HOST \ -p $POSTGRES_PORT \ -U $POSTGRES_USER \ $PGDUMP_EXTRA_OPTS \ > db.dump + gzip db.dump else echo "Creating backup of $POSTGRES_DATABASE database..." + file_ext="dump" pg_dump --format=custom \ -h $POSTGRES_HOST \ -p $POSTGRES_PORT \ @@ -25,17 +28,17 @@ else fi timestamp=$(date +"%Y-%m-%dT%H:%M:%S") -s3_uri_base="s3://${S3_BUCKET}/${S3_PREFIX}/${POSTGRES_DATABASE}_${timestamp}.dump" +s3_uri_base="s3://${S3_BUCKET}/${S3_PREFIX}/${POSTGRES_DATABASE}_${timestamp}.${file_ext}" if [ -n "$PASSPHRASE" ]; then echo "Encrypting backup..." - rm -f db.dump.gpg - gpg --symmetric --batch --passphrase "$PASSPHRASE" db.dump - rm db.dump - local_file="db.dump.gpg" + rm -f db.$file_ext.gpg + gpg --symmetric --batch --passphrase "$PASSPHRASE" db.$file_ext + rm db.$file_ext + local_file="db.${file_ext}.gpg" s3_uri="${s3_uri_base}.gpg" else - local_file="db.dump" + local_file="db.${file_ext}" s3_uri="$s3_uri_base" fi diff --git a/src/restore.sh b/src/restore.sh index 4113b2d..9c1b899 100644 --- a/src/restore.sh +++ b/src/restore.sh @@ -7,10 +7,16 @@ source ./env.sh s3_uri_base="s3://${S3_BUCKET}/${S3_PREFIX}" +if [ "$BACKUP_ALL" = "true" ]; then + file_ext=".dump.gz" +else + file_ext=".dump" +fi + if [ -z "$PASSPHRASE" ]; then - file_type=".dump" + file_type=$file_ext else - file_type=".dump.gpg" + file_type="${file_ext}.gpg" fi if [ $# -eq 1 ]; then @@ -31,13 +37,15 @@ aws $aws_args s3 cp "${s3_uri_base}/${key_suffix}" "db${file_type}" if [ -n "$PASSPHRASE" ]; then echo "Decrypting backup..." - gpg --decrypt --batch --passphrase "$PASSPHRASE" db.dump.gpg > db.dump - rm db.dump.gpg + gpg --decrypt --batch --passphrase "$PASSPHRASE" db$file_type > db$file_ext + rm db$file_type fi conn_opts="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER" if [ "$BACKUP_ALL" = "true" ]; then + gunzip db.dump.gz + echo "Restoring all databases..." psql -f db.dump $conn_opts else echo "Restoring from backup..."