diff --git a/src/jena/fuseki-docker/docker-compact-entrypoint.sh b/src/jena/fuseki-docker/docker-compact-entrypoint.sh index 38a01f0a1..064808102 100644 --- a/src/jena/fuseki-docker/docker-compact-entrypoint.sh +++ b/src/jena/fuseki-docker/docker-compact-entrypoint.sh @@ -18,8 +18,15 @@ for operation in "${operations[@]}"; do if [ -d "$dir" ]; then if [ "$operation" == "compact" ]; then echo "Compacting ${dir}..." - # TODO use --deleteOld command available in higher Fuseki versions - /jena-fuseki/bin/tdb2.tdbcompact --loc=${dir} + { + # TODO use --deleteOld command available in higher Fuseki versions + /jena-fuseki/bin/tdb2.tdbcompact --loc=${dir} + } || { + # We immediately delete any newly-created directory, to avoid potentially correct data to be removed during the deleteOld operation + echo "Compact job failed. Deleting new directories from ${dir}..." + cd "${dir}" + find . -iname 'Data*' ! -wholename $(find . -iname 'Data*' -type d | sort -n -r | tail -n 1) -type d -exec rm -rf {} + + } else echo "Deleting old directories from ${dir}..." cd "${dir}" diff --git a/src/middleware/packages/backup/index.js b/src/middleware/packages/backup/index.js index 9560d5581..a2f2b7c8b 100644 --- a/src/middleware/packages/backup/index.js +++ b/src/middleware/packages/backup/index.js @@ -1,5 +1,6 @@ const { CronJob } = require('cron'); const fs = require('fs'); +const { emptyDirSync } = require('fs-extra'); const pathJoin = require('path').join; const fsCopy = require('./utils/fsCopy'); const ftpCopy = require('./utils/ftpCopy'); @@ -18,6 +19,7 @@ const BackupService = { otherDirsPaths: {} }, copyMethod: 'rsync', // rsync, ftp, or fs + deleteFusekiBackupsAfterCopy: false, remoteServer: { path: null, // Required user: null, // Required by rsync and ftp @@ -35,16 +37,21 @@ const BackupService = { started() { const { cronJob, + copyMethod, localServer: { fusekiBase } } = this.settings; - if (cronJob.time) { - this.cronJob = new CronJob(cronJob.time, this.actions.backupAll, null, true, cronJob.timeZone); - } - if (!fusekiBase) { throw new Error('Backup service requires `localServer.fusekiBase` setting to be set to the FUSEKI_BASE path.'); } + + if (!['rsync', 'ftp', 'fs'].includes(copyMethod)) { + throw new Error(`The copyMethod setting must be either rysnc, ftp or fs. Provided: ${copyMethod}`); + } + + if (cronJob.time) { + this.cronJob = new CronJob(cronJob.time, this.actions.backupAll, null, true, cronJob.timeZone); + } }, actions: { async backupAll(ctx) { @@ -59,10 +66,17 @@ const BackupService = { await ctx.call('triplestore.dataset.backup', { dataset }); } - await this.actions.copyToRemoteServer( - { path: pathJoin(this.settings.localServer.fusekiBase, 'backups'), subDir: 'datasets' }, + const backupsDirPath = pathJoin(this.settings.localServer.fusekiBase, 'backups'); + + const copied = await this.actions.copyToRemoteServer( + { path: backupsDirPath, subDir: 'datasets' }, { parentCtx: ctx } ); + + // If there was an error on copy, don't delete the backups + if (copied && this.settings.deleteFusekiBackupsAfterCopy) { + emptyDirSync(backupsDirPath); + } }, async backupOtherDirs(ctx) { const { otherDirsPaths } = this.settings.localServer; @@ -84,24 +98,27 @@ const BackupService = { // Path is mandatory for all copy methods if (!remoteServer.path) { this.logger.info('No remote server config defined, skipping remote backup...'); - return; + return false; } - switch (copyMethod) { - case 'rsync': - await rsyncCopy(path, subDir, remoteServer); - break; - - case 'ftp': - await ftpCopy(path, subDir, remoteServer); - break; + try { + switch (copyMethod) { + case 'rsync': + await rsyncCopy(path, subDir, remoteServer, false); + break; - case 'fs': - await fsCopy(path, subDir, remoteServer); - break; + case 'ftp': + await ftpCopy(path, subDir, remoteServer); + break; - default: - throw new Error(`Unknown copy method: ${copyMethod}`); + case 'fs': + await fsCopy(path, subDir, remoteServer); + break; + } + return true; + } catch (e) { + this.logger.error(`Failed to copy ${path} to remote server with ${copyMethod}. Error: ${e.message}`); + return false; } }, deleteDataset: { @@ -147,7 +164,7 @@ const BackupService = { } } }, - /** Returns an array of file paths to the backups relative to `this.settings.localServer.fusekiBase`. */ + // Returns an array of file paths to the backups relative to `this.settings.localServer.fusekiBase`. async listBackupsForDataset(ctx) { const { dataset } = ctx.params; diff --git a/src/middleware/packages/backup/utils/rsyncCopy.js b/src/middleware/packages/backup/utils/rsyncCopy.js index 573075266..000645ec7 100644 --- a/src/middleware/packages/backup/utils/rsyncCopy.js +++ b/src/middleware/packages/backup/utils/rsyncCopy.js @@ -12,12 +12,12 @@ const rsyncCopy = (path, subDir, remoteServer, syncDelete = false) => { if (syncDelete) rsync.set('delete'); return new Promise((resolve, reject) => { - console.log(`Rsync started with command: ${rsync.command()}`); + this.logger.info(`Rsync started with command: ${rsync.command()}`); rsync.execute(error => { if (error) { reject(error); } else { - console.log('Rsync finished !'); + this.logger.info('Rsync finished !'); resolve(); } });