From d4cbbf18e0c5ae5713dce354a08252a300bba4a8 Mon Sep 17 00:00:00 2001 From: SamTV12345 <40429738+SamTV12345@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:06:16 +0200 Subject: [PATCH] Added new database rustydb (#6632) Co-authored-by: SamTv12345 --- bin/migrateDB.ts | 83 +++++++++++++++++++++++++ bin/package.json | 5 +- doc/cli.md | 29 +++++++++ pnpm-lock.yaml | 124 ++++++++++++++++++++++++++++++++++--- src/node/utils/Settings.ts | 9 ++- src/package.json | 5 +- var/.gitignore | 4 +- 7 files changed, 245 insertions(+), 14 deletions(-) create mode 100644 bin/migrateDB.ts create mode 100644 doc/cli.md diff --git a/bin/migrateDB.ts b/bin/migrateDB.ts new file mode 100644 index 00000000000..37f1cda230d --- /dev/null +++ b/bin/migrateDB.ts @@ -0,0 +1,83 @@ +// DB migration +import {readFileSync} from 'node:fs' +import {Database, DatabaseType} from "ueberdb2"; +import path from "node:path"; +const settings = require('ep_etherpad-lite/node/utils/Settings'); + + +// file1 = source, file2 = target +// pnpm run migrateDB --file1 --file2 +const arg = process.argv.slice(2); + +if (arg.length != 4) { + console.error('Wrong number of arguments!. Call with pnpm run migrateDB --file1 source.json target.json') + process.exit(1) +} + +type SettingsConfig = { + dbType: string, + dbSettings: any +} + +/* + { + "dbType": "", + "dbSettings": { + + } + } + */ + +let firstDBSettingsFile: string +let secondDBSettingsFile: string + + +if (arg[0] == "--file1") { + firstDBSettingsFile = arg[1] +} else if (arg[0] === "--file2") { + secondDBSettingsFile = arg[1] +} + +if (arg[2] == "--file1") { + firstDBSettingsFile = arg[3] +} else if (arg[2] === "--file2") { + secondDBSettingsFile = arg[3] +} + + + +const settingsfile = JSON.parse(readFileSync(path.join(settings.root,firstDBSettingsFile!)).toString()) as SettingsConfig +const settingsfile2 = JSON.parse(readFileSync(path.join(settings.root,secondDBSettingsFile!)).toString()) as SettingsConfig + +console.log(settingsfile2) +if ("filename" in settingsfile.dbSettings) { + settingsfile.dbSettings.filename = path.join(settings.root, settingsfile.dbSettings.filename) + console.log(settingsfile.dbType + " location is "+ settingsfile.dbSettings.filename) +} + +if ("filename" in settingsfile2.dbSettings) { + settingsfile2.dbSettings.filename = path.join(settings.root, settingsfile2.dbSettings.filename) + console.log(settingsfile2.dbType + " location is "+ settingsfile2.dbSettings.filename) +} + +const ueberdb1 = new Database(settingsfile.dbType as DatabaseType, settingsfile.dbSettings) +const ueberdb2 = new Database(settingsfile2.dbType as DatabaseType, settingsfile2.dbSettings) + +const handleSync = async ()=>{ + await ueberdb1.init() + await ueberdb2.init() + + const allKeys = await ueberdb1.findKeys('*','') + for (const key of allKeys) { + const foundVal = await ueberdb1.get(key)! + await ueberdb2.set(key, foundVal) + } +} + +handleSync().then(()=>{ + console.log("Done syncing dbs") +}).catch(e=>{ + console.log(`Error syncing db ${e}`) +}) + + diff --git a/bin/package.json b/bin/package.json index e4b15bd6cb5..c7ba3b15f84 100644 --- a/bin/package.json +++ b/bin/package.json @@ -12,7 +12,7 @@ "log4js": "^6.9.1", "semver": "^7.6.3", "tsx": "^4.19.0", - "ueberdb2": "^4.2.94" + "ueberdb2": "^4.2.100" }, "devDependencies": { "@types/node": "^22.5.3", @@ -34,7 +34,8 @@ "stalePlugins": "node --import tsx ./plugins/stalePlugins.ts", "checkPlugin": "node --import tsx ./plugins/checkPlugin.ts", "plugins": "node --import tsx ./plugins.ts", - "generateChangelog": "node --import tsx generateReleaseNotes.ts" + "generateChangelog": "node --import tsx generateReleaseNotes.ts", + "migrateDB": "node --import tsx migrateDB.ts" }, "author": "", "license": "ISC" diff --git a/doc/cli.md b/doc/cli.md new file mode 100644 index 00000000000..b80191e146a --- /dev/null +++ b/doc/cli.md @@ -0,0 +1,29 @@ +# CLI + +You can find different tools for migrating things, checking your Etherpad health in the bin directory. +One of these is the migrateDB command. It takes two settings.json files and copies data from one source to another one. +In this example we migrate from the old dirty db to the new rustydb engine. So we copy these files to the root of the etherpad-directory. + +````json +{ + "dbType": "dirty", + "dbSettings": { + "filename": "./var/rusty.db" + } +} +```` + + + +````json +{ + "dbType": "rustydb", + "dbSettings": { + "filename": "./var/rusty2.db" + } +} +```` + + +After that we need to move the data from dirty to rustydb. +Therefore, we call `pnpm run migrateDB --file1 test1.json --file2 test2.json` with these two files in our root directories. After some time the data should be copied over to the new database. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e336ac928c7..94b94f356d7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -119,8 +119,8 @@ importers: specifier: ^4.19.0 version: 4.19.0 ueberdb2: - specifier: ^4.2.94 - version: 4.2.94 + specifier: ^4.2.100 + version: 4.2.101 devDependencies: '@types/node': specifier: ^22.5.3 @@ -261,8 +261,8 @@ importers: specifier: 4.19.0 version: 4.19.0 ueberdb2: - specifier: ^4.2.94 - version: 4.2.94 + specifier: ^4.2.100 + version: 4.2.101 underscore: specifier: 1.13.7 version: 1.13.7 @@ -348,6 +348,9 @@ importers: openapi-schema-validation: specifier: ^0.4.2 version: 0.4.2 + rusty-store-kv: + specifier: ^1.1.4 + version: 1.1.4 set-cookie-parser: specifier: ^2.7.0 version: 2.7.0 @@ -3951,6 +3954,70 @@ packages: run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + rusty-store-kv-darwin-arm64@1.1.4: + resolution: {integrity: sha512-Pwedf7Fe0BBVmuM6Z/7JYGtwjVLI+rCEEUNoZ9jA31QtfWHHFYPk0XULk4mLnngP14eBy9opEpJO2wYAUra5bg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + rusty-store-kv-darwin-x64@1.1.4: + resolution: {integrity: sha512-R1vuflgIjP6JtFGnxRoDgZaw9kVF+m9FYEIZH0dG3v4AU55Zh53lQBflHLhqsC10OuCo3ZO0w++bSVmcGXmSJg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + rusty-store-kv-freebsd-x64@1.1.4: + resolution: {integrity: sha512-GbYN5+k44RJS2n6mqdDQz64ZLXl2mEZFIpdk5KblmUZaf7JPai6o6d/2kaldw+htTlVabdXVrmVRhytsHtmdcQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + rusty-store-kv-linux-arm-gnueabihf@1.1.4: + resolution: {integrity: sha512-hdPJTo/UR8kYSf6m6g530hzCRFCewyKhae/Dtol12fRqHlzr+RhygVkrxPblqwUFt8XJGG2A9xkc9CFJwbS7lg==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + rusty-store-kv-linux-arm64-gnu@1.1.4: + resolution: {integrity: sha512-KEsvvdbYHf6o01RY6DPGLt1x3BbDUnLvkJuMr6NLcX2mDgtvDtC2auNGuslb+mzVG3GqtCfiEesz4JRrJkfimA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + rusty-store-kv-linux-arm64-musl@1.1.4: + resolution: {integrity: sha512-o1PMDeMFc3nWHksO0pBHNdr5n2ZRFje8WmK6fwRMcicdsiK+fqv2zJyAjdDfVoP0RS4QXkM9DwCyqN+bziBz6A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + rusty-store-kv-linux-x64-gnu@1.1.4: + resolution: {integrity: sha512-EG7qO20CLY5F5kki0CFQY+p+CXyCVYtcah3vWI/Opbd6HcVlcoaZJ24KYWuoUCpRS4qIHAue/JtZ7kANlRUwCQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + rusty-store-kv-linux-x64-musl@1.1.4: + resolution: {integrity: sha512-oJ5AJVjnTaWwvahGWceH73mVOjolE+BcBghJsxmlRgw0y4sYwk7qaF0r2heh1GDeMZsYMhZV0o8GAa3FH0W3lQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + rusty-store-kv-win32-arm64-msvc@1.1.4: + resolution: {integrity: sha512-kqvaa4DqNNmOMQDBOoRBbiesTkE4E/nrghIk5eSalLaUHt1TNZpuOeWGS1+xRIYuRDBeHAqD72AkcqrP+9dzcQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + rusty-store-kv-win32-x64-msvc@1.1.4: + resolution: {integrity: sha512-SPI/Up8QF46MA2PkADeyIUd82PTa89i91TjkE81vVyKNIwcfXb9lFDOJThWds8gwMjiWcsMiMA1TclyCDZ0HoQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + rusty-store-kv@1.1.4: + resolution: {integrity: sha512-B8ibg3j9Wq2uY0FysjSpZ71GPj9x+oJUTvjzh0YUF4m+y7/4Lo+DwNLNU/4u5DTnGiXeAhWmXypjp+mWIN3GiQ==} + engines: {node: '>= 10'} + safe-array-concat@1.1.2: resolution: {integrity: sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q==} engines: {node: '>=0.4'} @@ -4306,8 +4373,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - ueberdb2@4.2.94: - resolution: {integrity: sha512-cfrMR3+wrUwGTFdujZb/BhqAC+pKHNJeqcCzF9ktn/y+8eG9I1xJXAzJvYWn6RxxjE/HosWXt36h0ZrxhbFOfg==} + ueberdb2@4.2.101: + resolution: {integrity: sha512-4WyfwvewluxElZ/FqoSupi/88FQANEmD0Cjy2cpSSExca3skZfe1thqw3kkpBvPOsyvdSekEkxDZbVe12wzn3A==} engines: {node: '>=16.20.1'} uid-safe@2.1.5: @@ -8571,6 +8638,49 @@ snapshots: dependencies: queue-microtask: 1.2.3 + rusty-store-kv-darwin-arm64@1.1.4: + optional: true + + rusty-store-kv-darwin-x64@1.1.4: + optional: true + + rusty-store-kv-freebsd-x64@1.1.4: + optional: true + + rusty-store-kv-linux-arm-gnueabihf@1.1.4: + optional: true + + rusty-store-kv-linux-arm64-gnu@1.1.4: + optional: true + + rusty-store-kv-linux-arm64-musl@1.1.4: + optional: true + + rusty-store-kv-linux-x64-gnu@1.1.4: + optional: true + + rusty-store-kv-linux-x64-musl@1.1.4: + optional: true + + rusty-store-kv-win32-arm64-msvc@1.1.4: + optional: true + + rusty-store-kv-win32-x64-msvc@1.1.4: + optional: true + + rusty-store-kv@1.1.4: + optionalDependencies: + rusty-store-kv-darwin-arm64: 1.1.4 + rusty-store-kv-darwin-x64: 1.1.4 + rusty-store-kv-freebsd-x64: 1.1.4 + rusty-store-kv-linux-arm-gnueabihf: 1.1.4 + rusty-store-kv-linux-arm64-gnu: 1.1.4 + rusty-store-kv-linux-arm64-musl: 1.1.4 + rusty-store-kv-linux-x64-gnu: 1.1.4 + rusty-store-kv-linux-x64-musl: 1.1.4 + rusty-store-kv-win32-arm64-msvc: 1.1.4 + rusty-store-kv-win32-x64-msvc: 1.1.4 + safe-array-concat@1.1.2: dependencies: call-bind: 1.0.7 @@ -9009,7 +9119,7 @@ snapshots: typescript@5.5.4: {} - ueberdb2@4.2.94: {} + ueberdb2@4.2.101: {} uid-safe@2.1.5: dependencies: diff --git a/src/node/utils/Settings.ts b/src/node/utils/Settings.ts index d798e10bffc..0113499f580 100644 --- a/src/node/utils/Settings.ts +++ b/src/node/utils/Settings.ts @@ -169,11 +169,11 @@ exports.authenticationMethod = 'sso' /* * The Type of the database */ -exports.dbType = 'dirty'; +exports.dbType = 'rustydb'; /** * This setting is passed with dbType to ueberDB to set up the database */ -exports.dbSettings = {filename: path.join(exports.root, 'var/dirty.db')}; +exports.dbSettings = {filename: path.join(exports.root, 'var/rusty.db')}; /** * The default Text of a new pad @@ -941,6 +941,11 @@ exports.reloadSettings = () => { logger.warn(`${dirtyWarning} File location: ${exports.dbSettings.filename}`); } + if (exports.dbType === 'rustydb') { + exports.dbSettings.filename = absolutePaths.makeAbsolute(exports.dbSettings.filename); + logger.warn(`File location: ${exports.dbSettings.filename}`); + } + if (exports.ip === '') { // using Unix socket for connectivity logger.warn('The settings file contains an empty string ("") for the "ip" parameter. The ' + diff --git a/src/package.json b/src/package.json index e4fbf9c6151..71e3577996e 100644 --- a/src/package.json +++ b/src/package.json @@ -70,7 +70,7 @@ "superagent": "10.1.0", "tinycon": "0.6.8", "tsx": "4.19.0", - "ueberdb2": "^4.2.94", + "ueberdb2": "^4.2.100", "underscore": "1.13.7", "unorm": "1.6.0", "wtfnode": "^0.9.3" @@ -110,7 +110,8 @@ "split-grid": "^1.0.11", "supertest": "^7.0.0", "typescript": "^5.5.4", - "vitest": "^2.0.5" + "vitest": "^2.0.5", + "rusty-store-kv": "^1.1.4" }, "engines": { "node": ">=18.18.2", diff --git a/var/.gitignore b/var/.gitignore index 1dea48383fd..d75cb9e42b2 100644 --- a/var/.gitignore +++ b/var/.gitignore @@ -1,3 +1,5 @@ sqlite.db minified* -installed_plugins.json \ No newline at end of file +installed_plugins.json +dirty.db +rusty.db