Added new database rustydb (#6632)

Co-authored-by: SamTv12345 <[email protected]>

bin/migrateDB.ts

@@ -0,0 +1,83 @@
+// DB migration
+import {readFileSync} from 'node:fs'
+import {Database, DatabaseType} from "ueberdb2";
+import path from "node:path";
+const settings = require('ep_etherpad-lite/node/utils/Settings');
+
+
+// file1 = source, file2 = target
+// pnpm run migrateDB --file1 <db1.json> --file2 <db2.json>
+const arg = process.argv.slice(2);
+
+if (arg.length != 4) {
+  console.error('Wrong number of arguments!. Call with pnpm run migrateDB --file1 source.json target.json')
+  process.exit(1)
+}
+
+type SettingsConfig = {
+  dbType: string,
+  dbSettings: any
+}
+
+/*
+  {
+    "dbType": "<your-db-type>",
+    "dbSettings": {
+      <your-db-settings>
+     }
+  }
+ */
+
+let firstDBSettingsFile: string
+let secondDBSettingsFile: string
+
+
+if (arg[0] == "--file1") {
+    firstDBSettingsFile = arg[1]
+} else if (arg[0] === "--file2") {
+  secondDBSettingsFile = arg[1]
+}
+
+if (arg[2] == "--file1") {
+  firstDBSettingsFile = arg[3]
+} else if (arg[2] === "--file2") {
+  secondDBSettingsFile = arg[3]
+}
+
+
+
+const settingsfile = JSON.parse(readFileSync(path.join(settings.root,firstDBSettingsFile!)).toString()) as SettingsConfig
+const settingsfile2 = JSON.parse(readFileSync(path.join(settings.root,secondDBSettingsFile!)).toString()) as SettingsConfig
+
+console.log(settingsfile2)
+if ("filename" in settingsfile.dbSettings) {
+  settingsfile.dbSettings.filename = path.join(settings.root, settingsfile.dbSettings.filename)
+  console.log(settingsfile.dbType + " location is "+ settingsfile.dbSettings.filename)
+}
+
+if ("filename" in settingsfile2.dbSettings) {
+  settingsfile2.dbSettings.filename = path.join(settings.root, settingsfile2.dbSettings.filename)
+  console.log(settingsfile2.dbType + " location is "+ settingsfile2.dbSettings.filename)
+}
+
+const ueberdb1 = new Database(settingsfile.dbType as DatabaseType, settingsfile.dbSettings)
+const ueberdb2 = new Database(settingsfile2.dbType as DatabaseType, settingsfile2.dbSettings)
+
+const handleSync = async ()=>{
+  await ueberdb1.init()
+  await ueberdb2.init()
+
+  const allKeys = await ueberdb1.findKeys('*','')
+  for (const key of allKeys) {
+    const foundVal = await ueberdb1.get(key)!
+    await ueberdb2.set(key, foundVal)
+  }
+}
+
+handleSync().then(()=>{
+  console.log("Done syncing dbs")
+}).catch(e=>{
+  console.log(`Error syncing db ${e}`)
+})
+
+

bin/package.json

@@ -12,7 +12,7 @@
     "log4js": "^6.9.1",
     "semver": "^7.6.3",
     "tsx": "^4.19.0",
-    "ueberdb2": "^4.2.94"
+    "ueberdb2": "^4.2.100"
   },
   "devDependencies": {
     "@types/node": "^22.5.3",
@@ -34,7 +34,8 @@
     "stalePlugins": "node --import tsx ./plugins/stalePlugins.ts",
     "checkPlugin": "node --import tsx ./plugins/checkPlugin.ts",
     "plugins": "node --import tsx ./plugins.ts",
-    "generateChangelog": "node --import tsx generateReleaseNotes.ts"
+    "generateChangelog": "node --import tsx generateReleaseNotes.ts",
+    "migrateDB": "node --import tsx migrateDB.ts"
   },
   "author": "",
   "license": "ISC"

doc/cli.md

@@ -0,0 +1,29 @@
+# CLI
+
+You can find different tools for migrating things, checking your Etherpad health in the bin directory.
+One of these is the migrateDB command. It takes two settings.json files and copies data from one source to another one.
+In this example we migrate from the old dirty db to the new rustydb engine. So we copy these files to the root of the etherpad-directory.
+
+````json
+{
+  "dbType": "dirty",
+  "dbSettings": {
+    "filename": "./var/rusty.db"
+  }
+}
+````
+
+
+
+````json
+{
+  "dbType": "rustydb",
+  "dbSettings": {
+    "filename": "./var/rusty2.db"
+  }
+}
+````
+
+
+After that we need to move the data from dirty to rustydb.
+Therefore, we call `pnpm run migrateDB --file1 test1.json --file2 test2.json` with these two files in our root directories. After some time the data should be copied over to the new database.

src/node/utils/Settings.ts

@@ -169,11 +169,11 @@ exports.authenticationMethod = 'sso'
 /*
  * The Type of the database
  */
-exports.dbType = 'dirty';
+exports.dbType = 'rustydb';
 /**
  * This setting is passed with dbType to ueberDB to set up the database
  */
-exports.dbSettings = {filename: path.join(exports.root, 'var/dirty.db')};
+exports.dbSettings = {filename: path.join(exports.root, 'var/rusty.db')};
 
 /**
  * The default Text of a new pad
@@ -941,6 +941,11 @@ exports.reloadSettings = () => {
         logger.warn(`${dirtyWarning} File location: ${exports.dbSettings.filename}`);
     }
 
+    if (exports.dbType === 'rustydb') {
+      exports.dbSettings.filename = absolutePaths.makeAbsolute(exports.dbSettings.filename);
+      logger.warn(`File location: ${exports.dbSettings.filename}`);
+    }
+
     if (exports.ip === '') {
         // using Unix socket for connectivity
         logger.warn('The settings file contains an empty string ("") for the "ip" parameter. The ' +

src/package.json

@@ -70,7 +70,7 @@
     "superagent": "10.1.0",
     "tinycon": "0.6.8",
     "tsx": "4.19.0",
-    "ueberdb2": "^4.2.94",
+    "ueberdb2": "^4.2.100",
     "underscore": "1.13.7",
     "unorm": "1.6.0",
     "wtfnode": "^0.9.3"
@@ -110,7 +110,8 @@
     "split-grid": "^1.0.11",
     "supertest": "^7.0.0",
     "typescript": "^5.5.4",
-    "vitest": "^2.0.5"
+    "vitest": "^2.0.5",
+    "rusty-store-kv": "^1.1.4"
   },
   "engines": {
     "node": ">=18.18.2",

var/.gitignore

@@ -1,3 +1,5 @@
 sqlite.db
 minified*
-installed_plugins.json
+installed_plugins.json
+dirty.db
+rusty.db