add sqlite2mysql.sh and sqlite2mysql.py

scripts/sqlite2mysql.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+"""Lifted from:
+http://stackoverflow.com/questions/18671/quick-easy-way-to-migrate-sqlite3-to-mysql
+
+Run like so:
+
+sqlite3 <your db>.db .dump | python sqlite2mysql.py > <your db>.sql
+
+Then you can import the .sql file into MySql
+
+Note - you need to add foreign key constrains manually since sqlite doesn't actually support them
+"""
+import re
+import fileinput
+
+def this_line_is_useless(line):
+    useless_es = [
+        'BEGIN TRANSACTION',
+        'COMMIT',
+        'sqlite_sequence',
+        'CREATE UNIQUE INDEX',
+        'PRAGMA',
+        ]
+    for useless in useless_es:
+        if re.search(useless, line):
+            return True
+
+def has_primary_key(line):
+    return bool(re.search(r'PRIMARY KEY', line))
+
+for line in fileinput.input():
+    searching_for_end = False
+    if this_line_is_useless(line): continue
+
+    # this line was necessary because ''); was getting
+    # converted (inappropriately) to \');
+    if re.match(r".*, ''\);", line):
+        line = re.sub(r"''\);", r'``);', line)
+
+    if re.match(r'^CREATE TABLE.*', line):
+        searching_for_end = True
+
+    m = re.search('CREATE TABLE [`"]?(\w*)[`"]?(.*)', line)
+    if m:
+        name, sub = m.groups()
+        sub = sub.replace('"','`')
+        line = line % dict(name=name, sub=sub)
+    else:
+        m = re.search('INSERT INTO "(\w*)"(.*)', line)
+        if m:
+            name, sub = m.groups()
+            line = 'INSERT INTO `%s`%s\n' % m.groups()
+            line = line.replace('"', r'\"')
+            line = line.replace('"', "'")
+    # line = re.sub(r"([^'])'t'(.)", r"\1THIS_IS_TRUE\2", line)
+    # line = line.replace('THIS_IS_TRUE', '1')
+    # line = re.sub(r"([^'])'f'(.)", r"\1THIS_IS_FALSE\2", line)
+    # line = line.replace('THIS_IS_FALSE', '0')
+
+    # Add auto_increment if it's not there since sqlite auto_increments ALL
+    # primary keys
+    if searching_for_end:
+        # replace " and ' with ` because mysql doesn't like quotes in CREATE commands
+        line = line.replace('"', '`').replace("'", '`')
+
+    # And now we convert it back (see above)
+    if re.match(r".*, ``\);", line):
+        line = re.sub(r'``\);', r"'');", line)
+
+    if searching_for_end and re.match(r'.*\);', line):
+        searching_for_end = False
+
+    if re.match(r"CREATE INDEX", line):
+        line = re.sub('"', '`', line)
+
+    line = line.replace('"', '`')
+    line = line.replace('AUTOINCREMENT', 'AUTO_INCREMENT')
+    print(line)

scripts/sqlite2mysql.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+#
+# This shell script and corresponding sqlite2mysql.py are used to
+# migrate Seafile data from SQLite to MySQL.
+#
+# Setup:
+# 
+#  1. Move this file and sqlite2mysql.py to the top directory of your Seafile
+#     installation path (e.g. /data/haiwen).
+#  2. Run: ./sqlite2mysql.sh
+#  3. Three files(ccnet-db.sql, seafile-db.sql, seahub-db.sql) are created.
+#  4. Loads these files to MySQL
+#     (mysql> source ccnet-db.sql)
+# 
+
+CCNET_DB='ccnet-db.sql'
+SEAFILE_DB='seafile-db.sql'
+SEAHUB_DB='seahub-db.sql'
+
+########## ccnet
+seafile_path=$(pwd)
+if [ -f "${seafile_path}/conf/ccnet.conf" ]; then
+    USER_MGR_DB=${seafile_path}/ccnet/PeerMgr/usermgr.db
+    GRP_MGR_DB=${seafile_path}/ccnet/GroupMgr/groupmgr.db
+else
+    echo "${seafile_path}/conf/ccnet.conf does not exists."
+    read -p "Please provide your ccnet.conf path(e.g. /data/haiwen/conf/ccnet.conf): " ccnet_conf_path
+    if [ -f ${ccnet_conf_path} ]; then
+        USER_MGR_DB=$(dirname $(dirname "${ccnet_conf_path}"))/ccnet/PeerMgr/usermgr.db
+        GRP_MGR_DB=$(dirname $(dirname "${ccnet_conf_path}"))/ccnet/GroupMgr/groupmgr.db
+    else
+        echo "${ccnet_conf_path} does not exists, quit."
+        exit 1
+    fi
+fi
+
+rm -rf ${CCNET_DB}
+
+echo "sqlite3 ${USER_MGR_DB} .dump | python sqlite2mysql.py > ${CCNET_DB}"
+sqlite3 ${USER_MGR_DB} .dump | python sqlite2mysql.py > ${CCNET_DB}
+echo "sqlite3 ${GRP_MGR_DB} .dump | python sqlite2mysql.py >> ${CCNET_DB}"
+sqlite3 ${GRP_MGR_DB} .dump | python sqlite2mysql.py >> ${CCNET_DB}
+
+# change ctime from INTEGER to BIGINT in EmailUser table
+sed 's/ctime INTEGER/ctime BIGINT/g' ${CCNET_DB} > ${CCNET_DB}.tmp && mv ${CCNET_DB}.tmp ${CCNET_DB}
+
+# change email in UserRole from TEXT to VARCHAR(255)
+sed 's/email TEXT, role TEXT/email VARCHAR(255), role TEXT/g' ${CCNET_DB} > ${CCNET_DB}.tmp && mv ${CCNET_DB}.tmp ${CCNET_DB}
+
+########## seafile
+rm -rf ${SEAFILE_DB}
+
+if [ -f "${seafile_path}/seafile-data/seafile.db" ]; then
+    echo "sqlite3 ${seafile_path}/seafile-data/seafile.db .dump | python sqlite2mysql.py > ${SEAFILE_DB}"
+    sqlite3 ${seafile_path}/seafile-data/seafile.db .dump | python sqlite2mysql.py > ${SEAFILE_DB}
+else
+    echo "${seafile_path}/seafile-data/seafile.db does not exists."
+    read -p "Please provide your seafile.db path(e.g. /data/haiwen/seafile-data/seafile.db): " seafile_db_path
+    if [ -f ${seafile_db_path} ];then
+        echo "sqlite3 ${seafile_db_path} .dump | python sqlite2mysql.py > ${SEAFILE_DB}"
+        sqlite3 ${seafile_db_path} .dump | python sqlite2mysql.py > ${SEAFILE_DB}
+    else
+        echo "${seafile_db_path} does not exists, quit."
+        exit 1
+    fi
+fi
+
+# change owner_id in RepoOwner from TEXT to VARCHAR(255)
+sed 's/owner_id TEXT/owner_id VARCHAR(255)/g' ${SEAFILE_DB} > ${SEAFILE_DB}.tmp && mv ${SEAFILE_DB}.tmp ${SEAFILE_DB}
+
+# change user_name in RepoGroup from TEXT to VARCHAR(255)
+sed 's/user_name TEXT/user_name VARCHAR(255)/g' ${SEAFILE_DB} > ${SEAFILE_DB}.tmp && mv ${SEAFILE_DB}.tmp ${SEAFILE_DB}
+
+########## seahub
+rm -rf ${SEAHUB_DB}
+
+if [ -f "${seafile_path}/seahub.db" ]; then
+    echo "sqlite3 ${seafile_path}/seahub.db .dump | tr -d '\n' | sed 's/;/;\n/g' | python sqlite2mysql.py > ${SEAHUB_DB}"
+    sqlite3 ${seafile_path}/seahub.db .dump | tr -d '\n' | sed 's/;/;\n/g' | python sqlite2mysql.py > ${SEAHUB_DB}
+else
+    echo "${seafile_path}/seahub.db does not exists."
+    read -p "Please prove your seahub.db path(e.g. /data/haiwen/seahub.db): " seahub_db_path
+    if [ -f ${seahub_db_path} ]; then
+        echo "sqlite3 ${seahub_db_path} .dump | tr -d '\n' | sed 's/;/;\n/g' | python sqlite2mysql.py > ${SEAHUB_DB}"
+        sqlite3 ${seahub_db_path} .dump | tr -d '\n' | sed 's/;/;\n/g' | python sqlite2mysql.py > ${SEAHUB_DB}
+    else
+        echo "${seahub_db_path} does not exists, quit."
+        exit 1
+    fi
+fi
+
+# change username from VARCHAR(256) to VARCHAR(255) in wiki_personalwiki
+sed 's/varchar(256) NOT NULL UNIQUE/varchar(255) NOT NULL UNIQUE/g' ${SEAHUB_DB} > ${SEAHUB_DB}.tmp && mv ${SEAHUB_DB}.tmp ${SEAHUB_DB}
+
+# remove unique from contacts_contact
+sed 's/,    UNIQUE (`user_email`, `contact_email`)//g' ${SEAHUB_DB} > ${SEAHUB_DB}.tmp && mv ${SEAHUB_DB}.tmp ${SEAHUB_DB}
+
+# remove base_dirfileslastmodifiedinfo records to avoid json string parsing issue between sqlite and mysql
+sed '/INSERT INTO `base_dirfileslastmodifiedinfo`/d' ${SEAHUB_DB} > ${SEAHUB_DB}.tmp && mv ${SEAHUB_DB}.tmp ${SEAHUB_DB}
+
+# remove notifications_usernotification records to avoid json string parsing issue between sqlite and mysql
+sed '/INSERT INTO `notifications_usernotification`/d' ${SEAHUB_DB} > ${SEAHUB_DB}.tmp && mv ${SEAHUB_DB}.tmp ${SEAHUB_DB}
+
+
+########## common logic
+
+# add ENGIN=INNODB to create table statment
+for sql_file in $CCNET_DB $SEAFILE_DB $SEAHUB_DB
+do
+    sed -r 's/(CREATE TABLE.*);/\1 ENGINE=INNODB;/g' $sql_file > $sql_file.tmp && mv $sql_file.tmp $sql_file
+done
+
+# remove COLLATE NOCASE if possible
+for sql_file in $CCNET_DB $SEAFILE_DB $SEAHUB_DB
+do
+    sed 's/COLLATE NOCASE//g' $sql_file > $sql_file.tmp && mv $sql_file.tmp $sql_file
+done
+