- Add id servers for generating global object ids (to allow for movin…

…g objects between shards)

- Fix FK bug that was causing sync processes to stay locked after certain errors

include/DB.inc.php

@@ -101,9 +101,6 @@ protected function getShardLink($shardID, $forWriting=false) {
 
 	/**
 	 * Get an instance of the appropriate class
-	 *
-	 * Note: We only use one instance now, and the class might need a little
-	 * reworking to go back to multiple.
 	 */
 	protected static function getInstance() {
 		$class = get_called_class();
@@ -584,8 +581,8 @@ public static function bulkInsert($sql, $sets, $maxInsertGroups, $firstVal=false
 
 			if ($insertCounter == $maxInsertGroups - 1) {
 				$insertSQL = substr($insertSQL, 0, -1);
-				$stmt = Zotero_DB::getStatement($insertSQL, true, $shardID);
-				Zotero_DB::queryFromStatement($stmt, $insertParams);
+				$stmt = self::getStatement($insertSQL, true, $shardID);
+				self::queryFromStatement($stmt, $insertParams);
 				$insertSQL = $origInsertSQL;
 				$insertParams = array();
 				$insertCounter = -1;
@@ -596,8 +593,8 @@ public static function bulkInsert($sql, $sets, $maxInsertGroups, $firstVal=false
 
 		if ($insertCounter > 0 && $insertCounter < $maxInsertGroups) {
 			$insertSQL = substr($insertSQL, 0, -1);
-			$stmt = Zotero_DB::getStatement($insertSQL, true, $shardID);
-			Zotero_DB::queryFromStatement($stmt, $insertParams);
+			$stmt = self::getStatement($insertSQL, true, $shardID);
+			self::queryFromStatement($stmt, $insertParams);
 		}
 	}
 
@@ -788,6 +785,26 @@ public function __destruct() {
 	}
 }
 
+
+class Zotero_ID_DB_1 extends Zotero_DB {
+	protected $db = 'id1';
+
+	protected function __construct() {
+		parent::__construct();
+	}
+}
+
+
+class Zotero_ID_DB_2 extends Zotero_DB {
+	protected $db = 'id2';
+
+	protected function __construct() {
+		parent::__construct();
+	}
+}
+
+
+
 class Zotero_DB_Statement extends Zend_Db_Statement_Mysqli {
 	private $link;
 	private $sql;

include/config/dbconnect.inc.php-sample

@@ -7,7 +7,20 @@ function Zotero_dbConnectAuth($db) {
 		$user = '';
 		$pass = '';
 	}
-
+	else if ($db == 'id1') {
+		$host = '';
+		$port = 3306;
+		$db = 'ids';
+		$user = '';
+		$pass = '';
+	}
+	else if ($db == 'id2') {
+		$host = '';
+		$port = 3306;
+		$db = 'ids';
+		$user = '';
+		$pass = '';
+	}
 	return array('host'=>$host, 'port'=>$port, 'db'=>$db, 'user'=>$user, 'pass'=>$pass);
 }
 ?>

misc/ids.sql

@@ -0,0 +1,43 @@
+-- From http://code.flickr.com/blog/2010/02/08/ticket-servers-distributed-unique-primary-keys-on-the-cheap/
+
+CREATE TABLE `collections` (
+  `id` int(10) unsigned NOT NULL auto_increment,
+  `stub` char(1) NOT NULL default '',
+  PRIMARY KEY  (`id`),
+  UNIQUE KEY `stub` (`stub`)
+) ENGINE=MyISAM;
+
+CREATE TABLE `creators` (
+  `id` int(10) unsigned NOT NULL auto_increment,
+  `stub` char(1) NOT NULL default '',
+  PRIMARY KEY  (`id`),
+  UNIQUE KEY `stub` (`stub`)
+) ENGINE=MyISAM;
+
+CREATE TABLE `items` (
+  `id` int(10) unsigned NOT NULL auto_increment,
+  `stub` char(1) NOT NULL default '',
+  PRIMARY KEY  (`id`),
+  UNIQUE KEY `stub` (`stub`)
+) ENGINE=MyISAM;
+
+CREATE TABLE `relations` (
+  `id` int(10) unsigned NOT NULL auto_increment,
+  `stub` char(1) NOT NULL default '',
+  PRIMARY KEY  (`id`),
+  UNIQUE KEY `stub` (`stub`)
+) ENGINE=MyISAM;
+
+CREATE TABLE `savedSearches` (
+  `id` int(10) unsigned NOT NULL auto_increment,
+  `stub` char(1) NOT NULL default '',
+  PRIMARY KEY  (`id`),
+  UNIQUE KEY `stub` (`stub`)
+) ENGINE=MyISAM;
+
+CREATE TABLE `tags` (
+  `id` int(10) unsigned NOT NULL auto_increment,
+  `stub` char(1) NOT NULL default '',
+  PRIMARY KEY  (`id`),
+  UNIQUE KEY `stub` (`stub`)
+) ENGINE=MyISAM;

misc/test_reset

@@ -1,6 +1,8 @@
 #!/bin/sh
 MASTER="mysql -h 127.0.0.1 -P 3306 -u root"
 SLAVE="mysql -h 127.0.0.1 -P 3307 -u root"
+ID1="mysql -h 127.0.0.1 -P 3308 -u root"
+ID2="mysql -h 127.0.0.1 -P 3309 -u root"
 MYSQLDUMP="mysqldump -h 127.0.0.1 -P 3306 -u root"
 
 echo "STOP SLAVE;" | $SLAVE
@@ -10,23 +12,32 @@ echo "DROP DATABASE IF EXISTS zoterotest1" | $SLAVE
 echo "DROP DATABASE IF EXISTS zoterotest2" | $SLAVE
 echo "DROP DATABASE IF EXISTS zoterotest_master" | $SLAVE
 echo "DROP DATABASE IF EXISTS zoterotest_master" | $MASTER
+echo "DROP DATABASE IF EXISTS zoterotest_ids" | $ID1
+echo "DROP DATABASE IF EXISTS zoterotest_ids" | $ID2
 
 echo "CREATE DATABASE zoterotest_master" | $MASTER
 echo "CREATE DATABASE zoterotest_master" | $SLAVE
 echo "CREATE DATABASE zoterotest1" | $SLAVE
 echo "CREATE DATABASE zoterotest2" | $SLAVE
+echo "CREATE DATABASE zoterotest_ids" | $ID1
+echo "CREATE DATABASE zoterotest_ids" | $ID2
 
 echo "DROP USER zoterotest0@localhost;" | $MASTER
 echo "DROP USER zoterotestrepl@localhost;" | $MASTER
 echo "DROP USER zoterotest1@localhost;" | $SLAVE
 echo "DROP USER zoterotest2@localhost;" | $SLAVE
+echo "DROP USER zoterotest_id@localhost;" | $ID1
+echo "DROP USER zoterotest_id@localhost;" | $ID2
 
 echo "CREATE USER zoterotest0@localhost IDENTIFIED BY 'pass0';" | $MASTER
 echo "CREATE USER zoterotestrepl@localhost IDENTIFIED BY 'passrepl';" | $MASTER
 
 echo "CREATE USER zoterotest1@localhost IDENTIFIED BY 'pass1';" | $SLAVE
 echo "CREATE USER zoterotest2@localhost IDENTIFIED BY 'pass2';" | $SLAVE
 
+echo "CREATE USER zoterotest_id@localhost IDENTIFIED BY 'pass1';" | $ID1
+echo "CREATE USER zoterotest_id@localhost IDENTIFIED BY 'pass2';" | $ID2
+
 echo "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE TEMPORARY TABLES ON zoterotest_master.* TO zoterotest0@localhost;" | $MASTER
 echo "GRANT SELECT, INSERT, DELETE ON zotero_www_test.* TO zoterotest0@localhost;" | $MASTER
 echo "GRANT REPLICATION SLAVE ON *.* TO zoterotestrepl@localhost;" | $MASTER
@@ -36,6 +47,9 @@ echo "GRANT SELECT ON zoterotest_master.* TO zoterotest2@localhost;" | $SLAVE
 echo "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE TEMPORARY TABLES ON zoterotest1.* TO zoterotest1@localhost;" | $SLAVE
 echo "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE TEMPORARY TABLES ON zoterotest2.* TO zoterotest2@localhost;" | $SLAVE
 
+echo "GRANT SELECT,INSERT,DELETE ON zoterotest_ids.* TO zoterotest_id@localhost;" | $ID1
+echo "GRANT SELECT,INSERT,DELETE ON zoterotest_ids.* TO zoterotest_id@localhost;" | $ID2
+
 $MASTER zoterotest_master < master.sql
 $MASTER zoterotest_master < coredata.sql
 
@@ -65,6 +79,9 @@ cat shard.sql | sed 's/`master`/`zoterotest_master`/g' | $SLAVE zoterotest2
 cat triggers.sql | sed 's/`master`/`zoterotest_master`/g' | $SLAVE zoterotest1
 cat triggers.sql | sed 's/`master`/`zoterotest_master`/g' | $SLAVE zoterotest2
 
+$ID1 zoterotest_ids < ids.sql
+$ID2 zoterotest_ids < ids.sql
+
 # Master my.cnf:
 #
 # [mysqld]

misc/test_setup

@@ -12,6 +12,8 @@ Zotero_DB::query("DELETE FROM syncDownloadQueue");
 Zotero_DB::query("DELETE FROM syncUploadQueue");
 Zotero_DB::query("DELETE FROM libraries WHERE libraryID>3");
 Zotero_DB::query("DELETE FROM zotero_www_test.users WHERE userID>2");
+Zotero_ID_DB_1::query("DELETE FROM ids");
+Zotero_ID_DB_2::query("DELETE FROM ids");
 
 $shards = array(1,2);
 foreach ($shards as $shardID) {

model/ID.inc.php

@@ -30,20 +30,13 @@ class Zotero_ID {
 	*/             
 	public static function get($table) {
 		switch ($table) {
-			// Always use auto-increment
-			// TODO: purge these sometimes?
-			case 'tags':
-			case 'creators':
 			case 'collections':
+			case 'creators':
 			case 'items':
 			case 'relations':
 			case 'savedSearches':
 			case 'tags':
-				return null;
-
-			// Non-autoincrement tables
-			//case :
-				//return self::getNext($table);
+				return self::getNext($table);
 
 			default:
 				trigger_error("Unsupported table '$table'", E_USER_ERROR);
@@ -57,41 +50,36 @@ public static function getBigInt() {
 
 
 	/*              
-	* Get MAX(id) + 1 from table
+	* Get MAX(id) + 1 from ids databases
 	*/                     
-	private function getNext($table) {
-		throw new Exception("Unavailable");
-		$column = self::getTableColumn($table);
-		$where = self::getWhere($table);
-		$sql = "SELECT NEXT_ID($column) FROM $table" . $where;
-		return Zotero_DB::valueQuery($sql);
-	}
-
-
-	private function getTableColumn($table) {
-		switch ($table) {
-			case 'savedSearches':
-				return 'savedSearchID';
-				
-			default:
-				return substr($table, 0, -1) . 'ID';
-       }
-	}
-
-
-	private function getWhere($table) {
-		$where = ' WHERE ';
+	private static function getNext($table) {
+		$sql = "REPLACE INTO $table (stub) VALUES ('a')";
+		if (Z_Core::probability(2)) {
+			try {
+				Zotero_ID_DB_1::query($sql);
+				$id = Zotero_ID_DB_1::valueQuery("SELECT LAST_INSERT_ID()");
+			}
+			catch (Exception $e) {
+				Zotero_ID_DB_2::query($sql);
+				$id = Zotero_ID_DB_2::valueQuery("SELECT LAST_INSERT_ID()");
+			}
+		}
+		else {
+			try {
+				Zotero_ID_DB_2::query($sql);
+				$id = Zotero_ID_DB_2::valueQuery("SELECT LAST_INSERT_ID()");
+			}
+			catch (Exception $e) {
+				Zotero_ID_DB_1::query($sql);
+				$id = Zotero_ID_DB_1::valueQuery("SELECT LAST_INSERT_ID()");
+			}
+		}
 
-		switch ($table) {
-			case 'items':
-			case 'creators':
-				break;
-
-			default:
-				trigger_error("Invalid table '$table'", E_USER_ERROR);
+		if (!$id || !is_int($id)) {
+			throw new Exception("Invalid id $id");
 		}
 
-		return $where;
+		return $id;
 	}
 }
 ?>

model/Sync.inc.php

@@ -96,7 +96,7 @@ public static function userIsReadLocked($userID) {
 	/**
 	 * Check if any of a user's libraries are being written to
 	 *
-	 * Clients can't read (/updated) but can write (/upload) if this is true
+	 * Clients can't read (/updated) or write (/upload) if this is true
 	 */
 	public static function userIsWriteLocked($userID) {
 		Zotero_DB::beginTransaction();
@@ -1229,19 +1229,25 @@ private static function processUploadInternal($userID, SimpleXMLElement $xml, $s
 				$xmlElements = dom_import_simplexml($xml->creators);
 				$xmlElements = $xmlElements->getElementsByTagName('creator');
 				Zotero_DB::query("SET foreign_key_checks = 0");
-				$addedLibraryIDs = array();
-				$addedCreatorDataHashes = array();
-				foreach ($xmlElements as $xmlElement) {
-					$key = $xmlElement->getAttribute('key');
-					if (isset($keys[$key])) {
-						throw new Exception("Creator $key already processed");
+				try {
+					$addedLibraryIDs = array();
+					$addedCreatorDataHashes = array();
+					foreach ($xmlElements as $xmlElement) {
+						$key = $xmlElement->getAttribute('key');
+						if (isset($keys[$key])) {
+							throw new Exception("Creator $key already processed");
+						}
+						$keys[$key] = true;
+
+						$creatorObj = Zotero_Creators::convertXMLToCreator($xmlElement);
+						$creatorObj->save();
+						$addedLibraryIDs[] = $creatorObj->libraryID;
+						$addedCreatorDataHashes[] = $creatorObj->creatorDataHash;
 					}
-					$keys[$key] = true;
-
-					$creatorObj = Zotero_Creators::convertXMLToCreator($xmlElement);
-					$creatorObj->save();
-					$addedLibraryIDs[] = $creatorObj->libraryID;
-					$addedCreatorDataHashes[] = $creatorObj->creatorDataHash;
+				}
+				catch (Exception $e) {
+					Zotero_DB::query("SET foreign_key_checks = 1");
+					throw ($e);
 				}
 				Zotero_DB::query("SET foreign_key_checks = 1");
 				unset($keys);

tests/Tests.php

@@ -263,6 +263,14 @@ public function testBulkInsert() {
 
 		Zotero_DB::query("DROP TEMPORARY TABLE foo");
 	}
+
+	public function testIDDB() {
+		$id = Zotero_ID_DB_1::valueQuery("SELECT id FROM items");
+		$this->assertNotEquals(false, $id);
+
+		$id = Zotero_ID_DB_2::valueQuery("SELECT id FROM items");
+		$this->assertNotEquals(false, $id);
+	}
 }