Add annotations and embedded-image attachments [DB update]

1) Apply triggers.sql to all DB shards. 2) Run 'annotations' to make additional DB schema changes. 3) Push the new code with $ITL=false in Zotero_Items::search() to start populating the new itemTopLevel table on change without using it for search. 4) Run 'itemTopLevelPopulate' to populate itemTopLevel for existing items. It can be safely re-run. 5) Enable $ITL in Zotero_Items::search() to start using itemTopLevel for searches.
VivienDelmon · Feb 19, 2021 · 430f8d4 · 430f8d4
1 parent 3190eb1
commit 430f8d4
Show file tree

Hide file tree

Showing 26 changed files with 2,110 additions and 187 deletions.
diff --git a/controllers/ItemsController.php b/controllers/ItemsController.php
@@ -798,7 +798,7 @@ private function _handleFileRequest($item) {
 		}
 
 		else if ($this->method == 'POST' || $this->method == 'PATCH') {
-			if (!$item->isImportedAttachment()) {
+			if (!$item->isStoredFileAttachment()) {
 				$this->e400("Cannot upload file for linked file/URL attachment item");
 			}
 

diff --git a/controllers/MappingsController.php b/controllers/MappingsController.php
@@ -43,7 +43,7 @@ public function mappings() {
 			$itemType = $_GET['itemType'];
 
 			$itemTypeID = Zotero_ItemTypes::getID($itemType);
-			if (!$itemTypeID) {
+			if (!$itemTypeID || $itemType == 'annotation') {
 				$this->e400("Invalid item type '$itemType'");
 			}
 		}
@@ -56,7 +56,7 @@ public function mappings() {
 			$itemType = $_GET['itemType'];
 
 			$itemTypeID = Zotero_ItemTypes::getID($itemType);
-			if (!$itemTypeID) {
+			if (!$itemTypeID || $itemType == 'annotation') {
 				$this->e400("Invalid item type '$itemType'");
 			}
 
@@ -158,7 +158,22 @@ public function newItem() {
 				$linkMode = Zotero_Attachments::linkModeNameToNumber($linkModeName);
 			}
 			catch (Exception $e) {
-				$this->e400("Invalid linkMode '$linkModeName'");
+				$this->e400("Invalid linkMode '$linkModeName'", Z_ERROR_INVALID_INPUT);
+			}
+		}
+		else if ($itemType == 'annotation') {
+			if (empty($_GET['annotationType'])) {
+				throw new Exception("annotationType required for itemType=annotation", Z_ERROR_INVALID_INPUT);
+			}
+			$annotationType = $_GET['annotationType'];
+			switch ($annotationType) {
+				case 'highlight':
+				case 'note':
+				case 'image':
+					break;
+
+				default:
+					throw new Exception("Invalid annotationType '$annotationType'", Z_ERROR_INVALID_INPUT);
 			}
 		}
 
@@ -177,6 +192,9 @@ public function newItem() {
 		if ($itemType == 'attachment') {
 			$cacheKey .= "_" . $linkMode;
 		}
+		else if ($itemType == 'annotation') {
+			$cacheKey .= "_" . $annotationType;
+		}
 		$cacheKey .= '_' . $this->apiVersion;
 		$ttl = 60;
 		$json = Z_Core::$MC->get($cacheKey);
@@ -194,6 +212,7 @@ public function newItem() {
 		if ($itemType == 'attachment') {
 			$json['linkMode'] = $linkModeName;
 		}
+		$isEmbeddedImage = $itemType == 'attachment' && $json['linkMode'] == 'embedded_image';
 
 		$fieldIDs = Zotero_ItemFields::getItemTypeFields($itemTypeID);
 		$first = true;
@@ -225,6 +244,31 @@ public function newItem() {
 			}
 		}
 
+		if ($isEmbeddedImage) {
+			$json['parentItem'] = '';
+		}
+		else if ($itemType == 'annotation') {
+			$json['parentItem'] = '';
+			$json['annotationType'] = $annotationType;
+
+			if ($annotationType == 'highlight') {
+				$json['annotationText'] = '';
+			}
+
+			$json['annotationComment'] = '';
+			$json['annotationColor'] = '';
+			$json['annotationPageLabel'] = '';
+			$json['annotationSortIndex'] = '00000|000000|00000';
+			$json['annotationPosition'] = [
+				'pageIndex' => 0,
+				'rects' => []
+			];
+			if ($annotationType == 'image') {
+				$json['annotationPosition']['width'] = 0;
+				$json['annotationPosition']['height'] = 0;
+			}
+		}
+
 		if ($itemType == 'note' || $itemType == 'attachment') {
 			$json['note'] = '';
 		}
@@ -250,12 +294,25 @@ public function newItem() {
 				$json['path'] = '';
 			}
 
-			if (preg_match('/^imported_/', $linkModeName)) {
+			if (preg_match('/^imported_/', $linkModeName) || $isEmbeddedImage) {
 				$json['filename'] = '';
 				$json['md5'] = null;
 				$json['mtime'] = null;
 				//$json['zip'] = false;
 			}
+
+			if ($isEmbeddedImage) {
+				$toRemove = [
+					'title', 'url', 'accessDate', 'tags', 'collections', 'relations', 'note', 'charset'
+				];
+				foreach ($toRemove as $prop) {
+					unset($json[$prop]);
+				}
+			}
+		}
+		else if ($itemType == 'annotation') {
+			unset($json['collections']);
+			unset($json['relations']);
 		}
 
 		header("Content-Type: application/json");

diff --git a/include/DB.inc.php b/include/DB.inc.php
@@ -996,14 +996,16 @@ public static function bulkInsert($sql, $sets, $maxInsertGroups, $firstVal = fal
 	}
 
 
-/*	// Checks the existence of a table in DB
-	public static function table_exists($table) {
-		$instance = self::getInstance();
-		return $instance->link->tableExists($table);
+	/**
+	 * Check the existence of a table in DB
+	 */
+	public static function tableExists($table, $shardID) {
+		// There are faster ways to do this if we start using this somewhere where performance matters
+		return !!Zotero_DB::valueQuery("SHOW TABLES LIKE '$table'", false, $shardID);
 	}
 
 
-	// List fields in table
+/*	// List fields in table
 	public static function list_fields($table, $exclude=array()) {
 		$instance = self::getInstance();
 		

diff --git a/include/Shards.inc.php b/include/Shards.inc.php
@@ -246,13 +246,15 @@ public static function copyLibrary($libraryID, $newShardID, $overrideLock=false)
 			'collectionItems',
 			'deletedItems',
 			'groupItems',
+			'itemAnnotations',
 			'itemAttachments',
 			'itemCreators',
 			'itemData',
 			'itemNotes',
 			'itemRelated',
 			'itemSortFields',
 			'itemTags',
+			'itemTopLevel',
 			'publicationsItems',
 			'savedSearchConditions',
 			'storageFileItems',
@@ -283,13 +285,15 @@ public static function copyLibrary($libraryID, $newShardID, $overrideLock=false)
 				case 'collectionItems':
 				case 'deletedItems':
 				case 'groupItems':
+				case 'itemAnnotations':
 				case 'itemAttachments':
 				case 'itemCreators':
 				case 'itemData':
 				case 'itemNotes':
 				case 'itemRelated':
 				case 'itemSortFields':
 				case 'itemTags':
+				case 'itemTopLevel':
 				case 'publicationsItems':
 				case 'storageFileItems':
 					$sql = "SELECT T.* FROM $table T JOIN items USING (itemID) WHERE libraryID=?";

diff --git a/misc/coredata.sql b/misc/coredata.sql
@@ -65,6 +65,7 @@ INSERT INTO `itemTypes` (`itemTypeID`, `itemTypeName`, `custom`) VALUES
 (34, 'document', 0),
 (35, 'encyclopediaArticle', 0),
 (36, 'dictionaryEntry', 0),
+(37, 'annotation', 0),
 (10001, 'nsfReviewer', 1);
 
 INSERT INTO `creatorTypes` (`creatorTypeID`, `creatorTypeName`, `custom`) VALUES

diff --git a/misc/db-updates/2020-08-31/annotations b/misc/db-updates/2020-08-31/annotations
@@ -0,0 +1,15 @@
+#!/usr/bin/php -d mysqlnd.net_read_timeout=3600
+<?php
+set_include_path("../../../include");
+require("header.inc.php");
+
+$startShard = !empty($argv[1]) ? $argv[1] : 1;
+
+$shardIDs = Zotero_DB::columnQuery("SELECT shardID FROM shards WHERE shardID >= ? ORDER BY shardID", $startShard);
+foreach ($shardIDs as $shardID) {
+	echo "Shard: $shardID\n";
+
+	Zotero_Admin_DB::query("ALTER TABLE `itemAttachments` CHANGE `linkMode` `linkMode` ENUM( 'IMPORTED_FILE', 'IMPORTED_URL', 'LINKED_FILE', 'LINKED_URL', 'EMBEDDED_IMAGE' )", false, $shardID);
+	Zotero_Admin_DB::query("CREATE TABLE `itemAnnotations` ( `itemID` int(10) unsigned NOT NULL, `parentItemID` int(10) unsigned NOT NULL, `type` enum('highlight','note','image') CHARACTER SET ascii COLLATE ascii_bin NOT NULL, `text` varchar(10000) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, `comment` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL, `color` char(6) CHARACTER SET ascii NOT NULL, `pageLabel` varchar(50) NOT NULL, `sortIndex` varchar(18) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, `position` varchar(20000) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, PRIMARY KEY (`itemID`), KEY `parentItemID` (`parentItemID`), CONSTRAINT `itemAnnotations_ibfk_1` FOREIGN KEY (`itemID`) REFERENCES `items` (`itemID`) ON DELETE CASCADE, CONSTRAINT `itemAnnotations_ibfk_2` FOREIGN KEY (`parentItemID`) REFERENCES `itemAttachments` (`itemID`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", false, $shardID);
+	Zotero_Admin_DB::query("CREATE TABLE `itemTopLevel` ( `itemID` int(10) unsigned NOT NULL, `topLevelItemID` int(10) unsigned NOT NULL, PRIMARY KEY (`itemID`), KEY `itemTopLevel_ibfk_2` (`topLevelItemID`), CONSTRAINT `itemTopLevel_ibfk_1` FOREIGN KEY (`itemID`) REFERENCES `items` (`itemID`) ON DELETE CASCADE, CONSTRAINT `itemTopLevel_ibfk_2` FOREIGN KEY (`topLevelItemID`) REFERENCES `items` (`itemID`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", false, $shardID);
+}
diff --git a/misc/db-updates/2020-08-31/itemTopLevelPopulate b/misc/db-updates/2020-08-31/itemTopLevelPopulate
@@ -0,0 +1,14 @@
+#!/usr/bin/php -d mysqlnd.net_read_timeout=3600
+<?php
+set_include_path("../../../include");
+require("header.inc.php");
+
+$startShard = !empty($argv[1]) ? $argv[1] : 1;
+
+$shardIDs = Zotero_DB::columnQuery("SELECT shardID FROM shards WHERE shardID >= ? ORDER BY shardID", $startShard);
+foreach ($shardIDs as $shardID) {
+	echo "Shard: $shardID\n";
+
+	Zotero_DB::query("INSERT IGNORE INTO itemTopLevel SELECT itemID, sourceItemID FROM itemAttachments WHERE sourceItemID IS NOT NULL UNION SELECT itemID, sourceItemID FROM itemNotes WHERE sourceItemID IS NOT NULL", false, $shardID);
+	Zotero_DB::query("DELETE ITL FROM itemTopLevel ITL LEFT JOIN (SELECT itemID, sourceItemID FROM itemAttachments UNION SELECT itemID, sourceItemID FROM itemNotes) S ON (ITL.itemID=S.itemID AND ITL.topLevelItemID=S.sourceItemID) WHERE S.itemID IS NULL", false, $shardID);
+}
diff --git a/misc/db-updates/2020-08-31/triggers.sql b/misc/db-updates/2020-08-31/triggers.sql
@@ -0,0 +1,49 @@
+delimiter //
+
+DROP TRIGGER IF EXISTS fki_itemAttachments;//
+CREATE TRIGGER fki_itemAttachments
+  BEFORE INSERT ON itemAttachments
+  FOR EACH ROW BEGIN
+    -- itemAttachments libraryID
+    IF NEW.sourceItemID IS NOT NULL AND (SELECT libraryID FROM items WHERE itemID = NEW.itemID) != (SELECT libraryID FROM items WHERE itemID = NEW.sourceItemID) THEN
+    SELECT libraryIDs_do_not_match INTO @failure FROM itemAttachments;
+    END IF;
+
+    -- Make sure this is an attachment item
+    IF ((SELECT itemTypeID FROM items WHERE itemID = NEW.itemID) != 14) THEN
+    SELECT not_an_attachment INTO @failure FROM items;
+    END IF;
+
+    -- If there's a parent, reject if it's an attachment or it's a note and this isn't an embedded-image attachment
+    SET @parentItemTypeID = IF(NEW.sourceItemID IS NULL, NULL, (SELECT itemTypeID FROM items WHERE itemID = NEW.sourceItemID));
+    IF (@parentItemTypeID = 14 OR (@parentItemTypeID = 1 AND NEW.linkMode != 'EMBEDDED_IMAGE')) THEN
+    SELECT invalid_parent INTO @failure FROM items;
+    END IF;
+
+    -- If child, make sure attachment is not in a collection
+    IF (NEW.sourceItemID IS NOT NULL AND (SELECT COUNT(*) FROM collectionItems WHERE itemID=NEW.itemID)>0) THEN
+    SELECT collection_item_must_be_top_level INTO @failure FROM collectionItems;
+    END IF;
+  END;//
+
+DROP TRIGGER IF EXISTS fku_itemAttachments_libraryID;//
+CREATE TRIGGER fku_itemAttachments_libraryID
+  BEFORE UPDATE ON itemAttachments
+  FOR EACH ROW BEGIN
+    IF NEW.sourceItemID IS NOT NULL AND (SELECT libraryID FROM items WHERE itemID = NEW.itemID) != (SELECT libraryID FROM items WHERE itemID = NEW.sourceItemID) THEN
+    SELECT libraryIDs_do_not_match INTO @failure FROM itemAttachments;
+    END IF;
+
+    -- If there's a parent, reject if it's an attachment or it's a note and this isn't an embedded-image attachment
+    SET @parentItemTypeID = IF(NEW.sourceItemID IS NULL, NULL, (SELECT itemTypeID FROM items WHERE itemID = NEW.sourceItemID));
+    IF (@parentItemTypeID = 14 OR (@parentItemTypeID = 1 AND NEW.linkMode != 'EMBEDDED_IMAGE')) THEN
+    SELECT invalid_parent INTO @failure FROM items;
+    END IF;
+
+    -- If child, make sure attachment is not in a collection
+    IF (NEW.sourceItemID IS NOT NULL AND (SELECT COUNT(*) FROM collectionItems WHERE itemID=NEW.itemID)>0) THEN
+    SELECT collection_item_must_be_top_level INTO @failure FROM collectionItems;
+    END IF;
+  END;//
+
+delimiter ;
diff --git a/misc/shard.sql b/misc/shard.sql
@@ -23,7 +23,7 @@
 
 
 --
--- IMPORTANT: All tables added here must be added to Zotero_Shards::moveLibrary()!
+-- IMPORTANT: All tables added here must be added to Zotero_Shards::copyLibrary()!
 --
 
 CREATE TABLE `collectionItems` (
@@ -114,7 +114,7 @@ CREATE TABLE `publicationsItems` (
 CREATE TABLE `itemAttachments` (
   `itemID` int(10) unsigned NOT NULL,
   `sourceItemID` int(10) unsigned DEFAULT NULL,
-  `linkMode` enum('IMPORTED_FILE','IMPORTED_URL','LINKED_FILE','LINKED_URL') NOT NULL,
+  `linkMode` enum('IMPORTED_FILE','IMPORTED_URL','LINKED_FILE','LINKED_URL','EMBEDDED_IMAGE'),
   `mimeType` varchar(255) NOT NULL,
   `charsetID` tinyint(3) unsigned DEFAULT NULL,
   `path` blob NOT NULL,
@@ -126,6 +126,20 @@ CREATE TABLE `itemAttachments` (
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
 
+CREATE TABLE `itemAnnotations` (
+  `itemID` int(10) unsigned NOT NULL,
+  `parentItemID` int(10) unsigned NOT NULL,
+  `type` enum('highlight','note','image') CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+  `text` varchar(10000) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL,
+  `comment` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_520_ci NOT NULL,
+  `color` char(6) CHARACTER SET ascii NOT NULL,
+  `pageLabel` varchar(50) NOT NULL,
+  `sortIndex` varchar(18) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+  `position` varchar(20000) CHARACTER SET ascii COLLATE ascii_bin NOT NULL,
+  PRIMARY KEY (`itemID`),
+  KEY `parentItemID` (`parentItemID`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
 
 CREATE TABLE `itemCreators` (
   `itemID` int(10) unsigned NOT NULL,
@@ -216,6 +230,14 @@ CREATE TABLE `itemTags` (
 
 
 
+CREATE TABLE `itemTopLevel` (
+  `itemID` int(10) unsigned NOT NULL,
+  `topLevelItemID` int(10) unsigned NOT NULL,
+  PRIMARY KEY (`itemID`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+
 CREATE TABLE `relations` (
   `relationID` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `libraryID` int(10) unsigned NOT NULL,
@@ -376,6 +398,10 @@ ALTER TABLE `groupItems`
 ALTER TABLE `publicationsItems`
   ADD CONSTRAINT `publicationsItems_ibfk_1` FOREIGN KEY (`itemID`) REFERENCES `items` (`itemID`) ON DELETE CASCADE;
 
+ALTER TABLE `itemAnnotations`
+  ADD CONSTRAINT `itemAnnotations_ibfk_1` FOREIGN KEY (`itemID`) REFERENCES `items` (`itemID`) ON DELETE CASCADE,
+  ADD CONSTRAINT `itemAnnotations_ibfk_2` FOREIGN KEY (`parentItemID`) REFERENCES `itemAttachments` (`itemID`);
+
 ALTER TABLE `itemAttachments`
   ADD CONSTRAINT `itemAttachments_ibfk_1` FOREIGN KEY (`itemID`) REFERENCES `items` (`itemID`) ON DELETE CASCADE,
   ADD CONSTRAINT `itemAttachments_ibfk_2` FOREIGN KEY (`sourceItemID`) REFERENCES `items` (`itemID`) ON DELETE SET NULL;
@@ -408,6 +434,10 @@ ALTER TABLE `itemTags`
   ADD CONSTRAINT `itemTags_ibfk_1` FOREIGN KEY (`itemID`) REFERENCES `items` (`itemID`) ON DELETE CASCADE,
   ADD CONSTRAINT `itemTags_ibfk_2` FOREIGN KEY (`tagID`) REFERENCES `tags` (`tagID`) ON DELETE CASCADE;
 
+ALTER TABLE `itemTopLevel`
+  ADD CONSTRAINT `itemTopLevel_ibfk_1` FOREIGN KEY (`itemID`) REFERENCES `items` (`itemID`) ON DELETE CASCADE,
+  ADD CONSTRAINT `itemTopLevel_ibfk_2` FOREIGN KEY (`topLevelItemID`) REFERENCES `items` (`itemID`) ON DELETE CASCADE;
+
 ALTER TABLE `relations`
   ADD CONSTRAINT `relations_ibfk_1` FOREIGN KEY (`libraryID`) REFERENCES `shardLibraries` (`libraryID`) ON DELETE CASCADE;
 

diff --git a/misc/triggers.sql b/misc/triggers.sql
@@ -74,9 +74,10 @@ CREATE TRIGGER fki_itemAttachments
     SELECT not_an_attachment INTO @failure FROM items;
     END IF;
 
-    -- Make sure parent is a regular item
-    IF (NEW.sourceItemID IS NOT NULL AND (SELECT itemTypeID FROM items WHERE itemID = NEW.sourceItemID) IN (1,14)) THEN
-    SELECT parent_not_regular_item INTO @failure FROM items;
+    -- If there's a parent, reject if it's an attachment or it's a note and this isn't an embedded-image attachment
+    SET @parentItemTypeID = IF(NEW.sourceItemID IS NULL, NULL, (SELECT itemTypeID FROM items WHERE itemID = NEW.sourceItemID));
+    IF (@parentItemTypeID = 14 OR (@parentItemTypeID = 1 AND NEW.linkMode != 'EMBEDDED_IMAGE')) THEN
+    SELECT invalid_parent INTO @failure FROM items;
     END IF;
 
     -- If child, make sure attachment is not in a collection
@@ -93,9 +94,10 @@ CREATE TRIGGER fku_itemAttachments_libraryID
     SELECT libraryIDs_do_not_match INTO @failure FROM itemAttachments;
     END IF;
 
-    -- Make sure parent is a regular item
-    IF (NEW.sourceItemID IS NOT NULL AND (SELECT itemTypeID FROM items WHERE itemID = NEW.sourceItemID) IN (1,14)) THEN
-    SELECT parent_not_regular_item INTO @failure FROM items;
+    -- If there's a parent, reject if it's an attachment or it's a note and this isn't an embedded-image attachment
+    SET @parentItemTypeID = IF(NEW.sourceItemID IS NULL, NULL, (SELECT itemTypeID FROM items WHERE itemID = NEW.sourceItemID));
+    IF (@parentItemTypeID = 14 OR (@parentItemTypeID = 1 AND NEW.linkMode != 'EMBEDDED_IMAGE')) THEN
+    SELECT invalid_parent INTO @failure FROM items;
     END IF;
 
     -- If child, make sure attachment is not in a collection

diff --git a/model/Attachments.inc.php b/model/Attachments.inc.php
@@ -31,7 +31,8 @@ class Zotero_Attachments {
 		0 => "imported_file",
 		1 => "imported_url",
 		2 => "linked_file",
-		3 => "linked_url"
+		3 => "linked_url",
+		4 => "embedded_image"
 	);
 
 	public static function linkModeNumberToName($number) {

diff --git a/model/Collection.inc.php b/model/Collection.inc.php
@@ -156,7 +156,7 @@ public function save($userID=false) {
 			if (!empty($this->changed['parentKey'])) {
 				$objectsClass = $this->objectsClass;
 
-				// Add this item to the parent's cached item lists after commit,
+				// Add this to the parent's cached collection lists after commit,
 				// if the parent was loaded
 				if ($this->_parentKey) {
 					$parentCollectionID = $objectsClass::getIDFromLibraryAndKey(