Remove empty TDE files from tablespace

... and rebase tests

expected/tablespace.out

@@ -1 +1,41 @@
-// TODO
+\set tde_am tde_heap
+\i sql/tablespace.inc
+CREATE EXTENSION pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault');
+ pg_tde_set_principal_key 
+--------------------------
+ t
+(1 row)
+
+CREATE TABLE test(num1 bigint, num2 double precision, t text) USING :tde_am;
+INSERT INTO test(num1, num2, t)
+  SELECT round(random()*100), random(), 'text'
+  FROM generate_series(1, 10) s(i);
+CREATE INDEX test_idx ON test(num1);
+SET allow_in_place_tablespaces = true;
+CREATE TABLESPACE test_tblspace LOCATION '';
+ALTER TABLE test SET TABLESPACE test_tblspace;
+SELECT count(*) FROM test;
+ count 
+-------
+    10
+(1 row)
+
+ALTER TABLE test SET TABLESPACE pg_default;
+REINDEX (TABLESPACE test_tblspace, CONCURRENTLY) TABLE test;
+INSERT INTO test VALUES (110, 2);
+SELECT * FROM test WHERE num1=110;
+ num1 | num2 | t 
+------+------+---
+  110 |    2 | 
+(1 row)
+
+DROP TABLE test;
+DROP TABLESPACE test_tblspace;
+DROP EXTENSION pg_tde;

expected/tablespace_basic.out

@@ -0,0 +1,41 @@
+\set tde_am tde_heap_basic
+\i sql/tablespace.inc
+CREATE EXTENSION pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault');
+ pg_tde_set_principal_key 
+--------------------------
+ t
+(1 row)
+
+CREATE TABLE test(num1 bigint, num2 double precision, t text) USING :tde_am;
+INSERT INTO test(num1, num2, t)
+  SELECT round(random()*100), random(), 'text'
+  FROM generate_series(1, 10) s(i);
+CREATE INDEX test_idx ON test(num1);
+SET allow_in_place_tablespaces = true;
+CREATE TABLESPACE test_tblspace LOCATION '';
+ALTER TABLE test SET TABLESPACE test_tblspace;
+SELECT count(*) FROM test;
+ count 
+-------
+    10
+(1 row)
+
+ALTER TABLE test SET TABLESPACE pg_default;
+REINDEX (TABLESPACE test_tblspace, CONCURRENTLY) TABLE test;
+INSERT INTO test VALUES (110, 2);
+SELECT * FROM test WHERE num1=110;
+ num1 | num2 | t 
+------+------+---
+  110 |    2 | 
+(1 row)
+
+DROP TABLE test;
+DROP TABLESPACE test_tblspace;
+DROP EXTENSION pg_tde;

meson.build

@@ -96,6 +96,7 @@ sql_tests = [
       'trigger_on_view_basic',
       'change_access_method_basic',
       'insert_update_delete_basic',
+      'tablespace_basic',
       'vault_v2_test_basic',
 ]
 

sql/tablespace.inc

@@ -0,0 +1,26 @@
+CREATE EXTENSION pg_tde;
+
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault');
+
+CREATE TABLE test(num1 bigint, num2 double precision, t text) USING :tde_am;
+INSERT INTO test(num1, num2, t)
+  SELECT round(random()*100), random(), 'text'
+  FROM generate_series(1, 10) s(i);
+CREATE INDEX test_idx ON test(num1);
+
+SET allow_in_place_tablespaces = true;
+CREATE TABLESPACE test_tblspace LOCATION '';
+
+ALTER TABLE test SET TABLESPACE test_tblspace;
+SELECT count(*) FROM test;
+ALTER TABLE test SET TABLESPACE pg_default;
+
+REINDEX (TABLESPACE test_tblspace, CONCURRENTLY) TABLE test;
+INSERT INTO test VALUES (110, 2);
+
+SELECT * FROM test WHERE num1=110;
+
+DROP TABLE test;
+DROP TABLESPACE test_tblspace;
+DROP EXTENSION pg_tde;

sql/tablespace.sql

@@ -1,26 +1,2 @@
-CREATE EXTENSION pg_tde;
-
-SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
-SELECT pg_tde_set_principal_key('test-db-principal-key','file-vault');
-
-CREATE TABLE test(num1 bigint, num2 double precision, t text) using tde_heap_basic;;
-INSERT INTO test(num1, num2, t)
-  SELECT round(random()*100), random(), 'text'
-  FROM generate_series(1, 10) s(i);
-CREATE INDEX test_idx ON test(num1);
-
-SET allow_in_place_tablespaces = true;
-CREATE TABLESPACE test_tblspace LOCATION '';
-
-ALTER TABLE test SET TABLESPACE test_tblspace;
-SELECT count(*) FROM test;
-ALTER TABLE test SET TABLESPACE pg_default;
-
-REINDEX (TABLESPACE test_tblspace, CONCURRENTLY) TABLE test;
-INSERT INTO test VALUES (110, 2);
-
-SELECT * FROM test WHERE num1=110;
-
-DROP TABLE test;
-DROP TABLESPACE test_tblspace;
-DROP EXTENSION pg_tde;
+\set tde_am tde_heap
+\i sql/tablespace.inc

sql/tablespace_basic.sql

@@ -0,0 +1,2 @@
+\set tde_am tde_heap_basic
+\i sql/tablespace.inc

src/access/pg_tde_tdemap.c

@@ -561,24 +561,23 @@ pg_tde_delete_key_map_entry(const RelFileLocator *rlocator)
  *
  * The offset allows us to simply seek to the desired location and mark the entry
  * as MAP_ENTRY_FREE without needing any further processing.
+ * 
+ * A caller should hold an EXCLUSIVE tde_lwlock_enc_keys lock.
  */
 void
 pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset)
 {
 	int32	key_index = 0;
-	LWLock	*lock_files = tde_lwlock_enc_keys();
 	char	db_map_path[MAXPGPATH] = {0};
-	char	db_keydata_path[MAXPGPATH] = {0};
+	off_t	start = 0;
 
 	Assert(rlocator);
 
 	/* Get the file paths */
-	pg_tde_set_db_file_paths(rlocator, db_map_path, db_keydata_path);
+	pg_tde_set_db_file_paths(rlocator, db_map_path, NULL);
 
 	/* Remove the map entry if found */
-	LWLockAcquire(lock_files, LW_EXCLUSIVE);
 	key_index = pg_tde_process_map_entry(rlocator, db_map_path, &offset, true);
-	LWLockRelease(lock_files);
 
 	if (key_index == -1)
 	{
@@ -588,7 +587,16 @@ pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset)
 						rlocator->relNumber,
 						db_map_path)));
 
-		return;
+	}
+	/* 
+	 * Remove TDE files it was the last TDE relation in a custom tablespace.
+	 * DROP TABLESPACE needs an empty dir.
+	 */
+	if (rlocator->spcOid != GLOBALTABLESPACE_OID && 
+			rlocator->spcOid != DEFAULTTABLESPACE_OID &&
+			pg_tde_process_map_entry(NULL, db_map_path, &start, false) == -1)
+	{
+		pg_tde_delete_tde_files(rlocator->dbOid, rlocator->spcOid);
 	}
 }
 
@@ -843,24 +851,30 @@ pg_tde_move_rel_key(const RelFileLocator *newrlocator, const RelFileLocator *old
 	RelKeyData 	*rel_key;
 	RelKeyData 	*enc_key;
 	TDEPrincipalKey *principal_key;
-	XLogRelKey xlrec;
-	LWLock *lock_pk = tde_lwlock_enc_keys();
+	XLogRelKey	xlrec;
+	char		db_map_path[MAXPGPATH] = {0};
+	off_t		offset = 0;
 
-	LWLockAcquire(lock_pk, LW_EXCLUSIVE);
+	LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE);
+
 	principal_key = GetPrincipalKey(oldrlocator->dbOid, oldrlocator->spcOid, LW_EXCLUSIVE);
 	rel_key = GetRelationKey(*oldrlocator);
 	Assert(rel_key);
 	enc_key = tde_encrypt_rel_key(principal_key, rel_key, newrlocator);
 	pg_tde_write_key_map_entry(newrlocator, enc_key, &principal_key->keyInfo);
 	pg_tde_put_key_into_cache(newrlocator->relNumber, rel_key);
+
+	pg_tde_free_key_map_entry(oldrlocator, offset);
+
+	LWLockRelease(tde_lwlock_enc_keys());
+
 
 	xlrec.rlocator = *newrlocator;
 	xlrec.relKey = *enc_key;
 	XLogBeginInsert();
 	XLogRegisterData((char *) &xlrec, sizeof(xlrec));
 	XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_RELATION_KEY);
 
-	LWLockRelease(lock_pk);
 	pfree(enc_key);
 }
 
@@ -1209,7 +1223,6 @@ pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int fla
 	return found;
 }
 
-
 /*
  * Reads a single keydata from the file.
  */

src/transam/pg_tde_xact_handler.c

@@ -101,6 +101,8 @@ do_pending_deletes(bool isCommit)
     PendingMapEntryDelete *prev;
     PendingMapEntryDelete *next;
 
+    LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE);
+
     prev = NULL;
     for (pending = pendingDeletes; pending != NULL; pending = next)
     {
@@ -129,6 +131,8 @@ do_pending_deletes(bool isCommit)
         /* prev does not change */
 
     }
+
+    LWLockRelease(tde_lwlock_enc_keys());
 }
 
 
@@ -153,6 +157,8 @@ reassign_pending_deletes_to_parent_xact(void)
         if (pending->nestLevel == nestLevel)
             pending->nestLevel--;
     }
+
+    LWLockRelease(tde_lwlock_enc_keys());
 }
 
 /*

src16/access/pg_tdeam_handler.c

@@ -707,7 +707,7 @@ pg_tdeam_relation_copy_data(Relation rel, const RelFileLocator *newrlocator)
 		}
 	}
 
-	pg_tde_move_rel_key(newrlocator, rel->rd_locator);
+	pg_tde_move_rel_key(newrlocator, &rel->rd_locator);
 
 	/* drop old relation, and close new one */
 	RelationDropStorage(rel);

src17/access/pg_tdeam_handler.c

@@ -702,7 +702,7 @@ pg_tdeam_relation_copy_data(Relation rel, const RelFileLocator *newrlocator)
 		}
 	}
 
-	pg_tde_move_rel_key(newrlocator, rel->rd_locator);
+	pg_tde_move_rel_key(newrlocator, &rel->rd_locator);
 
 	/* drop old relation, and close new one */
 	RelationDropStorage(rel);