percona · codeforall · Feb 28, 2024 · Feb 21, 2024 · Feb 21, 2024 · Feb 21, 2024
diff --git a/Makefile.in b/Makefile.in
@@ -34,6 +34,10 @@ src/keyring/keyring_config.o \
 src/keyring/keyring_file.o \
 src/keyring/keyring_vault.o \
 src/keyring/keyring_api.o \
+src/catalog/tde_keyring.o \
+src/catalog/tde_master_key.o \
+src/common/pg_tde_shmem.o \
+src/common/pg_tde_utils.o \
 src/pg_tde.o
 
 override PG_CPPFLAGS += @tde_CPPFLAGS@

diff --git a/expected/move_large_tuples.out b/expected/move_large_tuples.out
@@ -1,5 +1,17 @@
 -- test pg_tde_move_encrypted_data()
 CREATE EXTENSION pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+ pg_tde_set_master_key 
+-----------------------
+
+(1 row)
+
 CREATE TABLE sbtest2(
 	  id SERIAL,
 	  k TEXT STORAGE PLAIN,

diff --git a/expected/multi_insert.out b/expected/multi_insert.out
@@ -1,6 +1,18 @@
 -- trigger multi_insert path
 -- 
 CREATE EXTENSION pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+ pg_tde_set_master_key 
+-----------------------
+
+(1 row)
+
 CREATE TABLE albums (
     album_id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
     artist_id INTEGER,

diff --git a/expected/non_sorted_off_compact.out b/expected/non_sorted_off_compact.out
@@ -1,6 +1,18 @@
 -- A test case for https://github.com/Percona-Lab/pg_tde/pull/21
 -- 
 CREATE EXTENSION pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+ pg_tde_set_master_key 
+-----------------------
+
+(1 row)
+
 DROP TABLE IF EXISTS sbtest1;
 NOTICE:  table "sbtest1" does not exist, skipping
 CREATE TABLE sbtest1(

diff --git a/expected/pgtde_is_encrypted.out b/expected/pgtde_is_encrypted.out
@@ -1,4 +1,16 @@
 CREATE EXTENSION pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+ pg_tde_set_master_key 
+-----------------------
+
+(1 row)
+
 CREATE TABLE test_enc(
 	  id SERIAL,
 	  k INTEGER DEFAULT '0' NOT NULL,

diff --git a/expected/toast_decrypt.out b/expected/toast_decrypt.out
@@ -1,4 +1,16 @@
 CREATE EXTENSION pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+ pg_tde_set_master_key 
+-----------------------
+
+(1 row)
+
 CREATE TABLE src (f1 TEXT STORAGE EXTERNAL) USING pg_tde;
 INSERT INTO src VALUES(repeat('abcdeF',1000));
 SELECT * FROM src;

diff --git a/expected/toast_extended_storage.out b/expected/toast_extended_storage.out
@@ -1,5 +1,17 @@
 -- test https://github.com/Percona-Lab/pg_tde/issues/63
 CREATE EXTENSION pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+ pg_tde_set_master_key 
+-----------------------
+
+(1 row)
+
 CREATE TEMP TABLE src (f1 text) USING pg_tde;
 -- Crash on INSERT
 INSERT INTO src

diff --git a/expected/trigger_on_view.out b/expected/trigger_on_view.out
@@ -1,4 +1,16 @@
 CREATE extension pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+ pg_tde_set_master_key 
+-----------------------
+
+(1 row)
+
 --
 -- 2 -- Test triggers on a join view
 --

diff --git a/expected/update_compare_indexes.out b/expected/update_compare_indexes.out
@@ -1,4 +1,16 @@
 CREATE EXTENSION pg_tde;
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+ pg_tde_add_key_provider_file 
+------------------------------
+                            1
+(1 row)
+
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+ pg_tde_set_master_key 
+-----------------------
+
+(1 row)
+
 DROP TABLE IF EXISTS pvactst;
 NOTICE:  table "pvactst" does not exist, skipping
 CREATE TABLE pvactst (i INT, a INT[], p POINT) USING pg_tde;

diff --git a/meson.build b/meson.build
@@ -26,6 +26,10 @@ pg_tde_sources = files(
         'src/keyring/keyring_vault.c',
         'src/keyring/keyring_api.c',
 
+        'src/catalog/tde_keyring.c',
+        'src/catalog/tde_master_key.c',
+        'src/common/pg_tde_shmem.c',
+        'src/common/pg_tde_utils.c',
         'src/pg_tde.c',
 )
 

diff --git a/pg_tde--1.0.sql b/pg_tde--1.0.sql
@@ -3,6 +3,62 @@
 -- complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "CREATE EXTENSION pg_tde" to load this file. \quit
 
+-- pg_tde catalog tables
+CREATE SCHEMA percona_tde;
+-- Note: The table is created using heap storage becasue we do not want this table
+-- to be encrypted by pg_tde. This table is used to store key provider information
+-- and we do not want to encrypt this table using pg_tde.
+CREATE TABLE percona_tde.pg_tde_key_provider(provider_id SERIAL,
+        keyring_type VARCHAR(10) CHECK (keyring_type IN ('file', 'vault-v2')),
+        provider_name VARCHAR(255) UNIQUE NOT NULL, options JSON, PRIMARY KEY(provider_id)) using heap;
+
+-- If you want to add new provider types, you need to make appropriate changes
+-- in include/catalog/tde_keyring.h and src/catalog/tde_keyring.c files.
+
+SELECT pg_catalog.pg_extension_config_dump('percona_tde.pg_tde_key_provider', '');
+-- Key Provider Management
+
+CREATE OR REPLACE FUNCTION pg_tde_add_key_provider(provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON)
+RETURNS INT
+AS $$
+    INSERT INTO percona_tde.pg_tde_key_provider (keyring_type, provider_name, options) VALUES (provider_type, provider_name, options) RETURNING provider_id;
+$$
+LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_file(provider_name VARCHAR(128), file_path TEXT)
+RETURNS INT
+AS $$
+-- JSON keys in the options must be matched to the keys in
+-- load_file_keyring_provider_options function.
+
+    SELECT pg_tde_add_key_provider('file', provider_name,
+                json_object('type' VALUE 'file', 'path' VALUE file_path));
+$$
+LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_vault_v2(provider_name VARCHAR(128),
+                                                        vault_token TEXT,
+                                                        vault_url TEXT,
+                                                        vault_mount_path TEXT,
+                                                        vault_ca_path TEXT)
+RETURNS INT
+AS $$
+-- JSON keys in the options must be matched to the keys in
+-- load_vaultV2_keyring_provider_options function.
+    SELECT pg_tde_add_key_provider('vault-v2', provider_name,
+                            json_object('type' VALUE 'vault-v2',
+                            'url' VALUE vault_url,
+                            'token' VALUE vault_token,
+                            'mountPath' VALUE vault_mount_path,
+                            'caPath' VALUE vault_ca_path));
+$$
+LANGUAGE SQL;
+
+CREATE FUNCTION pg_tde_get_keyprovider(provider_name text)
+RETURNS VOID
+AS 'MODULE_PATHNAME'
+LANGUAGE C;
+-- Table access method
 CREATE FUNCTION pg_tdeam_handler(internal)
 RETURNS table_am_handler
 AS 'MODULE_PATHNAME'
@@ -18,9 +74,20 @@ RETURNS boolean
 AS 'MODULE_PATHNAME'
 LANGUAGE C;
 
+CREATE FUNCTION pg_tde_set_master_key(master_key_name VARCHAR(255), provider_name VARCHAR(255))
+RETURNS VOID
+AS 'MODULE_PATHNAME'
+LANGUAGE C;
+
+CREATE FUNCTION pg_tde_extension_initialize()
+RETURNS VOID
+AS 'MODULE_PATHNAME'
+LANGUAGE C;
+
+
 -- Access method
 CREATE ACCESS METHOD pg_tde TYPE TABLE HANDLER pg_tdeam_handler;
 COMMENT ON ACCESS METHOD pg_tde IS 'pg_tde table access method';
 
--- Opclasses
-
+-- Per database extension initialization
+SELECT pg_tde_extension_initialize();
diff --git a/sql/move_large_tuples.sql b/sql/move_large_tuples.sql
@@ -1,6 +1,9 @@
 -- test pg_tde_move_encrypted_data()
 CREATE EXTENSION pg_tde;
 
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+
 CREATE TABLE sbtest2(
 	  id SERIAL,
 	  k TEXT STORAGE PLAIN,

diff --git a/sql/multi_insert.sql b/sql/multi_insert.sql
@@ -2,6 +2,9 @@
 -- 
 CREATE EXTENSION pg_tde;
 
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+
 CREATE TABLE albums (
     album_id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
     artist_id INTEGER,

diff --git a/sql/non_sorted_off_compact.sql b/sql/non_sorted_off_compact.sql
@@ -1,6 +1,10 @@
 -- A test case for https://github.com/Percona-Lab/pg_tde/pull/21
 -- 
 CREATE EXTENSION pg_tde;
+
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+
 DROP TABLE IF EXISTS sbtest1;
 CREATE TABLE sbtest1(
 	  id SERIAL,

diff --git a/sql/pgtde_is_encrypted.sql b/sql/pgtde_is_encrypted.sql
@@ -1,5 +1,8 @@
 CREATE EXTENSION pg_tde;
 
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+
 CREATE TABLE test_enc(
 	  id SERIAL,
 	  k INTEGER DEFAULT '0' NOT NULL,

diff --git a/sql/toast_decrypt.sql b/sql/toast_decrypt.sql
@@ -1,5 +1,8 @@
 CREATE EXTENSION pg_tde;
 
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+
 CREATE TABLE src (f1 TEXT STORAGE EXTERNAL) USING pg_tde;
 INSERT INTO src VALUES(repeat('abcdeF',1000));
 SELECT * FROM src;

diff --git a/sql/toast_extended_storage.sql b/sql/toast_extended_storage.sql
@@ -1,6 +1,9 @@
 -- test https://github.com/Percona-Lab/pg_tde/issues/63
 CREATE EXTENSION pg_tde;
 
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+
 CREATE TEMP TABLE src (f1 text) USING pg_tde;
 -- Crash on INSERT
 INSERT INTO src

diff --git a/sql/trigger_on_view.sql b/sql/trigger_on_view.sql
@@ -1,5 +1,8 @@
 CREATE extension pg_tde;
 
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+
 --
 -- 2 -- Test triggers on a join view
 --

diff --git a/sql/update_compare_indexes.sql b/sql/update_compare_indexes.sql
@@ -1,5 +1,8 @@
 CREATE EXTENSION pg_tde;
 
+SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
+SELECT pg_tde_set_master_key('test-db-master-key','file-vault');
+
 DROP TABLE IF EXISTS pvactst;
 CREATE TABLE pvactst (i INT, a INT[], p POINT) USING pg_tde;
 INSERT INTO pvactst SELECT i, array[1,2,3], point(i, i+1) FROM generate_series(1,1000) i;