From 5ca8ede1851d342f952a877c85e842a6c0707db4 Mon Sep 17 00:00:00 2001 From: Muhammad Usama Date: Mon, 21 Oct 2024 17:17:01 +0500 Subject: [PATCH] Add key type to internal relation keys in pg_tde: (#317) This commit introduces a key type to internal relation keys, allowing segregation of SMGR and heap_basic internal keys. It resolves the issue of double encryption when both tde_heap and tde_heap_basic access methods are in use. With this enhancement, pg_tde now supports three types of internal keys: SMGR keys, heap_basic keys, and global keys, improving flexibility and preventing redundant encryption operations. --- src/access/pg_tde_ddl.c | 4 +- src/access/pg_tde_slot.c | 2 +- src/access/pg_tde_tdemap.c | 148 ++++++++++++++++++----------- src/access/pg_tde_xlog.c | 2 +- src/access/pg_tde_xlog_encrypt.c | 4 +- src/catalog/tde_global_space.c | 8 +- src/encryption/enc_tde.c | 2 +- src/include/access/pg_tde_tdemap.h | 30 ++++-- src/pg_tde_event_capture.c | 4 +- src/smgr/pg_tde_smgr.c | 6 +- src/transam/pg_tde_xact_handler.c | 2 +- src16/access/pg_tde_prune.c | 2 +- src16/access/pg_tdeam.c | 10 +- src16/access/pg_tdeam_handler.c | 2 +- src16/access/pg_tdetoast.c | 4 +- src17/access/pg_tde_prune.c | 2 +- src17/access/pg_tdeam.c | 10 +- src17/access/pg_tdeam_handler.c | 2 +- src17/access/pg_tdetoast.c | 4 +- 19 files changed, 151 insertions(+), 97 deletions(-) diff --git a/src/access/pg_tde_ddl.c b/src/access/pg_tde_ddl.c index 3916fd99..ee04af34 100644 --- a/src/access/pg_tde_ddl.c +++ b/src/access/pg_tde_ddl.c @@ -29,7 +29,7 @@ void SetupTdeDDLHooks(void) } static void - tdeheap_object_access_hook(ObjectAccessType access, Oid classId, Oid objectId, +tdeheap_object_access_hook(ObjectAccessType access, Oid classId, Oid objectId, int subId, void *arg) { Relation rel = NULL; @@ -49,7 +49,7 @@ static void rel->rd_rel->relkind == RELKIND_MATVIEW) && (subId == 0) && is_tdeheap_rel(rel)) { - pg_tde_delete_key_map_entry(&rel->rd_locator); + pg_tde_delete_key_map_entry(&rel->rd_locator, MAP_ENTRY_VALID); } relation_close(rel, AccessShareLock); } diff --git a/src/access/pg_tde_slot.c b/src/access/pg_tde_slot.c index 8db45d2a..1f5e29ec 100644 --- a/src/access/pg_tde_slot.c +++ b/src/access/pg_tde_slot.c @@ -579,6 +579,6 @@ get_current_slot_relation_key(TDEBufferHeapTupleTableSlot *bslot, Relation rel) { Assert(bslot != NULL); if (bslot->cached_relation_key == NULL) - bslot->cached_relation_key = GetRelationKey(rel->rd_locator); + bslot->cached_relation_key = GetHeapBaiscRelationKey(rel->rd_locator); return bslot->cached_relation_key; } \ No newline at end of file diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index f409f81f..57e67ff2 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -60,8 +60,6 @@ #define PG_TDE_FILEMAGIC 0x01454454 /* version ID value = TDE 01 */ -#define MAP_ENTRY_FREE 0x00 -#define MAP_ENTRY_VALID 0x01 #define MAP_ENTRY_SIZE sizeof(TDEMapEntry) #define TDE_FILE_HEADER_SIZE sizeof(TDEFileHeader) @@ -75,7 +73,7 @@ typedef struct TDEFileHeader typedef struct TDEMapEntry { RelFileNumber relNumber; - int32 flags; + uint32 flags; int32 key_index; } TDEMapEntry; @@ -88,7 +86,8 @@ typedef struct TDEMapFilePath typedef struct RelKeyCacheRec { - Oid rel_id; + RelFileNumber rel_number; + uint32 key_type; RelKeyData key; } RelKeyCacheRec; @@ -114,30 +113,48 @@ typedef struct RelKeyCache RelKeyCache *tde_rel_key_cache = NULL; -static int32 pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t *offset, bool should_delete); +static int32 pg_tde_process_map_entry(const RelFileLocator *rlocator, uint32 key_type, char *db_map_path, off_t *offset, bool should_delete); static RelKeyData* pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEPrincipalKey *principal_key); static int pg_tde_open_file_basic(char *tde_filename, int fileFlags, bool ignore_missing); static int pg_tde_file_header_read(char *tde_filename, int fd, TDEFileHeader *fheader, bool *is_new_file, off_t *bytes_read); static bool pg_tde_read_one_map_entry(int fd, const RelFileLocator *rlocator, int flags, TDEMapEntry *map_entry, off_t *offset); static RelKeyData* pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEPrincipalKey *principal_key); static int pg_tde_open_file(char *tde_filename, TDEPrincipalKeyInfo *principal_key_info, bool should_fill_info, int fileFlags, bool *is_new_file, off_t *offset); -static RelKeyData *pg_tde_get_key_from_cache(Oid rel_id); +static RelKeyData *pg_tde_get_key_from_cache(RelFileNumber rel_number, uint32 key_type); #ifndef FRONTEND static int pg_tde_file_header_write(char *tde_filename, int fd, TDEPrincipalKeyInfo *principal_key_info, off_t *bytes_written); -static int32 pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEPrincipalKeyInfo *principal_key_info); -static off_t pg_tde_write_one_map_entry(int fd, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset); +static int32 pg_tde_write_map_entry(const RelFileLocator *rlocator, uint32 entry_type, char *db_map_path, TDEPrincipalKeyInfo *principal_key_info); +static off_t pg_tde_write_one_map_entry(int fd, const RelFileLocator *rlocator, uint32 flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset); static void pg_tde_write_keydata(char *db_keydata_path, TDEPrincipalKeyInfo *principal_key_info, int32 key_index, RelKeyData *enc_rel_key_data); static void pg_tde_write_one_keydata(int keydata_fd, int32 key_index, RelKeyData *enc_rel_key_data); static int keyrotation_init_file(TDEPrincipalKeyInfo *new_principal_key_info, char *rotated_filename, char *filename, bool *is_new_file, off_t *curr_pos); static void finalize_key_rotation(char *m_path_old, char *k_path_old, char *m_path_new, char *k_path_new); +RelKeyData * +pg_tde_create_smgr_key(const RelFileLocator *newrlocator) +{ + return pg_tde_create_key_map_entry(newrlocator, TDE_KEY_TYPE_SMGR); +} + +RelKeyData * +pg_tde_create_global_key(const RelFileLocator *newrlocator) +{ + return pg_tde_create_key_map_entry(newrlocator, TDE_KEY_TYPE_GLOBAL); +} + +RelKeyData * +pg_tde_create_heap_basic_key(const RelFileLocator *newrlocator) +{ + return pg_tde_create_key_map_entry(newrlocator, TDE_KEY_TYPE_HEAP_BASIC); +} + /* * Generate an encrypted key for the relation and store it in the keymap file. */ -RelKeyData* -pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) +RelKeyData * +pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type) { InternalKey int_key; RelKeyData *rel_key_data; @@ -171,12 +188,13 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) } /* Encrypt the key */ - rel_key_data = tde_create_rel_key(newrlocator->relNumber, &int_key, &principal_key->keyInfo); + rel_key_data = tde_create_rel_key(newrlocator->relNumber, entry_type, &int_key, &principal_key->keyInfo); enc_rel_key_data = tde_encrypt_rel_key(principal_key, rel_key_data, newrlocator); /* * XLOG internal key */ + xlrec.entry_type = entry_type; xlrec.rlocator = *newrlocator; xlrec.relKey = *enc_rel_key_data; @@ -185,9 +203,9 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_RELATION_KEY); /* - * Add the encyrpted key to the key map data file structure. + * Add the encrypted key to the key map data file structure. */ - pg_tde_write_key_map_entry(newrlocator, enc_rel_key_data, &principal_key->keyInfo); + pg_tde_write_key_map_entry(newrlocator, entry_type, enc_rel_key_data, &principal_key->keyInfo); LWLockRelease(lock_pk); pfree(enc_rel_key_data); return rel_key_data; @@ -210,7 +228,7 @@ tde_sprint_key(InternalKey *k) * created key. */ RelKeyData * -tde_create_rel_key(Oid rel_id, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info) +tde_create_rel_key(RelFileNumber rel_num, uint32 key_type, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info) { RelKeyData rel_key_data; memcpy(&rel_key_data.principal_key_id, &principal_key_info->keyId, sizeof(TDEPrincipalKeyId)); @@ -218,7 +236,7 @@ tde_create_rel_key(Oid rel_id, InternalKey *key, TDEPrincipalKeyInfo *principal_ rel_key_data.internal_key.ctx = NULL; /* Add to the decrypted key to cache */ - return pg_tde_put_key_into_cache(rel_id, &rel_key_data); + return pg_tde_put_key_into_cache(rel_num, key_type, &rel_key_data); } /* * Encrypts a given key and returns the encrypted one. @@ -339,7 +357,7 @@ pg_tde_file_header_write(char *tde_filename, int fd, TDEPrincipalKeyInfo *princi * concurrent in place updates leading to data conflicts. */ static int32 -pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEPrincipalKeyInfo *principal_key_info) +pg_tde_write_map_entry(const RelFileLocator *rlocator, uint32 entry_type, char *db_map_path, TDEPrincipalKeyInfo *principal_key_info) { int map_fd = -1; int32 key_index = 0; @@ -349,7 +367,7 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEPri off_t prev_pos = 0; bool found = false; - /* Open and vaidate file for basic correctness. */ + /* Open and validate file for basic correctness. */ map_fd = pg_tde_open_file(db_map_path, principal_key_info, false, O_RDWR | O_CREAT, &is_new_file, &curr_pos); prev_pos = curr_pos; @@ -361,7 +379,7 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEPri while(1) { prev_pos = curr_pos; - found = pg_tde_read_one_map_entry(map_fd, NULL, MAP_ENTRY_FREE, &map_entry, &curr_pos); + found = pg_tde_read_one_map_entry(map_fd, NULL, MAP_ENTRY_EMPTY, &map_entry, &curr_pos); /* We either reach EOF or found an empty slot in the middle of the file */ if (prev_pos == curr_pos || found) @@ -373,7 +391,7 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEPri /* Write the given entry at the location pointed by prev_pos; i.e. the free entry */ curr_pos = prev_pos; - pg_tde_write_one_map_entry(map_fd, rlocator, MAP_ENTRY_VALID, key_index, &map_entry, &prev_pos); + pg_tde_write_one_map_entry(map_fd, rlocator, entry_type, key_index, &map_entry, &prev_pos); /* Let's close the file. */ close(map_fd); @@ -389,7 +407,7 @@ pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, TDEPri * map file. */ static off_t -pg_tde_write_one_map_entry(int fd, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset) +pg_tde_write_one_map_entry(int fd, const RelFileLocator *rlocator, uint32 flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset) { int bytes_written = 0; @@ -488,7 +506,7 @@ pg_tde_write_one_keydata(int fd, int32 key_index, RelKeyData *enc_rel_key_data) * The caller must hold an exclusive lock tde_lwlock_enc_keys. */ void -pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info) +pg_tde_write_key_map_entry(const RelFileLocator *rlocator, uint32 entry_type, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info) { int32 key_index = 0; char db_map_path[MAXPGPATH] = {0}; @@ -500,7 +518,7 @@ pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_k pg_tde_set_db_file_paths(rlocator->dbOid, rlocator->spcOid, db_map_path, db_keydata_path); /* Create the map entry and then add the encrypted key to the data file */ - key_index = pg_tde_write_map_entry(rlocator, db_map_path, principal_key_info); + key_index = pg_tde_write_map_entry(rlocator, entry_type, db_map_path, principal_key_info); /* Add the encrypted key to the data file. */ pg_tde_write_keydata(db_keydata_path, principal_key_info, key_index, enc_rel_key_data); @@ -510,8 +528,7 @@ pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_k * Deletes a map entry by setting marking it as unused. We don't have to delete * the actual key data as valid key data entries are identify by valid map entries. */ -void -pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) +void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator, uint32 key_type) { int32 key_index = 0; off_t offset = 0; @@ -527,7 +544,7 @@ pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) errno = 0; /* Remove the map entry if found */ LWLockAcquire(lock_files, LW_EXCLUSIVE); - key_index = pg_tde_process_map_entry(rlocator, db_map_path, &offset, false); + key_index = pg_tde_process_map_entry(rlocator, key_type, db_map_path, &offset, false); LWLockRelease(lock_files); if (key_index == -1) @@ -556,8 +573,7 @@ pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) * The offset allows us to simply seek to the desired location and mark the entry * as MAP_ENTRY_FREE without needing any further processing. */ -void -pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) +void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, uint32 key_type, off_t offset) { int32 key_index = 0; LWLock *lock_files = tde_lwlock_enc_keys(); @@ -571,7 +587,7 @@ pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) /* Remove the map entry if found */ LWLockAcquire(lock_files, LW_EXCLUSIVE); - key_index = pg_tde_process_map_entry(rlocator, db_map_path, &offset, true); + key_index = pg_tde_process_map_entry(rlocator, key_type, db_map_path, &offset, true); LWLockRelease(lock_files); if (key_index == -1) @@ -637,8 +653,6 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p int k_fd[PRINCIPAL_KEY_COUNT] = {-1}; char m_path[PRINCIPAL_KEY_COUNT][MAXPGPATH]; char k_path[PRINCIPAL_KEY_COUNT][MAXPGPATH]; - TDEMapEntry map_entry; - RelFileLocator rloc; bool found = false; off_t read_pos_tmp = 0; bool is_new_file; @@ -668,8 +682,11 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p /* Read all entries until EOF */ for(key_index[OLD_PRINCIPAL_KEY] = 0; ; key_index[OLD_PRINCIPAL_KEY]++) { + TDEMapEntry read_map_entry, write_map_entry; + RelFileLocator rloc; + prev_pos[OLD_PRINCIPAL_KEY] = curr_pos[OLD_PRINCIPAL_KEY]; - found = pg_tde_read_one_map_entry(m_fd[OLD_PRINCIPAL_KEY], NULL, MAP_ENTRY_VALID, &map_entry, &curr_pos[OLD_PRINCIPAL_KEY]); + found = pg_tde_read_one_map_entry(m_fd[OLD_PRINCIPAL_KEY], NULL, MAP_ENTRY_VALID, &read_map_entry, &curr_pos[OLD_PRINCIPAL_KEY]); /* We either reach EOF */ if (prev_pos[OLD_PRINCIPAL_KEY] == curr_pos[OLD_PRINCIPAL_KEY]) @@ -680,9 +697,9 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p continue; /* Set the relNumber of rlocator. Ignore the tablespace Oid since we only place our files under the default. */ - rloc.relNumber = map_entry.relNumber; + rloc.relNumber = read_map_entry.relNumber; rloc.dbOid = principal_key->keyInfo.databaseId; - rloc.spcOid = DEFAULTTABLESPACE_OID; + rloc.spcOid = DEFAULTTABLESPACE_OID; /* TODO. Revisit*/ /* Let's get the decrypted key and re-encrypt it with the new key. */ enc_rel_key_data[OLD_PRINCIPAL_KEY] = pg_tde_read_one_keydata(k_fd[OLD_PRINCIPAL_KEY], key_index[OLD_PRINCIPAL_KEY], principal_key); @@ -693,7 +710,7 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p /* Write the given entry at the location pointed by prev_pos */ prev_pos[NEW_PRINCIPAL_KEY] = curr_pos[NEW_PRINCIPAL_KEY]; - curr_pos[NEW_PRINCIPAL_KEY] = pg_tde_write_one_map_entry(m_fd[NEW_PRINCIPAL_KEY], &rloc, MAP_ENTRY_VALID, key_index[NEW_PRINCIPAL_KEY], &map_entry, &prev_pos[NEW_PRINCIPAL_KEY]); + curr_pos[NEW_PRINCIPAL_KEY] = pg_tde_write_one_map_entry(m_fd[NEW_PRINCIPAL_KEY], &rloc, read_map_entry.flags, key_index[NEW_PRINCIPAL_KEY], &write_map_entry, &prev_pos[NEW_PRINCIPAL_KEY]); pg_tde_write_one_keydata(k_fd[NEW_PRINCIPAL_KEY], key_index[NEW_PRINCIPAL_KEY], enc_rel_key_data[NEW_PRINCIPAL_KEY]); /* Increment the key index for the new principal key */ @@ -829,7 +846,7 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ * reads the key data from the keydata file. */ RelKeyData * -pg_tde_get_key_from_file(const RelFileLocator *rlocator) +pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type) { int32 key_index = 0; TDEPrincipalKey *principal_key; @@ -868,7 +885,7 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator) pg_tde_set_db_file_paths(rlocator->dbOid, rlocator->spcOid, db_map_path, db_keydata_path); /* Read the map entry and get the index of the relation key */ - key_index = pg_tde_process_map_entry(rlocator, db_map_path, &offset, false); + key_index = pg_tde_process_map_entry(rlocator, key_type, db_map_path, &offset, false); if (key_index == -1) { @@ -906,7 +923,7 @@ pg_tde_set_db_file_paths(Oid dbOid, Oid spcOid, char *map_path, char *keydata_pa * The function expects that the offset points to a valid map start location. */ static int32 -pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t *offset, bool should_delete) +pg_tde_process_map_entry(const RelFileLocator *rlocator, uint32 key_type, char *db_map_path, off_t *offset, bool should_delete) { File map_fd = -1; int32 key_index = 0; @@ -926,7 +943,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ /* * If we need to delete an entry, we expect an offset value to the start - * of the entry to speed up the operation. Otherwise, we'd be sequntially + * of the entry to speed up the operation. Otherwise, we'd be sequentially * scanning the entire map file. */ if (should_delete == true && *offset > 0) @@ -955,7 +972,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ while(1) { prev_pos = curr_pos; - found = pg_tde_read_one_map_entry(map_fd, rlocator, MAP_ENTRY_VALID, &map_entry, &curr_pos); + found = pg_tde_read_one_map_entry(map_fd, rlocator, key_type, &map_entry, &curr_pos); /* We've reached EOF */ if (curr_pos == prev_pos) @@ -968,7 +985,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ /* Mark the entry pointed by prev_pos as free */ if (should_delete) { - pg_tde_write_one_map_entry(map_fd, NULL, MAP_ENTRY_FREE, 0, &map_entry, &prev_pos); + pg_tde_write_one_map_entry(map_fd, NULL, MAP_ENTRY_EMPTY, 0, &map_entry, &prev_pos); } #endif break; @@ -1159,7 +1176,7 @@ pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int fla *offset += bytes_read; /* We found a valid entry for the relNumber */ - found = (map_entry->flags == flags); + found = (map_entry->flags & flags); /* If a valid rlocator is provided, let's compare and set found value */ found &= (rlocator == NULL) ? true : (map_entry->relNumber == rlocator->relNumber); @@ -1263,22 +1280,19 @@ pg_tde_get_principal_key_info(Oid dbOid, Oid spcOid) * the tde fork file and populates cache. */ RelKeyData * -GetRelationKey(RelFileLocator rel) +GetRelationKey(RelFileLocator rel, uint32 key_type) { RelKeyData *key; - Oid rel_id = rel.relNumber; - key = pg_tde_get_key_from_cache(rel_id); - if (key != NULL) - { + key = pg_tde_get_key_from_cache(rel.relNumber, key_type); + if (key) return key; - } - key = pg_tde_get_key_from_file(&rel); + key = pg_tde_get_key_from_file(&rel, key_type); if (key != NULL) { - RelKeyData* cached_key = pg_tde_put_key_into_cache(rel.relNumber, key); + RelKeyData* cached_key = pg_tde_put_key_into_cache(rel.relNumber, key_type, key); pfree(key); return cached_key; } @@ -1286,8 +1300,31 @@ GetRelationKey(RelFileLocator rel) return NULL; } +RelKeyData * +GetSMGRRelationKey(RelFileLocator rel) +{ + return GetRelationKey(rel, TDE_KEY_TYPE_SMGR); +} + +RelKeyData * +GetHeapBaiscRelationKey(RelFileLocator rel) +{ + return GetRelationKey(rel, TDE_KEY_TYPE_HEAP_BASIC); +} + +RelKeyData * +GetTdeGlobaleRelationKey(RelFileLocator rel) +{ + return GetRelationKey(rel, TDE_KEY_TYPE_GLOBAL); +} + +/* + * Returns TDE key for a given relation. + * First it looks in a cache. If nothing found in the cache, it reads data from + * the tde key file and populates cache. + */ static RelKeyData * -pg_tde_get_key_from_cache(Oid rel_id) +pg_tde_get_key_from_cache(RelFileNumber rel_number, uint32 key_type) { RelKeyCacheRec *rec; @@ -1297,7 +1334,9 @@ pg_tde_get_key_from_cache(Oid rel_id) for (int i = 0; i < tde_rel_key_cache->len; i++) { rec = tde_rel_key_cache->data+i; - if (rec != NULL && rec->rel_id == rel_id) + if (rec != NULL && + (rel_number == InvalidOid || (rec->rel_number == rel_number)) && + rec->key_type & key_type) { return &rec->key; } @@ -1311,7 +1350,7 @@ pg_tde_get_key_from_cache(Oid rel_id) * TODO: add tests. */ RelKeyData * -pg_tde_put_key_into_cache(Oid rel_id, RelKeyData *key) +pg_tde_put_key_into_cache(RelFileNumber rel_num, uint32 key_type, RelKeyData *key) { static long pageSize = 0; RelKeyCacheRec *rec; @@ -1319,7 +1358,7 @@ pg_tde_put_key_into_cache(Oid rel_id, RelKeyData *key) if (pageSize == 0) { - #ifndef _SC_PAGESIZE +#ifndef _SC_PAGESIZE pageSize = getpagesize(); #else pageSize = sysconf(_SC_PAGESIZE); @@ -1380,7 +1419,8 @@ pg_tde_put_key_into_cache(Oid rel_id, RelKeyData *key) rec = tde_rel_key_cache->data + tde_rel_key_cache->len; - rec->rel_id = rel_id; + rec->rel_number = rel_num; + rec->key_type = key_type; memcpy(&rec->key, key, sizeof(RelKeyCacheRec)); tde_rel_key_cache->len++; diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index e3039305..9f84b4c8 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -40,7 +40,7 @@ tdeheap_rmgr_redo(XLogReaderState *record) XLogRelKey *xlrec = (XLogRelKey *) XLogRecGetData(record); LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); - pg_tde_write_key_map_entry(&xlrec->rlocator, &xlrec->relKey, NULL); + pg_tde_write_key_map_entry(&xlrec->rlocator, xlrec->entry_type, &xlrec->relKey, NULL); LWLockRelease(tde_lwlock_enc_keys()); } else if (info == XLOG_TDE_ADD_PRINCIPAL_KEY) diff --git a/src/access/pg_tde_xlog_encrypt.c b/src/access/pg_tde_xlog_encrypt.c index 539fdec9..cd9f462f 100644 --- a/src/access/pg_tde_xlog_encrypt.c +++ b/src/access/pg_tde_xlog_encrypt.c @@ -124,7 +124,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) size_t data_size = 0; XLogPageHeader curr_page_hdr = &EncryptCurrentPageHrd; XLogPageHeader enc_buf_page; - RelKeyData *key = GetRelationKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID)); + RelKeyData *key = GetTdeGlobaleRelationKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID)); off_t enc_off; size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; uint32 iv_ctr = 0; @@ -229,7 +229,7 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) char iv_prefix[16] = {0,}; size_t data_size = 0; XLogPageHeader curr_page_hdr = &DecryptCurrentPageHrd; - RelKeyData *key = GetRelationKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID)); + RelKeyData *key = GetTdeGlobaleRelationKey(GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID)); size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; off_t dec_off; uint32 iv_ctr = 0; diff --git a/src/catalog/tde_global_space.c b/src/catalog/tde_global_space.c index 14ff128b..5ceab027 100644 --- a/src/catalog/tde_global_space.c +++ b/src/catalog/tde_global_space.c @@ -67,7 +67,7 @@ TDEInitGlobalKeys(const char *dir) if (dir != NULL) pg_tde_set_globalspace_dir(dir); - ikey = pg_tde_get_key_from_file(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID)); + ikey = pg_tde_get_key_from_file(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), TDE_KEY_TYPE_GLOBAL); /* * Internal Key should be in the TopMemmoryContext because of SSL @@ -78,7 +78,7 @@ TDEInitGlobalKeys(const char *dir) * backend. (see * https://github.com/percona-Lab/pg_tde/pull/214#discussion_r1648998317) */ - pg_tde_put_key_into_cache(XLOG_TDE_OID, ikey); + pg_tde_put_key_into_cache(XLOG_TDE_OID, TDE_KEY_TYPE_GLOBAL, ikey); } } @@ -156,9 +156,9 @@ init_keys(void) } rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); - rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &mkey->keyInfo); + rel_key_data = tde_create_rel_key(rlocator->relNumber, TDE_KEY_TYPE_GLOBAL, &int_key, &mkey->keyInfo); enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator); - pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo); + pg_tde_write_key_map_entry(rlocator, TDE_KEY_TYPE_GLOBAL, enc_rel_key_data, &mkey->keyInfo); pfree(enc_rel_key_data); pfree(mkey); } diff --git a/src/encryption/enc_tde.c b/src/encryption/enc_tde.c index d91fa04b..c9ed221d 100644 --- a/src/encryption/enc_tde.c +++ b/src/encryption/enc_tde.c @@ -214,7 +214,7 @@ PGTdePageAddItemExtended(RelFileLocator rel, uint32 data_len = size - header_size; /* ctid stored in item is incorrect (not set) at this point */ ItemPointerData ip; - RelKeyData *key = GetRelationKey(rel); + RelKeyData *key = GetHeapBaiscRelationKey(rel); ItemPointerSet(&ip, bn, off); diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 2d1448e7..0db1464a 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -14,6 +14,13 @@ #include "catalog/tde_principal_key.h" #include "storage/relfilelocator.h" +/* Map entry flags */ +#define MAP_ENTRY_EMPTY 0x00 +#define TDE_KEY_TYPE_HEAP_BASIC 0x01 +#define TDE_KEY_TYPE_SMGR 0x02 +#define TDE_KEY_TYPE_GLOBAL 0x04 +#define MAP_ENTRY_VALID (TDE_KEY_TYPE_HEAP_BASIC | TDE_KEY_TYPE_SMGR | TDE_KEY_TYPE_GLOBAL) + typedef struct InternalKey { uint8 key[INTERNAL_KEY_LEN]; @@ -30,15 +37,22 @@ typedef struct RelKeyData typedef struct XLogRelKey { RelFileLocator rlocator; + uint32 entry_type; RelKeyData relKey; } XLogRelKey; -extern RelKeyData* pg_tde_create_key_map_entry(const RelFileLocator *newrlocator); -extern void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info); -extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator); -extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset); +extern RelKeyData *pg_tde_create_smgr_key(const RelFileLocator *newrlocator); +extern RelKeyData *pg_tde_create_global_key(const RelFileLocator *newrlocator); +extern RelKeyData *pg_tde_create_heap_basic_key(const RelFileLocator *newrlocator); +extern RelKeyData *pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, uint32 entry_type); +extern void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, uint32 entry_type, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info); +extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator, uint32 key_type); +extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, uint32 key_type, off_t offset); -extern RelKeyData *GetRelationKey(RelFileLocator rel); +extern RelKeyData *GetRelationKey(RelFileLocator rel, uint32 entry_type); +extern RelKeyData *GetSMGRRelationKey(RelFileLocator rel); +extern RelKeyData *GetHeapBaiscRelationKey(RelFileLocator rel); +extern RelKeyData *GetTdeGlobaleRelationKey(RelFileLocator rel); extern void pg_tde_delete_tde_files(Oid dbOid, Oid spcOid); @@ -46,15 +60,15 @@ extern TDEPrincipalKeyInfo *pg_tde_get_principal_key_info(Oid dbOid, Oid spcOid) extern bool pg_tde_save_principal_key(TDEPrincipalKeyInfo *principal_key_info); extern bool pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_principal_key); extern bool pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_size, char *k_file_data); -extern RelKeyData* tde_create_rel_key(Oid rel_id, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info); +extern RelKeyData *tde_create_rel_key(RelFileNumber rel_num, uint32 key_type, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info); extern RelKeyData *tde_encrypt_rel_key(TDEPrincipalKey *principal_key, RelKeyData *rel_key_data, const RelFileLocator *rlocator); extern RelKeyData *tde_decrypt_rel_key(TDEPrincipalKey *principal_key, RelKeyData *enc_rel_key_data, const RelFileLocator *rlocator); -extern RelKeyData *pg_tde_get_key_from_file(const RelFileLocator *rlocator); +extern RelKeyData *pg_tde_get_key_from_file(const RelFileLocator *rlocator, uint32 key_type); extern void pg_tde_set_db_file_paths(Oid dbOid, Oid spcOid, char *map_path, char *keydata_path); const char * tde_sprint_key(InternalKey *k); -extern RelKeyData *pg_tde_put_key_into_cache(Oid rel_id, RelKeyData *key); +extern RelKeyData *pg_tde_put_key_into_cache(RelFileNumber rel_num, uint32 key_type, RelKeyData *key); #endif /*PG_TDE_MAP_H*/ diff --git a/src/pg_tde_event_capture.c b/src/pg_tde_event_capture.c index 54562b39..180eb49c 100644 --- a/src/pg_tde_event_capture.c +++ b/src/pg_tde_event_capture.c @@ -69,7 +69,7 @@ pg_tde_ddl_command_start_capture(PG_FUNCTION_ARGS) trigdata = (EventTriggerData *) fcinfo->context; parsetree = trigdata->parsetree; - elog(DEBUG2, "EVENT TRIGGER (%s) %s", trigdata->event, nodeToString(parsetree)); + elog(LOG, "EVENT TRIGGER (%s) %s", trigdata->event, nodeToString(parsetree)); reset_current_tde_create_event(); if (IsA(parsetree, IndexStmt)) @@ -184,7 +184,7 @@ pg_tde_ddl_command_end_capture(PG_FUNCTION_ARGS) ereport(ERROR, (errmsg("Function can only be fired by event trigger manager"))); - elog(DEBUG1, "Type:%s EncryptMode:%s, Oid:%d, Relation:%s ", + elog(LOG, "Type:%s EncryptMode:%s, Oid:%d, Relation:%s ", (tdeCurrentCreateEvent.eventType == TDE_INDEX_CREATE_EVENT) ? "CREATE INDEX" : (tdeCurrentCreateEvent.eventType == TDE_TABLE_CREATE_EVENT) ? "CREATE TABLE" : "UNKNOWN", tdeCurrentCreateEvent.encryptMode ? "true" : "false", diff --git a/src/smgr/pg_tde_smgr.c b/src/smgr/pg_tde_smgr.c index ca36ae10..39bb273f 100644 --- a/src/smgr/pg_tde_smgr.c +++ b/src/smgr/pg_tde_smgr.c @@ -34,7 +34,7 @@ tde_smgr_get_key(SMgrRelation reln) event = GetCurrentTdeCreateEvent(); // see if we have a key for the relation, and return if yes - rkd = GetRelationKey(reln->smgr_rlocator.locator); + rkd = GetSMGRRelationKey(reln->smgr_rlocator.locator); if(rkd != NULL) { @@ -44,7 +44,7 @@ tde_smgr_get_key(SMgrRelation reln) // if this is a CREATE TABLE, we have to generate the key if(event->encryptMode == true && event->eventType == TDE_TABLE_CREATE_EVENT) { - return pg_tde_create_key_map_entry(&reln->smgr_rlocator.locator); + return pg_tde_create_smgr_key(&reln->smgr_rlocator.locator); } // if this is a CREATE INDEX, we have to load the key based on the table @@ -52,7 +52,7 @@ tde_smgr_get_key(SMgrRelation reln) { // For now keep it simple and create separate key for indexes // Later we might modify the map infrastructure to support the same keys - return pg_tde_create_key_map_entry(&reln->smgr_rlocator.locator); + return pg_tde_create_smgr_key(&reln->smgr_rlocator.locator); } return NULL; diff --git a/src/transam/pg_tde_xact_handler.c b/src/transam/pg_tde_xact_handler.c index 2d2ce758..6fb02cd0 100644 --- a/src/transam/pg_tde_xact_handler.c +++ b/src/transam/pg_tde_xact_handler.c @@ -123,7 +123,7 @@ do_pending_deletes(bool isCommit) ereport(LOG, (errmsg("pg_tde_xact_callback: deleting entry at offset %d", (int)(pending->map_entry_offset)))); - pg_tde_free_key_map_entry(&pending->rlocator, pending->map_entry_offset); + pg_tde_free_key_map_entry(&pending->rlocator, MAP_ENTRY_VALID, pending->map_entry_offset); } pfree(pending); /* prev does not change */ diff --git a/src16/access/pg_tde_prune.c b/src16/access/pg_tde_prune.c index 552151c5..7c902f32 100644 --- a/src16/access/pg_tde_prune.c +++ b/src16/access/pg_tde_prune.c @@ -390,7 +390,7 @@ tdeheap_page_prune(Relation relation, Buffer buffer, * We need it here as there is `pgtde_compactify_tuples()` down in * the call stack wich reencrypt tuples. */ - GetRelationKey(relation->rd_locator); + GetHeapBaiscRelationKey(relation->rd_locator); /* Any error while applying the changes is critical */ START_CRIT_SECTION(); diff --git a/src16/access/pg_tdeam.c b/src16/access/pg_tdeam.c index 7a6b46bb..840aa6f0 100644 --- a/src16/access/pg_tdeam.c +++ b/src16/access/pg_tdeam.c @@ -1892,7 +1892,7 @@ tdeheap_insert(Relation relation, HeapTuple tup, CommandId cid, * Make sure relation keys in the cahce to avoid pallocs in * the critical section. */ - GetRelationKey(relation->rd_locator); + GetHeapBaiscRelationKey(relation->rd_locator); /* NO EREPORT(ERROR) from here till changes are logged */ START_CRIT_SECTION(); @@ -2233,7 +2233,7 @@ tdeheap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, * Make sure relation keys in the cahce to avoid pallocs in * the critical section. */ - GetRelationKey(relation->rd_locator); + GetHeapBaiscRelationKey(relation->rd_locator); /* NO EREPORT(ERROR) from here till changes are logged */ START_CRIT_SECTION(); @@ -2799,7 +2799,7 @@ tdeheap_delete(Relation relation, ItemPointer tid, * won't be able to extract varlen attributes. */ decrypted_tuple = tdeheap_copytuple(&tp); - PG_TDE_DECRYPT_TUPLE(&tp, decrypted_tuple, GetRelationKey(relation->rd_locator)); + PG_TDE_DECRYPT_TUPLE(&tp, decrypted_tuple, GetHeapBaiscRelationKey(relation->rd_locator)); old_key_tuple = ExtractReplicaIdentity(relation, decrypted_tuple, true, &old_key_copied); @@ -3157,7 +3157,7 @@ tdeheap_update(Relation relation, ItemPointer otid, HeapTuple newtup, oldtup_decrypted.t_data = (HeapTupleHeader)new_ptr; } PG_TDE_DECRYPT_TUPLE(&oldtup, &oldtup_decrypted, - GetRelationKey(relation->rd_locator)); + GetHeapBaiscRelationKey(relation->rd_locator)); // change field in oldtup now. // We can't do it before, as PG_TDE_DECRYPT_TUPLE uses t_data address in @@ -3809,7 +3809,7 @@ tdeheap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * Make sure relation keys in the cahce to avoid pallocs in * the critical section. */ - GetRelationKey(relation->rd_locator); + GetHeapBaiscRelationKey(relation->rd_locator); /* NO EREPORT(ERROR) from here till changes are logged */ START_CRIT_SECTION(); diff --git a/src16/access/pg_tdeam_handler.c b/src16/access/pg_tdeam_handler.c index 534b97bc..04df262c 100644 --- a/src16/access/pg_tdeam_handler.c +++ b/src16/access/pg_tdeam_handler.c @@ -648,7 +648,7 @@ pg_tdeam_relation_set_new_filelocator(Relation rel, ereport(DEBUG1, (errmsg("creating key file for relation %s", RelationGetRelationName(rel)))); - pg_tde_create_key_map_entry(newrlocator); + pg_tde_create_heap_basic_key(newrlocator); } } diff --git a/src16/access/pg_tdetoast.c b/src16/access/pg_tdetoast.c index 34dab84a..350c5291 100644 --- a/src16/access/pg_tdetoast.c +++ b/src16/access/pg_tdetoast.c @@ -657,7 +657,7 @@ tdeheap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, int validIndex; SnapshotData SnapshotToast; char decrypted_data[TOAST_MAX_CHUNK_SIZE]; - RelKeyData *key = GetRelationKey(toastrel->rd_locator); + RelKeyData *key = GetHeapBaiscRelationKey(toastrel->rd_locator); char iv_prefix[16] = {0,}; @@ -1093,7 +1093,7 @@ tdeheap_toast_save_datum(Relation rel, Datum value, /* * Encrypt toast data. */ - tdeheap_toast_encrypt(dval, toast_pointer.va_valueid, GetRelationKey(toastrel->rd_locator)); + tdeheap_toast_encrypt(dval, toast_pointer.va_valueid, GetHeapBaiscRelationKey(toastrel->rd_locator)); /* * Initialize constant parts of the tuple data diff --git a/src17/access/pg_tde_prune.c b/src17/access/pg_tde_prune.c index 56aee8ef..bd1e71a7 100644 --- a/src17/access/pg_tde_prune.c +++ b/src17/access/pg_tde_prune.c @@ -761,7 +761,7 @@ tdeheap_page_prune_and_freeze(Relation relation, Buffer buffer, * We need it here as there is `pgtde_compactify_tuples()` down in * the call stack wich reencrypt tuples. */ - GetRelationKey(relation->rd_locator); + GetHeapBaiscRelationKey(relation->rd_locator); /* Any error while applying the changes is critical */ START_CRIT_SECTION(); diff --git a/src17/access/pg_tdeam.c b/src17/access/pg_tdeam.c index 0848ec41..5f6cccfd 100644 --- a/src17/access/pg_tdeam.c +++ b/src17/access/pg_tdeam.c @@ -2050,7 +2050,7 @@ tdeheap_insert(Relation relation, HeapTuple tup, CommandId cid, * Make sure relation keys in the cahce to avoid pallocs in * the critical section. */ - GetRelationKey(relation->rd_locator); + GetHeapBaiscRelationKey(relation->rd_locator); /* NO EREPORT(ERROR) from here till changes are logged */ START_CRIT_SECTION(); @@ -2391,7 +2391,7 @@ tdeheap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, * Make sure relation keys in the cahce to avoid pallocs in * the critical section. */ - GetRelationKey(relation->rd_locator); + GetHeapBaiscRelationKey(relation->rd_locator); /* NO EREPORT(ERROR) from here till changes are logged */ START_CRIT_SECTION(); @@ -2957,7 +2957,7 @@ tdeheap_delete(Relation relation, ItemPointer tid, * won't be able to extract varlen attributes. */ decrypted_tuple = tdeheap_copytuple(&tp); - PG_TDE_DECRYPT_TUPLE(&tp, decrypted_tuple, GetRelationKey(relation->rd_locator)); + PG_TDE_DECRYPT_TUPLE(&tp, decrypted_tuple, GetHeapBaiscRelationKey(relation->rd_locator)); old_key_tuple = ExtractReplicaIdentity(relation, decrypted_tuple, true, &old_key_copied); @@ -3315,7 +3315,7 @@ tdeheap_update(Relation relation, ItemPointer otid, HeapTuple newtup, oldtup_decrypted.t_data = (HeapTupleHeader)new_ptr; } PG_TDE_DECRYPT_TUPLE(&oldtup, &oldtup_decrypted, - GetRelationKey(relation->rd_locator)); + GetHeapBaiscRelationKey(relation->rd_locator)); // change field in oldtup now. // We can't do it before, as PG_TDE_DECRYPT_TUPLE uses t_data address in @@ -3967,7 +3967,7 @@ tdeheap_update(Relation relation, ItemPointer otid, HeapTuple newtup, * Make sure relation keys in the cahce to avoid pallocs in * the critical section. */ - GetRelationKey(relation->rd_locator); + GetHeapBaiscRelationKey(relation->rd_locator); /* NO EREPORT(ERROR) from here till changes are logged */ START_CRIT_SECTION(); diff --git a/src17/access/pg_tdeam_handler.c b/src17/access/pg_tdeam_handler.c index 9c10d68b..dbbe62cf 100644 --- a/src17/access/pg_tdeam_handler.c +++ b/src17/access/pg_tdeam_handler.c @@ -645,7 +645,7 @@ pg_tdeam_relation_set_new_filelocator(Relation rel, ereport(DEBUG1, (errmsg("creating key file for relation %s", RelationGetRelationName(rel)))); - pg_tde_create_key_map_entry(newrlocator); + pg_tde_create_heap_basic_key(newrlocator); } } diff --git a/src17/access/pg_tdetoast.c b/src17/access/pg_tdetoast.c index a50ecae2..a24b21eb 100644 --- a/src17/access/pg_tdetoast.c +++ b/src17/access/pg_tdetoast.c @@ -657,7 +657,7 @@ tdeheap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, int validIndex; SnapshotData SnapshotToast; char decrypted_data[TOAST_MAX_CHUNK_SIZE]; - RelKeyData *key = GetRelationKey(toastrel->rd_locator); + RelKeyData *key = GetHeapBaiscRelationKey(toastrel->rd_locator); char iv_prefix[16] = {0,}; @@ -1093,7 +1093,7 @@ tdeheap_toast_save_datum(Relation rel, Datum value, /* * Encrypt toast data. */ - tdeheap_toast_encrypt(dval, toast_pointer.va_valueid, GetRelationKey(toastrel->rd_locator)); + tdeheap_toast_encrypt(dval, toast_pointer.va_valueid, GetHeapBaiscRelationKey(toastrel->rd_locator)); /* * Initialize constant parts of the tuple data