Vault KVv2 API support for key storage (#81)

This commit implements support for storing keys on a vault server
instead of locally. The current implementation only supports the KV
v2 engine, which is the default secrets engine in recent vault
versions.

To use vault for key storage, the following settings have to be
used in the keyring configuration file:

* `provider` set to `vault-v2`
* `url` set to the URL of the vault server
* `mountPath` is set to the mount point where the keyring should
  store the keys
* `token` is an access token with read and write access to the
  above mount point
* [optional] `caPath` is the path of the CA file used for SSL
  verification

Multiple servers can use the same vault server, with the following
restrictions:

* Servers in the same replication group should use the same
  'pg_tde.keyringKeyPrefix` to ensure that they see the same keys
* Unrelated servers should use different `pg_tde.keyringKeyPrefix`
  values to ensure that they use different keys without conflicts

The source also contains a sample keyring configuration file,
`keyring-vault.json`. This configuration matches the settings of
the vault development server (`vault server -dev`), only the
ROOT_TOKEN has to be replaced to the token of the actual server
process.

.github/workflows/postgresql-16-pgdg-package-pgxs.yml

@@ -24,7 +24,7 @@ jobs:
         run: |
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev wget \
             zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl \
-            libjson-c-dev
+            libjson-c-dev libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
 

.github/workflows/postgresql-16-src-make.yml

@@ -26,7 +26,7 @@ jobs:
             libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
             llvm-11 llvm-11-dev libselinux1-dev python3-dev \
-            uuid-dev liblz4-dev libjson-c-dev
+            uuid-dev liblz4-dev libjson-c-dev libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
 

.github/workflows/postgresql-16-src-meson-perf.yml

@@ -30,7 +30,7 @@ jobs:
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
             llvm-11 llvm-11-dev libselinux1-dev python3-dev \
             uuid-dev liblz4-dev meson ninja-build libjson-c-dev \
-            sysbench
+            sysbench libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
 

.github/workflows/postgresql-16-src-meson.yml

@@ -26,9 +26,13 @@ jobs:
             libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
             llvm-11 llvm-11-dev libselinux1-dev python3-dev \
-            uuid-dev liblz4-dev meson ninja-build libjson-c-dev
+            uuid-dev liblz4-dev meson ninja-build libjson-c-dev \
+            gpg wget libcurl4-openssl-dev
           sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
           sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
 
       - name: Clone postgres repository
         uses: actions/checkout@v2
@@ -52,7 +56,7 @@ jobs:
           cd build && ninja && ninja install
         working-directory: src
 
-      - name: Test postgres-tde-ext
+      - name: Test postgres-tde-ext with keyring_file
         run: |
           cp ../contrib/postgres-tde-ext/keyring.json /tmp/keyring.json
           meson test --suite setup -v
@@ -68,3 +72,26 @@ jobs:
             src/build/testrun/postgres-tde-ext/regress/
           retention-days: 3
 
+      - name: Test postgres-tde-ext with keyring_vault
+        run: |
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
+          cp ../contrib/postgres-tde-ext/keyring-vault.json /tmp/keyring.json
+          sed -i "s/ROOT_TOKEN/$ROOT_TOKEN/g" /tmp/keyring.json
+          cat /tmp/keyring.json
+          meson test --suite setup -v
+          meson test --suite postgres-tde-ext -v --num-processes 1
+        working-directory: src/build
+
+      - name: Report on test fail
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: Regressions diff and postgresql log
+          path: |
+            src/build/testrun/postgres-tde-ext/regress/
+          retention-days: 3
+

Makefile.in

@@ -32,6 +32,7 @@ src/access/pg_tde_ddl.o \
 src/transam/pg_tde_xact_handler.o \
 src/keyring/keyring_config.o \
 src/keyring/keyring_file.o \
+src/keyring/keyring_vault.o \
 src/keyring/keyring_api.o \
 src/pg_tde.o
 
@@ -50,4 +51,4 @@ include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
 
-override SHLIB_LINK += @tde_LDFLAGS@ -lcrypto -lssl
+override SHLIB_LINK += @tde_LDFLAGS@ -lcrypto -lssl -lcurl

configure

@@ -710,6 +710,7 @@ ac_subst_files=''
 ac_user_opts='
 enable_option_checking
 with_jsonc
+with_libcurl
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1341,6 +1342,7 @@ Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
   --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
   --with-jsonc=<path>     Location where json_object.h is installed
+  --with-libcurl=<path>   Location where curl/curl.h is installed
 
 Some influential environment variables:
   CC          C compiler command
@@ -2407,7 +2409,7 @@ case $host_os in *\ *) host_os=`echo "$host_os" | sed 's/ /-/g'`;; esac
 
 
 
-# REQUIRE_LIB(name,lib,testfn,description)
+# REQUIRE_LIB(name,lib,testfn,test_include.h)
 # name = The complete name of the library file without the extension.
 # lib = The name of the library file without the 'lib' prefix and without the extension.
 # testfn = One function included in the library that can be used for a test compilation.
@@ -3531,6 +3533,96 @@ else $as_nop
 
 fi
 
+}
+ {
+
+# Check whether --with-libcurl was given.
+if test ${with_libcurl+y}
+then :
+  withval=$with_libcurl;
+else $as_nop
+  with_libcurl=default
+fi
+
+	if test "x$with_libcurl" == xdefault
+then :
+
+		case $host_os in
+			darwin*) libpathx=($HOMEBREW_CELLAR/curl/*)
+						tde_CPPFLAGS="$tde_CPPFLAGS -I$libpathx/include/curl"
+						tde_LDFLAGS="$tde_LDFLAGS -L$libpathx/lib -lcurl" ;;
+				  *) tde_CPPFLAGS="$tde_CPPFLAGS -I/usr/include/curl"
+				  		tde_LDFLAGS="$tde_LDFLAGS -lcurl" ;;
+		esac
+
+else $as_nop
+   #AS_ELSE
+		tde_CPPFLAGS="$tde_CPPFLAGS -I${with_libcurl}/include"
+		tde_LDFLAGS="$tde_LDFLAGS -L${with_libcurl}/lib -lcurl"
+
+fi
+
+	LDFLAGS="$LDFLAGS $tde_LDFLAGS"
+	CPPFLAGS="$CPPFLAGS $tde_CPPFLAGS"
+
+	ac_fn_c_check_header_compile "$LINENO" "curl/curl.h" "ac_cv_header_curl_curl_h" "$ac_includes_default"
+if test "x$ac_cv_header_curl_curl_h" = xyes
+then :
+
+else $as_nop
+
+		as_fn_error $? "header file <curl/curl.h> is required, try specifying --with-libcurl" "$LINENO" 5
+
+fi
+
+
+	{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curl_easy_setopt in -lcurl" >&5
+printf %s "checking for curl_easy_setopt in -lcurl... " >&6; }
+if test ${ac_cv_lib_curl_curl_easy_setopt+y}
+then :
+  printf %s "(cached) " >&6
+else $as_nop
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-lcurl  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+char curl_easy_setopt ();
+int
+main (void)
+{
+return curl_easy_setopt ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"
+then :
+  ac_cv_lib_curl_curl_easy_setopt=yes
+else $as_nop
+  ac_cv_lib_curl_curl_easy_setopt=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.beam \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curl_curl_easy_setopt" >&5
+printf "%s\n" "$ac_cv_lib_curl_curl_easy_setopt" >&6; }
+if test "x$ac_cv_lib_curl_curl_easy_setopt" = xyes
+then :
+  printf "%s\n" "#define HAVE_LIBCURL 1" >>confdefs.h
+
+  LIBS="-lcurl $LIBS"
+
+else $as_nop
+
+		as_fn_error $? "curl was not found, try specifying --with-libcurl" "$LINENO" 5
+
+fi
+
 }
 
 
@@ -4689,3 +4781,4 @@ if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then
 printf "%s\n" "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;}
 fi
 
+

configure.ac

@@ -44,10 +44,11 @@ AC_ARG_WITH([$1], AS_HELP_STRING([--with-$1=<path>],[Location where $4 is instal
 #=======================================
 
 REQUIRE_LIB(jsonc, json-c, json_object_get, json_object.h)
+REQUIRE_LIB(libcurl, curl, curl_easy_setopt, curl/curl.h)
 
 AC_SUBST(tde_CPPFLAGS)
 AC_SUBST(tde_LDFLAGS)
 
 AC_CONFIG_FILES([Makefile])
 
-AC_OUTPUT
+AC_OUTPUT

keyring-vault.json

@@ -0,0 +1,6 @@
+{
+        'provider': 'vault-v2',
+        'token': 'ROOT_TOKEN',
+        'url': 'http://127.0.0.1:8200',
+        'mountPath': 'secret'
+}

meson.build

@@ -1,6 +1,7 @@
 
 # libjson-c-dev on ubuntu
 jsondep = dependency('json-c')
+curldep = dependency('libcurl')
 
 pg_tde_sources = files(
         'src/pg_tde.c',
@@ -22,14 +23,15 @@ pg_tde_sources = files(
 
         'src/keyring/keyring_config.c',
         'src/keyring/keyring_file.c',
+        'src/keyring/keyring_vault.c',
         'src/keyring/keyring_api.c',
 
         'src/pg_tde.c',
 )
 
 incdir = include_directories('src/include')
 
-deps_update = {'dependencies': contrib_mod_args.get('dependencies') + [jsondep]}
+deps_update = {'dependencies': contrib_mod_args.get('dependencies') + [jsondep, curldep]}
 
 mod_args = contrib_mod_args + deps_update
 

src/include/keyring/keyring_config.h

@@ -6,6 +6,14 @@
 
 #include <json.h>
 
+enum KeyringProvider
+{
+	PROVIDER_UNKNOWN,
+	PROVIDER_FILE,
+	PROVIDER_VAULT_V2,
+} ;
+
+extern enum KeyringProvider keyringProvider;
 extern char* keyringConfigFile;
 extern char* keyringKeyPrefix;
 

src/include/keyring/keyring_vault.h

@@ -0,0 +1,19 @@
+
+#ifndef KEYRING_VAULT_H
+#define KEYRING_VAULT_H
+
+#include "postgres.h"
+
+#include <json.h>
+
+#include "keyring_api.h"
+
+int keyringVaultPreloadCache(void);
+
+int keyringVaultParseConfiguration(json_object* configRoot);
+
+int keyringVaultStoreKey(const keyInfo* ki);
+
+int keyringVaultGetKey(keyName name, keyData* outData);
+
+#endif // KEYRING_FILE_H

src/keyring/keyring_api.c

@@ -1,6 +1,7 @@
 
 #include "keyring/keyring_api.h"
 #include "keyring/keyring_file.h"
+#include "keyring/keyring_vault.h"
 #include "keyring/keyring_config.h"
 
 #include "postgres.h"
@@ -29,8 +30,18 @@ void keyringInitCache(void)
 		memset(cache, 0, keyringCacheMemorySize());
 	}
 
-	// TODO: HARDCODED FILE PROVIDER
-	keyringFilePreloadCache();
+	switch(keyringProvider)
+	{
+		case PROVIDER_FILE:
+		keyringFilePreloadCache();
+		break;
+		case PROVIDER_VAULT_V2:
+		keyringVaultPreloadCache();
+		break;
+		case PROVIDER_UNKNOWN:
+		// nop
+		break;
+	}
 }
 
 const keyInfo* keyringCacheStoreKey(keyName name, keyData data)
@@ -57,7 +68,28 @@ const keyInfo* keyringGetKey(keyName name)
 			return &cache->keys[i];
 		}
 	}
-	// TODO: HARDCODED FILE PROVIDER
+	// not found in cache, try to look up
+	switch(keyringProvider)
+	{
+		case PROVIDER_FILE:
+			// nop, not implmeneted
+		break;
+		case PROVIDER_VAULT_V2:
+		{
+			keyData data;
+			data.len = 0;
+			keyringVaultGetKey(name, &data);
+			if(data.len > 0)
+			{
+				return keyringCacheStoreKey(name, data);
+			}
+			break;
+		}
+		case PROVIDER_UNKNOWN:
+		// nop
+		break;
+	}
+
 #if KEYRING_DEBUG
 	fprintf(stderr, " -- not found\n");
 #endif
@@ -66,18 +98,27 @@ const keyInfo* keyringGetKey(keyName name)
 
 const keyInfo* keyringStoreKey(keyName name, keyData data)
 {
-	// TODO: HARDCODED FILE PROVIDER
-	// Todo: we should first call the provider, and if it succeeds, add the key to the cache
-	// But as the current file implementation just dumps the cache to disk, this is a good first prototype
 	const keyInfo* ki = keyringCacheStoreKey(name, data);
 #if KEYRING_DEBUG
 	fprintf(stderr, "Storing key: %s\n", name.name);
 #endif
-	if(!keyringFileStoreKey(ki))
+	switch(keyringProvider)
 	{
-		return NULL;
+		case PROVIDER_FILE:
+		if(keyringFileStoreKey(ki)) return ki;
+		break;
+		case PROVIDER_VAULT_V2:
+		if(keyringVaultStoreKey(ki)) return ki;
+		break;
+		case PROVIDER_UNKNOWN:
+		// nop
+		break;
 	}
-	return ki;
+
+	//  if we are here, storeKey failed, remove from cache
+	cache->keyCount--;
+
+	return NULL;
 }
 
 keyName keyringConstructKeyName(const char* internalName, unsigned version)