Cryptfs fstrim + FDE fix

common/cryptfs.h

@@ -36,6 +36,17 @@
 
 #define CRYPTFS_FDE_KEY_LEN 64
 
+/**
+ * Mode of encryption/integrity setup which should be used
+ */
+typedef enum {
+	CRYPTFS_MODE_NOT_IMPLEMENTED = 1,
+	CRYPTFS_MODE_AUTHENC,
+	CRYPTFS_MODE_ENCRYPT_ONLY,
+	CRYPTFS_MODE_INTEGRITY_ENCRYPT,
+	CRYPTFS_MODE_INTEGRITY_ONLY
+} cryptfs_mode_t;
+
 /**
  * Get the full path of a cryptfs device with the specified name
  *
@@ -52,20 +63,23 @@ cryptfs_get_device_path_new(const char *label);
  * @param real_blk_dev The name of the loop device
  * @param ascii_key The key for the volume
  * @param meta_blk_dev The meta loop device
+ * @param mode mode used for encryption, e.g. stacked use of dm-crypt on dm-integrity
+ *     with AEAD algorithm or individual dm-integrity and dm-crypt usage.
  * @return char* The path of the newly created volume
  */
 char *
 cryptfs_setup_volume_new(const char *label, const char *real_blk_dev, const char *ascii_key,
-			 const char *meta_blk_dev);
+			 const char *meta_blk_dev, cryptfs_mode_t mode);
 
 /**
  * Close a device-mapper volume
  *
  * @param fd The filedescriptor of the device
  * @param name The name of the device
+ * @param mode cryptfs mode which was used for the device with name 'name'
  * @return int 0 if successful, otherwise -1
  */
 int
-cryptfs_delete_blk_dev(int fd, const char *name);
+cryptfs_delete_blk_dev(int fd, const char *name, cryptfs_mode_t mode);
 
 #endif /* CRYPTFS_H */

daemon/c_vol.c

@@ -92,6 +92,7 @@ typedef struct c_vol {
 	const guestos_t *os;
 	mount_t *mnt;
 	mount_t *mnt_setup;
+	cryptfs_mode_t mode;
 } c_vol_t;
 
 /******************************************************************************/
@@ -140,7 +141,7 @@ c_vol_image_path_new(c_vol_t *vol, const mount_entry_t *mntent)
 }
 
 static char *
-c_vol_meta_image_path_new(c_vol_t *vol, const mount_entry_t *mntent)
+c_vol_meta_image_path_new(c_vol_t *vol, const mount_entry_t *mntent, const char *suffix)
 {
 	const char *dir;
 
@@ -162,7 +163,8 @@ c_vol_meta_image_path_new(c_vol_t *vol, const mount_entry_t *mntent)
 		return NULL;
 	}
 
-	return mem_printf("%s/%s.meta.img", dir, mount_entry_get_img(mntent));
+	return mem_printf("%s/%s.meta.img%s", dir, mount_entry_get_img(mntent),
+			  suffix ? suffix : "");
 }
 
 static char *
@@ -347,7 +349,7 @@ c_vol_create_image(c_vol_t *vol, const char *img, const mount_entry_t *mntent)
 		return 0;
 	case MOUNT_TYPE_OVERLAY_RW:
 	case MOUNT_TYPE_EMPTY: {
-		char *img_meta = c_vol_meta_image_path_new(vol, mntent);
+		char *img_meta = c_vol_meta_image_path_new(vol, mntent, NULL);
 		int ret = c_vol_create_image_empty(img, img_meta, mount_entry_get_size(mntent));
 		mem_free0(img_meta);
 		return ret;
@@ -858,16 +860,17 @@ c_vol_mount_image(c_vol_t *vol, const char *root, const mount_entry_t *mntent)
 		if (file_is_blk(crypt) || file_links_to_blk(crypt)) {
 			INFO("Using existing mapper device: %s", crypt);
 		} else {
-			DEBUG("Setting up cryptfs volume %s for %s", label, dev);
+			DEBUG("Setting up cryptfs volume %s for %s (%s)", label, dev,
+			      vol->mode == CRYPTFS_MODE_AUTHENC ? "AUTHENC" : "INTEGRITY_ENCRYPT");
 
-			img_meta = c_vol_meta_image_path_new(vol, mntent);
+			img_meta = c_vol_meta_image_path_new(vol, mntent, NULL);
 			dev_meta = loopdev_create_new(&fd_meta, img_meta, 0, 0);
 
 			IF_NULL_GOTO(dev_meta, error);
 
 			mem_free0(crypt);
 			crypt = cryptfs_setup_volume_new(
-				label, dev, container_get_key(vol->container), dev_meta);
+				label, dev, container_get_key(vol->container), dev_meta, vol->mode);
 
 			// release loopdev fd (crypt device should keep it open now)
 			close(fd_meta);
@@ -1094,12 +1097,12 @@ c_vol_cleanup_dm(c_vol_t *vol)
 
 		DEBUG("Cleanup: removing block device %s of type %s\n", label, type);
 
-		if (!strcmp(type, "crypt")) {
-			if (cryptfs_delete_blk_dev(fd, label) < 0)
-				DEBUG("Could not delete dm-crypt dev %s", label);
+		if (!strcmp(type, "crypt") || !strcmp(type, "integrity")) {
+			if (cryptfs_delete_blk_dev(fd, label, vol->mode) < 0)
+				WARN("Could not delete dm-%s dev %s", type, label);
 		} else if (!strcmp(type, "verity")) {
 			if (verity_delete_blk_dev(label) < 0)
-				DEBUG("Could not delete dm-verity dev %s", label);
+				WARN("Could not delete dm-verity dev %s", label);
 		}
 		mem_free0(label);
 		mem_free0(type);
@@ -1430,6 +1433,40 @@ c_vol_verify_mount_entries_bg(const c_vol_t *vol)
 	return true;
 }
 
+/*
+ * If images_dir does not have stacked images, persist policy without stacking
+ * using cryptfs_mode INTEGRITY_ENCRYPT to support TRIM on SSDs.
+ * Call this function on container start to allow switching the policy by container
+ * wipe.
+ */
+static void
+c_vol_set_dm_mode(c_vol_t *vol)
+{
+	ASSERT(vol);
+
+	const char *images_dir = container_get_images_dir(vol->container);
+	ASSERT(images_dir);
+
+	bool is_c0 = container_uuid_is_c0id(container_get_uuid(vol->container));
+
+	char *not_stacked_file = mem_printf("%s/not-stacked", images_dir);
+	if (file_exists(not_stacked_file)) {
+		TRACE("file exists %s %s", not_stacked_file,
+		      is_c0 ? "(c0) -> CRYPTFS_MODE_INTEGRITY_ONLY" :
+			      "-> CRYPTFS_MODE_INTEGRITY_ENCRYPT");
+		vol->mode = is_c0 ? CRYPTFS_MODE_INTEGRITY_ONLY : CRYPTFS_MODE_INTEGRITY_ENCRYPT;
+	} else if (container_images_dir_contains_image(vol->container)) {
+		TRACE("previous image files exists -> CRYPTFS_MODE_AUTHENC");
+		vol->mode = CRYPTFS_MODE_AUTHENC;
+	} else {
+		TRACE("new image files %s", is_c0 ? "(c0) -> CRYPTFS_MODE_INTEGRITY_ONLY" :
+						    "-> CRYPTFS_MODE_INTEGRITY_ENCRYPT");
+		vol->mode = is_c0 ? CRYPTFS_MODE_INTEGRITY_ONLY : CRYPTFS_MODE_INTEGRITY_ENCRYPT;
+		file_touch(not_stacked_file);
+	}
+	mem_free0(not_stacked_file);
+}
+
 /******************************************************************************/
 
 static void *
@@ -1603,6 +1640,18 @@ c_vol_start_child_early(void *volp)
 	return -COMPARTMENT_ERROR_VOL;
 }
 
+static int
+c_vol_start_pre_clone(void *volp)
+{
+	c_vol_t *vol = volp;
+	ASSERT(vol);
+
+	// set device mapper mode for data integrity and encryption
+	c_vol_set_dm_mode(vol);
+
+	return 0;
+}
+
 static int
 c_vol_start_post_clone(void *volp)
 {
@@ -1963,6 +2012,18 @@ c_vol_is_encrypted(void *volp)
 	return false;
 }
 
+static cryptfs_mode_t
+c_vol_get_mode(void *volp)
+{
+	c_vol_t *vol = volp;
+	ASSERT(vol);
+
+	// update internal mode variable if container did not run or was wiped
+	c_vol_set_dm_mode(vol);
+
+	return vol->mode;
+}
+
 static void
 c_vol_cleanup(void *volp, bool is_rebooting)
 {
@@ -1984,7 +2045,7 @@ static compartment_module_t c_vol_module = {
 	.compartment_destroy = NULL,
 	.start_post_clone_early = NULL,
 	.start_child_early = c_vol_start_child_early,
-	.start_pre_clone = NULL,
+	.start_pre_clone = c_vol_start_pre_clone,
 	.start_post_clone = c_vol_start_post_clone,
 	.start_pre_exec = c_vol_start_pre_exec,
 	.start_post_exec = NULL,
@@ -2006,4 +2067,5 @@ c_vol_init(void)
 	container_register_get_rootdir_handler(MOD_NAME, c_vol_get_rootdir);
 	container_register_get_mnt_handler(MOD_NAME, c_vol_get_mnt);
 	container_register_is_encrypted_handler(MOD_NAME, c_vol_is_encrypted);
+	container_register_get_cryptfs_mode_handler(MOD_NAME, c_vol_get_mode);
 }

daemon/cc_mode/container.proto

@@ -164,6 +164,14 @@ enum ContainerTrust {
 	UNSIGNED = 3;
 }
 
+enum CryptfsMode {
+	NOT_IMPLEMENTED = 1;
+	AUTHENC = 2;
+	ENCRYPT_ONLY = 3;
+	INTEGRITY_ENCRYPT = 4;
+	INTEGRITY_ONLY = 5;
+}
+
 /**
  * Represents the status of a single container.
  */
@@ -176,5 +184,6 @@ message ContainerStatus {
 	required uint64 created = 6;
 	required string guestos = 7;
 	required ContainerTrust trust_level = 8;
+	required CryptfsMode cryptfs_mode = 9;
 	/* TBD more state values */
 }

daemon/cmld.c

@@ -42,6 +42,7 @@
 #include "common/dir.h"
 #include "common/network.h"
 #include "common/reboot.h"
+#include "common/hex.h"
 #include "mount.h"
 #include "device_config.h"
 #include "device_id.h"
@@ -60,6 +61,7 @@
 #include "container.h"
 #include "input.h"
 #include "oci.h"
+#include "crypto.h"
 
 #include <inttypes.h>
 #include <stdio.h>
@@ -96,12 +98,14 @@
 #define CMLD_KSM_AGGRESSIVE_TIME_AFTER_CONTAINER_BOOT 70000
 
 /*
- * dummy key used for unecnrypted c0 and for reboots where the real key
- * is already in kernel
+ * dummy key used for unencrypted c0 (legacy dm ondisk format)
+ * and for reboots where the real key is already in kernel
  */
 #define DUMMY_KEY                                                                                  \
 	"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
 
+#define CMLD_C0_KEY_BUF_SIZE 256
+#define CMLD_C0_KEY_LEN 32
 #define CMLD_C0_UUID "00000000-0000-0000-0000-000000000000"
 
 static const char *cmld_path = DEFAULT_BASE_PATH;
@@ -1283,6 +1287,67 @@ cmld_init_c0(const char *path, const char *c0os)
 	return 0;
 }
 
+static int
+cmld_set_key_c0(container_t *new_c0)
+{
+	int ret = -1;
+
+	int c0_key_len;
+	char *c0_key, *c0_ascii_key;
+
+	char *c0_key_file = mem_printf("%s/%s.key", cmld_get_wrapped_keys_dir(), CMLD_C0_UUID);
+	if (file_exists(c0_key_file)) {
+		c0_key = mem_alloc0(CMLD_C0_KEY_BUF_SIZE);
+		c0_key_len = file_read(c0_key_file, c0_key, CMLD_C0_KEY_BUF_SIZE);
+		if (c0_key_len < CMLD_C0_KEY_LEN) {
+			ERROR("Failed to read key from %s for c0", c0_key_file);
+			goto out;
+		}
+	} else {
+		if (container_images_dir_contains_image(new_c0)) {
+			// set legacy mode where we used the dummy key for c0
+			c0_key_len = strlen(DUMMY_KEY) / 2;
+			c0_key = mem_alloc0(c0_key_len);
+			if (convert_hex_to_bin(DUMMY_KEY, strlen(DUMMY_KEY), (uint8_t *)c0_key,
+					       c0_key_len) < 0) {
+				ERROR("Failed to generate key for c0 (using legacy DUMMY_KEY)");
+				goto out;
+			}
+		} else {
+			// for new images we use a random key for integrity protection of c0
+			c0_key_len = CMLD_C0_KEY_LEN;
+			c0_key = mem_alloc0(c0_key_len);
+			int bytes_read = crypto_random_get_bytes((uint8_t *)c0_key, c0_key_len);
+			if (bytes_read != c0_key_len) {
+				ERROR("Failed to generate key for c0");
+				goto out;
+			}
+		}
+
+		int bytes_written = file_write(c0_key_file, c0_key, c0_key_len);
+		if (bytes_written != c0_key_len) {
+			ERROR("Failed to write c0 key to file, bytes written: %d", bytes_written);
+			goto out;
+		}
+	}
+
+	c0_ascii_key = convert_bin_to_hex_new((uint8_t *)c0_key, c0_key_len);
+	container_set_key(new_c0, c0_ascii_key);
+
+	mem_memset0(c0_key, c0_key_len);
+	mem_memset0(c0_ascii_key, strlen(c0_ascii_key));
+	mem_free0(c0_ascii_key);
+
+	ret = 0;
+
+out:
+	if (c0_key)
+		mem_free0(c0_key);
+	mem_free0(c0_key_file);
+
+	return ret;
+}
+
 static int
 cmld_start_c0(container_t *new_c0)
 {
@@ -1306,7 +1371,8 @@ cmld_start_c0(container_t *new_c0)
 		return -1;
 	}
 
-	container_set_key(new_c0, DUMMY_KEY);
+	IF_TRUE_RETVAL_ERROR(cmld_set_key_c0(new_c0), -1);
+
 	if (container_start(new_c0)) {
 		audit_log_event(container_get_uuid(new_c0), FSA, CMLD, CONTAINER_MGMT, "c0-start",
 				uuid_string(container_get_uuid(new_c0)), 0);
@@ -1533,7 +1599,7 @@ cmld_init(const char *path)
 		FATAL_ERRNO("Could not mkdir containers directory %s", containers_path);
 
 	cmld_wrapped_keys_path = mem_printf("%s/%s", path, CMLD_PATH_CONTAINER_KEYS_DIR);
-	if (mkdir(containers_path, 0700) < 0 && errno != EEXIST)
+	if (mkdir(cmld_wrapped_keys_path, 0700) < 0 && errno != EEXIST)
 		FATAL_ERRNO("Could not mkdir container keys directory %s", containers_path);
 
 	if (cmld_init_c0(containers_path, device_config_get_c0os(device_config)) < 0)

daemon/container.c

@@ -548,6 +548,20 @@ container_get_images_dir(const container_t *container)
 	return container->images_dir;
 }
 
+static int
+container_images_exist_cb(UNUSED const char *path, const char *name, UNUSED void *data)
+{
+	int len = strlen(name);
+	return (len >= 4 && !strcmp(name + len - 4, ".img")) ? 1 : 0;
+}
+
+bool
+container_images_dir_contains_image(const container_t *container)
+{
+	ASSERT(container);
+	return dir_foreach(container->images_dir, container_images_exist_cb, NULL) > 0;
+}
+
 static int
 container_wipe_image_cb(const char *path, const char *name, UNUSED void *data)
 {
@@ -970,6 +984,9 @@ CONTAINER_MODULE_REGISTER_WRAPPER_IMPL(get_rootdir, char *, void *)
 CONTAINER_MODULE_FUNCTION_WRAPPER_IMPL(get_rootdir, char *, NULL)
 CONTAINER_MODULE_REGISTER_WRAPPER_IMPL(get_mnt, void *, void *)
 CONTAINER_MODULE_FUNCTION_WRAPPER_IMPL(get_mnt, void *, NULL)
+CONTAINER_MODULE_REGISTER_WRAPPER_IMPL(get_cryptfs_mode, cryptfs_mode_t, void *)
+CONTAINER_MODULE_FUNCTION_WRAPPER_IMPL(get_cryptfs_mode, cryptfs_mode_t,
+				       CRYPTFS_MODE_NOT_IMPLEMENTED)
 CONTAINER_MODULE_REGISTER_WRAPPER_IMPL(is_encrypted, bool, void *)
 CONTAINER_MODULE_FUNCTION_WRAPPER_IMPL(is_encrypted, bool, false)