LU-10070 lod: SEL: Add FLR support

Add FLR support for self-extending layouts.

The basic model is that when a layout intent would modify
an FLR replica, we first run the extent_update code to
perform any layout extent changes for self extending
layouts.

This treats the FLR operations (stale, resync, etc)
similarly to i/o, creating initialized layout where those
operations need it.  This makes the interaction between SEL
and FLR fairly simple.

Add FLR tests for self-extending layouts

Cray-bug-id: LUS-2528
Signed-off-by: Patrick Farrell <[email protected]>
Change-Id: Ia23df8e226955f64e9b19df993b66d2d4f820f33
Reviewed-on: https://review.whamcloud.com/33785
Reviewed-by: Patrick Farrell <[email protected]>
Tested-by: jenkins <[email protected]>
Tested-by: Maloo <[email protected]>
Reviewed-by: Alexey Lyashkov <[email protected]>
Reviewed-by: Oleg Drokin <[email protected]>

Author: paf-49

lustre/lod/lod_object.c

@@ -6786,15 +6786,21 @@ static inline int lod_comp_index(struct lod_object *lo,
 /**
  * Stale other mirrors by writing extent.
  */
-static void lod_stale_components(struct lod_object *lo, int primary,
-				 struct lu_extent *extent)
+static int lod_stale_components(const struct lu_env *env, struct lod_object *lo,
+				int primary, struct lu_extent *extent,
+				struct thandle *th)
 {
 	struct lod_layout_component *pri_comp, *lod_comp;
+	struct lu_extent pri_extent;
+	int rc = 0;
 	int i;
+	ENTRY;
 
 	/* The writing extent decides which components in the primary
 	 * are affected... */
 	CDEBUG(D_LAYOUT, "primary mirror %d, "DEXT"\n", primary, PEXT(extent));
+
+restart:
 	lod_foreach_mirror_comp(pri_comp, lo, primary) {
 		if (!lu_extent_is_overlapped(extent, &pri_comp->llc_extent))
 			continue;
@@ -6803,15 +6809,27 @@ static void lod_stale_components(struct lod_object *lo, int primary,
 		       lod_comp_index(lo, pri_comp),
 		       PEXT(&pri_comp->llc_extent));
 
+		pri_extent.e_start = pri_comp->llc_extent.e_start;
+		pri_extent.e_end = pri_comp->llc_extent.e_end;
+
 		for (i = 0; i < lo->ldo_mirror_count; i++) {
 			if (i == primary)
 				continue;
+			rc = lod_declare_update_extents(env, lo, &pri_extent,
+							th, i, 0);
+			/* if update_extents changed the layout, it may have
+			 * reallocated the component array, so start over to
+			 * avoid using stale pointers */
+			if (rc == 1)
+				goto restart;
+			if (rc < 0)
+				RETURN(rc);
 
 			/* ... and then stale other components that are
 			 * overlapping with primary components */
 			lod_foreach_mirror_comp(lod_comp, lo, i) {
 				if (!lu_extent_is_overlapped(
-							&pri_comp->llc_extent,
+							&pri_extent,
 							&lod_comp->llc_extent))
 					continue;
 
@@ -6823,6 +6841,8 @@ static void lod_stale_components(struct lod_object *lo, int primary,
 			}
 		}
 	}
+
+	RETURN(rc);
 }
 
 /**
@@ -7058,6 +7078,7 @@ static int lod_declare_update_rdonly(const struct lu_env *env,
 
 	if (mlc->mlc_opc == MD_LAYOUT_WRITE) {
 		struct layout_intent *layout = mlc->mlc_intent;
+		int write = layout->li_opc == LAYOUT_INTENT_WRITE;
 		int picked;
 
 		extent = layout->li_extent;
@@ -7072,6 +7093,12 @@ static int lod_declare_update_rdonly(const struct lu_env *env,
 		       PFID(lod_object_fid(lo)),
 		       lo->ldo_mirrors[picked].lme_id);
 
+		/* Update extents of primary before staling */
+		rc = lod_declare_update_extents(env, lo, &extent, th, picked,
+						write);
+		if (rc < 0)
+			GOTO(out, rc);
+
 		if (layout->li_opc == LAYOUT_INTENT_TRUNC) {
 			/**
 			 * trunc transfers [0, size) in the intent extent, we'd
@@ -7082,7 +7109,9 @@ static int lod_declare_update_rdonly(const struct lu_env *env,
 		}
 
 		/* stale overlapping components from other mirrors */
-		lod_stale_components(lo, picked, &extent);
+		rc = lod_stale_components(env, lo, picked, &extent, th);
+		if (rc < 0)
+			GOTO(out, rc);
 
 		/* restore truncate intent extent */
 		if (layout->li_opc == LAYOUT_INTENT_TRUNC)
@@ -7222,13 +7251,22 @@ static int lod_declare_update_write_pending(const struct lu_env *env,
 	 * 2. transfer layout version to all objects to close write era. */
 
 	if (mlc->mlc_opc == MD_LAYOUT_WRITE) {
+		struct layout_intent *layout = mlc->mlc_intent;
+		int write = layout->li_opc == LAYOUT_INTENT_WRITE;
+
 		LASSERT(mlc->mlc_intent != NULL);
 
 		extent = mlc->mlc_intent->li_extent;
 
 		CDEBUG(D_LAYOUT, DFID": intent to write: "DEXT"\n",
 		       PFID(lod_object_fid(lo)), PEXT(&extent));
 
+		/* 1. Update extents of primary before staling */
+		rc = lod_declare_update_extents(env, lo, &extent, th, primary,
+						write);
+		if (rc < 0)
+			GOTO(out, rc);
+
 		if (mlc->mlc_intent->li_opc == LAYOUT_INTENT_TRUNC) {
 			/**
 			 * trunc transfers [0, size) in the intent extent, we'd
@@ -7237,10 +7275,13 @@ static int lod_declare_update_write_pending(const struct lu_env *env,
 			extent.e_start = extent.e_end;
 			extent.e_end = OBD_OBJECT_EOF;
 		}
-		/* 1. stale overlapping components */
-		lod_stale_components(lo, primary, &extent);
 
-		/* 2. find out the components need instantiating.
+		/* 2. stale overlapping components */
+		rc = lod_stale_components(env, lo, primary, &extent, th);
+		if (rc < 0)
+			GOTO(out, rc);
+
+		/* 3. find the components which need instantiating.
 		 * instantiate [0, mlc->mlc_intent->e_end) */
 
 		/* restore truncate intent extent */
@@ -7380,9 +7421,9 @@ static int lod_declare_update_sync_pending(const struct lu_env *env,
 		GOTO(out, rc = -EINVAL);
 	}
 
-	CDEBUG(D_LAYOUT, DFID": resynced %u/%zu components\n",
+	CDEBUG(D_LAYOUT, DFID": synced %u resynced %u/%zu components\n",
 	       PFID(lod_object_fid(lo)),
-	       resync_components, mlc->mlc_resync_count);
+	       sync_components, resync_components, mlc->mlc_resync_count);
 
 	lo->ldo_flr_state = LCM_FL_RDONLY;
 	lod_obj_inc_layout_gen(lo);

lustre/tests/sanity-flr.sh

@@ -2286,6 +2286,186 @@ test_203() {
 }
 run_test 203 "mirror file preserve mirror ID"
 
+# Simple test of FLR + self-extending layout, SEL in non-primary mirror
+test_204a() {
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code $SEL_VER) ] &&
+		skip "skipped for lustre < $SEL_VER"
+
+	local comp_file=$DIR/$tdir/$tfile
+	local flg_opts=""
+	local found=""
+
+	test_mkdir $DIR/$tdir
+
+	# first mirror is 0-10M, then 10M-(-1), second mirror is 1M followed
+	# by extension space to -1
+	$LFS setstripe -N -E 10M -E-1 -N -E 1M -E-1 -z64M $comp_file ||
+		error "Create $comp_file failed"
+
+	# Write to first component, extending & staling second mirror
+	dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc ||
+		error "dd to extend + stale failed"
+
+	$LFS getstripe $comp_file
+
+	flg_opts="--component-flags init,stale"
+	found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "write: Second comp end incorrect"
+
+	flg_opts="--component-flags extension"
+	found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "write: Third comp start incorrect"
+
+	# mirror resync should not change the extents
+	$LFS mirror resync $comp_file
+
+	flg_opts="--component-flags init"
+	found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "resync: Second comp end incorrect"
+
+	flg_opts="--component-flags extension"
+	found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "resync: Third comp start incorrect"
+
+	sel_layout_sanity $comp_file 5
+
+	rm -f $comp_file
+}
+run_test 204a "FLR write/stale/resync tests with self-extending mirror"
+
+# Simple test of FLR + self-extending layout, SEL in primary mirror
+test_204b() {
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code $SEL_VER) ] &&
+		skip "skipped for lustre < $SEL_VER"
+
+	local comp_file=$DIR/$tdir/$tfile
+	local flg_opts=""
+	local found=""
+
+	test_mkdir $DIR/$tdir
+
+	# first mirror is 1M followed by extension space to -1, second mirror
+	# is 0-10M, then 10M-(-1),
+	$LFS setstripe -N -E 1M -E-1 -z64M -N -E 10M -E-1 $comp_file ||
+		error "Create $comp_file failed"
+
+	# Write to first component, extending first component & staling
+	# other mirror
+	dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc ||
+		error "dd to extend + stale failed"
+
+	$LFS getstripe $comp_file
+
+	flg_opts="--component-flags init"
+	found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "write: First comp end incorrect"
+
+	flg_opts="--component-flags extension"
+	found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "write: Second comp start incorrect"
+
+	flg_opts="--component-flags init,stale"
+	found=$($LFS find --component-end 10M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "write: First mirror comp flags incorrect"
+
+	# This component is staled because it overlaps the extended first
+	# component of the primary mirror, even though it doesn't overlap
+	# the actual write - thus not inited.
+	flg_opts="--component-flags stale"
+	found=$($LFS find --component-start 10M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "write: Second mirror comp flags incorrect"
+
+	# mirror resync should not change the extents
+	$LFS mirror resync $comp_file
+
+	$LFS getstripe $comp_file
+
+	flg_opts="--component-flags init"
+	found=$($LFS find --component-end 65M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "resync: First comp end incorrect"
+
+	flg_opts="--component-flags extension"
+	found=$($LFS find --component-start 65M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "resync: Second comp start incorrect"
+
+	flg_opts="--component-flags init"
+	found=$($LFS find --component-end 10M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "resync: First mirror comp flags incorrect"
+
+	flg_opts="--component-flags init"
+	found=$($LFS find --component-start 10M $flg_opts $comp_file | wc -l)
+	[ $found -eq 1 ] || error "resync: Second mirror comp flags incorrect"
+
+	sel_layout_sanity $comp_file 5
+
+	rm -f $comp_file
+}
+run_test 204b "FLR write/stale/resync tests with self-extending primary"
+
+# FLR + SEL failed extension & component removal
+# extension space in second mirror
+test_204c() {
+	[ $OSTCOUNT -lt 2 ] && skip "needs >= 2 OSTs" && return
+	[ $(lustre_version_code $SINGLEMDS) -lt $(version_code $SEL_VER) ] &&
+		skip "skipped for lustre < $SEL_VER"
+
+	local comp_file=$DIR/$tdir/$tfile
+	local found=""
+	local ost_idx1=0
+	local ost_name=$(ostname_from_index $ost_idx1)
+
+	test_mkdir $DIR/$tdir
+
+	# first mirror is is 0-10M, then 10M-(-1), second mirror is 0-1M, then
+	# extension space from 1M to 1G, then normal space to -1
+	$LFS setstripe -N -E 10M -E-1 -N -E 1M -E 1G -i $ost_idx1 -z 64M \
+		-E -1 $comp_file || error "Create $comp_file failed"
+
+	do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=1
+	sleep_maxage
+
+	# write to first comp (0 - 10M) of mirror 1, extending + staling
+	# first + second comp of mirror 2
+	dd if=/dev/zero bs=2M count=1 of=$comp_file conv=notrunc
+	RC=$?
+
+	do_facet ost1 $LCTL set_param -n obdfilter.$ost_name.degraded=0
+	sleep_maxage
+
+	[ $RC -eq 0 ] || error "dd to extend + stale failed"
+
+	$LFS getstripe $comp_file
+
+	found=$($LFS find --component-start 0m --component-end 1m \
+		--comp-flags init,stale $comp_file | wc -l)
+	[ $found -eq 1 ] || error "write: First mirror comp incorrect"
+
+	found=$($LFS find --component-start 1m --component-end EOF \
+		--comp-flags stale,^init $comp_file | wc -l)
+	[ $found -eq 1 ] || error "write: Second mirror comp incorrect"
+
+	local mirror_id=$($LFS getstripe --component-start=1m	\
+			 --component-end=EOF $comp_file |	\
+			 grep lcme_mirror_id | awk '{ print $2 }')
+
+	[[ $mirror_id -eq 2 ]] ||
+		error "component not in correct mirror? $mirror_id"
+
+	$LFS mirror resync $comp_file
+
+	$LFS getstripe $comp_file
+
+	# component dimensions should not change from resync
+	found=$($LFS find --component-start 1m --component-end EOF \
+		--component-flags init $comp_file | wc -l)
+	[ $found -eq 1 ] || error "resync: Second mirror comp incorrect"
+
+	sel_layout_sanity $comp_file 4
+
+	rm -f $comp_file
+}
+run_test 204c "FLR write/stale/resync test with component removal"
+
 complete $SECONDS
 check_and_cleanup_lustre
 exit_status