LU-12131 tests: properly handle GSS in server failover

In case of server failover, a number of aspects must be handled when
GSS based features (SSK or Kerberos) are activated:
- lsvcgssd daemon must be restarted;
- targets must be mounted with proper skpath option;
- permissions on keys must be adjusted.
When service is initially started, all that is managed in setupall().
fail() and facet_failover() have to be improved to take GSS aspects
into account.

Test-Parameters: envdefinitions=SHARED_KEY=true testlist=sanity,recovery-small,sanity-sec
Signed-off-by: Sebastien Buisson <[email protected]>
Change-Id: I8db686f406629c7eec655496cf83c0539c1bfb33
Reviewed-on: https://review.whamcloud.com/35041
Tested-by: jenkins <[email protected]>
Tested-by: Maloo <[email protected]>
Reviewed-by: Andreas Dilger <[email protected]>
Reviewed-by: Wei Liu <[email protected]>
Reviewed-by: Oleg Drokin <[email protected]>

lustre/tests/test-framework.sh

@@ -3517,6 +3517,10 @@ facet_failover() {
 		change_active ${affecteds[index]}
 
 		wait_for_facet ${affecteds[index]}
+		if $GSS_SK; then
+			init_gss
+			init_facets_vars_simple
+		fi
 		# start mgs first if it is affected
 		if ! combined_mgs_mds &&
 			list_member ${affecteds[index]} mgs; then
@@ -3526,6 +3530,12 @@ facet_failover() {
 		affected=$(exclude_items_from_list ${affecteds[index]} mgs)
 		echo mount facets: ${affecteds[index]}
 		mount_facets ${affecteds[index]}
+		if $GSS_SK; then
+			do_nodes $(comma_list $(all_nodes)) \
+				"keyctl show | grep lustre | cut -c1-11 |
+				sed -e 's/ //g;' |
+				xargs -IX keyctl setperm X 0x3f3f3f3f"
+		fi
 	done
 }
 
@@ -3619,7 +3629,12 @@ fail() {
 	local facets=$1
 	local clients=${CLIENTS:-$HOSTNAME}
 
+	SK_NO_KEY_save=$SK_NO_KEY
+	if $GSS_SK; then
+		export SK_NO_KEY=false
+	fi
 	facet_failover $* || error "failover: $?"
+	export SK_NO_KEY=$SK_NO_KEY_save
 	# to initiate all OSC idling connections
 	clients_up
 	wait_clients_import_state "$clients" "$facets" "\(FULL\|IDLE\)"
@@ -4984,6 +4999,31 @@ init_facets_vars () {
 	fi
 }
 
+init_facets_vars_simple () {
+	local devname
+
+	if ! remote_mds_nodsh; then
+		for num in $(seq $MDSCOUNT); do
+			devname=$(mdsdevname $num)
+			eval export mds${num}_dev=${devname}
+			eval export mds${num}_opt=\"${MDS_MOUNT_OPTS}\"
+		done
+	fi
+
+	if ! combined_mgs_mds ; then
+		eval export mgs_dev=$(mgsdevname)
+		eval export mgs_opt=\"${MGS_MOUNT_OPTS}\"
+	fi
+
+	if ! remote_ost_nodsh; then
+		for num in $(seq $OSTCOUNT); do
+			devname=$(ostdevname $num)
+			eval export ost${num}_dev=${devname}
+			eval export ost${num}_opt=\"${OST_MOUNT_OPTS}\"
+		done
+	fi
+}
+
 osc_ensure_active () {
     local facet=$1
     local timeout=$2