b=15255

i=Adilger

config for several remote clients; replay-single test_70a, test_70b

lustre/tests/Makefile.am

@@ -13,7 +13,7 @@ noinst_SCRIPTS += llog-test.sh recovery-small.sh replay-dual.sh sanity-quota.sh
 noinst_SCRIPTS += replay-ost-single.sh replay-single.sh run-llog.sh sanityN.sh
 noinst_SCRIPTS += lockorder.sh socketclient socketserver runmultiop_bg_pause
 nobase_noinst_SCRIPTS = cfg/insanity-local.sh
-nobase_noinst_SCRIPTS += cfg/local.sh acl/make-tree acl/run
+nobase_noinst_SCRIPTS += cfg/local.sh acl/make-tree acl/run cfg/ncli.sh
 nobase_noinst_DATA = acl/cp.test acl/getfacl-noacl.test acl/inheritance.test
 nobase_noinst_DATA += acl/misc.test acl/permissions.test acl/setfacl.test
 

lustre/tests/cfg/ncli.sh

+. $LUSTRE/tests/cfg/local.sh
+
+CLIENT1=${CLIENT1:-`hostname`}
+SINGLECLIENT=$CLIENT1
+RCLIENTS=${RCLIENTS:-""}
+CLIENTS=`comma_list $SINGLECLIENT $RCLIENTS`
+REMOTECLIENTS=($RCLIENTS)
+for ((i=0; $i<${#REMOTECLIENTS[@]}; i++)); do
+	varname=CLIENT$((i + 2))
+	eval $varname=${REMOTECLIENTS[i]}
+done
+
+CLIENTCOUNT=$((${#REMOTECLIENTS[@]} + 1))
+
+[ -n "$RCLIENTS" -a "$PDSH" = "no_dsh" ] && \
+                error "tests for remote clients $RCLIENTS needs pdsh != do_dsh " || true
+

lustre/tests/replay-single.sh

@@ -1563,6 +1563,76 @@ fi
 
 # end of AT tests includes above lines
 
+# start multi-client tests
+test_70a () {
+	[ -z "$CLIENTS" ] && \
+		{ skip "Need two or more clients." && return; }
+	[ $CLIENTCOUNT -lt 2 ] && \
+		{ skip "Need two or more clients, have $CLIENTCOUNT" && return; }
+
+	echo "mount clients $CLIENTS ..."
+	zconf_mount_clients $CLIENTS $DIR
+
+	local clients=${CLIENTS//,/ }
+	echo "Write/read files on $DIR ; clients $CLIENTS ... "
+	for CLIENT in $clients; do
+		do_node $CLIENT dd bs=1M count=10 if=/dev/zero \
+			of=$DIR/${tfile}_${CLIENT} 2>/dev/null || \
+				error "dd failed on $CLIENT"
+	done
+
+	local prev_client=$(echo $clients | sed 's/^.* \(\w\+\)$/\1/') 
+	for C in ${CLIENTS//,/ }; do
+		do_node $prev_client dd if=$DIR/${tfile}_${C} of=/dev/null 2>/dev/null || \
+			error "dd if=$DIR/${tfile}_${C} failed on $prev_client"
+		prev_client=$C
+	done
+	
+	ls $DIR
+
+	zconf_umount_clients $CLIENTS $DIR
+}
+run_test 70a "check multi client t-f"
+
+test_70b () {
+	[ -z "$CLIENTS" ] && \
+		{ skip "Need two or more clients." && return; }
+	[ $CLIENTCOUNT -lt 2 ] && \
+		{ skip "Need two or more clients, have $CLIENTCOUNT" && return; }
+
+	zconf_mount_clients $CLIENTS $DIR
+	
+	local duration="-t 60"
+	local cmd="rundbench 1 $duration "
+	local PID=""
+	for CLIENT in ${CLIENTS//,/ }; do
+		$PDSH $CLIENT "set -x; PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests/:${DBENCH_LIB} DBENCH_LIB=${DBENCH_LIB} $cmd" &
+		PID=$!
+		echo $PID >pid.$CLIENT
+		echo "Started load PID=`cat pid.$CLIENT`"
+	done
+
+	replay_barrier mds 
+	sleep 3 # give clients a time to do operations
+
+	log "$TESTNAME fail mds 1"
+	fail mds
+
+# wait for client to reconnect to MDS
+	sleep $TIMEOUT
+
+	for CLIENT in ${CLIENTS//,/ }; do
+		PID=`cat pid.$CLIENT`
+		wait $PID
+		rc=$?
+		echo "load on ${CLIENT} returned $rc"
+	done
+
+	zconf_umount_clients $CLIENTS $DIR 
+}
+run_test 70b "mds recovery; $CLIENTCOUNT clients"
+# end multi-client tests
+
 equals_msg `basename $0`: test complete, cleaning up
 check_and_cleanup_lustre
 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true

lustre/tests/test-framework.sh

@@ -330,7 +330,7 @@ zconf_mount() {
         exit 1
     fi
 
-    echo "Starting client: $OPTIONS $device $mnt" 
+    echo "Starting client: $client: $OPTIONS $device $mnt" 
     do_node $client mkdir -p $mnt
     do_node $client mount -t lustre $OPTIONS $device $mnt || return 1
 
@@ -342,17 +342,40 @@ zconf_mount() {
 }
 
 zconf_umount() {
-    client=$1
-    mnt=$2
+    local client=$1
+    local mnt=$2
     [ "$3" ] && force=-f
     local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true
     if [ $running -ne 0 ]; then
-        echo "Stopping client $mnt (opts:$force)"
+        echo "Stopping client $client $mnt (opts:$force)"
         lsof | grep "$mnt" || true
         do_node $client umount $force $mnt
     fi
 }
 
+zconf_mount_clients() {
+    local clients=$1
+    local mnt=$2
+
+    echo "Mounting clients: $clients"
+    local client
+    for client in ${clients//,/ }; do
+        zconf_mount $client $mnt  || true
+    done
+}
+
+zconf_umount_clients() {
+    local clients=$1
+    local mnt=$2
+    [ "$3" ] && force=-f
+
+    echo "Umounting clients: $clients"
+    local client
+    for client in ${clients//,/ }; do
+        zconf_umount $client $mnt $force || true
+    done
+}
+
 shutdown_facet() {
     facet=$1
     if [ "$FAILURE_MODE" = HARD ]; then
@@ -733,6 +756,12 @@ stopall() {
     # assume client mount is local 
     grep " $MOUNT " /proc/mounts && zconf_umount $HOSTNAME $MOUNT $*
     grep " $MOUNT2 " /proc/mounts && zconf_umount $HOSTNAME $MOUNT2 $*
+
+    if [ -n "$CLIENTS" ]; then
+            zconf_umount_clients $CLIENTS $MOUNT "$*" || true
+            zconf_umount_clients $CLIENTS $MOUNT2 "$*" || true
+    fi
+
     [ "$CLIENTONLY" ] && return
     stop mds -f
     for num in `seq $OSTCOUNT`; do
@@ -805,8 +834,11 @@ setupall() {
     fi
     [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
     mount_client $MOUNT
+    [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT
+
     if [ "$MOUNT_2" ]; then
         mount_client $MOUNT2
+        [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2
     fi
     sleep 5
 }
@@ -1384,6 +1416,9 @@ nodes_list () {
     local myNODES=$HOSTNAME
     local myNODES_sort
 
+    # CLIENTS (if specified) contains the local client
+    [ -n "$CLIENTS" ] && myNODES=${CLIENTS//,/ }
+
     if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then
         myNODES="$myNODES $(osts_nodes) $mds_HOST"
     fi