From 7389031e787cf55a5858fad8bf5d241705bf7cdd Mon Sep 17 00:00:00 2001
From: grev <grev>
Date: Mon, 26 May 2008 14:17:31 +0000
Subject: [PATCH] b=15255 i=Adilger
config for several remote clients; replay-single test_70a, test_70b
---
lustre/tests/Makefile.am | 2 +-
lustre/tests/cfg/ncli.sh | 17 +++++++++
lustre/tests/replay-single.sh | 70 ++++++++++++++++++++++++++++++++++
lustre/tests/test-framework.sh | 43 +++++++++++++++++++--
4 files changed, 127 insertions(+), 5 deletions(-)
create mode 100644 lustre/tests/cfg/ncli.sh
diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index 305ff0e500..60dedf1d6a 100644
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -13,7 +13,7 @@ noinst_SCRIPTS += llog-test.sh recovery-small.sh replay-dual.sh sanity-quota.sh
noinst_SCRIPTS += replay-ost-single.sh replay-single.sh run-llog.sh sanityN.sh
noinst_SCRIPTS += lockorder.sh socketclient socketserver runmultiop_bg_pause
nobase_noinst_SCRIPTS = cfg/insanity-local.sh
-nobase_noinst_SCRIPTS += cfg/local.sh acl/make-tree acl/run
+nobase_noinst_SCRIPTS += cfg/local.sh acl/make-tree acl/run cfg/ncli.sh
nobase_noinst_DATA = acl/cp.test acl/getfacl-noacl.test acl/inheritance.test
nobase_noinst_DATA += acl/misc.test acl/permissions.test acl/setfacl.test
diff --git a/lustre/tests/cfg/ncli.sh b/lustre/tests/cfg/ncli.sh
new file mode 100644
index 0000000000..2735b9ad1c
--- /dev/null
+++ b/lustre/tests/cfg/ncli.sh
@@ -0,0 +1,17 @@
+. $LUSTRE/tests/cfg/local.sh
+
+CLIENT1=${CLIENT1:-`hostname`}
+SINGLECLIENT=$CLIENT1
+RCLIENTS=${RCLIENTS:-""}
+CLIENTS=`comma_list $SINGLECLIENT $RCLIENTS`
+REMOTECLIENTS=($RCLIENTS)
+for ((i=0; $i<${#REMOTECLIENTS[@]}; i++)); do
+ varname=CLIENT$((i + 2))
+ eval $varname=${REMOTECLIENTS[i]}
+done
+
+CLIENTCOUNT=$((${#REMOTECLIENTS[@]} + 1))
+
+[ -n "$RCLIENTS" -a "$PDSH" = "no_dsh" ] && \
+ error "tests for remote clients $RCLIENTS needs pdsh != do_dsh " || true
+
diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index ea3c6226b5..5142e45f96 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -1563,6 +1563,76 @@ fi
# end of AT tests includes above lines
+# start multi-client tests
+test_70a () {
+ [ -z "$CLIENTS" ] && \
+ { skip "Need two or more clients." && return; }
+ [ $CLIENTCOUNT -lt 2 ] && \
+ { skip "Need two or more clients, have $CLIENTCOUNT" && return; }
+
+ echo "mount clients $CLIENTS ..."
+ zconf_mount_clients $CLIENTS $DIR
+
+ local clients=${CLIENTS//,/ }
+ echo "Write/read files on $DIR ; clients $CLIENTS ... "
+ for CLIENT in $clients; do
+ do_node $CLIENT dd bs=1M count=10 if=/dev/zero \
+ of=$DIR/${tfile}_${CLIENT} 2>/dev/null || \
+ error "dd failed on $CLIENT"
+ done
+
+ local prev_client=$(echo $clients | sed 's/^.* \(\w\+\)$/\1/')
+ for C in ${CLIENTS//,/ }; do
+ do_node $prev_client dd if=$DIR/${tfile}_${C} of=/dev/null 2>/dev/null || \
+ error "dd if=$DIR/${tfile}_${C} failed on $prev_client"
+ prev_client=$C
+ done
+
+ ls $DIR
+
+ zconf_umount_clients $CLIENTS $DIR
+}
+run_test 70a "check multi client t-f"
+
+test_70b () {
+ [ -z "$CLIENTS" ] && \
+ { skip "Need two or more clients." && return; }
+ [ $CLIENTCOUNT -lt 2 ] && \
+ { skip "Need two or more clients, have $CLIENTCOUNT" && return; }
+
+ zconf_mount_clients $CLIENTS $DIR
+
+ local duration="-t 60"
+ local cmd="rundbench 1 $duration "
+ local PID=""
+ for CLIENT in ${CLIENTS//,/ }; do
+ $PDSH $CLIENT "set -x; PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests/:${DBENCH_LIB} DBENCH_LIB=${DBENCH_LIB} $cmd" &
+ PID=$!
+ echo $PID >pid.$CLIENT
+ echo "Started load PID=`cat pid.$CLIENT`"
+ done
+
+ replay_barrier mds
+ sleep 3 # give clients a time to do operations
+
+ log "$TESTNAME fail mds 1"
+ fail mds
+
+# wait for client to reconnect to MDS
+ sleep $TIMEOUT
+
+ for CLIENT in ${CLIENTS//,/ }; do
+ PID=`cat pid.$CLIENT`
+ wait $PID
+ rc=$?
+ echo "load on ${CLIENT} returned $rc"
+ done
+
+ zconf_umount_clients $CLIENTS $DIR
+}
+run_test 70b "mds recovery; $CLIENTCOUNT clients"
+# end multi-client tests
+
equals_msg `basename $0`: test complete, cleaning up
check_and_cleanup_lustre
[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 913f7920b4..e2f6e9deb3 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -330,7 +330,7 @@ zconf_mount() {
exit 1
fi
- echo "Starting client: $OPTIONS $device $mnt"
+ echo "Starting client: $client: $OPTIONS $device $mnt"
do_node $client mkdir -p $mnt
do_node $client mount -t lustre $OPTIONS $device $mnt || return 1
@@ -342,17 +342,40 @@ zconf_mount() {
}
zconf_umount() {
- client=$1
- mnt=$2
+ local client=$1
+ local mnt=$2
[ "$3" ] && force=-f
local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true
if [ $running -ne 0 ]; then
- echo "Stopping client $mnt (opts:$force)"
+ echo "Stopping client $client $mnt (opts:$force)"
lsof | grep "$mnt" || true
do_node $client umount $force $mnt
fi
}
+zconf_mount_clients() {
+ local clients=$1
+ local mnt=$2
+
+ echo "Mounting clients: $clients"
+ local client
+ for client in ${clients//,/ }; do
+ zconf_mount $client $mnt || true
+ done
+}
+
+zconf_umount_clients() {
+ local clients=$1
+ local mnt=$2
+ [ "$3" ] && force=-f
+
+ echo "Umounting clients: $clients"
+ local client
+ for client in ${clients//,/ }; do
+ zconf_umount $client $mnt $force || true
+ done
+}
+
shutdown_facet() {
facet=$1
if [ "$FAILURE_MODE" = HARD ]; then
@@ -733,6 +756,12 @@ stopall() {
# assume client mount is local
grep " $MOUNT " /proc/mounts && zconf_umount $HOSTNAME $MOUNT $*
grep " $MOUNT2 " /proc/mounts && zconf_umount $HOSTNAME $MOUNT2 $*
+
+ if [ -n "$CLIENTS" ]; then
+ zconf_umount_clients $CLIENTS $MOUNT "$*" || true
+ zconf_umount_clients $CLIENTS $MOUNT2 "$*" || true
+ fi
+
[ "$CLIENTONLY" ] && return
stop mds -f
for num in `seq $OSTCOUNT`; do
@@ -805,8 +834,11 @@ setupall() {
fi
[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
mount_client $MOUNT
+ [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT
+
if [ "$MOUNT_2" ]; then
mount_client $MOUNT2
+ [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2
fi
sleep 5
}
@@ -1384,6 +1416,9 @@ nodes_list () {
local myNODES=$HOSTNAME
local myNODES_sort
+ # CLIENTS (if specified) contains the local client
+ [ -n "$CLIENTS" ] && myNODES=${CLIENTS//,/ }
+
if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then
myNODES="$myNODES $(osts_nodes) $mds_HOST"
fi
--
GitLab