diff --git a/lustre/tests/acceptance-small.sh b/lustre/tests/acceptance-small.sh
index 6f4e42580e888301627659b6a7f866a53017096c..8d25297009f787586904d3da9e53a599dba18e04 100755
--- a/lustre/tests/acceptance-small.sh
+++ b/lustre/tests/acceptance-small.sh
@@ -76,7 +76,7 @@ for NAME in $CONFIGS; do
 
 	if [ "$SANITY" != "no" ]; then
 	        title sanity
-		SANITYLOG=/tmp/sanity.log bash sanity.sh
+		bash sanity.sh
 		$CLEANUP
 		$SETUP
 	fi
@@ -189,7 +189,7 @@ for NAME in $CONFIGS; do
 		mkdir -p $MOUNT2
 		mount_client $MOUNT2
 		#echo "can't mount2 for '$NAME', skipping sanityN.sh"
-		SANITYLOG=$TMP/sanity.log START=: CLEAN=: bash sanityN.sh
+		START=: CLEAN=: bash sanityN.sh
 		umount $MOUNT2
 
 		$DEBUG_ON
@@ -219,7 +219,10 @@ for NAME in $CONFIGS; do
 		$CLEANUP
 		unload_modules
 		# Liblustre needs accept=all, noacl
-		LNETOPTS="accept=all" MDS_MOUNT_OPTS="${MDS_MOUNT_OPTS},noacl" $SETUP
+		[ -f /etc/modprobe.conf ] && MODPROBECONF=/etc/modprobe.conf
+		[ -f /etc/modprobe.d/Lustre ] && MODPROBECONF=/etc/modprobe.d/Lustre
+
+		LNETOPTS="$(awk '/^options lnet/ { print $0}' $MODPROBECONF | sed 's/^options lnet //g') accept=all" MDS_MOUNT_OPTS="${MDS_MOUNT_OPTS},noacl" $SETUP
 		export LIBLUSTRE_MOUNT_POINT=$MOUNT2
 		export LIBLUSTRE_MOUNT_TARGET=$MGSNID:/$FSNAME
 		export LIBLUSTRE_TIMEOUT=`cat /proc/sys/lustre/timeout`
diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh
index f0536637e3071fc5409232dc89f776c493a067e0..c5aedced855c58713b9f6cdcc206558a1dbe4de8 100644
--- a/lustre/tests/conf-sanity.sh
+++ b/lustre/tests/conf-sanity.sh
@@ -613,7 +613,7 @@ test_15() {
 		echo "save $MOUNTLUSTRE to $MOUNTLUSTRE.sav"
 		mv $MOUNTLUSTRE $MOUNTLUSTRE.sav && trap cleanup_15 EXIT INT
 		if [ -f $MOUNTLUSTRE ]; then
-			echo "$MOUNTLUSTRE cannot be moved, skipping test"
+			skip "$MOUNTLUSTRE cannot be moved, skipping test"
 			return 0
 		fi
 	fi
@@ -985,7 +985,7 @@ test_28() {
 run_test 28 "permanent parameter setting"
 
 test_29() {
-	[ "$OSTCOUNT" -lt "2" ] && echo "skipping deactivate test" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "$OSTCOUNT < 2, skipping" && return
         setup > /dev/null 2>&1
 	start_ost2
 	sleep 10
@@ -1074,9 +1074,9 @@ test_32a() {
         #       there appears to be a lot of assumption here about loopback
         #       devices
         # or maybe this test is just totally useless on a client-only system
-        [ -z "$TUNEFS" ] && echo "No tunefs" && return
-        [ ! -r disk1_4.zip ] && echo "Cant find disk1_4.zip, skipping" && return
-	unzip -o -j -d $TMP/$tdir disk1_4.zip || { echo "Cant unzip disk1_4, skipping" && return ; }
+        [ -z "$TUNEFS" ] && skip "No tunefs" && return
+        [ ! -r disk1_4.zip ] && skip "Cant find disk1_4.zip, skipping" && return
+	unzip -o -j -d $TMP/$tdir disk1_4.zip || { skip "Cant unzip disk1_4, skipping" && return ; }
 	load_modules
 	sysctl lnet.debug=$PTLDEBUG
 
@@ -1123,9 +1123,9 @@ test_32b() {
         #       there appears to be a lot of assumption here about loopback
         #       devices
         # or maybe this test is just totally useless on a client-only system
-        [ -z "$TUNEFS" ] && echo "No tunefs" && return
-        [ ! -r disk1_4.zip ] && echo "Cant find disk1_4.zip, skipping" && return
-	unzip -o -j -d $TMP/$tdir disk1_4.zip || { echo "Cant unzip disk1_4, skipping" && return ; }
+        [ -z "$TUNEFS" ] && skip "No tunefs" && return
+        [ ! -r disk1_4.zip ] && skip "Cant find disk1_4.zip, skipping" && return
+	unzip -o -j -d $TMP/$tdir disk1_4.zip || { skip "Cant unzip disk1_4, skipping" && return ; }
 	load_modules
 	sysctl lnet.debug=$PTLDEBUG
 
@@ -1300,5 +1300,6 @@ test_35() { # bug 12459
 }
 run_test 35 "Reconnect to the last active server first"
 
-equals_msg "Done"
-echo "$0: completed"
+equals_msg `basename $0`: test complete
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
+
diff --git a/lustre/tests/insanity.sh b/lustre/tests/insanity.sh
index fd875b3c812dd9e387e41c820bf4cace8e5b0994..32e6ad94aa5c82307ff6f568610a572b812655e3 100755
--- a/lustre/tests/insanity.sh
+++ b/lustre/tests/insanity.sh
@@ -317,7 +317,7 @@ run_test 4 "Fourth Failure Mode: OST/MDS `date`"
 
 ############### Fifth Failure Mode ###############
 test_5() {
-    [ $OSTCOUNT -lt 1 ] && echo "skipping test_5, not enough OSTs" && return 0
+    [ $OSTCOUNT -lt 1 ] && skip "$OSTCOUNT < 1, not enough OSTs" && return 0
 
     echo "Fifth Failure Mode: OST/OST `date`"
 
@@ -575,7 +575,6 @@ test_10() {
 }
 run_test 10 "Running Availability for 6 hours..."
 
-equals_msg "Done, cleaning up"
+equals_msg `basename $0`: test complete, cleaning up
 $CLEANUP
-echo "$0: completed"
-
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 8adc969fed913607dfa73364374e8a7b60842e3b..89c2acb0b5030b7ac0e644473ad28d9132198566 100755
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -213,7 +213,7 @@ test_17() {
 run_test 17 "timeout bulk get, don't evict client (2732)"
 
 test_18a() {
-    [ -z ${ost2_svc} ] && echo Skipping, needs 2 osts && return 0
+    [ -z ${ost2_svc} ] && skip "needs 2 osts" && return 0
 
     do_facet client mkdir -p $MOUNT/$tdir
     f=$MOUNT/$tdir/$tfile
@@ -563,10 +563,8 @@ run_test 24 "fsync error (should return error)"
 
 test_26() {      # bug 5921 - evict dead exports by pinger
 # this test can only run from a client on a separate node.
-	[ "`lsmod | grep obdfilter`" ] && \
-	    echo "skipping test 26 (local OST)" && return
-	[ "`lsmod | grep mds`" ] && \
-	    echo "skipping test 26 (local MDS)" && return
+	remote_ost || skip "local OST" && return
+	remote_mds || skip "local MDS" && return
 	OST_FILE=$LPROC/obdfilter/${ost1_svc}/num_exports
         OST_EXP="`do_facet ost1 cat $OST_FILE`"
 	OST_NEXP1=`echo $OST_EXP | cut -d' ' -f2`
@@ -611,8 +609,7 @@ test_26b() {      # bug 10140 - evict dead exports by pinger
 run_test 26b "evict dead exports"
 
 test_27() {
-	[ "`lsmod | grep mds`" ] || \
-	    { echo "skipping test 27 (non-local MDS)" && return 0; }
+	remote_mds && { skip "remote MDS" && return 0; }
 	mkdir -p $DIR/$tdir
 	writemany -q -a $DIR/$tdir/$tfile 0 5 &
 	CLIENT_PID=$!
@@ -758,8 +755,7 @@ run_test 54 "back in time"
 
 # bug 11330 - liblustre application death during I/O locks up OST
 test_55() {
-	[ "`lsmod | grep obdfilter`" ] || \
-	    { echo "skipping test 55 (non-local OST)" && return 0; }	
+	remote_ost && { skip "remote OST" && return 0; }
 
 	mkdir -p $DIR/$tdir
 
@@ -877,5 +873,6 @@ test_59() { # bug 10589
 }
 run_test 59 "Read cancel race on client eviction"
 
+equals_msg `basename $0`: test complete, cleaning up
 check_and_cleanup_lustre
-echo "$0: completed"
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index bbd2687ad3f4df2fd7cdf19d254682e56cafd425..ab25f6b3cd3cd9ab7cf3b29384acf0f5a18d5c7a 100755
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -434,9 +434,8 @@ test_19() { # Bug 10991 - resend of open request does not fail assertion.
 }
 run_test 19 "resend of open request"
 
-equals_msg test complete, cleaning up
+equals_msg `basename $0`: test complete, cleaning up
 SLEEP=$((`date +%s` - $NOW))
 [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
 check_and_cleanup_lustre
-
-echo "$0: completed"
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/replay-ost-single.sh b/lustre/tests/replay-ost-single.sh
index 68cf1fe1d3fb07b34938fb1c882f17c7b742d287..c73eb8d8bc9b9c99bd9a213be816261c2933f10b 100755
--- a/lustre/tests/replay-ost-single.sh
+++ b/lustre/tests/replay-ost-single.sh
@@ -199,7 +199,6 @@ test_7() {
 }
 run_test 7 "Fail OST before obd_destroy"
 
-equals_msg test complete, cleaning up
+equals_msg `basename $0`: test complete, cleaning up
 $CLEANUP
-echo "$0: completed"
-
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index c8b02e091e46e2b61c36424ad47e6dc4a6bc630f..da1b1d3d9b726f75fc7033d009d876a6b457b8cd 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -860,7 +860,7 @@ run_test 40 "cause recovery in ptlrpc, ensure IO continues"
 # assert on trying to unlock the unlocked page.
 test_41() {
     [ $OSTCOUNT -lt 2 ] && \
-	echo "skipping test 41: we don't have a second OST to test with" && \
+	skip "skipping test 41: we don't have a second OST to test with" && \
 	return
 
     local f=$MOUNT/$tfile
@@ -1170,5 +1170,5 @@ test_61c() {
 run_test 61c "test race mds llog sync vs llog cleanup"
 
 equals_msg `basename $0`: test complete, cleaning up
-
 check_and_cleanup_lustre
+[ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index bccee987dfe385c2f1c6aaa3aa04fefc06537722..4cc9cf9b1de1cb3ceca9a4de10883c011d83a49d 100644
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -61,6 +61,9 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
+QUOTALOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
+FAIL_ON_ERROR=false
+
 log() {
 	echo "$*"
 	$LCTL mark "$*" 2> /dev/null || true
@@ -126,12 +129,12 @@ run_test() {
  	fi
         testname=EXCEPT_$1
         if [ ${!testname}x != x ]; then
-                 echo "skipping excluded test $1"
+                 TESTNAME=test_$1 skip "skipping excluded test $1"
                  return 0
         fi
         testname=EXCEPT_$base
         if [ ${!testname}x != x ]; then
-                 echo "skipping excluded test $1 (base $base)"
+                 TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
                  return 0
         fi
         run_one $1 "$2"
@@ -140,20 +143,6 @@ run_test() {
 
 [ "$QUOTALOG" ] && rm -f $QUOTALOG || true
 
-error() { 
-	sysctl -w lustre.fail_loc=0
-	log "FAIL: $TESTNAME $@"
-	if [ "$QUOTALOG" ]; then
-		echo "FAIL: $TESTNAME $@" >> $QUOTALOG
-	else
-		exit 1
-	fi
-}
-
-pass() { 
-	echo PASS $@
-}
-
 mounted_lustre_filesystems() {
 	awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts | sed -n ${1}p
 }
@@ -623,7 +612,7 @@ run_test 5 "Chown & chgrp successfully even out of block/file quota ==="
 # block quota acquire & release
 test_6() {
 	if [ $OSTCOUNT -lt 2 ]; then
-		echo "WARN: too few osts, skip this test."
+		skip "$OSTCOUNT < 2, too few osts"
 		return 0;
 	fi
 
@@ -684,10 +673,7 @@ run_test 6 "Block quota acquire & release ========="
 # quota recovery (block quota only by now)
 test_7()
 {
-	if [ -z "`lsmod|grep mds`" ]; then 
-		echo "WARN: no local mds, skip this test"
-		return 0
-	fi
+	remote_mds && skip "remote mds"
 
 	LIMIT=$(( $BUNIT_SZ * $(($OSTCOUNT + 1)) * 10)) # 10 bunits each sever
 	TESTFILE="$TSTDIR/quota_tst70"
@@ -739,7 +725,7 @@ test_8() {
 	FILE_LIMIT=1000000
 	DBENCH_LIB=${DBENCH_LIB:-/usr/lib/dbench}
 	
-	[ ! -d $DBENCH_LIB ] && echo "dbench not installed, skip this test" && return 0
+	[ ! -d $DBENCH_LIB ] && skip "dbench not installed" && return 0
 	
 	echo "  Set enough high limit for user: $TSTUSR"
 	$LFS setquota -u $TSTUSR 0 $BLK_LIMIT 0 $FILE_LIMIT $MOUNT
@@ -770,7 +756,7 @@ test_9() {
         size_file=$((1024 * 1024 * 9 / 2 * $OSTCOUNT))
         echo "lustrefs_size:$lustrefs_size  size_file:$size_file"
         if [ $lustrefs_size -lt $size_file ]; then
-	    echo "WARN: less than $size_file free, skip this test."
+	    skip "less than $size_file free"
 	    return 0;
         fi
 
@@ -818,12 +804,12 @@ test_10() {
 	size_file=$((1024 * 1024 * 9 / 2 * $OSTCOUNT))
 	echo "lustrefs_size:$lustrefs_size  size_file:$size_file"
 	if [ $lustrefs_size -lt $size_file ]; then
-		echo "WARN: less than $size_file free, skip this test."
+		skip "less than $size_file free"
 		return 0;
 	fi
 
 	if [ ! -d /proc/fs/lustre/ost/ -o ! -d /proc/fs/lustre/mds ]; then
-		echo "WARN: mds or ost isn't local, skip this test."
+		skip "mds or ost isn't local"
 		return 0;
 	fi
 
@@ -1134,5 +1120,5 @@ if [ "`mount | grep ^$NAME`" ]; then
 fi
 
 echo '=========================== finished ==============================='
-[ -f "$QUOTALOG" ] && cat $QUOTALOG && exit 1
+[ -f "$QUOTALOG" ] && cat $QUOTALOG && grep -q FAIL $QUOTALOG && exit 1 || true
 echo "$0: completed"
diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 73adfe358c08163895fcb1aeb7ea4760b1fa6b37..785a68556897756b026ef752d22fe4aaffba3dc8 100644
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -73,8 +73,6 @@ else
 	fi
 fi
 
-SANITYLOG=${SANITYLOG:-$TMP/sanity.log}
-
 export NAME=${NAME:-local}
 
 SAVE_PWD=$PWD
@@ -87,6 +85,9 @@ LUSTRE=${LUSTRE:-`dirname $0`/..}
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
+SANITYLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
+FAIL_ON_ERROR=false
+
 cleanup() {
 	echo -n "cln.."
 	cleanupall ${FORCE} $* || { echo "FAILed to clean up"; exit 20; }
@@ -128,7 +129,6 @@ STRIPECOUNT=`cat $LPROC/lov/$LOVNAME/stripecount`
 STRIPESIZE=`cat $LPROC/lov/$LOVNAME/stripesize`
 ORIGFREE=`cat $LPROC/lov/$LOVNAME/kbytesavail`
 MAXFREE=${MAXFREE:-$((200000 * $OSTCOUNT))}
-MDS=$(\ls $LPROC/mds 2> /dev/null | grep -v num_refs | tail -n 1)
 
 [ -f $DIR/d52a/foo ] && chattr -a $DIR/d52a/foo
 [ -f $DIR/d52b/foo ] && chattr -i $DIR/d52b/foo
@@ -927,8 +927,8 @@ exhaust_all_precreations() {
 }
 
 test_27n() {
-	[ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	reset_enospc
 	rm -f $DIR/d27/f27n
@@ -941,8 +941,8 @@ test_27n() {
 run_test 27n "create file with some full OSTs =================="
 
 test_27o() {
-	[ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	reset_enospc
 	rm -f $DIR/d27/f27o
@@ -956,8 +956,8 @@ test_27o() {
 run_test 27o "create file with all full OSTs (should error) ===="
 
 test_27p() {
-	[ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	reset_enospc
 	rm -f $DIR/d27/f27p
@@ -975,8 +975,8 @@ test_27p() {
 run_test 27p "append to a truncated file with some full OSTs ==="
 
 test_27q() {
-	[ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	reset_enospc
 	rm -f $DIR/d27/f27q
@@ -995,8 +995,8 @@ test_27q() {
 run_test 27q "append to truncated file with all OSTs full (should error) ==="
 
 test_27r() {
-	[ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	reset_enospc
 	rm -f $DIR/d27/f27r
@@ -1044,8 +1044,8 @@ test_27x() { # bug 10997
 run_test 27x "check lfs setstripe -c -s -i options ============="
 
 test_27u() { # bug 4900
-        [ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
         #define OBD_FAIL_MDS_OSC_PRECREATE      0x139
 
@@ -1063,8 +1063,8 @@ test_27u() { # bug 4900
 run_test 27u "skip object creation on OSC w/o objects =========="
 
 test_27v() { # bug 4900
-        [ "$OSTCOUNT" -lt "2" -o -z "$MDS" ] && \
-		skip "too few OSTs, or remote MDS" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
         exhaust_all_precreations
 
@@ -1561,7 +1561,7 @@ test_36f() {
 }
 run_test 36f "utime on file racing with OST BRW write =========="
 
-export FMD_MAX_AGE=`cat $LPROC/obdfilter/*/client_cache_seconds 2> /dev/null | head -n 1`
+export FMD_MAX_AGE=`do_facet ost1 cat $LPROC/obdfilter/*/client_cache_seconds 2> /dev/null | head -n 1`
 test_36g() {
 	[ -z "$FMD_MAX_AGE" ] && skip "skip test for remote OST" && return
 	FMD_BEFORE="`awk '/ll_fmd_cache/ { print $2 }' /proc/slabinfo`"
@@ -2155,7 +2155,7 @@ test_52c() { # 12848 simulating client < 1.4.7
 run_test 52c "immutable flag test for client < 1.4.7 ======="
 
 test_53() {
-	[ -z "$MDS" ] && skip "skipping test with remote MDS" && return
+	remote_mds && skip "remote MDS" && return
 	
         for i in `ls -d $LPROC/osc/*-osc 2> /dev/null` ; do
                 ostname=`basename $i | cut -d - -f 1-2`
@@ -2368,7 +2368,8 @@ test_56h() {
 run_test 56h "check lfs find ! -name ============================="
 
 test_57a() {
-	[ -z "$MDS" ] && skip "skipping test for remote MDS" && return
+	remote_mds && skip "remote MDS" && return
+
 	for DEV in `cat $LPROC/mds/*/mntdev`; do
 		dumpe2fs -h $DEV > $TMP/t57a.dump || error "can't access $DEV"
 		DEVISIZE=`awk '/Inode size:/ { print $3 }' $TMP/t57a.dump`
@@ -2673,6 +2674,7 @@ test_67() { # bug 3285 - supplementary group fails on MDS, passes on client
 	chgrp $RUNAS_ID $DIR/$tdir
 	$RUNAS -u $RUNAS_ID -g $(($RUNAS_ID + 1)) -G1,2,$RUNAS_ID ls $DIR/$tdir
 	RC=$?
+	MDS=$(\ls $LPROC/mds 2> /dev/null | grep -v num_refs | tail -n 1)
 	if [ "$MDS" ]; then
 		# can't tell which is correct otherwise
 		GROUP_UPCALL=`cat $LPROC/mds/$MDS/group_upcall`
@@ -2694,7 +2696,7 @@ test_67b() { # bug 3285 - supplementary group fails on MDS, passes on client
 	T67_UID=${T67_UID:-1}	# needs to be in /etc/groups on MDS, gid == uid
 	[ "$UID" = "$T67_UID" ] && skip "UID = T67_UID = $UID -- skipping" && return
 	check_kernel_version 35 || return 0
-	[ -z "$MDS" ] && skip "no MDS" && return
+	remote_mds && skip "remote MDS" && return
 	GROUP_UPCALL=`cat $LPROC/mds/$MDS/group_upcall`
 	[ "$GROUP_UPCALL" != "NONE" ] && skip "skip test - upcall" &&return
 	set -vx
@@ -3740,9 +3742,8 @@ free_min_max () {
 }
 
 test_116() {
-	[ "$OSTCOUNT" -lt "2" ] && echo "not enough OSTs" && return
-	[ $(grep -c obdfilter $LPROC/devices) -eq 0 ] &&
-		skip "remote MDS, skipping test" && return
+	[ "$OSTCOUNT" -lt "2" ] && skip "too few OSTs" && return
+	remote_mds && skip "remote MDS" && return
 
 	echo -n "Free space priority "
 	cat $LPROC/lov/*/qos_prio_free
diff --git a/lustre/tests/sanityN.sh b/lustre/tests/sanityN.sh
index 50dde49148857b3ed37b944437f813217cd7e6c9..5cd12861b07d3e64ed9cb95350820688d9633c98 100644
--- a/lustre/tests/sanityN.sh
+++ b/lustre/tests/sanityN.sh
@@ -49,6 +49,10 @@ SETUP=${SETUP:-:}
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
+SANITYLOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
+FAIL_ON_ERROR=false
+
+SETUP=${SETUP:-:}
 TRACE=${TRACE:-""}
 
 LPROC=/proc/fs/lustre
@@ -326,7 +330,7 @@ test_16() {
 run_test 16 "2500 iterations of dual-mount fsx ================="
 
 test_17() { # bug 3513, 3667
-	[ ! -d /proc/fs/lustre/ost ] && echo "skipping OST-only test" && return
+	[ ! -d /proc/fs/lustre/ost ] && skip "remote OST, skipping OST-only test" && return
 
 	cp /etc/termcap $DIR1/f17
 	cancel_lru_locks osc > /dev/null
@@ -469,7 +473,8 @@ test_24() {
 run_test 24 "lfs df [-ih] [path] test ========================="
 
 test_25() {
-	[ `cat $LPROC/mdc/*-mdc-*/connect_flags | grep -c acl` -lt 2 ] && echo "skipping $TESTNAME (must have acl)" && return
+	[ `cat $LPROC/mdc/*-mdc-*/connect_flags | grep -c acl` -lt 2 ] && \
+	    skip "must have acl, skipping" && return
 
 	mkdir $DIR1/$tdir || error "mkdir $DIR1/$tdir"
 	touch $DIR1/$tdir/f1 || error "touch $DIR1/$tdir/f1"
@@ -583,6 +588,6 @@ log "cleanup: ======================================================"
 check_and_cleanup_lustre
 
 echo '=========================== finished ==============================='
-[ -f "$SANITYLOG" ] && cat $SANITYLOG && exit 1 || true
+[ -f "$SANITYLOG" ] && cat $SANITYLOG && grep -q FAIL $SANITYLOG && exit 1 || true
 echo "$0: completed"
 
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 163470e8cd73477f1f3b11740b6ca61567c3d3cf..79bef78b3ed66996cc9f6ffe1fae1b5db4f1890d 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -38,6 +38,7 @@ init_test_env() {
 
     [ -d /r ] && export ROOT=${ROOT:-/r}
     export TMP=${TMP:-$ROOT/tmp}
+    export TESTSUITELOG=${TMP}/${TESTSUITE}.log
 
     export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests
     export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
@@ -76,6 +77,9 @@ init_test_env() {
 
     shift $((OPTIND - 1))
     ONLY=${ONLY:-$*}
+
+    [ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
+
 }
 
 case `uname -r` in
@@ -863,6 +867,7 @@ debugrestore() {
     DEBUGSAVE=""
 }
 
+FAIL_ON_ERROR=true
 ##################################
 # Test interface 
 error() {
@@ -871,9 +876,20 @@ error() {
     log "${TESTSUITE} ${TESTNAME}: **** FAIL:" $@
     ERRLOG=$TMP/lustre_${TESTSUITE}_${TESTNAME}.$(date +%s)
     echo "Dumping lctl log to $ERRLOG"
+    # We need to dump the logs on all nodes
     $LCTL dk $ERRLOG
+    [ ! "$mds_HOST" = "$(hostname)" ] && do_node $mds_HOST $LCTL dk $ERRLOG
+    [ ! "$ost_HOST" = "$(hostname)" -a ! "$ost_HOST" = "$mds_HOST" ] && do_node $ost_HOST $LCTL dk $ERRLOG
     debugrestore
-    exit 1
+    [ "$TESTSUITELOG" ] && echo "$0: FAIL: $TESTNAME $@" >> $TESTSUITELOG
+    if $FAIL_ON_ERROR; then
+	exit 1
+    fi
+}
+
+skip () {
+	log " SKIP: ${TESTSUITE} ${TESTNAME} $@"
+	[ "$TESTSUITELOG" ] && echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG
 }
 
 build_test_filter() {
@@ -917,12 +933,12 @@ run_test() {
     fi
     testname=EXCEPT_$1
     if [ ${!testname}x != x ]; then
-        log "skipping excluded test $1"
+        TESTNAME=test_$1 skip "skipping excluded test $1"
         return 0
     fi
     testname=EXCEPT_$base
     if [ ${!testname}x != x ]; then
-        log "skipping excluded test $1 (base $base)"
+        TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
         return 0
     fi
     run_one $1 "$2"
@@ -953,11 +969,6 @@ trace() {
 	return 1
 }
 
-skip () {
-        log "$0: SKIP: $TESTNAME $@"
-        [ "$SANITYLOG" ] && echo "$0: SKIP: $TESTNAME $@" >> $SANITYLOG
-} 
-
 pass() {
     echo PASS $@
 }
@@ -1054,3 +1065,13 @@ osc_to_ost()
     fi
     echo $ost
 }
+
+remote_mds ()
+{
+    [ ! -e /proc/fs/lustre/mds/*MDT* ]
+}
+
+remote_ost ()
+{
+    [ $(grep -c obdfilter $LPROC/devices) -eq 0 ]
+}
\ No newline at end of file