diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh
index 4a2b57a86a8c0d5be02b46b30384e87a4c0f954f..ee6ebd769143815e48f0a88b502af35413881d61 100755
--- a/lustre/tests/recovery-small.sh
+++ b/lustre/tests/recovery-small.sh
@@ -15,32 +15,10 @@ build_test_filter
 
 # Allow us to override the setup if we already have a mounted system by
 # setting SETUP=" " and CLEANUP=" "
-SETUP=${SETUP:-"setup"}
-CLEANUP=${CLEANUP:-"cleanup"}
+SETUP=${SETUP:-""}
+CLEANUP=${CLEANUP:-""}
 
-setup() {
-    [ "$REFORMAT" ] && formatall
-    setupall
-}
-
-cleanup() {
-	cleanupall || { echo "FAILed to clean up"; exit 20; }
-}
-
-if [ ! -z "$EVAL" ]; then
-    eval "$EVAL"
-    exit $?
-fi
-
-if [ "$ONLY" == "cleanup" ]; then
-    sysctl -w lnet.debug=0 || true
-    cleanup
-    exit
-fi
-
-$SETUP
-
-[ "$ONLY" == "setup" ] && exit
+cleanup_and_setup_lustre
 
 test_1() {
     drop_request "mcreate $MOUNT/1"  || return 1
@@ -877,5 +855,5 @@ test_58() { # bug 11546
 run_test 58 "Eviction in the middle of open RPC reply processing"
 
 equals_msg `basename $0`: test complete, cleaning up
-$CLEANUP
+check_and_cleanup_lustre
 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh
index 7c5a908930be3646cc996a57a62ebd6933d36e34..f1dba24b48c99aabb47c6bdd7503381cf4091db8 100755
--- a/lustre/tests/replay-dual.sh
+++ b/lustre/tests/replay-dual.sh
@@ -5,37 +5,23 @@ set -e
 # bug number:  6088 10124 10800
 ALWAYS_EXCEPT="8    15c   17    $REPLAY_DUAL_EXCEPT"
 
+SAVE_PWD=$PWD
 PTLDEBUG=${PTLDEBUG:--1}
 LUSTRE=${LUSTRE:-`dirname $0`/..}
+SETUP=${SETUP:-""}
+CLEANUP=${CLEANUP:-""}
+MOUNT_2=${MOUNT_2:-"yes"}
 . $LUSTRE/tests/test-framework.sh
 
 init_test_env $@
 
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
-SETUP=${SETUP:-"setup"}
-CLEANUP=${CLEANUP:-"cleanup"}
 
 build_test_filter
 
-cleanup() {
-    stopall
-}
-
-if [ "$ONLY" == "cleanup" ]; then
-    sysctl -w lnet.debug=0
-    cleanup
-    exit
-fi
-
-setup() {
-    cleanup
-    [ "$REFORMAT" ] && formatall
-    setupall
-    mount_client $MOUNT2
-}
+cleanup_and_setup_lustre
 
-$SETUP
 [ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE
 
 test_1() {
@@ -444,11 +430,9 @@ test_19() { # Bug 10991 - resend of open request does not fail assertion.
 }
 run_test 19 "resend of open request"
 
-if [ "$ONLY" != "setup" ]; then
-   equals_msg `basename $0`: test complete, cleaning up
-   SLEEP=$((`date +%s` - $NOW))
-   [ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
-   $CLEANUP
-fi
+equals_msg `basename $0`: test complete, cleaning up
+SLEEP=$((`date +%s` - $NOW))
+[ $SLEEP -lt $TIMEOUT ] && sleep $SLEEP
+check_and_cleanup_lustre
 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
 
diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 7455edc8b6e7c3fec9c422e16be09d2cdf652df1..2d75155e2f3c78d02304f6f5cb48eeb3514522d6 100755
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -6,52 +6,24 @@ set -e
 #
 # This test needs to be run on the client
 #
-
+SAVE_PWD=$PWD
 LUSTRE=${LUSTRE:-`dirname $0`/..}
+SETUP=${SETUP:-}
+CLEANUP=${CLEANUP:-}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+CHECK_GRANT=${CHECK_GRANT:-"yes"}
+GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 
 
 # Skip these tests
 # bug number: 2766 4176
 ALWAYS_EXCEPT="0b  39   $REPLAY_SINGLE_EXCEPT"
 
-gen_config() {
-    rm -f $XMLCONFIG
-    add_mds $SINGLEMDS --dev $MDSDEV --size $MDSSIZE
-    if [ ! -z "$mdsfailover_HOST" ]; then
-	 add_mdsfailover $SINGLEMDS --dev $MDSDEV --size $MDSSIZE
-    fi
-    
-    add_lov lov1 $SINGLEMDS --stripe_sz $STRIPE_BYTES \
-	--stripe_cnt $STRIPES_PER_OBJ --stripe_pattern 0
-    add_ost ost --lov lov1 --dev `ostdevname 1` --size $OSTSIZE
-    add_ost ost2 --lov lov1 --dev `ostdevname 2` --size $OSTSIZE
-    add_client client $SINGLEMDS --lov lov1 --path $MOUNT
-}
-
 build_test_filter
 
-SETUP=${SETUP:-"setup"}
-CLEANUP=${CLEANUP:-"cleanupall"}
-
-if [ "$ONLY" == "cleanup" ]; then
-    sysctl -w lnet.debug=0 || true
-    $CLEANUP
-    exit 0
-fi
-
-setup() {
-    [ "$REFORMAT" ] && formatall
-    setupall
-}
-
-$SETUP
-
-if [ "$ONLY" == "setup" ]; then
-    exit 0
-fi
+cleanup_and_setup_lustre
 
 mkdir -p $DIR
 
@@ -1201,5 +1173,5 @@ test_60() {
 run_test 60 "test llog post recovery init vs llog unlink"
 
 equals_msg `basename $0`: test complete, cleaning up
-$CLEANUP
+check_and_cleanup_lustre
 [ -f "$TESTSUITELOG" ] && cat $TESTSUITELOG || true
diff --git a/lustre/tests/sanity-quota.sh b/lustre/tests/sanity-quota.sh
index 0e1e3f1c7789195ce58ed56b6aa7e153e3f2ff05..ab43295586c955af61869bab9e74f1c481340c88 100644
--- a/lustre/tests/sanity-quota.sh
+++ b/lustre/tests/sanity-quota.sh
@@ -48,85 +48,14 @@ ITUNE_SZ=${ITUNE_SZ:-5}		# default 50% of IUNIT_SZ
 MAX_DQ_TIME=604800
 MAX_IQ_TIME=604800
 
-QUOTALOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
-FAIL_ON_ERROR=false
-
-log() {
-	echo "$*"
-	$LCTL mark "$*" 2> /dev/null || true
-}
-
-trace() {
-	log "STARTING: $*"
-	strace -o $TMP/$1.strace -ttt $*
-	RC=$?
-	log "FINISHED: $*: rc $RC"
-	return 1
-}
 TRACE=${TRACE:-""}
+LUSTRE=${LUSTRE:-`dirname $0`/..}
+. $LUSTRE/tests/test-framework.sh
+init_test_env $@
+. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
-run_one() {
-	BEFORE=`date +%s`
-	log "== test $1: $2= `date +%H:%M:%S` ($BEFORE)"
-	export TESTNAME=test_$1
-	test_$1 || error "exit with rc=$?"
-	unset TESTNAME
-	pass "($((`date +%s` - $BEFORE))s)"
-	cd $SAVE_PWD
-}
-
-build_test_filter() {
-	[ "$ALWAYS_EXCEPT$EXCEPT$SANITY_EXCEPT" ] && \
-	    echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT $SANITY_QUOTA_EXCEPT`"
-
-        for O in $ONLY; do
-            eval ONLY_${O}=true
-        done
-        for E in $EXCEPT $ALWAYS_EXCEPT $SANITY_QUOTA_EXCEPT; do
-            eval EXCEPT_${E}=true
-        done
-	# turn on/off quota tests must be included
-	eval ONLY_0=true
-	eval ONLY_99=true
-}
-
-_basetest() {
-	echo $*
-}
-
-basetest() {
-	IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
-}
-
-run_test() {
-         base=`basetest $1`
-         if [ "$ONLY" ]; then
-                 testname=ONLY_$1
-                 if [ ${!testname}x != x ]; then
- 			run_one $1 "$2"
- 			return $?
-                 fi
-                 testname=ONLY_$base
-                 if [ ${!testname}x != x ]; then
-                         run_one $1 "$2"
-                         return $?
-                 fi
-                 echo -n "."
-                 return 0
- 	fi
-        testname=EXCEPT_$1
-        if [ ${!testname}x != x ]; then
-                 TESTNAME=test_$1 skip "skipping excluded test $1"
-                 return 0
-        fi
-        testname=EXCEPT_$base
-        if [ ${!testname}x != x ]; then
-                 TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
-                 return 0
-        fi
-        run_one $1 "$2"
- 	return $?
-}
+QUOTALOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
+FAIL_ON_ERROR=false
 
 [ "$QUOTALOG" ] && rm -f $QUOTALOG || true
 
@@ -162,7 +91,6 @@ DIR=${DIR:-$MOUNT}
 [ -z "`echo $DIR | grep $MOUNT`" ] && echo "$DIR not in $MOUNT" && exit 99
 DIR2=${DIR2:-$MOUNT_2}
 
-LPROC=/proc/fs/lustre
 LOVNAME=`cat $LPROC/llite/*/lov/common_name | tail -n 1`
 OSTCOUNT=`cat $LPROC/lov/$LOVNAME/numobd`
 STRIPECOUNT=`cat $LPROC/lov/$LOVNAME/stripecount`
@@ -178,6 +106,8 @@ SHOW_QUOTA_INFO="$LFS quota -t $MOUNT"
 
 build_test_filter
 
+eval ONLY_0=true
+eval ONLY_99=true
 
 # set_blk_tunables(btune_sz)
 set_blk_tunesz() {
diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh
index 75e46b4fab2309464b9c8843edca9abfe7f96644..e1dcda2d2d755b91f7d54362019c23973fe80542 100644
--- a/lustre/tests/sanity.sh
+++ b/lustre/tests/sanity.sh
@@ -60,6 +60,8 @@ DIRECTIO=${DIRECTIO:-directio}
 ACCEPTOR_PORT=${ACCEPTOR_PORT:-988}
 UMOUNT=${UMOUNT:-"umount -d"}
 STRIPES_PER_OBJ=-1
+CHECK_GRANT=${CHECK_GRANT:-"yes"}
+GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 
 if [ $UID -ne 0 ]; then
 	echo "Warning: running as non-root uid $UID"
@@ -80,6 +82,9 @@ export NAME=${NAME:-local}
 
 SAVE_PWD=$PWD
 
+CLEANUP=${CLEANUP:-:}
+SETUP=${SETUP:-:}
+TRACE=${TRACE:-""}
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
 init_test_env $@
@@ -97,29 +102,12 @@ cleanup() {
 	echo -n "cln.."
 	cleanupall ${FORCE} $* || { echo "FAILed to clean up"; exit 20; }
 }
-CLEANUP=${CLEANUP:-:}
-
 setup() {
 	echo -n "mnt.."
         load_modules
 	setupall || exit 10
 	echo "done"
 }
-SETUP=${SETUP:-:}
-
-log() {
-	echo "$*"
-	$LCTL mark "$*" 2> /dev/null || true
-}
-
-trace() {
-	log "STARTING: $*"
-	strace -o $TMP/$1.strace -ttt $*
-	RC=$?
-	log "FINISHED: $*: rc $RC"
-	return 1
-}
-TRACE=${TRACE:-""}
 
 check_kernel_version() {
 	VERSION_FILE=$LPROC/version
@@ -132,96 +120,14 @@ check_kernel_version() {
 	return 1
 }
 
-_basetest() {
-    echo $*
-}
-
-basetest() {
-    IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
-}
-
-run_one() {
-	if ! grep -q $DIR /proc/mounts; then
-		$SETUP
-	fi
-	testnum=$1
-	message=$2
-	BEFORE=`date +%s`
-	log "== test $testnum: $message= `date +%H:%M:%S` ($BEFORE)"
-	export TESTNAME=test_$testnum
-	export tfile=f${testnum}
-	export tdir=d${base}
-	test_${testnum} || error "exit with rc=$?"
-	unset TESTNAME
-	pass "($((`date +%s` - $BEFORE))s)"
-	cd $SAVE_PWD
-	$CLEANUP
-}
-
-build_test_filter() {
-	[ "$ALWAYS_EXCEPT$EXCEPT$SANITY_EXCEPT" ] && \
-	    echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT $SANITY_EXCEPT`"
-
-        for O in $ONLY; do
-            eval ONLY_${O}=true
-        done
-        for E in $EXCEPT $ALWAYS_EXCEPT $SANITY_EXCEPT; do
-            eval EXCEPT_${E}=true
-        done
-}
-
-_basetest() {
-	echo $*
-}
-
-basetest() {
-	IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
-}
-
-run_test() {
-         export base=`basetest $1`
-         if [ "$ONLY" ]; then
-                 testname=ONLY_$1
-                 if [ ${!testname}x != x ]; then
- 			run_one $1 "$2"
- 			return $?
-                 fi
-                 testname=ONLY_$base
-                 if [ ${!testname}x != x ]; then
-                         run_one $1 "$2"
-                         return $?
-                 fi
-                 echo -n "."
-                 return 0
- 	fi
-        testname=EXCEPT_$1
-        if [ ${!testname}x != x ]; then
-                 TESTNAME=test_$1 skip "skipping excluded test $1"
-                 return 0
-        fi
-        testname=EXCEPT_$base
-        if [ ${!testname}x != x ]; then
-                 TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
-                 return 0
-        fi
-        run_one $1 "$2"
- 	return $?
-}
+if [ "$ONLY" == "cleanup" ]; then
+       sh llmountcleanup.sh
+       exit 0
+fi
 
 [ "$SANITYLOG" ] && rm -f $SANITYLOG || true
 
-mounted_lustre_filesystems() {
-	awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
-}
-
-MOUNTED="`mounted_lustre_filesystems`"
-if [ -z "$MOUNTED" ]; then
-        formatall
-	setupall
-	MOUNTED="`mounted_lustre_filesystems`"
-	[ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"
-	I_MOUNTED=yes
-fi
+check_and_setup_lustre
 
 DIR=${DIR:-$MOUNT}
 [ -z "`echo $DIR | grep $MOUNT`" ] && echo "$DIR not in $MOUNT" && exit 99
@@ -4235,14 +4141,11 @@ TMP=$OLDTMP
 HOME=$OLDHOME
 
 log "cleanup: ======================================================"
-if [ "`mount | grep $MOUNT`" ]; then
-    rm -rf $DIR/[Rdfs][1-9]*
-fi
-if [ "$I_MOUNTED" = "yes" ]; then
-    cleanupall -f || error "cleanup failed"
+check_and_cleanup_lustre
+if [ "$I_MOUNTED" != "yes" ]; then
+	sysctl -w lnet.debug="$OLDDEBUG" 2> /dev/null || true
 fi
 
-
 echo '=========================== finished ==============================='
 [ -f "$SANITYLOG" ] && cat $SANITYLOG && grep -q FAIL $SANITYLOG && exit 1 || true
 echo "$0: completed"
diff --git a/lustre/tests/sanityN.sh b/lustre/tests/sanityN.sh
index e1d00aa459a27810979856c0c527e850d91d1fe6..beeeba3823a357c356f8c39ffae596b6d6a47ca5 100644
--- a/lustre/tests/sanityN.sh
+++ b/lustre/tests/sanityN.sh
@@ -24,16 +24,15 @@ CHECKSTAT=${CHECKSTAT:-"checkstat -v"}
 CREATETEST=${CREATETEST:-createtest}
 GETSTRIPE=${GETSTRIPE:-lfs getstripe}
 SETSTRIPE=${SETSTRIPE:-lstripe}
-LFS=${LFS:-lfs}
-LCTL=${LCTL:-lctl}
 MCREATE=${MCREATE:-mcreate}
 OPENFILE=${OPENFILE:-openfile}
 OPENUNLINK=${OPENUNLINK:-openunlink}
 TOEXCL=${TOEXCL:-toexcl}
 TRUNCATE=${TRUNCATE:-truncate}
 export TMP=${TMP:-/tmp}
-CHECK_GRANT=${CHECK_GRANT:-"no"}
-
+MOUNT_2=${MOUNT_2:-"yes"}
+CHECK_GRANT=${CHECK_GRANT:-"yes"}
+GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
 
 if [ $UID -ne 0 ]; then
 	RUNAS_ID="$UID"
@@ -49,6 +48,8 @@ export NAME=${NAME:-local}
 
 LUSTRE=${LUSTRE:-`dirname $0`/..}
 . $LUSTRE/tests/test-framework.sh
+CLEANUP=${CLEANUP:-:}
+SETUP=${SETUP:-:}
 init_test_env $@
 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
 
@@ -59,213 +60,14 @@ if [ ! -z "$USING_KRB5" ]; then
     $RUNAS krb5_login.sh || exit 1
 fi
 
-cleanup() {
-	echo -n "cln.."
-	grep " $MOUNT2 " /proc/mounts && zconf_umount `hostname` $MOUNT2 ${FORCE}
-	cleanupall ${FORCE} > /dev/null || { echo "FAILed to clean up"; exit 20; }
-}
-CLEANUP=${CLEANUP:-:}
-
-setup() {
-	echo -n "mnt.."
-	setupall || exit 10
-	echo "done"
-}
 SETUP=${SETUP:-:}
-
-log() {
-	echo "$*"
-	lctl mark "$*" 2> /dev/null || true
-}
-
-trace() {
-	log "STARTING: $*"
-	strace -o $TMP/$1.strace -ttt $*
-	RC=$?
-	log "FINISHED: $*: rc $RC"
-	return 1
-}
 TRACE=${TRACE:-""}
 
 LPROC=/proc/fs/lustre
 
-run_one() {
-	if ! grep -q $DIR /proc/mounts; then
-		$SETUP
-	fi
-	testnum=$1
-	message=$2
-	BEFORE=`date +%s`
-	log "== test $testnum: $message= `date +%H:%M:%S` ($BEFORE)"
-	export TESTNAME=test_$testnum
-	export tfile=f${testnum}
-	export tdir=d${base}
-	test_$1 || error "exit with rc=$?"
-	check_grant || error "check grant fail"
-	unset TESTNAME
-	pass "($((`date +%s` - $BEFORE))s)"
-	cd $SAVE_PWD
-	$CLEANUP
-}
-
-build_test_filter() {
-	[ "$ALWAYS_EXCEPT$EXCEPT$SANITYN_EXCEPT" ] && \
-	    echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT $SANITYN_EXCEPT`"
-
-        for O in $ONLY; do
-            eval ONLY_${O}=true
-        done
-        for E in $EXCEPT $ALWAYS_EXCEPT $SANITY_EXCEPT; do
-            eval EXCEPT_${E}=true
-        done
-}
-
-_basetest() {
-    echo $*
-}
-
-basetest() {
-    IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
-}
-
-build_test_filter() {
-	[ "$ALWAYS_EXCEPT$EXCEPT$SANITYN_EXCEPT" ] && \
-	    echo "Skipping tests: `echo $ALWAYS_EXCEPT $EXCEPT $SANITYN_EXCEPT`"
-
-        for O in $ONLY; do
-            eval ONLY_${O}=true
-        done
-        for E in $EXCEPT $ALWAYS_EXCEPT $SANITYN_EXCEPT; do
-            eval EXCEPT_${E}=true
-        done
-}
-
-_basetest() {
-    echo $*
-}
-
-basetest() {
-    IFS=abcdefghijklmnopqrstuvwxyz _basetest $1
-}
-
-sync_clients() {
-	cd $DIR1
-	sync; sleep 1; sync
-	cd $DIR2
-	sync; sleep 1; sync
-
-	cd $SAVE_PWD
-}
-
-check_grant() {
-	[ "$CHECK_GRANT" == "no" ] && return 0
-
-	echo -n "checking grant......"
-	cd $SAVE_PWD
-	# write some data to sync client lost_grant
-	rm -f $DIR1/${tfile}_check_grant_* 2>&1
-	for i in `seq $OSTCOUNT`; do
-		$LFS setstripe $DIR1/${tfile}_check_grant_$i 0 $(($i -1)) 1
-		dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
-					      count=1 > /dev/null 2>&1 
-	done
-	# sync all the data and make sure no pending data on server
-	sync_clients
-	
-	#get client grant and server grant 
-	client_grant=0
-       	for d in /proc/fs/lustre/osc/*/cur_grant_bytes; do 
-		client_grant=$(($client_grant + `cat $d`))
-	done
-	server_grant=0
-	for d in /proc/fs/lustre/obdfilter/*/tot_granted; do
-		server_grant=$(($server_grant + `cat $d`))
-	done
-
-	# cleanup the check_grant file
-	for i in `seq $OSTCOUNT`; do
-	        rm $DIR1/${tfile}_check_grant_$i
-	done
-
-	#check whether client grant == server grant 
-	if [ $client_grant != $server_grant ]; then
-		echo "failed: client:${client_grant} server: ${server_grant}"
-		return 1
-	else
-		echo "pass"
-	fi
-}
-
-run_test() {
-         export base=`basetest $1`
-         if [ "$ONLY" ]; then
-                 testname=ONLY_$1
-                 if [ ${!testname}x != x ]; then
- 			run_one $1 "$2"
- 			return $?
-                 fi
-                 testname=ONLY_$base
-                 if [ ${!testname}x != x ]; then
-                         run_one $1 "$2"
-                         return $?
-                 fi
-                 echo -n "."
-                 return 0
- 	fi
-        testname=EXCEPT_$1
-        if [ ${!testname}x != x ]; then
-                 echo "skipping excluded test $1"
-                 return 0
-        fi
-        testname=EXCEPT_$base
-        if [ ${!testname}x != x ]; then
-                 echo "skipping excluded test $1 (base $base)"
-                 return 0
-        fi
-        run_one $1 "$2"
- 	return $?
-}
-
 [ "$SANITYLOG" ] && rm -f $SANITYLOG || true
 
-error () {
-	sysctl -w lustre.fail_loc=0 2> /dev/null || true
-	log "$0: FAIL: $TESTNAME $@"
-	$LCTL dk $TMP/lustre-log-$TESTNAME.log
-        if [ "$SANITYLOG" ]; then
-		echo "$0: FAIL: $TESTNAME $@" >> $SANITYLOG
-	else
-		exit 1
-	fi
-}
-
-pass() {
-	echo PASS $@
-}
-
-mounted_lustre_filesystems() {
-	awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
-}
-MOUNTED="`mounted_lustre_filesystems`"
-if [ -z "$MOUNTED" ]; then
-    formatall
-    setupall
-    mount_client $MOUNT2
-    MOUNTED="`mounted_lustre_filesystems`"
-    [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"
-    I_MOUNTED=yes
-fi
-export MOUNT1=`mounted_lustre_filesystems | head -n 1`
-[ -z "$MOUNT1" ] && error "NAME=$NAME not mounted once"
-export MOUNT2=`mounted_lustre_filesystems | tail -n 1`
-[ "$MOUNT1" = "$MOUNT2" ] && error "NAME=$NAME not mounted twice"
-[ `mounted_lustre_filesystems | wc -l` -ne 2 ] && \
-	error "NAME=$NAME mounted more than twice"
-
-export DIR1=${DIR1:-$MOUNT1}
-export DIR2=${DIR2:-$MOUNT2}
-[ -z "`echo $DIR1 | grep $MOUNT1`" ] && echo "$DIR1 not in $MOUNT1" && exit 96
-[ -z "`echo $DIR2 | grep $MOUNT2`" ] && echo "$DIR2 not in $MOUNT2" && exit 95
+check_and_setup_lustre
 
 LPROC=/proc/fs/lustre
 LOVNAME=`cat $LPROC/llite/*/lov/common_name | tail -n 1`
@@ -805,10 +607,8 @@ test_30() { #bug #11110
 run_test 30 "recreate file race ========="
 
 log "cleanup: ======================================================"
-rm -rf $DIR1/[df][0-9]* $DIR1/lnk || true
-if [ "$I_MOUNTED" = "yes" ]; then
-    cleanup
-fi
+
+check_and_cleanup_lustre
 
 echo '=========================== finished ==============================='
 [ -f "$SANITYLOG" ] && cat $SANITYLOG && grep -q FAIL $SANITYLOG && exit 1 || true
diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh
index 4b28769ca25d675e59d63a3393d3e7e7638bc35f..b7d5e138a406e483047465d39ca81c54d0122c2a 100644
--- a/lustre/tests/test-framework.sh
+++ b/lustre/tests/test-framework.sh
@@ -42,6 +42,7 @@ init_test_env() {
 
     export PATH=:$PATH:$LUSTRE/utils:$LUSTRE/utils/gss:$LUSTRE/tests
     export LCTL=${LCTL:-"$LUSTRE/utils/lctl"}
+    export LFS=${LFS:-"$LUSTRE/utils/lfs"}
     [ ! -f "$LCTL" ] && export LCTL=$(which lctl) 
     export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"}
     [ ! -f "$MKFS" ] && export MKFS=$(which mkfs.lustre) 
@@ -69,6 +70,7 @@ init_test_env() {
     # Paths on remote nodes, if different 
     export RLUSTRE=${RLUSTRE:-$LUSTRE}
     export RPWD=${RPWD:-$PWD}
+    export I_MOUNTED=${I_MOUNTED:-"no"}
 
     # command line
     
@@ -808,6 +810,44 @@ setupall() {
     sleep 5
 }
 
+mounted_lustre_filesystems() {
+	awk '($3 ~ "lustre" && $1 ~ ":") { print $2 }' /proc/mounts
+}
+
+check_and_setup_lustre() {
+    MOUNTED="`mounted_lustre_filesystems`"
+    if [ -z "$MOUNTED" ]; then
+        [ "$REFORMAT" ] && formatall
+        setupall
+        MOUNTED="`mounted_lustre_filesystems`"
+        [ -z "$MOUNTED" ] && error "NAME=$NAME not mounted"
+        export I_MOUNTED=yes
+    fi
+    if [ "$ONLY" == "setup" ]; then
+        exit 0
+    fi
+}
+
+cleanup_and_setup_lustre() {
+    if [ "$ONLY" == "cleanup" -o "`mount | grep $MOUNT`" ]; then
+        sysctl -w lnet.debug=0 || true
+        cleanupall
+        if [ "$ONLY" == "cleanup" ]; then 
+    	    exit 0
+        fi
+    fi
+    check_and_setup_lustre
+}
+
+check_and_cleanup_lustre() {
+    if [ "`mount | grep $MOUNT`" ]; then
+        rm -rf $DIR/[Rdfs][1-9]*
+    fi
+    if [ "$I_MOUNTED" = "yes" ]; then
+        cleanupall -f || error "cleanup failed"
+    fi
+    unset I_MOUNTED
+}
 
 ####### 
 # General functions
@@ -983,6 +1023,9 @@ build_test_filter() {
     for E in $EXCEPT $ALWAYS_EXCEPT; do
         eval EXCEPT_${E}=true
     done
+    for G in $GRANT_CHECK_LIST; do
+        eval GCHECK_ONLY_${G}=true
+   	done
 }
 
 _basetest() {
@@ -1039,6 +1082,14 @@ log() {
     $LCTL mark "$*" 2> /dev/null || true
 }
 
+trace() {
+	log "STARTING: $*"
+	strace -o $TMP/$1.strace -ttt $*
+	RC=$?
+	log "FINISHED: $*: rc $RC"
+	return 1
+}
+
 pass() {
     echo PASS $@
 }
@@ -1055,25 +1106,74 @@ run_one() {
     tfile=f${testnum}
     tdir=d${base}
 
-    # Pretty tests run faster.
-    equals_msg $testnum: $message
-
     BEFORE=`date +%s`
     log "== test $testnum: $message ============ `date +%H:%M:%S` ($BEFORE)"
     #check_mds
     export TESTNAME=test_$testnum
     test_${testnum} || error "test_$testnum failed with $?"
     #check_mds
+    check_grant ${testnum} || error "check_grant $testnum failed with $?"
     [ -f $CATASTROPHE ] && [ `cat $CATASTROPHE` -ne 0 ] && \
         error "LBUG/LASSERT detected"
     pass "($((`date +%s` - $BEFORE))s)"
     unset TESTNAME
+    cd $SAVE_PWD
+    $CLEANUP
 }
 
 canonical_path() {
     (cd `dirname $1`; echo $PWD/`basename $1`)
 }
 
+sync_clients() {
+    [ -d $DIR1 ] && cd $DIR1 && sync; sleep 1; sync 
+    [ -d $DIR2 ] && cd $DIR2 && sync; sleep 1; sync 
+	cd $SAVE_PWD
+}
+
+check_grant() {
+    export base=`basetest $1`
+    [ "$CHECK_GRANT" == "no" ] && return 0
+
+	testname=GCHECK_ONLY_${base}
+        [ ${!testname}x == x ] && return 0
+
+	echo -n "checking grant......"
+	cd $SAVE_PWD
+	# write some data to sync client lost_grant
+	rm -f $DIR1/${tfile}_check_grant_* 2>&1
+	for i in `seq $OSTCOUNT`; do
+		$LFS setstripe $DIR1/${tfile}_check_grant_$i 0 $(($i -1)) 1
+		dd if=/dev/zero of=$DIR1/${tfile}_check_grant_$i bs=4k \
+					      count=1 > /dev/null 2>&1 
+	done
+	# sync all the data and make sure no pending data on server
+	sync_clients
+	
+	#get client grant and server grant 
+	client_grant=0
+    for d in ${LPROC}/osc/*/cur_grant_bytes; do 
+		client_grant=$((client_grant + `cat $d`))
+	done
+	server_grant=0
+	for d in ${LPROC}/obdfilter/*/tot_granted; do
+		server_grant=$((server_grant + `cat $d`))
+	done
+
+	# cleanup the check_grant file
+	for i in `seq $OSTCOUNT`; do
+	        rm $DIR1/${tfile}_check_grant_$i
+	done
+
+	#check whether client grant == server grant 
+	if [ $client_grant != $server_grant ]; then
+		echo "failed: client:${client_grant} server: ${server_grant}"
+		return 1
+	else
+		echo "pass"
+	fi
+}
+
 ########################
 # helper functions