Newer
Older
#
# This test needs to be run on the client
#
. $LUSTRE/tests/test-framework.sh
init_test_env $@
CHECK_GRANT=${CHECK_GRANT:-"yes"}
GRANT_CHECK_LIST=${GRANT_CHECK_LIST:-""}
# 63 min 7 min AT AT AT AT"
[ "$SLOW" = "no" ] && EXCEPT_SLOW="1 2 3 4 6 12 16 44a 44b 65 66 67 68"
build_test_filter
test_0b() {
# this test attempts to trigger a race in the precreation code,
# and must run before any other objects are created on the filesystem
createmany -o $DIR/$tfile 20 || return 1
unlinkmany $DIR/$tfile 20 || return 2
}
run_test 0b "ensure object created after recover exists. (3284)"
$CHECKSTAT -t file $DIR/$tfile || return 1
rm $DIR/$tfile
run_test 1 "simple create"
$CHECKSTAT -t file $DIR/$tfile || return 1
rm $DIR/$tfile
$CHECKSTAT -t file $DIR/$tfile || return 1
rm $DIR/$tfile
mcreate $DIR/$tfile
o_directory $DIR/$tfile
$CHECKSTAT -t file $DIR/$tfile || return 2
rm $DIR/$tfile
run_test 3a "replay failed open(O_DIRECTORY)"
test_3b() {
#define OBD_FAIL_MDS_OPEN_PACK | OBD_FAIL_ONCE
$CHECKSTAT -t file $DIR/$tfile && return 2
return 0
}
run_test 3b "replay failed open -ENOMEM"
test_3c() {
#define OBD_FAIL_MDS_ALLOC_OBDO | OBD_FAIL_ONCE
$CHECKSTAT -t file $DIR/$tfile && return 2
return 0
}
run_test 3c "replay failed open -ENOMEM"
for i in `seq 10`; do
echo "tag-$i" > $DIR/$tfile-$i
done
for i in `seq 10`; do
grep -q "tag-$i" $DIR/$tfile-$i || error "$tfile-$i"
rm -rf $DIR/$tfile-*
$CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
}
run_test 4b "|x| rm 10 files"
# The idea is to get past the first block of precreated files on both
# osts, and then replay.
test_5() {
for i in `seq 220`; do
echo "tag-$i" > $DIR/$tfile-$i
done
for i in `seq 220`; do
grep -q "tag-$i" $DIR/$tfile-$i || error "f1c-$i"
done
rm -rf $DIR/$tfile-*
sleep 3
# waiting for commitment of removal
}
run_test 5 "|x| 220 open(O_CREAT)"
mcreate $DIR/$tdir/$tfile
$CHECKSTAT -t dir $DIR/$tdir || return 1
$CHECKSTAT -t file $DIR/$tdir/$tfile || return 2
sleep 2
# waiting for log process thread
}
$CHECKSTAT -t dir $DIR/$tdir && return 1 || true
}
run_test 6b "|X| rmdir"
test_7() {
mcreate $DIR/$tdir/$tfile
$CHECKSTAT -t dir $DIR/$tdir || return 1
$CHECKSTAT -t file $DIR/$tdir/$tfile || return 2
rm -fr $DIR/$tdir
}
run_test 7 "mkdir |X| contained create"
test_8() {
multiop_bg_pause $DIR/$tfile mo_c || return 4
ls $DIR/$tfile
$CHECKSTAT -t file $DIR/$tfile || return 1
kill -USR1 $MULTIPID || return 2
wait $MULTIPID || return 3
rm $DIR/$tfile
}
run_test 8 "creat open |X| close"
test_9() {
mcreate $DIR/$tfile
local old_inum=`ls -i $DIR/$tfile | awk '{print $1}'`
local new_inum=`ls -i $DIR/$tfile | awk '{print $1}'`
echo " old_inum == $old_inum, new_inum == $new_inum"
if [ $old_inum -eq $new_inum ] ;
then
echo " old_inum and new_inum match"
else
echo "!!!! old_inum and new_inum NOT match"
return 1
fi
rm $DIR/$tfile
}
run_test 9 "|X| create (same inum/gen)"
test_10() {
mcreate $DIR/$tfile
mv $DIR/$tfile $DIR/$tfile-2
rm -f $DIR/$tfile
$CHECKSTAT $DIR/$tfile && return 1
rm $DIR/$tfile-2
return 0
}
run_test 10 "create |X| rename unlink"
test_11() {
mcreate $DIR/$tfile
echo "old" > $DIR/$tfile
mv $DIR/$tfile $DIR/$tfile-2
echo "new" > $DIR/$tfile
grep new $DIR/$tfile
grep old $DIR/$tfile-2
grep new $DIR/$tfile || return 1
grep old $DIR/$tfile-2 || return 2
}
run_test 11 "create open write rename |X| create-old-name read"
test_12() {
mcreate $DIR/$tfile
multiop_bg_pause $DIR/$tfile o_tSc || return 3
pid=$!
rm -f $DIR/$tfile
kill -USR1 $pid
wait $pid || return 1
[ -e $DIR/$tfile ] && return 2
return 0
}
run_test 12 "open, unlink |X| close"
# 1777 - replay open after committed chmod that would make
# a regular open a failure
test_13() {
mcreate $DIR/$tfile
multiop_bg_pause $DIR/$tfile O_wc || return 3
pid=$!
chmod 0 $DIR/$tfile
$CHECKSTAT -p 0 $DIR/$tfile
kill -USR1 $pid
wait $pid || return 1
$CHECKSTAT -s 1 -p 0 $DIR/$tfile || return 2
return 0
}
run_test 13 "open chmod 0 |x| write close"
test_14() {
multiop_bg_pause $DIR/$tfile O_tSc || return 4
pid=$!
rm -f $DIR/$tfile
kill -USR1 $pid || return 1
wait $pid || return 2
[ -e $DIR/$tfile ] && return 3
return 0
}
run_test 14 "open(O_CREAT), unlink |X| close"
test_15() {
multiop_bg_pause $DIR/$tfile O_tSc || return 5
pid=$!
rm -f $DIR/$tfile
touch $DIR/g11 || return 1
kill -USR1 $pid
wait $pid || return 2
[ -e $DIR/$tfile ] && return 3
touch $DIR/h11 || return 4
return 0
}
run_test 15 "open(O_CREAT), unlink |X| touch new, close"
test_16() {
mcreate $DIR/$tfile
munlink $DIR/$tfile
mcreate $DIR/$tfile-2
[ -e $DIR/$tfile ] && return 1
[ -e $DIR/$tfile-2 ] || return 2
munlink $DIR/$tfile-2 || return 3
}
run_test 16 "|X| open(O_CREAT), unlink, touch new, unlink new"
test_17() {
multiop_bg_pause $DIR/$tfile O_c || return 4
kill -USR1 $pid || return 1
wait $pid || return 2
$CHECKSTAT -t file $DIR/$tfile || return 3
rm $DIR/$tfile
}
run_test 17 "|X| open(O_CREAT), |replay| close"
test_18() {
multiop_bg_pause $DIR/$tfile O_tSc || return 8
pid=$!
rm -f $DIR/$tfile
touch $DIR/$tfile-2 || return 1
kill -USR1 $pid
wait $pid || return 2
[ -e $DIR/$tfile ] && return 3
[ -e $DIR/$tfile-2 ] || return 4
# this touch frequently fails
touch $DIR/$tfile-3 || return 5
munlink $DIR/$tfile-2 || return 6
munlink $DIR/$tfile-3 || return 7
return 0
}
run_test 18 "|X| open(O_CREAT), unlink, touch new, close, touch, unlink"
# bug 1855 (a simpler form of test_11 above)
test_19() {
mcreate $DIR/$tfile
echo "old" > $DIR/$tfile
mv $DIR/$tfile $DIR/$tfile-2
grep old $DIR/$tfile-2
grep old $DIR/$tfile-2 || return 2
}
run_test 19 "|X| mcreate, open, write, rename "
multiop_bg_pause $DIR/$tfile O_tSc || return 3
pid=$!
rm -f $DIR/$tfile
kill -USR1 $pid
wait $pid || return 1
[ -e $DIR/$tfile ] && return 2
return 0
}
run_test 20a "|X| open(O_CREAT), unlink, replay, close (test mds_cleanup_orphans)"
test_20b() { # bug 10480
BEFOREUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'`
dd if=/dev/zero of=$DIR/$tfile bs=4k count=10000 &
pid=$!
while [ ! -e $DIR/$tfile ] ; do
done
lfs getstripe $DIR/$tfile || return 1
rm -f $DIR/$tfile || return 2 # make it an orphan
mds_evict_client
df -P $DIR || df -P $DIR || true # reconnect
fail mds # start orphan recovery
df -P $DIR || df -P $DIR || true # reconnect
AFTERUSED=`df -P $DIR | tail -1 | awk '{ print $3 }'`
log "before $BEFOREUSED, after $AFTERUSED"
[ $AFTERUSED -gt $((BEFOREUSED + 20)) ] && \
error "after $AFTERUSED > before $BEFOREUSED" && return 5
return 0
}
run_test 20b "write, unlink, eviction, replay, (test mds_cleanup_orphans)"
test_20c() { # bug 10480
multiop $DIR/$tfile Ow_c &
pid=$!
# give multiop a chance to open
sleep 1
ls -la $DIR/$tfile
mds_evict_client
df -P $DIR || df -P $DIR || true # reconnect
test -s $DIR/$tfile || error "File was truncated"
return 0
}
run_test 20c "check that client eviction does not affect file content"
multiop_bg_pause $DIR/$tfile O_tSc || return 5
pid=$!
rm -f $DIR/$tfile
touch $DIR/g11 || return 1
kill -USR1 $pid
wait $pid || return 2
[ -e $DIR/$tfile ] && return 3
touch $DIR/h11 || return 4
return 0
}
run_test 21 "|X| open(O_CREAT), unlink touch new, replay, close (test mds_cleanup_orphans)"
test_22() {
multiop_bg_pause $DIR/$tfile O_tSc || return 3
rm -f $DIR/$tfile
kill -USR1 $pid
wait $pid || return 1
[ -e $DIR/$tfile ] && return 2
return 0
}
run_test 22 "open(O_CREAT), |X| unlink, replay, close (test mds_cleanup_orphans)"
test_23() {
multiop_bg_pause $DIR/$tfile O_tSc || return 5
rm -f $DIR/$tfile
touch $DIR/g11 || return 1
kill -USR1 $pid
wait $pid || return 2
[ -e $DIR/$tfile ] && return 3
touch $DIR/h11 || return 4
return 0
}
run_test 23 "open(O_CREAT), |X| unlink touch new, replay, close (test mds_cleanup_orphans)"
test_24() {
multiop_bg_pause $DIR/$tfile O_tSc || return 3
rm -f $DIR/$tfile
kill -USR1 $pid
wait $pid || return 1
[ -e $DIR/$tfile ] && return 2
return 0
}
run_test 24 "open(O_CREAT), replay, unlink, close (test mds_cleanup_orphans)"
test_25() {
multiop_bg_pause $DIR/$tfile O_tSc || return 3
pid=$!
rm -f $DIR/$tfile
kill -USR1 $pid
wait $pid || return 1
[ -e $DIR/$tfile ] && return 2
return 0
}
run_test 25 "open(O_CREAT), unlink, replay, close (test mds_cleanup_orphans)"
test_26() {
multiop_bg_pause $DIR/$tfile-1 O_tSc || return 5
multiop_bg_pause $DIR/$tfile-2 O_tSc || return 6
pid2=$!
rm -f $DIR/$tfile-1
rm -f $DIR/$tfile-2
kill -USR1 $pid2
wait $pid2 || return 1
kill -USR1 $pid1
wait $pid1 || return 2
[ -e $DIR/$tfile-1 ] && return 3
[ -e $DIR/$tfile-2 ] && return 4
return 0
}
run_test 26 "|X| open(O_CREAT), unlink two, close one, replay, close one (test mds_cleanup_orphans)"
test_27() {
multiop_bg_pause $DIR/$tfile-1 O_tSc || return 5
multiop_bg_pause $DIR/$tfile-2 O_tSc || return 6
pid2=$!
rm -f $DIR/$tfile-1
rm -f $DIR/$tfile-2
kill -USR1 $pid1
wait $pid1 || return 1
kill -USR1 $pid2
wait $pid2 || return 2
[ -e $DIR/$tfile-1 ] && return 3
[ -e $DIR/$tfile-2 ] && return 4
return 0
}
run_test 27 "|X| open(O_CREAT), unlink two, replay, close two (test mds_cleanup_orphans)"
test_28() {
multiop_bg_pause $DIR/$tfile-1 O_tSc || return 5
multiop_bg_pause $DIR/$tfile-2 O_tSc || return 6
rm -f $DIR/$tfile-1
rm -f $DIR/$tfile-2
kill -USR1 $pid2
wait $pid2 || return 1
kill -USR1 $pid1
wait $pid1 || return 2
[ -e $DIR/$tfile-1 ] && return 3
[ -e $DIR/$tfile-2 ] && return 4
return 0
}
run_test 28 "open(O_CREAT), |X| unlink two, close one, replay, close one (test mds_cleanup_orphans)"
test_29() {
multiop_bg_pause $DIR/$tfile-1 O_tSc || return 5
multiop_bg_pause $DIR/$tfile-2 O_tSc || return 6
rm -f $DIR/$tfile-1
rm -f $DIR/$tfile-2
kill -USR1 $pid1
wait $pid1 || return 1
kill -USR1 $pid2
wait $pid2 || return 2
[ -e $DIR/$tfile-1 ] && return 3
[ -e $DIR/$tfile-2 ] && return 4
return 0
}
run_test 29 "open(O_CREAT), |X| unlink two, replay, close two (test mds_cleanup_orphans)"
test_30() {
multiop_bg_pause $DIR/$tfile-1 O_tSc || return 5
multiop_bg_pause $DIR/$tfile-2 O_tSc || return 6
pid2=$!
rm -f $DIR/$tfile-1
rm -f $DIR/$tfile-2
kill -USR1 $pid1
wait $pid1 || return 1
kill -USR1 $pid2
wait $pid2 || return 2
[ -e $DIR/$tfile-1 ] && return 3
[ -e $DIR/$tfile-2 ] && return 4
return 0
}
run_test 30 "open(O_CREAT) two, unlink two, replay, close two (test mds_cleanup_orphans)"
test_31() {
multiop_bg_pause $DIR/$tfile-1 O_tSc || return 5
multiop_bg_pause $DIR/$tfile-2 O_tSc || return 6
pid2=$!
rm -f $DIR/$tfile-1
rm -f $DIR/$tfile-2
kill -USR1 $pid1
wait $pid1 || return 1
kill -USR1 $pid2
wait $pid2 || return 2
[ -e $DIR/$tfile-1 ] && return 3
[ -e $DIR/$tfile-2 ] && return 4
return 0
}
run_test 31 "open(O_CREAT) two, unlink one, |X| unlink one, close two (test mds_cleanup_orphans)"
# tests for bug 2104; completion without crashing is success. The close is
# stale, but we always return 0 for close, so the app never sees it.
test_32() {
multiop_bg_pause $DIR/$tfile O_c || return 2
multiop_bg_pause $DIR/$tfile O_c || return 3
pid2=$!
mds_evict_client
df $MOUNT || sleep 1 && df $MOUNT || return 1
kill -USR1 $pid1
kill -USR1 $pid2
return 0
}
run_test 32 "close() notices client eviction; close() after client eviction"
# Abort recovery before client complete
# this file should be gone, because the replay was aborted
multiop_bg_pause $DIR/$tfile O_c || return 2
pid=$!
rm -f $DIR/$tfile
kill -USR1 $pid
[ -e $DIR/$tfile ] && return 1
return 0
}
run_test 34 "abort recovery before client does replay (test mds_cleanup_orphans)"
# bug 2278 - generate one orphan on OST, then destroy it during recovery from llog
test_35() {
touch $DIR/$tfile
rm -f $DIR/$tfile &
sleep 1
# give a chance to remove from MDS
$CHECKSTAT -t file $DIR/$tfile && return 1 || true
}
run_test 35 "test recovery from llog for unlink op"
# b=2432 resent cancel after replay uses wrong cookie,
# so don't resend cancels
test_36() {
if dmesg | grep "unknown lock cookie"; then
echo "cancel after replay failed"
return 1
fi
}
run_test 36 "don't resend cancel"
# b=2368
# directory orphans can't be unlinked from PENDING directory
test_37() {
rmdir $DIR/$tfile 2>/dev/null
multiop_bg_pause $DIR/$tfile dD_c || return 2
# clear the dmesg buffer so we only see errors from this recovery
dmesg -c >/dev/null
kill -USR1 $pid
dmesg | grep "mds_unlink_orphan.*error .* unlinking orphan" && return 1
sync
return 0
}
run_test 37 "abort recovery before client does replay (test mds_cleanup_orphans for directories)"
createmany -o $DIR/$tfile-%d 800
unlinkmany $DIR/$tfile-%d 0 400
sleep 2
$CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
}
run_test 38 "test recovery from unlink llog (test llog_gen_rec) "
$CHECKSTAT -t file $DIR/$tfile-* && return 1 || true
}
run_test 39 "test recovery from unlink llog (test llog_gen_rec) "
lctl get_param -n osc.*.stats | awk -vwrites=0 '/ost_write/ { writes += $2 } END { print writes; }'
}
#b=2477,2532
test_40(){
$LCTL mark multiop $MOUNT/$tfile OS_c
multiop $MOUNT/$tfile OS_c &
PID=$!
writeme -s $MOUNT/${tfile}-2 &
WRITE_PID=$!
sleep 1
kill -USR1 $PID
stat1=`count_ost_writes`
sleep $TIMEOUT
stat2=`count_ost_writes`
echo "$stat1, $stat2"
if [ $stat1 -lt $stat2 ]; then
echo "writes continuing during recovery"
RC=0
else
echo "writes not continuing during recovery, bug 2477"
RC=4
fi
echo "waiting for writeme $WRITE_PID"
kill $WRITE_PID
wait $WRITE_PID
echo "waiting for multiop $PID"
wait $PID || return 2
do_facet client munlink $MOUNT/$tfile || return 3
do_facet client munlink $MOUNT/${tfile}-2 || return 3
return $RC
}
run_test 40 "cause recovery in ptlrpc, ensure IO continues"
#b=2814
# make sure that a read to one osc doesn't try to double-unlock its page just
# because another osc is invalid. trigger_group_io used to mistakenly return
# an error if any oscs were invalid even after having successfully put rpcs
# on valid oscs. This was fatal if the caller was ll_readpage who unlocked
# the page, guarnateeing that the unlock from the RPC completion would
# assert on trying to unlock the unlocked page.
test_41() {
skip "skipping test 41: we don't have a second OST to test with" && \
local f=$MOUNT/$tfile
# make sure the start of the file is ost1
do_facet client dd if=/dev/zero of=$f bs=4k count=1 || return 3
local osc2dev=`lctl get_param -n devices | grep ${ost2_svc}-osc- | awk '{print $1}'`
[ "$osc2dev" ] || return 4
$LCTL --device $osc2dev deactivate || return 1
do_facet client dd if=$f of=/dev/null bs=4k count=1 || return 3
return 0
}
run_test 41 "read from a valid osc while other oscs are invalid"
# test MDS recovery after ost failure
test_42() {
blocks=`df -P $MOUNT | tail -n 1 | awk '{ print $2 }'`
# osc is evicted, fs is smaller (but only with failout OSTs (bug 7287)
#blocks_after=`df -P $MOUNT | tail -n 1 | awk '{ print $2 }'`
#[ $blocks_after -lt $blocks ] || return 1
echo wait for MDS to timeout and recover
sleep $((TIMEOUT * 2))
unlinkmany $DIR/$tfile-%d 400 400
$CHECKSTAT -t file $DIR/$tfile-* && return 2 || true
# timeout in MDS/OST recovery RPC will LBUG MDS
# OBD_FAIL_OST_CREATE_NET 0x204
sleep 10
return 0
}
run_test 43 "mds osc import failure during recovery; don't LBUG"
if at_is_valid && at_is_enabled; then
at_max_saved=$(at_max_get mds)
at_max_set 40 mds
fi
$LCTL --device $mdcdev recover
df $MOUNT
done
$LCTL --device $mdcdev recover
df $MOUNT
done
return 0
}
run_test 44b "race in target handle connect"
multiop_bg_pause $DIR/$tfile O_c || return 1
pid=$!
# This will cause the CLOSE to fail before even
# allocating a reply buffer
# try the close
kill -USR1 $pid
wait $pid || return 1
$CHECKSTAT -t file $DIR/$tfile || return 2
return 0
}
run_test 45 "Handle failed close"
test_46() {
dmesg -c >/dev/null
drop_reply "touch $DIR/$tfile"
# ironically, the previous test, 45, will cause a real forced close,
# so just look for one for this test
dmesg | grep -i "force closing client file handle for $tfile" && return 1
return 0
}
run_test 46 "Don't leak file handle after open resend (3325)"
# create some files to make sure precreate has been done on all
# OSTs. (just in case this test is run independently)
createmany -o $DIR/$tfile 20 || return 1
# OBD_FAIL_OST_CREATE_NET 0x204
df $MOUNT || return 2
# let the MDS discover the OST failure, attempt to recover, fail
# and recover again.
sleep $((3 * TIMEOUT))
# Without 2824, this createmany would hang
createmany -o $DIR/$tfile 20 || return 3
unlinkmany $DIR/$tfile 20 || return 4
return 0
}
run_test 47 "MDS->OSC failure during precreate cleanup (2824)"
replay_barrier mds
createmany -o $DIR/$tfile 20 || return 1
# OBD_FAIL_OST_EROFS 0x216
fail mds
createmany -o $DIR/$tfile 20 20 || return 2
unlinkmany $DIR/$tfile 40 || return 3
run_test 48 "MDS->OSC failure during precreate cleanup (2824)"
local oscdev=`do_facet mds lctl get_param -n devices | grep ${ost1_svc}-osc | awk '{print $1}' | head -1`
do_facet mds $LCTL --device $oscdev recover || return 2
do_facet mds $LCTL --device $oscdev recover || return 3
# give the mds_lov_sync threads a chance to run
sleep 5
}
run_test 50 "Double OSC recovery, don't LASSERT (3812)"
# b3764 timed out lock replay
test_52() {
touch $DIR/$tfile
multiop $DIR/$tfile s || return 1
replay_barrier mds
#define OBD_FAIL_LDLM_REPLY 0x30c
$CHECKSTAT -t file $DIR/$tfile-* && return 3 || true
}
run_test 52 "time out lock replay (3764)"
# bug 3462 - simultaneous MDC requests
test_53a() {
mkdir -p $DIR/${tdir}-1
mkdir -p $DIR/${tdir}-2
multiop $DIR/${tdir}-1/f O_c &
close_pid=$!
# give multiop a chance to open
sleep 1
#define OBD_FAIL_MDS_CLOSE_NET 0x115
mcreate $DIR/${tdir}-2/f || return 1
# close should still be here
[ -d /proc/$close_pid ] || return 2
replay_barrier_nodf mds
fail mds
wait $close_pid || return 3
$CHECKSTAT -t file $DIR/${tdir}-1/f || return 4
$CHECKSTAT -t file $DIR/${tdir}-2/f || return 5
rm -rf $DIR/${tdir}-*
}
run_test 53a "|X| close request while two MDC requests in flight"
test_53b() {
mkdir -p $DIR/$tdir-1
mkdir -p $DIR/$tdir-2
multiop $DIR/$tdir-1/f O_c &
close_pid=$!
#define OBD_FAIL_MDS_REINT_NET 0x107
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
kill -USR1 $close_pid
cancel_lru_locks MDC # force the close
wait $close_pid || return 1
# open should still be here
[ -d /proc/$open_pid ] || return 2
replay_barrier_nodf mds
fail mds
wait $open_pid || return 3
$CHECKSTAT -t file $DIR/${tdir}-1/f || return 4
$CHECKSTAT -t file $DIR/${tdir}-2/f || return 5
rm -rf $DIR/${tdir}-*
}
run_test 53b "|X| open request while two MDC requests in flight"
test_53c() {
mkdir -p $DIR/${tdir}-1
mkdir -p $DIR/${tdir}-2
multiop $DIR/${tdir}-1/f O_c &
close_pid=$!
kill -USR1 $close_pid
cancel_lru_locks MDC # force the close
replay_barrier_nodf mds
fail_nodf mds
wait $open_pid || return 1
sleep 2
# close should be gone
[ -d /proc/$close_pid ] && return 2
$CHECKSTAT -t file $DIR/${tdir}-1/f || return 3
$CHECKSTAT -t file $DIR/${tdir}-2/f || return 4
rm -rf $DIR/${tdir}-*
}
run_test 53c "|X| open request and close request while two MDC requests in flight"
test_53d() {
mkdir -p $DIR/${tdir}-1
mkdir -p $DIR/${tdir}-2
multiop $DIR/${tdir}-1/f O_c &
close_pid=$!
# give multiop a chance to open
sleep 1
# define OBD_FAIL_MDS_CLOSE_NET_REP 0X138
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
mcreate $DIR/${tdir}-2/f || return 1
# close should still be here
[ -d /proc/$close_pid ] || return 2
replay_barrier_nodf mds
fail mds
wait $close_pid || return 3
$CHECKSTAT -t file $DIR/${tdir}-1/f || return 4
$CHECKSTAT -t file $DIR/${tdir}-2/f || return 5
rm -rf $DIR/${tdir}-*
}
run_test 53d "|X| close reply while two MDC requests in flight"
test_53e() {
mkdir -p $DIR/$tdir-1
mkdir -p $DIR/$tdir-2
multiop $DIR/$tdir-1/f O_c &
close_pid=$!
#define OBD_FAIL_MDS_REINT_NET_REP 0x119
kill -USR1 $close_pid
cancel_lru_locks MDC # force the close
wait $close_pid || return 1
# open should still be here
[ -d /proc/$open_pid ] || return 2
replay_barrier_nodf mds
fail mds
wait $open_pid || return 3
$CHECKSTAT -t file $DIR/${tdir}-1/f || return 4
$CHECKSTAT -t file $DIR/${tdir}-2/f || return 5
rm -rf $DIR/${tdir}-*
}
run_test 53e "|X| open reply while two MDC requests in flight"
test_53f() {
mkdir -p $DIR/${tdir}-1
mkdir -p $DIR/${tdir}-2
multiop $DIR/${tdir}-1/f O_c &
close_pid=$!
kill -USR1 $close_pid
cancel_lru_locks MDC
replay_barrier_nodf mds
fail_nodf mds
wait $open_pid || return 1
sleep 2
#close should be gone
[ -d /proc/$close_pid ] && return 2
$CHECKSTAT -t file $DIR/${tdir}-1/f || return 3
$CHECKSTAT -t file $DIR/${tdir}-2/f || return 4
rm -rf $DIR/${tdir}-*
}
run_test 53f "|X| open reply and close reply while two MDC requests in flight"
test_53g() {
mkdir -p $DIR/${tdir}-1
mkdir -p $DIR/${tdir}-2
multiop $DIR/${tdir}-1/f O_c &
close_pid=$!
kill -USR1 $close_pid
cancel_lru_locks MDC # force the close
replay_barrier_nodf mds
fail_nodf mds
wait $open_pid || return 1
sleep 2
# close should be gone
[ -d /proc/$close_pid ] && return 2
$CHECKSTAT -t file $DIR/${tdir}-1/f || return 3
$CHECKSTAT -t file $DIR/${tdir}-2/f || return 4
rm -rf $DIR/${tdir}-*
}
run_test 53g "|X| drop open reply and close request while close and open are both in flight"
test_53h() {
mkdir -p $DIR/${tdir}-1
mkdir -p $DIR/${tdir}-2
multiop $DIR/${tdir}-1/f O_c &
close_pid=$!
kill -USR1 $close_pid
cancel_lru_locks MDC # force the close
sleep 1
replay_barrier_nodf mds
fail_nodf mds
wait $open_pid || return 1
sleep 2
# close should be gone
[ -d /proc/$close_pid ] && return 2
$CHECKSTAT -t file $DIR/${tdir}-1/f || return 3
$CHECKSTAT -t file $DIR/${tdir}-2/f || return 4
rm -rf $DIR/${tdir}-*
}
run_test 53h "|X| open request and close reply while two MDC requests in flight"
#b3761 ASSERTION(hash != 0) failed
test_55() {
# OBD_FAIL_MDS_OPEN_CREATE | OBD_FAIL_ONCE
touch $DIR/$tfile &
# give touch a chance to run
sleep 5
run_test 55 "let MDS_CHECK_RESENT return the original return code instead of 0"
#b3440 ASSERTION(rec->ur_fid2->id) failed
run_test 56 "don't replay a symlink open request (3440)"
#recovery one mds-ost setattr from llog
test_57() {
#define OBD_FAIL_MDS_OST_SETATTR 0x12c
replay_barrier mds
fail mds
sleep 1
$CHECKSTAT -t file $DIR/$tfile || return 1
run_test 57 "test recovery from llog for setattr op"
createmany -o $DIR/$tdir/$tfile-%d 2500
replay_barrier mds
fail mds
sleep 2
$CHECKSTAT -t file $DIR/$tdir/$tfile-* >/dev/null || return 1
unlinkmany $DIR/$tdir/$tfile-%d 2500
rmdir $DIR/$tdir
}
run_test 58 "test recovery from llog for setattr op (test llog_gen_rec)"
# log_commit_thread vs filter_destroy race used to lead to import use after free
# bug 11658
test_59() {
createmany -o $DIR/$tdir/$tfile-%d 200
sync
unlinkmany $DIR/$tdir/$tfile-%d 200
#define OBD_FAIL_PTLRPC_DELAY_RECOV 0x507
sleep 20
rmdir $DIR/$tdir
}
run_test 59 "test log_commit_thread vs filter_destroy race"
# race between add unlink llog vs cat log init in post_recovery (only for b1_6)
# bug 12086: should no oops and No ctxt error for this test
test_60() {
createmany -o $DIR/$tdir/$tfile-%d 200
replay_barrier mds
unlinkmany $DIR/$tdir/$tfile-%d 0 100
fail mds
unlinkmany $DIR/$tdir/$tfile-%d 100 100
local no_ctxt=`dmesg | grep "No ctxt"`
[ -z "$no_ctxt" ] || error "ctxt is not initialized in recovery"
}
run_test 60 "test llog post recovery init vs llog unlink"
createmany -o $DIR/$tdir/$tfile-%d 800
replay_barrier ost1
# OBD_FAIL_OST_LLOG_RECOVERY_TIMEOUT 0x221
unlinkmany $DIR/$tdir/$tfile-%d 800
$CHECKSTAT -t file $DIR/$tdir/$tfile-* && return 1
rmdir $DIR/$tdir
}
#test race mds llog sync vs llog cleanup
test_61b() {
# OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT 0x13a
facet_failover mds
sleep 10
fail mds
do_facet client dd if=/dev/zero of=$DIR/$tfile bs=4k count=1 || return 1
}
run_test 61b "test race mds llog sync vs llog cleanup"
#test race cancel cookie cb vs llog cleanup
test_61c() {
# OBD_FAIL_OST_CANCEL_COOKIE_TIMEOUT 0x222
touch $DIR/$tfile
rm $DIR/$tfile
sleep 10
fail ost1
}
run_test 61c "test race mds llog sync vs llog cleanup"
test_62() { # Bug 15756 - don't mis-drop resent replay
replay_barrier mds
createmany -o $DIR/$tdir/$tfile- 25
#define OBD_FAIL_TGT_REPLAY_DROP 0x707
unlinkmany $DIR/$tdir/$tfile- 25 || return 2
return 0
}
run_test 62 "don't mis-drop resent replay"
#Adaptive Timeouts (bug 3055)
AT_MAX_SET=0
at_start()
if ! at_is_valid; then
skip "AT env is invalid"
return 1
fi
if ! at_is_enabled; then
echo "AT is disabled, enable it by force temporarily"
at_max_set 600 mds ost client
AT_MAX_SET=1
fi
local at_history=$(do_facet mds "find /sys/ -name at_history")
[ -z "$at_history" ] && skip "missing /sys/.../at_history " && return 1
ATOLDBASE=$(do_facet mds "cat $at_history")
do_facet mds "echo 8 >> $at_history"
do_facet ost1 "echo 8 >> $at_history"
createmany -o $DIR/$tfile 10 > /dev/null
unlinkmany $DIR/$tfile 10 > /dev/null
# check for log message
$LCTL dk | grep "Early reply #" || error "No early reply"
run_test 65a "AT: verify early replies"
test_65b() #bug 3055
{
rm -f $DIR/$tfile
lfs setstripe $DIR/$tfile --index=0 --count=1
# force some real bulk transfer
# check for log message
$LCTL dk | grep "Early reply #" || error "No early reply"
debugrestore
# client should show 30s estimates
}
run_test 65b "AT: verify early replies on packed reply / bulk"
lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
# adjust 5s at a time so no early reply is sent (within deadline)
createmany -o $DIR/$tfile 20 > /dev/null
unlinkmany $DIR/$tfile 20 > /dev/null
lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
do_facet mds "lctl set_param fail_val=10000"
do_facet mds "lctl set_param fail_loc=0x8000050a"
createmany -o $DIR/$tfile 20 > /dev/null
unlinkmany $DIR/$tfile 20 > /dev/null
lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep "portal 12"
sleep 9
createmany -o $DIR/$tfile 20 > /dev/null
unlinkmany $DIR/$tfile 20 > /dev/null
lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | grep portal | grep "portal 12"
CUR=$(lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | awk '/portal 12/ {print $5}')
WORST=$(lctl get_param -n mdc.${FSNAME}-MDT0000-mdc-*.timeouts | awk '/portal 12/ {print $7}')
echo "Current MDT timeout $CUR, worst $WORST"
[ $CUR -lt $WORST ] || error "Current $CUR should be less than worst $WORST"
}
run_test 66a "AT: verify MDT service time adjusts with no early replies"
test_66b() #bug 3055
{
ORIG=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $4}')
CUR=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $4}')
WORST=$(lctl get_param -n mdc.${FSNAME}-*.timeouts | awk '/network/ {print $6}')
echo "network timeout orig $ORIG, cur $CUR, worst $WORST"
[ $WORST -gt $ORIG ] || error "Worst $WORST should be worse than orig $ORIG"
}
run_test 66b "AT: verify net latency adjusts"
test_67a() #bug 3055
{
CONN1=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
createmany -o $DIR/$tfile 20 > /dev/null
unlinkmany $DIR/$tfile 20 > /dev/null
CONN2=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
ATTEMPTS=$(($CONN2 - $CONN1))
echo "$ATTEMPTS osc reconnect attemps on gradual slow"
[ $ATTEMPTS -gt 0 ] && error_ignore 13721 "AT should have prevented reconnect"
return 0
}
run_test 67a "AT: verify slow request processing doesn't induce reconnects"
test_67b() #bug 3055
{
CONN1=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
do_facet ost1 "lctl set_param fail_val=20000"
do_facet ost1 "lctl set_param fail_loc=0x80000223"
cp /etc/profile $DIR/$tfile || error "cp failed"
client_reconnect
CONN2=$(lctl get_param -n osc.*.stats | awk '/_connect/ {total+=$2} END {print total}')
ATTEMPTS=$(($CONN2 - $CONN1))
echo "$ATTEMPTS osc reconnect attemps on instant slow"
# do it again; should not timeout
lctl get_param -n ost.OSS.ost_create.timeouts
CONN3=$(`lctl get_param -n osc.*.stats` | awk '/_connect/ {total+=$2} END {print total}')
ATTEMPTS=$(($CONN3 - $CONN2))
echo "$ATTEMPTS osc reconnect attemps on 2nd slow"
[ $ATTEMPTS -gt 0 ] && error "AT should have prevented reconnect"
return 0
}
run_test 67b "AT: verify instant slowdown doesn't induce reconnects"
at_start || return 0
local ldlm_enqueue_min=$(find /sys -name ldlm_enqueue_min)
[ -z "$ldlm_enqueue_min" ] && skip "missing /sys/.../ldlm_enqueue_min" && return 0
local ENQ_MIN=$(cat $ldlm_enqueue_min)
echo $TIMEOUT >> $ldlm_enqueue_min
rm -f $DIR/${tfile}_[1-2]
lfs setstripe $DIR/$tfile --index=0 --count=1
#define OBD_FAIL_LDLM_PAUSE_CANCEL 0x312
lctl set_param fail_val=$(($TIMEOUT - 1))
lctl set_param fail_loc=0x80000312
lctl set_param fail_val=$((TIMEOUT * 3 / 2))
lctl set_param fail_loc=0x80000312
return 0
}
run_test 68 "AT: verify slowing locks"
at_history=$(do_facet mds "find /sys/ -name at_history")
do_facet mds "echo $ATOLDBASE >> $at_history" || true
do_facet ost1 "echo $ATOLDBASE >> $at_history" || true
if [ $AT_MAX_SET -ne 0 ]; then
echo "restore AT status to be disabled"
at_max_set 0 mds ost client
fi
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
# start multi-client tests
test_70a () {
[ -z "$CLIENTS" ] && \
{ skip "Need two or more clients." && return; }
[ $CLIENTCOUNT -lt 2 ] && \
{ skip "Need two or more clients, have $CLIENTCOUNT" && return; }
echo "mount clients $CLIENTS ..."
zconf_mount_clients $CLIENTS $DIR
local clients=${CLIENTS//,/ }
echo "Write/read files on $DIR ; clients $CLIENTS ... "
for CLIENT in $clients; do
do_node $CLIENT dd bs=1M count=10 if=/dev/zero \
of=$DIR/${tfile}_${CLIENT} 2>/dev/null || \
error "dd failed on $CLIENT"
done
local prev_client=$(echo $clients | sed 's/^.* \(\w\+\)$/\1/')
for C in ${CLIENTS//,/ }; do
do_node $prev_client dd if=$DIR/${tfile}_${C} of=/dev/null 2>/dev/null || \
error "dd if=$DIR/${tfile}_${C} failed on $prev_client"
prev_client=$C
done
ls $DIR
zconf_umount_clients $CLIENTS $DIR
}
run_test 70a "check multi client t-f"
test_70b () {
[ -z "$CLIENTS" ] && \
{ skip "Need two or more clients." && return; }
[ $CLIENTCOUNT -lt 2 ] && \
{ skip "Need two or more clients, have $CLIENTCOUNT" && return; }
zconf_mount_clients $CLIENTS $DIR
local duration="-t 60"
local cmd="rundbench 1 $duration "
local PID=""
for CLIENT in ${CLIENTS//,/ }; do
$PDSH $CLIENT "set -x; PATH=:$PATH:$LUSTRE/utils:$LUSTRE/tests/:${DBENCH_LIB} DBENCH_LIB=${DBENCH_LIB} $cmd" &
PID=$!
echo $PID >pid.$CLIENT
echo "Started load PID=`cat pid.$CLIENT`"
done
replay_barrier mds
sleep 3 # give clients a time to do operations
log "$TESTNAME fail mds 1"
fail mds
# wait for client to reconnect to MDS
sleep $TIMEOUT
for CLIENT in ${CLIENTS//,/ }; do
PID=`cat pid.$CLIENT`
wait $PID
rc=$?
echo "load on ${CLIENT} returned $rc"
done
zconf_umount_clients $CLIENTS $DIR
}
run_test 70b "mds recovery; $CLIENTCOUNT clients"
# end multi-client tests
equals_msg `basename $0`: test complete, cleaning up