diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index b3b7aa9c86a1fb9bb3ea15b4d7b5e0a408b5e7e5..cfaec51a580e28786196445fc708a8cd4299a4b8 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -4205,10 +4205,10 @@ test_118b() reset_async #define OBD_FAIL_OST_ENOENT 0x217 - do_facet ost sysctl -w lustre.fail_loc=0x217 + set_nodes_failloc "$(osts_nodes)" 0x217 multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c RC=$? - do_facet ost sysctl -w lustre.fail_loc=0 + set_nodes_failloc "$(osts_nodes)" 0 DIRTY=$(lctl get_param -n llite.*.dump_page_cache | grep -c dirty) WRITEBACK=$(lctl get_param -n llite.*.dump_page_cache | grep -c writeback) @@ -4241,7 +4241,7 @@ test_118c() reset_async #define OBD_FAIL_OST_EROFS 0x216 - do_facet ost sysctl -w lustre.fail_loc=0x216 + set_nodes_failloc "$(osts_nodes)" 0x216 # multiop should block due to fsync until pages are written multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c & @@ -4258,7 +4258,7 @@ test_118c() error "No page in writeback, writeback=$WRITEBACK" fi - do_facet ost sysctl -w lustre.fail_loc=0 + set_nodes_failloc "$(osts_nodes)" 0 wait $MULTIPID RC=$? if [[ $RC -ne 0 ]]; then @@ -4285,7 +4285,7 @@ test_118d() reset_async #define OBD_FAIL_OST_BRW_PAUSE_BULK - do_facet ost sysctl -w lustre.fail_loc=0x214 + set_nodes_failloc "$(osts_nodes)" 0x214 # multiop should block due to fsync until pages are written multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c & MULTIPID=$! @@ -4302,7 +4302,7 @@ test_118d() fi wait $MULTIPID || error "Multiop fsync failed, rc=$?" - do_facet ost sysctl -w lustre.fail_loc=0 + set_nodes_failloc "$(osts_nodes)" 0 DIRTY=$(lctl get_param -n llite.*.dump_page_cache | grep -c dirty) WRITEBACK=$(lctl get_param -n llite.*.dump_page_cache | @@ -4393,12 +4393,12 @@ test_118h() { reset_async #define OBD_FAIL_OST_BRW_WRITE_BULK 0x20e - do_facet ost sysctl -w lustre.fail_loc=0x20e + set_nodes_failloc "$(osts_nodes)" 0x20e # Should simulate ENOMEM error which is recoverable and should be handled by timeout multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c RC=$? - do_facet ost sysctl -w lustre.fail_loc=0 + set_nodes_failloc "$(osts_nodes)" 0 if [[ $RC -eq 0 ]]; then error "Must return error due to dropped pages, rc=$RC" fi @@ -4428,13 +4428,13 @@ test_118i() { reset_async #define OBD_FAIL_OST_BRW_WRITE_BULK 0x20e - do_facet ost sysctl -w lustre.fail_loc=0x20e + set_nodes_failloc "$(osts_nodes)" 0x20e # Should simulate ENOMEM error which is recoverable and should be handled by timeout multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c & PID=$! sleep 5 - do_facet ost sysctl -w lustre.fail_loc=0 + set_nodes_failloc "$(osts_nodes)" 0 wait $PID RC=$? @@ -4466,12 +4466,12 @@ test_118j() { reset_async #define OBD_FAIL_OST_BRW_WRITE_BULK2 0x220 - do_facet ost sysctl -w lustre.fail_loc=0x220 + set_nodes_failloc "$(osts_nodes)" 0x220 # return -EIO from OST multiop $DIR/$tfile oO_CREAT:O_RDWR:O_SYNC:w4096c RC=$? - do_facet ost sysctl -w lustre.fail_loc=0x0 + set_nodes_failloc "$(osts_nodes)" 0x0 if [[ $RC -eq 0 ]]; then error "Must return error due to dropped pages, rc=$RC" fi @@ -4498,7 +4498,7 @@ run_test 118j "Simulate unrecoverable OST side error ==========" test_118k() { #define OBD_FAIL_OST_BRW_WRITE_BULK 0x20e - do_facet ost sysctl -w lustre.fail_loc=0x20e + set_nodes_failloc "$(osts_nodes)" 0x20e mkdir -p $DIR/$tdir for ((i=0;i<10;i++)); do @@ -4510,7 +4510,7 @@ test_118k() wait $SLEEPPID done - sysctl -w lustre.fail_loc=0 + set_nodes_failloc "$(osts_nodes)" 0 } run_test 118k "bio alloc -ENOMEM and IO TERM handling =========" diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index a92a828e15b80719a5156bac23a0a2d8fcf82069..6426799762e69a212dd85b09aba94326b6cc7f89 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -1184,6 +1184,15 @@ clear_failloc() { do_facet $facet "sysctl -e -w lustre.fail_loc=0" } +set_nodes_failloc () { + local nodes=$1 + local node + + for node in $nodes ; do + do_node $node sysctl -w lustre.fail_loc=$2 + done +} + cancel_lru_locks() { $LCTL mark "cancel_lru_locks $1 start" for d in `find $LPROC/ldlm/namespaces | egrep -i $1`; do