diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index 8e9605047c3b2caa18bc0f945c8055f323d57a71..990198fbcd62d61f82849410ecfb4b031c300e61 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -11,6 +11,9 @@ noinst_SCRIPTS += sanity.sh rundbench acceptance-small.sh compile.sh noinst_SCRIPTS += conf-sanity.sh insanity.sh lfscktest.sh oos.sh oos2.sh noinst_SCRIPTS += llog-test.sh recovery-small.sh replay-dual.sh sanity-quota.sh noinst_SCRIPTS += replay-ost-single.sh replay-single.sh run-llog.sh sanityN.sh +noinst_SCRIPTS += performance-sanity.sh mdsrate-create-small.sh +noinst_SCRIPTS += mdsrate-create-large.sh mdsrate-lookup-1dir.sh +noinst_SCRIPTS += mdsrate-stat-small.sh mdsrate-stat-large.sh noinst_SCRIPTS += lockorder.sh socketclient socketserver runmultiop_bg_pause nobase_noinst_SCRIPTS = cfg/local.sh nobase_noinst_SCRIPTS += acl/make-tree acl/run cfg/ncli.sh diff --git a/lustre/tests/acceptance-small.sh b/lustre/tests/acceptance-small.sh index 38975bd04bdaa97b3b6caf06106801afe00b1201..aa53fc1f038204e7512ba7f2cdabbc3d18237811 100755 --- a/lustre/tests/acceptance-small.sh +++ b/lustre/tests/acceptance-small.sh @@ -23,7 +23,7 @@ fi [ "$DEBUG_OFF" ] || DEBUG_OFF="eval lctl set_param debug=\"$DEBUG_LVL\"" [ "$DEBUG_ON" ] || DEBUG_ON="eval lctl set_param debug=0x33f0484" -export TESTSUITE_LIST="RUNTESTS SANITY DBENCH BONNIE IOZONE FSX SANITYN LFSCK LIBLUSTRE REPLAY_SINGLE CONF_SANITY RECOVERY_SMALL REPLAY_OST_SINGLE REPLAY_DUAL INSANITY SANITY_QUOTA" +export TESTSUITE_LIST="RUNTESTS SANITY DBENCH BONNIE IOZONE FSX SANITYN LFSCK LIBLUSTRE REPLAY_SINGLE CONF_SANITY RECOVERY_SMALL REPLAY_OST_SINGLE REPLAY_DUAL INSANITY SANITY_QUOTA PERFORMANCE_SANITY" if [ "$ACC_SM_ONLY" ]; then for O in $TESTSUITE_LIST; do @@ -356,6 +356,15 @@ if [ "$SANITY_QUOTA" != "no" ]; then fi +[ "$SLOW" = no ] && PERFORMANCE_SANITY="no" +[ -x "$MDSRATE" ] || PERFORMANCE_SANITY="no" +which mpirun > /dev/null 2>&1 || PERFORMANCE_SANITY="no" +if [ "$PERFORMANCE_SANITY" != "no" ]; then + title performance-sanity + bash performance-sanity.sh + PERFORMANCE_SANITY="done" +fi + RC=$? title FINISHED echo "Finished at `date` in $((`date +%s` - $STARTTIME))s" diff --git a/lustre/tests/cfg/ncli.sh b/lustre/tests/cfg/ncli.sh index 6dfae9b18b063e978cb52a184f8dba186ba026a0..ce95c68d6e1954e56f2b76f0baa86084e595af9f 100644 --- a/lustre/tests/cfg/ncli.sh +++ b/lustre/tests/cfg/ncli.sh @@ -16,3 +16,8 @@ CLIENTCOUNT=$((${#REMOTECLIENTS[@]} + 1)) error "tests for remote clients $RCLIENTS needs pdsh != do_dsh " || true [ -n "$FUNCTIONS" ] && . $FUNCTIONS || true + +MPIBIN=${MPIBIN:-/testsuite/tests/`arch`/bin} +export PATH=:$PATH:$MPIBIN +MPIRUN=$(which mpirun) + diff --git a/lustre/tests/mdsrate-create-large.sh b/lustre/tests/mdsrate-create-large.sh new file mode 100644 index 0000000000000000000000000000000000000000..faaf6bd198c5574dfa332e0efba2dbc6189e31b6 --- /dev/null +++ b/lustre/tests/mdsrate-create-large.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# This test was used in a set of CMD3 tests (cmd3-4 test). + +LUSTRE=${LUSTRE:-`dirname $0`/..} +. $LUSTRE/tests/test-framework.sh +init_test_env $@ +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} + +assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN + +MACHINEFILE=${MACHINEFILE:-$(basename $0 .sh).machines} +TESTDIR=$MOUNT + +# Requirements +TIME_PERIOD=${TIME_PERIOD:-600} # seconds +SINGLE_TARGET_RATE=$((1300 / OSTCOUNT)) # ops/sec +AGGREGATE_TARGET_RATE=$((7000 / OSTCOUNT)) # ops/sec + +# Local test variables +TESTDIR_SINGLE="${TESTDIR}/single" +TESTDIR_MULTI="${TESTDIR}/multi" + +LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} +CLIENT=$SINGLECLIENT +NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} +NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) + +[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." + +log "===== $0 ====== " + +check_and_setup_lustre + +generate_machine_file $NODES_TO_USE $MACHINEFILE + +$LFS setstripe $TESTDIR -c -1 +get_stripe $TESTDIR + +# Make sure we start with a clean slate +rm -f ${LOG} PI* + +if [ -n "$NOSINGLE" ]; then + echo "NO Test for creates for a single client." +else + log "===== $0 ### 1 NODE CREATE ###" + echo "Running creates on 1 node(s)." + + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD} + --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'" + echo "+ ${COMMAND}" + $MPIRUN -np 1 -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + + check_rate create ${SINGLE_TARGET_RATE} 1 ${LOG} || true + + log "===== $0 ### 1 NODE UNLINK ###" + echo "Running unlinks on 1 node(s)." + + let NUM_FILES=${SINGLE_TARGET_RATE}\*${TIME_PERIOD} + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --time ${TIME_PERIOD} + --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'" + echo "+ ${COMMAND}" + $MPIRUN -np 1 -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + + check_rate unlink ${SINGLE_TARGET_RATE} 1 ${LOG} || true +fi + +if [ -n "$NOMULTI" ]; then + echo "NO test for create on multiple nodes." +else + + log "===== $0 ### $NUM_CLIENTS NODES CREATE ###" + echo "Running creates on ${NUM_CLIENTS} node(s)." + + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD} + --dir ${TESTDIR_MULTI} --filefmt 'f%%d'" + echo "+ ${COMMAND}" + $MPIRUN -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + + check_rate create ${AGGREGATE_TARGET_RATE} ${NUM_CLIENTS} ${LOG} || true + + echo "Running unlinks on ${NUM_CLIENTS} node(s)." + + let NUM_FILES=${AGGREGATE_TARGET_RATE}\*${TIME_PERIOD} + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --time ${TIME_PERIOD} + --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'" + echo "+ ${COMMAND}" + $MPIRUN -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + + check_rate unlink ${AGGREGATE_TARGET_RATE} ${NUM_CLIENTS} ${LOG} || true +fi + +equals_msg `basename $0`: test complete, cleaning up +zconf_umount_clients $NODES_TO_USE $MOUNT +check_and_cleanup_lustre +#rm -f $LOG + +exit 0 diff --git a/lustre/tests/mdsrate-create-small.sh b/lustre/tests/mdsrate-create-small.sh new file mode 100644 index 0000000000000000000000000000000000000000..9a74c49a9143ba9784adbc5206cda3f39cda6542 --- /dev/null +++ b/lustre/tests/mdsrate-create-small.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# +# This test was used in a set of CMD3 tests (cmd3-3 test). + +LUSTRE=${LUSTRE:-`dirname $0`/..} +. $LUSTRE/tests/test-framework.sh +init_test_env $@ +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} + +assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN + +MACHINEFILE=${MACHINEFILE:-$(basename $0 .sh).machines} +TESTDIR=$MOUNT + +# Requirements +# The default number of stripes per file is set to 1 in test3/run_test.sh. +TIME_PERIOD=${TIME_PERIOD:-600} # seconds +SINGLE_TARGET_RATE=1400 # ops/sec +AGGREGATE_TARGET_RATE=10000 # ops/sec + +# Local test variables +TESTDIR_SINGLE="${TESTDIR}/single" +TESTDIR_MULTI="${TESTDIR}/multi" + +LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} +CLIENT=$SINGLECLIENT +NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} +NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) +# XXX - this needs to be determined given the number of MDTs and the number +# of clients. +THREADS_PER_CLIENT=3 # threads/client for multi client test +if [ $NUM_CLIENTS -gt 50 ]; then + THREADS_PER_CLIENT=1 +fi + +[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." + +# Make sure we start with a clean slate +rm -f ${LOG} PI* + +log "===== $0 ====== " + +check_and_setup_lustre + +generate_machine_file $NODES_TO_USE $MACHINEFILE + +$LFS setstripe $TESTDIR -i 0 -c 1 +get_stripe $TESTDIR + +if [ -n "$NOSINGLE" ]; then + echo "NO Tests on single client." +else + if [ -n "$NOCREATE" ]; then + echo "NO Test for creates for a single client." + else + do_node ${CLIENT} "rm -rf $TESTDIR_SINGLE" + + log "===== $0 ### 1 NODE CREATE ###" + echo "Running creates on 1 node(s)." + + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD} + --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'" + echo "+ ${COMMAND}" + $MPIRUN -np 1 -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + check_rate create ${SINGLE_TARGET_RATE} 1 ${LOG} || true + fi + + if [ -n "$NOUNLINK" ]; then + echo "NO Test for unlinks for a single client." + else + log "===== $0 ### 1 NODE UNLINK ###" + echo "Running unlinks on 1 node(s)." + + let NUM_FILES=${SINGLE_TARGET_RATE}\*${TIME_PERIOD} + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --time ${TIME_PERIOD} + --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'" + echo "+ ${COMMAND}" + $MPIRUN -np 1 -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + check_rate unlink ${SINGLE_TARGET_RATE} 1 ${LOG} || true + fi +fi + +if [ -n "$NOMULTI" ]; then + echo "NO tests on multiple nodes." +else + if [ -n "$NOCREATE" ]; then + echo "NO test for create on multiple nodes." + else + do_node $CLIENT rm -rf $TESTDIR_MULTI + + log "===== $0 ### $NUM_CLIENTS NODES CREATE ###" + echo "Running creates on ${NUM_CLIENTS} node(s) with $THREADS_PER_CLIENT threads per client." + + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD} + --dir ${TESTDIR_MULTI} --filefmt 'f%%d'" + echo "+ ${COMMAND}" + $MPIRUN -np $((${NUM_CLIENTS}*THREADS_PER_CLIENT)) -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + check_rate create ${AGGREGATE_TARGET_RATE} ${NUM_CLIENTS} ${LOG} || true + fi + + if [ -n "$NOUNLINK" ]; then + echo "NO Test for unlinks multiple nodes." + else + log "===== $0 ### $NUM_CLIENTS NODES UNLINK ###" + echo "Running unlinks on ${NUM_CLIENTS} node(s) with $THREADS_PER_CLIENT threads per client." + + let NUM_FILES=${AGGREGATE_TARGET_RATE}\*${TIME_PERIOD} + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --time ${TIME_PERIOD} + --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'" + echo "+ ${COMMAND}" + $MPIRUN -np $((${NUM_CLIENTS}*THREADS_PER_CLIENT)) -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + check_rate unlink ${AGGREGATE_TARGET_RATE} ${NUM_CLIENTS} ${LOG} || true + fi +fi + +equals_msg `basename $0`: test complete, cleaning up +zconf_umount_clients $NODES_TO_USE $MOUNT +check_and_cleanup_lustre +#rm -f $LOG + +exit 0 diff --git a/lustre/tests/mdsrate-lookup-1dir.sh b/lustre/tests/mdsrate-lookup-1dir.sh new file mode 100644 index 0000000000000000000000000000000000000000..6ecf4a86d34d6fb1974f7c5cd977c3f1f7051c9b --- /dev/null +++ b/lustre/tests/mdsrate-lookup-1dir.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# +# This test was used in a set of CMD3 tests (cmd3-5 test). + +# Directory lookup retrieval rate single directory 10 million files +# 5900 random lookups/sec per client node 62,000 random lookups/sec aggregate +# +# In a dir containing 10 million non-striped files the mdsrate Test Program will +# perform lookups for 10 minutes. This test can be run from a single node for +# #1 and from all nodes for #2 aggregate test to measure lookup performance. + +LUSTRE=${LUSTRE:-`dirname $0`/..} +. $LUSTRE/tests/test-framework.sh +init_test_env $@ +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} +assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN + +MACHINEFILE=${MACHINEFILE:-$(basename $0 .sh).machines} +TESTDIR=$MOUNT + +# Requirements +NUM_FILES=${NUM_FILES:-1000000} +TIME_PERIOD=${TIME_PERIOD:-600} # seconds +SINGLE_TARGET_RATE=5900 # ops/sec +AGGREGATE_TARGET_RATE=62000 # ops/sec + +LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} +CLIENT=$SINGLECLIENT +NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} +NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) + +rm -f $LOG + +[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." + +log "===== $0 ====== " + +check_and_setup_lustre + +generate_machine_file $NODES_TO_USE $MACHINEFILE + +$LFS setstripe $TESTDIR -c 1 +get_stripe $TESTDIR + +if [ -n "$NOCREATE" ]; then + echo "NOCREATE=$NOCREATE => no file creation." +else + log "===== $0 Test preparation: creating ${NUM_FILES} files." + echo "Test preparation: creating ${NUM_FILES} files." + + MDSCOUNT=1 + NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) + NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT)) + if [ $NUM_CLIENTS -gt 50 ]; then + NUM_THREADS=$NUM_CLIENTS + fi + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir ${TESTDIR} + --nfiles ${NUM_FILES} --filefmt 'f%%d'" + echo "+" ${COMMAND} + $MPIRUN -np ${NUM_THREADS} -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} 2>&1 + + # No lockup if error occurs on file creation, abort. + [ ${PIPESTATUS[0]} != 0 ] && error "mpirun ... mdsrate ... file creation failed, aborting" +fi + +COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --lookup --time ${TIME_PERIOD} ${SEED_OPTION} + --dir ${TESTDIR} --nfiles ${NUM_FILES} --filefmt 'f%%d'" + +# 1 +if [ -n "$NOSINGLE" ]; then + echo "NO Test for lookups on a single client." +else + log "===== $0 ### 1 NODE LOOKUPS ###" + echo "Running lookups on 1 node(s)." + echo "+" ${COMMAND} + $MPIRUN -np 1 -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + check_rate lookup ${SINGLE_TARGET_RATE} 1 ${LOG} || true +fi + +# 2 +if [ -n "$NOMULTI" ]; then + echo "NO test for lookups on multiple nodes." +else + log "===== $0 ### ${NUM_CLIENTS} NODES LOOKUPS ###" + echo "Running lookups on ${NUM_CLIENTS} node(s)." + echo "+" ${COMMAND} + $MPIRUN -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + check_rate lookup ${AGGREGATE_TARGET_RATE} ${NUM_CLIENTS} ${LOG} || true +fi + +equals_msg `basename $0`: test complete, cleaning up +zconf_umount_clients $NODES_TO_USE $MOUNT +check_and_cleanup_lustre +#rm -f $LOG + +exit 0 diff --git a/lustre/tests/mdsrate-stat-large.sh b/lustre/tests/mdsrate-stat-large.sh new file mode 100644 index 0000000000000000000000000000000000000000..bd1fd8f1e9150f051f5dc891e65a4e195c27d7c6 --- /dev/null +++ b/lustre/tests/mdsrate-stat-large.sh @@ -0,0 +1,117 @@ +#!/bin/sh +# +# This test was used in a set of CMD3 tests (cmd3-8 test). + +# File attribute retrieval rate for large file creation +# 3300 ops/sec/OST for single node 28500 ops/sec/OST aggregate + +# In a dir containing 10 million striped files, the mdsrate Test Program will +# perform directory ordered stat's (readdir) for 10 minutes. This test will be +# run from a single node for #1 and from all nodes for #2 aggregate test to +# measure stat performance. + +LUSTRE=${LUSTRE:-`dirname $0`/..} +. $LUSTRE/tests/test-framework.sh +init_test_env $@ +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} + +assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN + +MACHINEFILE=${MACHINEFILE:-$(basename $0 .sh).machines} +TESTDIR=$MOUNT + +# Requirements +NUM_FILES=${NUM_FILES:-1000000} +TIME_PERIOD=${TIME_PERIOD:-600} # seconds +SINGLE_TARGET_RATE=$((3300 / OSTCOUNT)) # ops/sec +AGGREGATE_TARGET_RATE=$((28500 / OSTCOUNT)) # ops/sec + +# --random_order (default) -OR- --readdir_order +DIR_ORDER=${DIR_ORDER:-"--readdir_order"} + +LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} +CLIENT=$SINGLECLIENT +NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} +NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) + +rm -f $LOG + +[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." + +log "===== $0 ====== " + +check_and_setup_lustre + +generate_machine_file $NODES_TO_USE $MACHINEFILE + +$LFS setstripe $TESTDIR -c -1 +get_stripe $TESTDIR + +if [ -n "$NOCREATE" ]; then + echo "NOCREATE=$NOCREATE => no file creation." +else + log "===== $0 Test preparation: creating ${NUM_FILES} files." + echo "Test preparation: creating ${NUM_FILES} files." + + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --dir ${TESTDIR} + --nfiles ${NUM_FILES} --filefmt 'f%%d'" + echo "+" ${COMMAND} + + MDSCOUNT=1 + NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) + NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT)) + if [ $NUM_CLIENTS -gt 50 ]; then + NUM_THREADS=$NUM_CLIENTS + fi + + $MPIRUN -np ${NUM_THREADS} -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} 2>&1 + [ ${PIPESTATUS[0]} != 0 ] && error "mpirun ... mdsrate ... file creation failed, aborting" + +fi + +COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --stat --time ${TIME_PERIOD} + --dir ${TESTDIR} --nfiles ${NUM_FILES} --filefmt 'f%%d' + ${DIR_ORDER} ${SEED_OPTION}" + +# 1 +if [ -n "$NOSINGLE" ]; then + echo "NO Test for stats on a single client." +else + log "===== $0 ### 1 NODE STAT ###" + echo "Running stats on 1 node(s)." + echo "+" ${COMMAND} + + $MPIRUN -np 1 -machinefile ${MACHINEFILE} ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + check_rate stat ${SINGLE_TARGET_RATE} 1 ${LOG} || true +fi + +# 2 +if [ -n "$NOMULTI" ]; then + echo "NO test for stats on multiple nodes." +else + log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###" + echo "Running stats on ${NUM_CLIENTS} node(s)." + echo "+" ${COMMAND} + + NUM_THREADS=$(get_node_count ${NODES_TO_USE//,/ }) + $MPIRUN -np ${NUM_THREADS} -machinefile ${MACHINEFILE} ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + check_rate stat ${AGGREGATE_TARGET_RATE} ${NUM_CLIENTS} ${LOG} || true +fi + +equals_msg `basename $0`: test complete, cleaning up +zconf_umount_clients $NODES_TO_USE $MOUNT +check_and_cleanup_lustre +#rm -f $LOG + +exit 0 diff --git a/lustre/tests/mdsrate-stat-small.sh b/lustre/tests/mdsrate-stat-small.sh new file mode 100644 index 0000000000000000000000000000000000000000..bc058950764127af1e9f07fd0c638d6b4d3a50b1 --- /dev/null +++ b/lustre/tests/mdsrate-stat-small.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# +# This test was used in a set of CMD3 tests (cmd3-7 test). + +# File attribute retrieval rate for small file creation +# 3200 ops/sec for single node 29,000 ops/sec aggregate + +# In a dir containing 10 million non-striped files, the mdsrate Test Program +# will perform directory ordered stat's (readdir) for 10 minutes. This test +# will be run from a single node for #1 and from all nodes for #2 +# aggregate test to measure stat performance. + +LUSTRE=${LUSTRE:-`dirname $0`/..} +. $LUSTRE/tests/test-framework.sh +init_test_env $@ +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} + +assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN + +MACHINEFILE=${MACHINEFILE:-$(basename $0 .sh).machines} +TESTDIR=$MOUNT + +# Requirements +NUM_FILES=${NUM_FILES:-1000000} +TIME_PERIOD=${TIME_PERIOD:-600} # seconds +SINGLE_TARGET_RATE=3200 # ops/sec +AGGREGATE_TARGET_RATE=29000 # ops/sec + +# --random_order (default) -OR- --readdir_order +DIR_ORDER=${DIR_ORDER:-"--readdir_order"} + +LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log} +CLIENT=$SINGLECLIENT +NODES_TO_USE=${NODES_TO_USE:-$CLIENTS} +NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) + +rm -f $LOG + +[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built." + +log "===== $0 ====== " + +check_and_setup_lustre + +generate_machine_file $NODES_TO_USE $MACHINEFILE + +$LFS setstripe $TESTDIR -i 0 -c 1 +get_stripe $TESTDIR + +if [ -n "$NOCREATE" ]; then + echo "NOCREATE=$NOCREATE => no file creation." +else + log "===== $0 Test preparation: creating ${NUM_FILES} files." + echo "Test preparation: creating ${NUM_FILES} files." + + COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir ${TESTDIR} + --nfiles ${NUM_FILES} --filefmt 'f%%d'" + echo "+" ${COMMAND} + + MDSCOUNT=1 + NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ }) + NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT)) + if [ $NUM_CLIENTS -gt 50 ]; then + NUM_THREADS=$NUM_CLIENTS + fi + + $MPIRUN -np ${NUM_THREADS} -machinefile ${MACHINEFILE} \ + ${MPIRUN_OPTIONS} ${COMMAND} 2>&1 + [ ${PIPESTATUS[0]} != 0 ] && error "Error running mdsrate, aborting..." + +fi + +COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --stat --time ${TIME_PERIOD} + --dir ${TESTDIR} --nfiles ${NUM_FILES} --filefmt 'f%%d' + ${DIR_ORDER} ${SEED_OPTION}" + +# 1 +if [ -n "$NOSINGLE" ]; then + echo "NO Test for stats on a single client." +else + log "===== $0 ### 1 NODE STAT ###" + echo "Running stats on 1 node(s)." + echo "+" ${COMMAND} + + $MPIRUN -np 1 -machinefile ${MACHINEFILE} ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + check_rate stat ${SINGLE_TARGET_RATE} 1 ${LOG} || true +fi + +# 2 +if [ -n "$NOMULTI" ]; then + echo "NO test for stats on multiple nodes." +else + log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###" + echo "Running stats on ${NUM_CLIENTS} node(s)." + echo "+" ${COMMAND} + + $MPIRUN -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} ${MPIRUN_OPTIONS} ${COMMAND} | tee ${LOG} + + if [ ${PIPESTATUS[0]} != 0 ]; then + [ -f $LOG ] && cat $LOG + error "mpirun ... mdsrate ... failed, aborting" + fi + check_rate stat ${AGGREGATE_TARGET_RATE} ${NUM_CLIENTS} ${LOG} || true +fi + +equals_msg `basename $0`: test complete, cleaning up +zconf_umount_clients $NODES_TO_USE $MOUNT +check_and_cleanup_lustre +#rm -f $LOG + +exit 0 diff --git a/lustre/tests/performance-sanity.sh b/lustre/tests/performance-sanity.sh new file mode 100644 index 0000000000000000000000000000000000000000..0139f0c6b24e73a005396c1534b4ffa8fd94831f --- /dev/null +++ b/lustre/tests/performance-sanity.sh @@ -0,0 +1,81 @@ +#!/bin/bash +#set -vx +set -e + +TESTNAME=`basename $0 .sh` +TMP=${TMP:-/tmp} +LOG=${LOG:-"$TMP/${TESTNAME}.log"} + +LUSTRE=${LUSTRE:-`dirname $0`/..} +. $LUSTRE/tests/test-framework.sh +init_test_env $@ + +. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh} + +[ -x "$MDSRATE" ] || FAIL_ON_ERROR=true error "No mdsrate program. Aborting." +which mpirun > /dev/null 2>&1 || \ + FAIL_ON_ERROR=true error "No mpirun program. Aborting." + +# Skip these tests +# bug number: 15266 15266 15266 +ALWAYS_EXCEPT="1 2 6 $PERFORMANCE_SANITY_EXCEPT" + +build_test_filter + +# single-IOR-rates +test_1() { + echo "Single client I/O performance as a percentage of raw" +} +run_test 1 "single-client IO perf =====" + +# parallel-IOR-rates +test_2() { + echo "MPI coordinated test of parallel filesystem system calls and library functions" +} +run_test 2 "multi-client IO perf =====" + +# mdsrate-create-small +test_3() { + echo "File creation performance tests for file objects" + bash mdsrate-create-small.sh +} +run_test 3 "small file create/open/delete ======" + +# mdsrate-create-large +test_4() { + echo "Large file creation performance" + bash mdsrate-create-large.sh +} +run_test 4 "large file create/open/delete ======" + +# mdsrate-lookup-1dir +test_5() { + echo "Single directory lookup retrieval rate" + bash mdsrate-lookup-1dir.sh +} +run_test 5 "lookup rate 10M file dir ======" + +# mdsrate-lookup-10dir +test_6() { + echo "Directory lookup retrieval rate 10 directories, 1 million files each" + bash mdsrate-lookup-10dirs.sh +} +run_test 6 "lookup rate 10M file 10 dir ======" + +# mdsrate-stat-small +test_7() { + echo "File attribute retrieval rate for small file creation" + bash mdsrate-stat-small.sh +} +run_test 7 "getattr small file ======" + +# mdsrate-stat-large +test_8() { + echo "File attribute retrieval rate for large file creation" + bash mdsrate-stat-large.sh +} +run_test 8 "getattr large files ======" + +equals_msg `basename $0`: test complete, cleaning up +check_and_cleanup_lustre +[ -f "$LOG" ] && cat $LOG || true diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index cbaec79b7c8b0edf01f4c0ea1ed4fbe35340c466..3ee33cc5f8c998b2382b41c3dd62507628cdae38 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -89,6 +89,8 @@ init_test_env() { if ! echo $PATH | grep -q $LUSTRE/test; then export PATH=$PATH:$LUSTRE/tests fi + export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mdsrate"} + [ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate) export LCTL=${LCTL:-"$LUSTRE/utils/lctl"} export LFS=${LFS:-"$LUSTRE/utils/lfs"} [ ! -f "$LCTL" ] && export LCTL=$(which lctl) @@ -1575,6 +1577,22 @@ mixed_ost_devs () { [ ! "$OSTCOUNT" = "$osscount" ] } +generate_machine_file() { + local nodes=${1//,/ } + local machinefile=$2 + rm -f $machinefile || error "can't rm $machinefile" + for node in $nodes; do + echo $node >>$machinefile + done +} + +get_stripe () { + local file=$1/stripe + touch $file + $LFS getstripe -v $file || error + rm -f $file +} + check_runas_id_ret() { local myRC=0 local myRUNAS_ID=$1 @@ -1630,6 +1648,27 @@ multiop_bg_pause() { return 0 } +check_rate() { + local OP=$1 + local TARGET_RATE=$2 + local NUM_CLIENTS=$3 + local LOG=$4 + + local RATE=$(awk '/^Rate: [0-9\.]+ '"${OP}"'s\/sec/ { print $2}' ${LOG}) + + # We need to use bc since the rate is a floating point number + local RES=$(echo "${RATE} < ${TARGET_RATE}" | bc -l ) + if [ ${RES} -eq 0 ]; then + echo "Success: ${RATE} ${OP}s/sec met target rate" \ + "${TARGET_RATE} ${OP}s/sec for ${NUM_CLIENTS} client(s)." + return 0 + else + echo "Failure: ${RATE} ${OP}s/sec did not meet target rate" \ + "${TARGET_RATE} ${OP}s/sec for ${NUM_CLIENTS} client(s)." + return 1 + fi +} + # reset llite stat counters clear_llite_stats(){ lctl set_param -n llite.*.stats 0