Skip to content
Snippets Groups Projects
Commit 51c037f2 authored by Cliff White's avatar Cliff White
Browse files 

b=10960
i=kmerrit
i=jitendra

Multiple fixes, adds support for yod, mpirun
parent 89b041a1
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
lustre-iokit/ior-survey/ior-survey
+ 133
− 69
View file @ 51c037f2
#!/bin/bash
# the value of these can be set in the environment.
# This test assumes a typical pdsh naming scheme, where
# node names can be expressed as a string
# forllowed by a number
# cluster name (all node names are this followed by the node number)
cluster=mdev
cluster=${cluster:-""}
if [ -z "$cluster" ];then echo "cluster not defined"; exit 1; fi
# client node numbers (individual numbers or inclusive ranges)
clients=(7-8)
clients=${clients:-""}
if [ -z "$clients" ]; then echo "clients not defined"; exit 1; fi
# numbers of clients to survey
clients_lo=1
clients_hi=2
clients_iterator="+=1"
clients_lo=${clients_lo:-1}
clients_hi=${clients_hi:-3}
clients_iterator=${clients_iterator:-"+=1"}
# numbers of tasks per client to survey
tasks_per_client_lo=1
tasks_per_client_hi=8
tasks_per_client_iterator="*=2"
tasks_per_client_lo=${task_per_client_lo:-1}
tasks_per_client_hi=${tasks_per_client_hi:-8}
tasks_per_client_iterator=${tasks_per_client_iterator:-"*=2"}
# record sizes to survey
rsize_lo=1M
rsize_hi=1M
rsize_iterator="*=2"
rsize_lo=${rsize_lo:-1M}
rsize_hi=${rsize_hi:-1M}
rsize_iterator=${rsize_iterator:-"*=2"}
## which tests to run (first must be write)
# clear_cache) not really a test; just uncache everything
# *write*) write
# *) read
#tests=(write rewrite read reread rewrite_again)
tests=(write rewrite clear_cache read reread)
# total # bytes written/read by any client node
min_per_client_size=4G
min_total_size=8G
min_per_client_size=${min_per_client_size:-"1G"}
min_total_size=${min_total_size:-"2G"}
# should each task do I/O to its own file?
file_per_task=1
file_per_task=${file_per_task:-1}
# the binaries
IOR=/usr/local/sbin/IOR
llmount=llmount
pdsh=pdsh
IOR=${IOR:-"/usr/local/sbin/IOR"}
llmount=${llmount:-"llmount"}
# Select mpirun, yod, or pdsh
fanout_cmd=${fanout_cmd:-"pdsh"}
# mpirun still uses pdsh for cleanup
pdsh=${pdsh:-"pdsh"}
pdsh_args="-R ssh -S -b -w "
# the result file prefix (date/time + hostname makes unique)
rslt=/tmp/ior_survey_`date +%F@%R`_`uname -n`
rslt_loc=${rslt_loc:-"/tmp"}
rslt=${rslt:-"$rslt_loc/ior_survey_`date +%F@%R`_`uname -n`"}
# where lustre is mounted on the clients
lustre=/mnt/lustre
lustre=${lustre:-"/mnt/lustre"}
# basename of the test file(s)
testfile=${lustre}/ior_survey_testfile
# pdsh args required to instantiate all instances of IOR in parallel
# the chosen module must support '-n <procs-per-node>'
# -R<module>, -f<fanout> etc
pdsh_mpiargs="-Rmqsh"
testfile=${testfile:-"${lustre}/ior/ior_survey_testfile"}
#don't spin for MPI completions
export LIBELAN_WAITTYPE=0
......@@ -60,6 +65,54 @@ export LIBELAN_WAITTYPE=0
################################################################################
# dont change stuff below here unless you know what you're doing...
# This is to allow use of yod, pdsh, etc.
fanout() {
local clients=$1; shift
local tmpfile=$1; shift
local result
case $fanout_cmd in
'pdsh')
$fanout_cmd $pdsh_args "$clients" "$@" >> $tmpfile 2>&1
echo $?
return
;;
'mpirun')
# horrible misuse of globals
$fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
echo $?
return
;;
'yod')
# and another
$fanout_cmd -np $((ntask*nclnt)) "$@" >> $tmpfile 2>&1
echo $?
return
;;
*)
echo "255"
return
;;
esac
}
dump_cache() {
# we are assuming mpi uses will also have pdsh
local clients=$1;shift
local tmpfile=$1;shift
clear_cache='for LRU in /proc/fs/lustre/ldlm/namespaces/*/lru_size;
do sudo /bin/bash -c "echo clear > $LRU"; done'
echo "=> $clear_cache" >> $tmpfile
$pdsh $pdsh_args "$test_clients" "$clear_cache" >> $tmpfile 2>&1
status=$?
echo "Completion Status: $status" >> $tmpfile
if ((status)); then
echo "ERROR"
else
echo "OK"
fi
}
count_range() {
echo $1 | awk '{ nvals=split($1, vals, "-");\
if (nvals == 1) print 1;\
......@@ -78,7 +131,7 @@ idx2nodenum() {
return
fi
chunk=`count_range $range`
if ((chunk > n)); then
if ((chunk > $n)); then
base=`base_range $range`
echo $((base + n))
return
......@@ -98,7 +151,7 @@ n2noderange() {
fi
local base=`base_range $range`
local chunk=`count_range $range`
if ((chunk > n)); then chunk=n; fi
if ((chunk > $n)); then chunk=$n; fi
local nodes="${nodes}${sep}${base}"; sep=","
if ((chunk > 1)); then nodes="${nodes}-$((base+chunk-1))"; fi
n=$((n-chunk))
......@@ -134,11 +187,11 @@ pp_number() {
local G=$((1024*1024*1024))
local M=$((1024*1024))
local K=$((1024))
if ((n%G == 0 && n >= G)); then
if ((n%G == 0 && n >= $G)); then
echo "$((n/G))G"
elif ((n%M == 0 && n >= M)); then
elif ((n%M == 0 && n >= $M)); then
echo "$((n/M))M"
elif ((n%K == 0 && n >= K)); then
elif ((n%K == 0 && n >= $K)); then
echo "$((n/K))K"
else
echo $n
......@@ -165,6 +218,21 @@ print_summary () {
echo $minusn "$*"
}
check_mount() {
local lustre=$1; shift
local tmpb=$1; shift
local clients=$1; shift
local tmpfile=${tmpb}_tmp
# check lustre is mounted everywhere it's needed
cmd="grep $lustre /proc/mounts"
$pdsh $pdsh_args "$clients" "$cmd" >> $tmpfile
status=$?
if (($status)); then
print_summary "Lustre NOT mounted on $lustre somewhere"
exit 1
fi
}
# convert params to actual numbers
min_per_client_size=`parse_number $min_per_client_size`
min_total_size=`parse_number $min_total_size`
......@@ -174,20 +242,30 @@ rsize_hi=`parse_number $rsize_hi`
# check on actual numbers of client nodes
nclients=`countnodes ${clients[@]}`
if ((clients_hi > nclients)); then clients_hi=$nclients; fi
if ((clients_hi > $nclients)); then clients_hi=$nclients; fi
cur_date=`date`
machine=`uname -n`
script_name=`echo $0 | cut -d "/" -f2`
echo "$cur_date $script_name on $lustre from $machine" >> $workf
for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
for ((rsize=$rsize_lo; rsize<=$rsize_hi; rsize$rsize_iterator)); do
pp_rsize=`pp_number $rsize`
for ((nclnt=clients_lo; nclnt<=clients_hi; nclnt$clients_iterator)); do
for ((nclnt=$clients_lo; nclnt<=$clients_hi; nclnt$clients_iterator)); do
test_clients="${cluster}`n2noderange $nclnt ${clients[@]}`"
echo $test_clients
if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ];then
check_mount $lustre $workf $test_clients
fi
per_client_size=$((min_total_size/nclnt))
if ((per_client_size < min_per_client_size)); then
if ((per_client_size < $min_per_client_size)); then
per_client_size=$min_per_client_size
fi
for ((ntask=tasks_per_client_lo; ntask <= tasks_per_client_hi; ntask$tasks_per_client_iterator)); do
for ((ntask=$tasks_per_client_lo; ntask <= $tasks_per_client_hi; \
ntask$tasks_per_client_iterator)); do
per_task_size=$((per_client_size/ntask))
if ((per_task_size%rsize != 0)); then
per_task_size=$(((per_task_size/rsize + 1)*rsize))
......@@ -207,32 +285,13 @@ for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
echo -n > $tmpf
if [ "$test" = "clear_cache" ]; then
clear_cache='for LRU in /proc/fs/lustre/ldlm/namespaces/*/lru_size; do; echo clear > $LRU; done'
echo "=> $clear_cache" >> $tmpf
$pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
"$clear_cache"
status=$?
echo "Completion Status: $status" >> $tmpf
if ((status)); then
result="ERROR"
else
result="OK"
fi
if [ "$fanout_cmd" = "pdsh" ] || [ "$fanout_cmd" = "mpirun" ]; then
result=`dump_cache $test_clients $tmpf`
else
echo "Haven't figured out how to clear cache" >> $tmpf
result="N/A"
fi
else
# check lustre is mounted everywhere it's needed
cmd="(mount -t lustre; mount -t lustre_lite) | grep $lustre"
echo "=> Mount Check: $cmd" >> $tmpf
$pdsh -S -b -w "$test_clients" >> $tmpf 2>&1 \
"$cmd"
status=$?
echo "Completion Status: $status" >> $tmpf
if ((status)); then
cat $tmpf >> $workf
rm $tmpf
print_summary "Lustre NOT mounted on $lustre somewhere"
exit 1
fi
cmdline=(
$IOR # the command
......@@ -246,13 +305,14 @@ for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
idx=${#cmdline[@]}
# keep the test file(s) unless this is the last test
((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
#((test_idx < ${#tests[@]}-1)) && cmdline[$((idx++))]="-k"
cmdline[$((idx++))]="-k"
# use the existing test file(s) unless this is the first test
((test_idx > 0)) && cmdline[$((idx++))]="-E"
# file-per-task
((file_per_task)) && cmdline[$((idx++))]="-F"
(($file_per_task)) && cmdline[$((idx++))]="-F"
case "$test" in
*write*) cmdline[$((idx++))]="-w"
......@@ -263,17 +323,21 @@ for ((rsize=rsize_lo; rsize<=rsize_hi; rsize$rsize_iterator)); do
echo "=> ${cmdline[@]}" >> $tmpf
$pdsh -S -b $pdsh_mpiargs -w "$test_clients" -n $ntask >> $tmpf 2>&1 \
"${cmdline[@]}"
status=$?
status=`fanout $test_clients $tmpf ${cmdline[@]}`
echo "Completion Status: $status" >> $tmpf
if ((status)); then
if (($status)); then
result="ERROR"
else
result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
# pdsh adds an extra field
if [ "$fanout_cmd" = "pdsh" ]; then
result=`awk < $tmpf "/$awkstr/ {print $ 4; found=1; exit}\
END {if (!found) print \"ERROR\"}"`
else
result=`awk < $tmpf "/$awkstr/ {print $ 3; found=1; exit}\
END {if (!found) print \"ERROR\"}"`
fi
fi
fi
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

https://git.gsi.de is provided by CIT→Linux&Web | GSI Helmholtzzentrum fuer Schwerionenforschung GmbH | Imprint (in German) | Privacy policy