b=16808

i=Adilger
check CATASTROPHE on all nodes

lustre/tests/test-framework.sh

@@ -1649,8 +1649,7 @@ run_one() {
     cd $SAVE_PWD
     reset_fail_loc
     check_grant ${testnum} || error "check_grant $testnum failed with $?"
-    [ -f $CATASTROPHE ] && [ `cat $CATASTROPHE` -ne 0 ] && \
-        error "LBUG/LASSERT detected"
+    check_catastrophe || error "LBUG/LASSERT detected"
     ps auxww | grep -v grep | grep -q multiop && error "multiop still running"
     pass "($((`date +%s` - $BEFORE))s)"
     unset TESTNAME
@@ -1793,6 +1792,12 @@ nodes_list () {
     echo $myNODES_sort
 }
 
+remote_nodes_list () {
+    local rnodes=$(nodes_list)
+    rnodes=$(echo " $rnodes " | sed -re "s/\s+$HOSTNAME\s+/ /g")
+    echo $rnodes 
+}
+
 init_clients_lists () {
     # Sanity check: exclude the local client from RCLIENTS
     local rclients=$(echo " $RCLIENTS " | sed -re "s/\s+$HOSTNAME\s+/ /g")
@@ -1920,3 +1925,12 @@ restore_lustre_params() {
         done
 }
 
+check_catastrophe () {
+    local rnodes=$(comma_list $(remote_nodes_list))
+
+    [ -f $CATASTROPHE ] && [ `cat $CATASTROPHE` -ne 0 ] && return 1
+    if [ $rnodes ]; then
+        do_nodes $rnodes "[ -f $CATASTROPHE ] && { [ `cat $CATASTROPHE` -eq 0 ] || false; } || true"
+    fi 
+}
+