From b94d4f7076b0a1ffd3c82c75505782c36073e8cb Mon Sep 17 00:00:00 2001
From: yujian <yujian>
Date: Thu, 31 Jul 2008 02:26:18 +0000
Subject: [PATCH] Branch HEAD b=13929 i=eric.mei i=grev
Deployment tool for setting up GSS/Kerberos environment on Lustre cluster.
---
lustre/tests/Makefile.am | 2 +-
lustre/tests/setup_kerberos.sh | 1073 ++++++++++++++++++++++++++++++++
2 files changed, 1074 insertions(+), 1 deletion(-)
create mode 100755 lustre/tests/setup_kerberos.sh
diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am
index f2b2c65240..156ca4c24d 100644
--- a/lustre/tests/Makefile.am
+++ b/lustre/tests/Makefile.am
@@ -12,7 +12,7 @@ noinst_SCRIPTS += conf-sanity.sh insanity.sh lfscktest.sh oos.sh oos2.sh
noinst_SCRIPTS += llog-test.sh recovery-small.sh replay-dual.sh sanity-quota.sh
noinst_SCRIPTS += replay-ost-single.sh replay-single.sh run-llog.sh sanityN.sh
noinst_SCRIPTS += lockorder.sh socketclient socketserver runmultiop_bg_pause
-noinst_SCRIPTS += sanity-sec.sh sanity-gss.sh krb5_login.sh
+noinst_SCRIPTS += sanity-sec.sh sanity-gss.sh krb5_login.sh setup_kerberos.sh
nobase_noinst_SCRIPTS = cfg/local.sh
nobase_noinst_SCRIPTS += acl/make-tree acl/run cfg/ncli.sh
nobase_noinst_SCRIPTS += rmtacl/make-tree rmtacl/run
diff --git a/lustre/tests/setup_kerberos.sh b/lustre/tests/setup_kerberos.sh
new file mode 100755
index 0000000000..7763fb2821
--- /dev/null
+++ b/lustre/tests/setup_kerberos.sh
@@ -0,0 +1,1073 @@
+#!/bin/bash
+# vim:expandtab:shiftwidth=4:softtabstop=4:tabstop=4:
+
+#
+# setup_kerberos.sh - setup the Kerberos environment on Lustre cluster
+#
+# Notes:
+# * Only one KDC involved, no slave KDC.
+# * Only one Kerberos realm involved, no multiple Kerberos realms.
+#
+###############################################################################
+
+# usage
+my_usage() {
+ cat <<EOF
+Usage: $(basename $0) <KDC_distro> <KDC_node> <MDS_node>[:MDS_node:...]
+ <OSS_node>[:OSS_node:...] <CLIENT_node>[:CLIENT_node:...]
+
+ This script is used to setup the Kerberos environment on Lustre cluster.
+
+ KDC_distro distribution on the KDC node (rhel5 or sles10)
+ KDC_node KDC node name
+ MDS_node Lustre MDS node name
+ OSS_node Lustre OSS node name
+ CLIENT_node Lustre client node name
+
+ e.g.: $(basename $0) rhel5 scsi2 sata2 sata3 client5
+ e.g.: $(basename $0) sles10 scsi2 scsi2 sata3:sata5 client2:client3
+ e.g.: $(basename $0) rhel5 scsi2 scsi2 scsi2 scsi2
+
+ Notes:
+ 1) The script will destroy all the old Kerberos settings by default. If you
+ want to reserve the original krb5.conf and KDC configuration, please set
+ "RESET_KDC=false".
+
+ 2) The script will create principals for some runas users and add them into
+ the Kerberos database by default. The UIDs of the runas users specified in
+ "LOCAL_UIDS" variable need exist on KDC, MDS and Client nodes. If you do not
+ need runas users, please set "CFG_RUNAS=false".
+
+ 3) The script will create idmap.conf and perm.conf under /etc/lustre dir on
+ MDS node for remote ACL by default. If you do not need remote ACL, please
+ set "CFG_IDMAP=false".
+
+EOF
+}
+
+# ************************ Parameters and Variables ************************ #
+MY_KDC_DISTRO=$1
+MY_KDCNODE=$2
+MY_MDSNODES=$3
+MY_OSSNODES=$4
+MY_CLIENTNODES=$5
+
+# translate to lower case letters
+MY_KDC_DISTRO=$(echo $MY_KDC_DISTRO | tr '[A-Z]' '[a-z]')
+
+if [ -z "$MY_KDC_DISTRO" -o -z "$MY_KDCNODE" -o -z "$MY_MDSNODES" -o \
+ -z "$MY_OSSNODES" -o -z "$MY_CLIENTNODES" ]; then
+ my_usage
+ exit 1
+fi
+
+LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
+. $LUSTRE/tests/test-framework.sh
+init_test_env
+. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+
+SCP=${SCP:-"scp -q"}
+ACCEPTOR_PORT=${ACCEPTOR_PORT:-988}
+
+# check and configure runas users
+CFG_RUNAS=${CFG_RUNAS:-true}
+# uids for local and remote users
+LOCAL_UIDS=${LOCAL_UIDS:-"500 501"}
+REMOTE_UIDS=${REMOTE_UIDS:-"500 501"} # for remote ACL testing
+
+# remove the original Kerberos and KDC settings
+RESET_KDC=${RESET_KDC:-true}
+
+# generate unique keytab for each client node
+SPLIT_KEYTAB=${SPLIT_KEYTAB:-true}
+
+# encryption types for generating keytab
+MDS_ENCTYPE=${MDS_ENCTYPE:-"des3-hmac-sha1"}
+OSS_ENCTYPE=${OSS_ENCTYPE:-"des3-hmac-sha1"}
+CLIENT_ENCTYPE=${CLIENT_ENCTYPE:-"des3-hmac-sha1"}
+
+# configuration file for Kerberos
+KRB5_CONF=${KRB5_CONF:-"/etc/krb5.conf"}
+KRB5_KEYTAB=${KRB5_KEYTAB:-"/etc/krb5.keytab"}
+KRB5_TICKET_LIFETIME=${KRB5_TICKET_LIFETIME:-"24h"}
+
+# configuration files for libgssapi and keyutils
+GSSAPI_MECH_CONF=${GSSAPI_MECH_CONF:-"/etc/gssapi_mech.conf"}
+REQUEST_KEY_CONF=${REQUEST_KEY_CONF:-"/etc/request-key.conf"}
+
+# create configuration files for remote ACL testing
+CFG_IDMAP=${CFG_IDMAP:-true}
+LUSTRE_CONF_DIR=${LUSTRE_CONF_DIR:-"/etc/lustre"}
+IDMAP_CONF=$LUSTRE_CONF_DIR/idmap.conf
+PERM_CONF=$LUSTRE_CONF_DIR/perm.conf
+
+# krb5 realm & domain
+KRB5_REALM=${KRB5_REALM:-"CO.CFS"}
+KRB5_DOMAIN=$(echo $KRB5_REALM | tr '[A-Z]' '[a-z]')
+
+MY_MDSNODES=${MY_MDSNODES//:/ }
+MY_OSSNODES=${MY_OSSNODES//:/ }
+MY_CLIENTNODES=${MY_CLIENTNODES//:/ }
+
+# set vars according to the KDC distribution
+KRB5PKG_SVR="krb5-server"
+KRB5PKG_DEV="krb5-devel"
+case $MY_KDC_DISTRO in
+ rhel5)
+ KRB5PKG_CLI="krb5-workstation"
+ KRB5PKG_LIB="krb5-libs"
+ KDC_CONF_DIR="/var/kerberos/krb5kdc"
+ ;;
+ sles10)
+ KRB5PKG_CLI="krb5-client"
+ KRB5PKG_LIB="krb5"
+ KDC_CONF_DIR="/var/lib/kerberos/krb5kdc"
+ ;;
+ *)
+ echo "Unsupported KDC distro: $MY_KDC_DISTRO!"
+ exit 1
+esac
+KDC_CONF="$KDC_CONF_DIR/kdc.conf"
+KDC_ACL="$KDC_CONF_DIR/kadm5.acl"
+
+# ******************************** Functions ******************************** #
+is_part_of() {
+ local name="$1"
+ shift
+ local list="$@"
+
+ if [ -z "$name" -o -z "$list" ]; then
+ false
+ return
+ fi
+
+ if [[ " $list " == *\ $name\ * ]]; then
+ true
+ else
+ false
+ fi
+
+ return
+}
+
+my_do_node() {
+ local node=$1
+ shift
+ local nodename=${node%.$KRB5_DOMAIN}
+ do_node $node "PATH=\$PATH:/usr/kerberos/sbin:/usr/kerberos/bin:\
+/usr/lib/mit/sbin:/usr/lib/mit/bin $@" | sed "s/^${nodename}: //"
+ return ${PIPESTATUS[0]}
+}
+
+do_node_mute() {
+ local output
+ output=$(my_do_node "$@" 2>&1)
+ return ${PIPESTATUS[0]}
+}
+
+do_kdc() {
+ my_do_node $MY_KDCNODE "$@"
+ return ${PIPESTATUS[0]}
+}
+
+do_kdc_mute() {
+ do_node_mute $MY_KDCNODE "$@"
+ return ${PIPESTATUS[0]}
+}
+
+#
+# convert a space-delimited node name list to a canonical name list
+#
+get_fqdn() {
+ local nodename_list="$@"
+ local fqdn_list=""
+ local name
+ local fqdn
+ local rc
+
+ for name in $nodename_list; do
+ fqdn=$(do_kdc "gethostip -n $name 2>&1")
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo "Can not get the FQDN of node $name: $fqdn"
+ return $rc
+ fi
+ [ -z "$fqdn_list" ] && fqdn_list="$fqdn" \
+ || fqdn_list="$fqdn_list $fqdn"
+
+ done
+
+ echo "$fqdn_list"
+ return 0
+}
+
+#
+# convert MDS/OSS nodes to their canonical name, it required by
+# kerberos. we also convert kdc and client too in order to make
+# node name comparison easier
+#
+normalize_names() {
+ local rc
+
+ # KDC
+ MY_KDCNODE=$(get_fqdn $MY_KDCNODE)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo $MY_KDCNODE
+ return $rc
+ fi
+
+ # MDS nodes
+ MY_MDSNODES=$(get_fqdn $MY_MDSNODES)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo $MY_MDSNODES
+ return $rc
+ fi
+
+ # OSS nodes
+ MY_OSSNODES=$(get_fqdn $MY_OSSNODES)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo $MY_OSSNODES
+ return $rc
+ fi
+
+ # client nodes
+ MY_CLIENTNODES=$(get_fqdn $MY_CLIENTNODES)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo $MY_CLIENTNODES
+ return $rc
+ fi
+
+ return 0
+}
+
+#
+# verify remote shell works on all nodes
+#
+check_rsh() {
+ local checked=""
+ local node
+
+ echo "+++ Checking remote shell"
+
+ for node in $MY_KDCNODE $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES; do
+ is_part_of $node $checked && continue
+
+ echo -n "Checking remote shell on $node..."
+ do_node_mute $node true || return ${PIPESTATUS[0]}
+ echo "OK!"
+
+ checked="$checked $node"
+ done
+}
+
+#
+# verify the entropy (random numbers) on the KDC node, which is
+# used by kdb5_util to create Kerberos database
+#
+check_entropy() {
+ local limit=170
+ local avail
+
+ echo "+++ Checking the entropy on the KDC"
+
+ echo -n "Checking $MY_KDCNODE..."
+ avail=$(do_kdc "sysctl -n kernel.random.entropy_avail")
+ local rc=${PIPESTATUS[0]}
+ if [ $rc -eq 0 ]; then
+ if [ $avail -lt $limit ]; then
+ echo -e "\nWarning: The entropy on the KDC node is only $avail, \
+which is not enough for kdb5_util to create Kerberos database! \
+Let's use /dev/urandom!"
+ do_kdc "rm -f /dev/random.bak && mv /dev/random{,.bak} && \
+mknod /dev/random c 1 9"
+ return ${PIPESTATUS[0]}
+ fi
+ else
+ echo "Can not get the entropy on the KDC node!"
+ return $rc
+ fi
+ echo "OK!"
+}
+
+#
+# verify runas users and groups
+#
+check_users() {
+ local checked=""
+ local node
+ local id
+ local user
+
+ echo "+++ Checking users and groups"
+
+ for node in $MY_KDCNODE $MY_MDSNODES $MY_CLIENTNODES; do
+ is_part_of $node $checked && continue
+
+ for id in $LOCAL_UIDS; do
+ echo -n "Checking uid/gid $id/$id on $node..."
+ user=$(my_do_node $node getent passwd | grep :$id:$id: | cut -d: -f1)
+ if [ -z "$user" ]; then
+ echo -e "\nPlease set LOCAL_UIDS and REMOTE_UIDS to some users \
+which exist on KDC, MDS and client or add user/group $id/$id on these nodes."
+ return 1
+ fi
+ echo "OK!"
+ done
+ checked="$checked $node"
+ done
+}
+
+cfg_mount() {
+ local node=$1
+ local dev=$2
+ local dir=$3
+
+ echo -n "Checking $dev mount on $node..."
+ if do_node_mute $node "grep -q $dir' ' /proc/mounts"; then
+ echo "OK!"
+ return 0
+ fi
+
+ if ! do_node_mute $node "grep -q ^$dev /etc/fstab"; then
+ my_do_node $node "echo '$dev $dir $dev defaults 0 0' >> /etc/fstab" || \
+ return ${PIPESTATUS[0]}
+ fi
+ my_do_node $node "mkdir -p $dir && mount $dir" || true
+
+ if ! do_node_mute $node "grep -q $dir' ' /proc/mounts"; then
+ echo "Failed to mount fs $dev at $dir!"
+ return 1
+ fi
+ echo "OK!"
+}
+
+#
+# configure nfsd mount on MDS and OSS nodes
+#
+cfg_nfs_mount() {
+ local checked=""
+ local node
+
+ echo "+++ Configuring nfsd mount"
+
+ for node in $MY_OSSNODES $MY_MDSNODES; do
+ is_part_of $node $checked && continue
+ cfg_mount $node nfsd /proc/fs/nfsd || return ${PIPESTATUS[0]}
+ checked="$checked $node"
+ done
+}
+
+get_pkgname() {
+ local node=$1
+ local pkg=$2
+
+ my_do_node $node "rpm -q $pkg 2>&1" | tail -n1
+ return ${PIPESTATUS[0]}
+}
+
+get_krb5pkgname() {
+ local node=$1
+ local flavor=$2
+
+ my_do_node $node cat /etc/SuSE-release 2>/dev/null | \
+ grep -q 'Enterprise Server 10'
+ if [ ${PIPESTATUS[1]} -eq 0 ]; then
+ case $flavor in
+ cli) echo "krb5-client";;
+ lib) echo "krb5";;
+ esac
+ else
+ case $flavor in
+ cli) echo "krb5-workstation";;
+ lib) echo "krb5-libs";;
+ esac
+ fi
+}
+
+check_kdc() {
+ local pkg
+ local rc
+
+ echo "+++ Checking KDC installation"
+
+ echo -n "Checking $MY_KDCNODE..."
+ pkg=$(get_pkgname $MY_KDCNODE $KRB5PKG_SVR)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo -e "\nCan not find $KRB5PKG_SVR package on $MY_KDCNODE: $pkg"
+ return $rc
+ fi
+ echo "OK!"
+}
+
+check_krb5() {
+ local checked=""
+ local pkg
+ local rc
+ local krb5pkg_cli
+
+ echo "+++ Checking Kerberos 5 installation"
+ for node in $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES; do
+ is_part_of $node $checked && continue
+
+ echo -n "Checking $node..."
+ krb5pkg_cli=$(get_krb5pkgname $node cli)
+
+ pkg=$(get_pkgname $node $krb5pkg_cli)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo -e "\nCan not find $krb5pkg_cli package on $node: $pkg"
+ return $rc
+ fi
+ echo "OK!"
+ checked="$checked $node"
+ done
+}
+
+check_libgssapi() {
+ local checked=""
+ local node
+ local pkg
+ local rc
+
+ echo "+++ Checking libgssapi installation"
+
+ LIBGSSAPI=$(get_pkgname $MY_KDCNODE libgssapi)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo "Can not find libgssapi package on $MY_KDCNODE: $LIBGSSAPI"
+ return $rc
+ fi
+
+ for node in $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES; do
+ is_part_of $node $checked && continue
+
+ echo -n "Checking $node..."
+ pkg=$(get_pkgname $node libgssapi)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo -e "\nCan not find libgssapi package on $node: $pkg"
+ return $rc
+ fi
+ echo "OK!"
+ checked="$checked $node"
+ done
+}
+
+#
+# check and update the /etc/gssapi_mech.conf file on each node
+# We only support MIT Kerberos 5 GSS-API mechanism.
+#
+cfg_libgssapi() {
+ local checked=""
+ local node
+ local pkg
+ local rc
+ local krb5pkg_lib
+ local krb5_lib
+
+ echo "+++ Updating $GSSAPI_MECH_CONF"
+
+ for node in $MY_KDCNODE $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES; do
+ is_part_of $node $checked && continue
+
+ krb5pkg_lib=$(get_krb5pkgname $node lib)
+ pkg=$(get_pkgname $node $krb5pkg_lib)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo -e "\nCan not find $krb5pkg_lib package on $node: $pkg"
+ return $rc
+ fi
+
+ krb5_lib=$(my_do_node $node "rpm -ql $pkg" | \
+ grep libgssapi_krb5.so | head -n1)
+
+ if ! do_node_mute $node \
+"egrep -q \\\"^$krb5_lib|^$(basename $krb5_lib)\\\" $GSSAPI_MECH_CONF"; then
+ do_node_mute $node \
+"echo '$krb5_lib mechglue_internal_krb5_init' >> $GSSAPI_MECH_CONF"
+ fi
+ checked="$checked $node"
+ done
+ echo "OK!"
+}
+
+#
+# check and update the /etc/request-key.conf file on each MDS and client node
+#
+cfg_keyutils() {
+ local checked=""
+ local node
+ local lgss_keyring
+
+ echo "+++ Updating $REQUEST_KEY_CONF"
+
+ for node in $MY_MDSNODES $MY_CLIENTNODES; do
+ is_part_of $node $checked && continue
+ lgss_keyring=$(my_do_node $node "which lgss_keyring") || \
+ return ${PIPESTATUS[0]}
+
+ if ! do_node_mute $node \
+"grep -q \\\"^create.*$lgss_keyring\\\" $REQUEST_KEY_CONF"; then
+ do_node_mute $node \
+"echo 'create lgssc * * $lgss_keyring %o %k %t %d %c %u %g %T %P %S' \
+>> $REQUEST_KEY_CONF"
+ fi
+ checked="$checked $node"
+ done
+ echo "OK!"
+}
+
+add_svc_princ() {
+ local fqdn=$1
+ local type=$2
+
+ echo -n "Creating service principal lustre_$type/$fqdn@$KRB5_REALM..."
+ do_kdc_mute "kadmin.local -r $KRB5_REALM <<EOF
+addprinc -randkey lustre_$type/$fqdn@$KRB5_REALM
+EOF"
+ local rc=${PIPESTATUS[0]}
+ [ $rc -ne 0 ] && echo "Failed!" || echo "OK!"
+
+ return $rc
+}
+
+add_svc_princ_root() {
+ echo -n "Creating service principal lustre_root@$KRB5_REALM..."
+ do_kdc_mute "kadmin.local -r $KRB5_REALM <<EOF
+addprinc -randkey lustre_root@$KRB5_REALM
+EOF"
+ local rc=${PIPESTATUS[0]}
+ [ $rc -ne 0 ] && echo "Failed!" || echo "OK!"
+
+ return $rc
+}
+
+add_user_princ() {
+ local user=$1
+
+ echo -n "Creating user principal $user@$KRB5_REALM..."
+ do_kdc_mute "kadmin.local -r $KRB5_REALM <<EOF
+addprinc -pw $user $user@$KRB5_REALM
+EOF"
+ local rc=${PIPESTATUS[0]}
+ [ $rc -ne 0 ] && echo "Failed!" || echo "OK!"
+
+ return $rc
+}
+
+add_test_princ_id() {
+ local id=$1
+ local user
+
+ user=$(do_kdc getent passwd $id | cut -d: -f1)
+ if [ -z "$user" ]; then
+ echo "Can not find the user with uid $id on the KDC!"
+ return 1
+ fi
+
+ add_user_princ $user || return ${PIPESTATUS[0]}
+}
+
+#
+# create principals for the client, MDS, OSS, runas users and add them to
+# the Kerberos database
+#
+cfg_kdc_princs() {
+ local node
+
+ for node in $MY_MDSNODES; do
+ add_svc_princ $node mds || return ${PIPESTATUS[0]}
+ done
+
+ for node in $MY_OSSNODES; do
+ add_svc_princ $node oss || return ${PIPESTATUS[0]}
+ done
+
+ for node in $MY_CLIENTNODES; do
+ if $SPLIT_KEYTAB; then
+ add_svc_princ $node root || return ${PIPESTATUS[0]}
+ else
+ add_svc_princ_root || return ${PIPESTATUS[0]}
+ fi
+ done
+
+ if ! $SPLIT_KEYTAB; then
+ add_user_princ lustre_root || return ${PIPESTATUS[0]}
+ fi
+ add_user_princ bin || return ${PIPESTATUS[0]}
+ add_user_princ daemon || return ${PIPESTATUS[0]}
+ add_user_princ games || return ${PIPESTATUS[0]}
+
+ if $CFG_RUNAS; then
+ for uid in $LOCAL_UIDS; do
+ add_test_princ_id $uid || return ${PIPESTATUS[0]}
+ done
+ fi
+}
+
+#
+# create and install the KDC configuration file kdc.conf on the KDC, which
+# will destroy the old KDC setting
+#
+cfg_kdc() {
+ local tmpdir="$TMP/krb5_cfg_tmp_$UID"
+ local tmpcfg=$tmpdir/kdc.conf
+ local tmpacl=$tmpdir/kadm5.acl
+
+ echo "+++ Configuring KDC on $MY_KDCNODE"
+ echo "Warning: old KDC setting on $MY_KDCNODE will be destroied!!!"
+
+ echo -n "Checking the existence of KDC config dir..."
+ do_kdc_mute "[ -d $KDC_CONF_DIR ]"
+ if [ ${PIPESTATUS[0]} -ne 0 ]; then
+ echo -e "\nUnrecognized krb5 distribution!"
+ return 1
+ else
+ echo "OK!"
+ fi
+
+ # stop KDC daemon
+ do_kdc_mute "/etc/init.d/krb5kdc stop < /dev/null" || true
+
+ echo -n "Removing old KDC configurations..."
+ do_kdc_mute "rm -f $KDC_CONF_DIR/*"
+ echo "OK!"
+
+ # create kdc.conf locally
+ rm -rf $tmpdir
+ mkdir -p $tmpdir || return ${PIPESTATUS[0]}
+ cat <<EOF > $tmpcfg
+[kdcdefaults]
+ acl_file = $KDC_ACL
+
+[realms]
+ $KRB5_REALM = {
+ master_key_type = des3-hmac-sha1
+ supported_enctypes = des3-hmac-sha1:normal aes128-cts:normal aes256-cts:normal des-cbc-md5:normal
+ }
+EOF
+
+ # install kdc.conf remotely
+ echo -n "Installing kdc.conf on $MY_KDCNODE..."
+ $SCP $tmpcfg root@$MY_KDCNODE:$KDC_CONF || return ${PIPESTATUS[0]}
+ echo "OK!"
+
+ # initialize KDC database
+ echo -n "Creating Kerberos database on $MY_KDCNODE..."
+ do_kdc_mute "kdb5_util create -r $KRB5_REALM -s -P 111111"
+ local rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo "Failed!"
+ return $rc
+ else
+ echo "OK!"
+ fi
+
+ # create ACL file locally & install remotely
+ cat <<EOF > $tmpacl
+*/admin@$KRB5_REALM *
+root@$KRB5_REALM *
+EOF
+ echo -n "Installing kadm5.acl on $MY_KDCNODE..."
+ $SCP $tmpacl root@$MY_KDCNODE:$KDC_ACL || return ${PIPESTATUS[0]}
+ echo "OK!"
+ rm -rf $tmpdir || true
+
+ # start KDC daemon
+ do_kdc "/etc/init.d/krb5kdc restart < /dev/null" || return ${PIPESTATUS[0]}
+}
+
+#
+# create and install the Kerberos configuration file krb5.conf on the KDC,
+# client, MDS and OSS
+#
+cfg_krb5_conf() {
+ local tmpdir="$TMP/krb5_cfg_tmp_$UID"
+ local tmpcfg="$tmpdir/krb5.conf"
+ local checked=""
+
+ echo "+++ Installing krb5.conf on all nodes"
+
+ # create krb5.conf locally
+ rm -rf $tmpdir
+ mkdir -p $tmpdir || return ${PIPESTATUS[0]}
+ cat <<EOF > $tmpcfg
+[libdefaults]
+ default_realm = $KRB5_REALM
+ dns_lookup_realm = false
+ dns_lookup_kdc = false
+ ticket_lifetime = $KRB5_TICKET_LIFETIME
+ forwardable = yes
+
+[realms]
+ $KRB5_REALM = {
+ kdc = $MY_KDCNODE:88
+ admin_server = $MY_KDCNODE:749
+ default_domain = $KRB5_DOMAIN
+ }
+
+[domain_realm]
+ .$KRB5_DOMAIN = $KRB5_REALM
+ $KRB5_DOMAIN = $KRB5_REALM
+
+[kdc]
+ profile = $KDC_CONF
+
+[appdefaults]
+ pam = {
+ debug = false
+ forwardable = true
+ krb4_convert = false
+ }
+EOF
+
+ # install krb5.conf remotely
+ for node in $MY_KDCNODE $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES; do
+ is_part_of $node $checked && continue
+
+ echo -n "Installing krb5.conf on $node..."
+ $SCP $tmpcfg root@$node:$KRB5_CONF || return ${PIPESTATUS[0]}
+ echo "OK!"
+
+ checked="$checked $node"
+ done
+ rm -rf $tmpdir || true
+}
+
+add_keytab() {
+ local tab=$1
+ local princ=$2
+ local enctype=$3
+
+ do_kdc_mute "kadmin.local -r $KRB5_REALM <<EOF
+ktadd -k $tab -e $enctype:normal $princ@$KRB5_REALM
+EOF"
+}
+
+add_keytab_svc() {
+ local tab=$1
+ local fqdn=$2
+ local type=$3
+ local enctype=$4
+
+ add_keytab $tab lustre_$type/$fqdn $enctype
+}
+
+add_keytab_root() {
+ local tab=$1
+ local enctype=$2
+
+ add_keytab $tab lustre_root $enctype
+}
+
+merge_keytab() {
+ local tab=$1
+ local node=$2
+
+ $SCP $tab root@$node:$tab || return ${PIPESTATUS[0]}
+ do_node_mute $node "ktutil <<EOF
+rkt $tab
+wkt $KRB5_KEYTAB
+EOF" || return ${PIPESTATUS[0]}
+ do_node_mute $node "rm -f $tab" || true
+}
+
+#
+# create and install the keytab file krb5.keytab on the client, MDS and OSS
+#
+cfg_keytab() {
+ local tmptab="$TMP/keytab.tmp"
+ local node
+
+ echo "+++ Generating keytabs"
+
+ # remove old keytabs
+ echo -n "Deleting old keytabs on all nodes..."
+ for node in $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES; do
+ do_node_mute $node "rm -f $KRB5_KEYTAB $TMP/krb5cc*"
+ done
+ echo "OK!"
+
+ # install for MDS nodes
+ for node in $MY_MDSNODES; do
+ echo -n "Preparing for MDS $node..."
+ do_kdc_mute "rm -f $tmptab"
+ add_keytab_svc $tmptab $node mds $MDS_ENCTYPE || return ${PIPESTATUS[0]}
+ if is_part_of $node $MY_OSSNODES; then
+ echo -n "also be an OSS..."
+ add_keytab_svc $tmptab $node oss $OSS_ENCTYPE || \
+ return ${PIPESTATUS[0]}
+ fi
+ echo "OK!"
+
+ echo -n "Installing krb5.keytab on $node..."
+ $SCP root@$MY_KDCNODE:$tmptab $tmptab || return ${PIPESTATUS[0]}
+ $SCP $tmptab root@$node:$KRB5_KEYTAB || return ${PIPESTATUS[0]}
+ echo "OK!"
+ rm -f $tmptab
+ done
+
+ # install for OSS nodes
+ for node in $MY_OSSNODES; do
+ echo -n "Preparing for OSS $node..."
+ if ! is_part_of $node $MY_MDSNODES; then
+ do_kdc_mute "rm -f $tmptab"
+ add_keytab_svc $tmptab $node oss $OSS_ENCTYPE || \
+ return ${PIPESTATUS[0]}
+ echo "OK!"
+
+ echo -n "Installing krb5.keytab on $node..."
+ $SCP root@$MY_KDCNODE:$tmptab $tmptab || return ${PIPESTATUS[0]}
+ $SCP $tmptab root@$node:$KRB5_KEYTAB || return ${PIPESTATUS[0]}
+ echo "OK!"
+ rm -f $tmptab
+ else
+ echo "also be an MDS, already done, skip"
+ fi
+ done
+
+ # install for client nodes
+ do_kdc_mute "rm -f $tmptab"
+ if ! $SPLIT_KEYTAB; then
+ echo -n "Preparing for client..."
+ add_keytab_root $tmptab $CLIENT_ENCTYPE || return ${PIPESTATUS[0]}
+ $SCP root@$MY_KDCNODE:$tmptab $tmptab || return ${PIPESTATUS[0]}
+ else
+ for node in $MY_CLIENTNODES; do
+ echo -n "Preparing for client $node..."
+ # don't generate keytabs if it's also an MDS
+ if is_part_of $node $MY_MDSNODES; then
+ echo "also be an MDS, already done, skip"
+ continue
+ fi
+
+ add_keytab_svc $tmptab $node root $CLIENT_ENCTYPE || \
+ return ${PIPESTATUS[0]}
+ $SCP root@$MY_KDCNODE:$tmptab $tmptab || return ${PIPESTATUS[0]}
+ done
+ fi
+ echo "OK!"
+ for node in $MY_CLIENTNODES; do
+ echo -n "Installing krb5.keytab on client $node..."
+
+ # don't install if it's also an MDS
+ if is_part_of $node $MY_MDSNODES; then
+ echo "also be an MDS, already done, skip"
+ continue
+ fi
+
+ # merge keytab if it's also an OSS
+ if is_part_of $node $MY_OSSNODES; then
+ echo -n "also be an OSS, merging keytab..."
+ merge_keytab $tmptab $node || return ${PIPESTATUS[0]}
+ echo "OK!"
+ continue
+ fi
+
+ # simply install otherwise
+ $SCP $tmptab root@$node:$KRB5_KEYTAB || return ${PIPESTATUS[0]}
+ echo "OK!"
+ done
+ rm -f $tmptab || true
+}
+
+check_acceptor_port() {
+ local node=$1
+ local port=$2
+
+ if [ -z "$port" ]; then
+ echo "Missing acceptor port!"
+ return 1
+ fi
+
+ local WAIT=0
+ local MAX_WAIT=300
+ while [ $WAIT -lt $MAX_WAIT ]; do
+ sleep 5
+ my_do_node $node "netstat -tpan" | grep -q ":$port .*TIME_WAIT"
+ if [ ${PIPESTATUS[1]} -ne 0 ]; then
+ return 0
+ fi
+ WAIT=$((WAIT + 5))
+ done
+
+ echo "LNET acceptor port $port is in use on node $node!"
+ return 2
+}
+
+get_client_nids() {
+ local client_nids=""
+ local node
+ local nid
+ local local_fqdn
+ local rc
+
+ # get the fqdn of the local host
+ local_fqdn=$(get_fqdn $HOSTNAME)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo $local_fqdn
+ return $rc
+ fi
+
+ for node in $MY_CLIENTNODES; do
+ my_do_node $node lsmod | grep -q lnet || \
+ my_do_node $node "modprobe lnet" || {
+ if [ "$node" = "$local_fqdn" ]; then
+ lsmod | grep -q lnet || load_modules
+ else
+ echo "Failed to load lnet module on node $node!"
+ return 1
+ fi
+ }
+
+ check_acceptor_port $node $ACCEPTOR_PORT || return ${PIPESTATUS[0]}
+
+ nid=$(set +x; my_do_node $node \
+"$LCTL net up 1>/dev/null && $LCTL list_nids" 2>&1 | head -n1
+exit ${PIPESTATUS[0]})
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo "Failed to get the nid for node $node: $nid"
+ return $rc
+ fi
+ [ -z "$client_nids" ] && client_nids="$nid" \
+ || client_nids="$client_nids $nid"
+
+ my_do_node $node "$LCTL net down 1>/dev/null" || true
+ done
+
+ echo "$client_nids"
+ return 0
+}
+
+#
+# create and install idmap.conf on the MDS
+#
+cfg_idmap_conf() {
+ local tmpcfg="$TMP/idmap.conf"
+ local fqdn
+ local user
+ local uid
+ local client_nids client_nid
+ local rc
+
+ echo "+++ Installing idmap.conf on MDS"
+ echo "Getting Client NID..."
+ client_nids=$(get_client_nids)
+ rc=${PIPESTATUS[0]}
+ if [ $rc -ne 0 ]; then
+ echo $client_nids
+ return $rc
+ fi
+
+ rm -f $tmpcfg
+ if $SPLIT_KEYTAB; then
+ for fqdn in $MY_CLIENTNODES; do
+ echo "lustre_root/$fqdn@$KRB5_REALM * 0" >> $tmpcfg
+ done
+ else
+ echo "lustre_root@$KRB5_REALM * 0" >> $tmpcfg
+ fi
+ cat <<EOF >> $tmpcfg
+bin@$KRB5_REALM * 1
+daemon@$KRB5_REALM * 2
+games@$KRB5_REALM * 12
+EOF
+
+ for node in $MY_MDSNODES; do
+ for uid in $LOCAL_UIDS; do
+ user=$(my_do_node $node getent passwd $uid | cut -d: -f1)
+ for client_nid in $client_nids; do
+ echo "$user@$KRB5_REALM $client_nid $uid" >> $tmpcfg
+ done
+ done
+ done
+
+ for node in $MY_MDSNODES; do
+ my_do_node $node "mkdir -p $LUSTRE_CONF_DIR" || return ${PIPESTATUS[0]}
+ $SCP $tmpcfg root@$node:$IDMAP_CONF || return ${PIPESTATUS[0]}
+ done
+ rm -f $tmpcfg
+ echo "OK!"
+}
+
+#
+# create and install perm.conf on the MDS for remote ACL testing
+#
+cfg_perm_conf() {
+ local tmpcfg="$TMP/perm.conf"
+ local uid
+
+ echo "+++ Installing perm.conf on MDS"
+
+ rm -f $tmpcfg
+ for node in $MY_MDSNODES; do
+ my_do_node $node "mkdir -p $LUSTRE_CONF_DIR" || return ${PIPESTATUS[0]}
+
+ for uid in $LOCAL_UIDS $REMOTE_UIDS; do
+ if ! grep -q " $uid " $tmpcfg 2>/dev/null; then
+ echo "* $uid rmtacl" >> $tmpcfg
+ fi
+ done
+
+ echo "* 0 setgid" >> $tmpcfg
+
+ $SCP $tmpcfg root@$node:$PERM_CONF || return ${PIPESTATUS[0]}
+ done
+ rm -f $tmpcfg
+ echo "OK!"
+}
+
+# ******************************** Main Flow ******************************** #
+normalize_names || exit ${PIPESTATUS[0]}
+check_rsh || exit ${PIPESTATUS[0]}
+check_entropy || exit ${PIPESTATUS[0]}
+
+if $CFG_RUNAS; then
+ check_users || exit ${PIPESTATUS[0]}
+elif $CFG_IDMAP; then
+ echo "Remote ACL operations need local and remote users!"
+ exit 1
+fi
+
+check_kdc || exit ${PIPESTATUS[0]}
+check_krb5 || exit ${PIPESTATUS[0]}
+check_libgssapi || exit ${PIPESTATUS[0]}
+
+echo "===================================================================="
+echo " Configure Kerberos testing environment for Lustre"
+echo " KDC: $MY_KDCNODE"
+echo " realm: $KRB5_REALM, domain: $KRB5_DOMAIN"
+echo " Using gssapi package: $LIBGSSAPI"
+echo " OSS nodes:"
+for i in $MY_OSSNODES; do echo " $i"; done
+echo " MDS nodes:"
+for i in $MY_MDSNODES; do echo " $i"; done
+echo " CLIENT nodes:"
+for i in $MY_CLIENTNODES; do echo " $i"; done
+echo "===================================================================="
+
+cfg_nfs_mount || exit ${PIPESTATUS[0]}
+cfg_libgssapi || exit ${PIPESTATUS[0]}
+cfg_keyutils || exit ${PIPESTATUS[0]}
+
+if $RESET_KDC; then
+ cfg_krb5_conf || exit ${PIPESTATUS[0]}
+ cfg_kdc || exit ${PIPESTATUS[0]}
+fi
+
+cfg_kdc_princs || exit ${PIPESTATUS[0]}
+cfg_keytab || exit ${PIPESTATUS[0]}
+
+if $CFG_IDMAP; then
+ cfg_idmap_conf || exit ${PIPESTATUS[0]}
+ cfg_perm_conf || exit ${PIPESTATUS[0]}
+fi
+
+echo "Complete successfully!"
--
GitLab