lbuild

#!/bin/sh

TOPDIR=$PWD

# CVSROOT is inherited from the environment
KERNELDIR=
LINUX=
LUSTRE=
PUBLISH=0
RELEASE=0
DO_SRC=0
DOWNLOAD=1
TAG=
TARGET=
TARGET_ARCHS=
CONFIGURE_FLAGS=
EXTERNAL_PATCHES=
EXTRA_VERSION=

# from target file
KERNEL=
SERIES=
CONFIG=
VERSION=

RHBUILD=0
SUSEBUILD=0
LINUX26=0
SUSEBUILD=0

BASE_ARCHS=
BIGMEM_ARCHS=
BOOT_ARCHS=
JENSEN_ARCHS=
SMP_ARCHS=
BIGSMP_ARCHS=
PSERIES64_ARCHS=
UP_ARCHS=

DATE=$(date)

USE_DATESTAMP=1
RPMBUILD=

export CC=${CC:-gcc}

# Readlink is not present on some older distributions: emulate it.
readlink() {
    local path=$1 ll

    if [ -L "$path" ]; then
	ll="$(LC_ALL=C ls -l "$path" 2> /dev/null)" &&
	echo "${ll/* -> }"
    else
	return 1
    fi
}

cleanup()
{
    true
}

error()
{
    [ "$1" ] && echo -e "\n${0##*/}: $1"
}

fatal()
{
    cleanup
    error "$2"
    exit $1
}

publishing()
{
    (( $PUBLISH )) || return 0
}

is_release()
{
    (( $RELEASE )) || return 0
}

list_targets()
{
    echo -n "Available targets:"
    for target in $TOPDIR/lustre/lustre/kernel_patches/targets/*.target ; do
	target_file=${target##*/}
	echo -n " ${target_file%%.target}"
    done
    echo
}

usage()
{
    cat <<EOF
Usage: ${0##*/} [OPTION]... [-- <lustre configure options>]

  -d CVSROOT
    Specifies the CVS Root to use when pulling files from CVS.  The
    environment variable \$CVSROOT is used if this option is not
    present.

  --external-patches=EXTERNAL_PATCHES
    Directory similar to lustre/lustre/kernel_patches/ that lbuild should
    look for seres and config files in before looking in the lustre
    tree.

  --extraversion=EXTRAVERSION
    Text to use for the rpm release and kernel extraversion.

  --kerneldir=KERNELDIR
    Directory containing Linux source tarballs referenced by target
    files.

  --linux=LINUX
    Directory of Linux kernel sources.  When this option is used, only
    Lustre modules and userspace are built.

  --lustre=LUSTRE
    Path to an existing lustre source tarball to use instead of
    pulling from CVS.

  --nodownload
    Do not try to download a kernel from ftp.lustre.org

  --nosrc
    Do not build a .src.rpm, a full kernel patch, or a patched kernel
    tarball.

  --publish
    Publish the packages, patches, and tarballs on the ftp server.

  --release
    Specifies that the files generated do not include timestamps, and
    that this is an official release.

  --src
    Build a .src.rpm, a full kernel patch, and a patched kernel tarball.

  --tag=TAG
    A CVS branch/tag name to build from when pulling from CVS.

  --target=TARGET
    The name of the target to build.  The available targets are listed
    below.

  --target-archs=TARGET_ARCHS
    A (space delimited) list of architectures to build.  By default,
    all of the archs supported by the TARGET will be built, in
    addition to a .src.rpm.  This option can limit those, for machines
    that can only build certain archs or if you only want a certain
    arch built (for testing, or a one-off kernel).

    Also note that by using a non-"base" arch (eg, i386) only kernels
    will be built - there will be no lustre-lite-utils package.

  --disable-datestamp
    Prevents the datestamp flag (-D) from being passed to cvs for 
    checkouts. This is a workaround for a problem encountered when 
    using lbuild with tinderbox.

EOF

#   list_targets

    fatal "$1" "$2"
}

check_options()
{
    if [ "$LUSTRE" ] ; then
	[ -r "$LUSTRE" ] || \
	    usage 1 "Could not find Lustre source tarball '$LUSTRE'."
    else
	[ "$CVSROOT" ] || \
	    usage 1 "Either specify a CVS Root with -d, or a Lustre source tarball with --lustre."
	[ "$TAG" ] || \
	    usage 1 "A branch/tag name must be specified with --tag when not building from a tarball."
    fi

    if [ -z "$LINUX" ] ; then
	[ "$KERNELDIR" ] || \
	    usage 1 "A kernel directory must be specified with --kerneldir."

	[ -d "$KERNELDIR" ] || \
	    usage 1 "$KERNELDIR is not a directory."

	if ! (( $RELEASE )) ; then
	    [ "$TAG" ] || \
		usage 1 "When building a snapshot, a tag name must be used."
	fi

	[ "$TARGET" ] || usage 1 "A target must be specified with --target."
#       TARGET_FILE="$TOPDIR/lustre/kernel_patches/targets/$TARGET.target"
#       [ -r "$TARGET_FILE" ] || \
#       	usage 1 "Target '$TARGET' was not found."
    fi

    TIMESTAMP=$(date -d "$DATE" "+%Y%m%d%H%M")

    RPMBUILD=$(which rpmbuild 2>/dev/null | head -1)
    if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then
	RPMBUILD=$(which rpm 2>/dev/null | head -1)
	if [ ! "$RPMBUILD" -o "$RPMBUILD" == "" ]; then
	    usage 1 "Could not find binary for making rpms (tried rpmbuild and rpm)."
	fi
    fi
}

uniqify()
{
    echo $(echo "$*" | xargs -n 1 | sort -u)
}

load_target()
{
    EXTRA_VERSION_save="$EXTRA_VERSION"
    for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches" ; do
	TARGET_FILE="$patchesdir/targets/$TARGET.target"
	[ -r "$TARGET_FILE" ] && break
    done
    [ -r "$TARGET_FILE" ] || \
	fatal 1 "Target $TARGET was not found."

    echo "Loading target config file $TARGET.target..."	

    . "$TARGET_FILE"

    [ "$KERNEL"  ] || fatal 1 "Target $TARGET did not specify a kernel."
#    [ "$SERIES"  ] || fatal 1 "Target $TARGET did not specify a kernel patch series."
#    [ "$CONFIG"  ] || fatal 1 "Target $TARGET did not specify a kernel config."
    [ "$VERSION" ] || fatal 1 "Target $TARGET did not specify a kernel version."

    if [ "$KERNELDIR" ] ; then
	KERNEL_FILE="$KERNELDIR/$KERNEL"
	if [ ! -r "$KERNELDIR/$KERNEL" ] ; then
	    case $TARGET in
		2.6-rhel4)
		    dldir="rhel-2.6"
		    ;;
		2.6-suse)
		    dldir="suse-2.6"
		    ;;
		hp_pnnl-2.4)
		    dldir="hp-pnnl-2.4"
		    ;;
		2.6-vanilla \
		| suse-2.4.21-2 \
		| rh-2.4 \
		| rhel-2.4 \
		| sles-2.4)
		    dldir="$TARGET"
		    ;;
	    esac
	    if (( $DOWNLOAD )) ; then
		echo "Downloading http://ftp.lustre.org/kernels/$dldir/old/$KERNEL..."
		if ! wget -nv "http://ftp.lustre.org/kernels/$dldir/old/$KERNEL" \
		    -O "$KERNELDIR/$KERNEL" ; then
		    fatal 1 "Could not download target $TARGET's kernel file $KERNEL from ftp.lustre.org."
		fi
	    else
		fatal 1 "Target $TARGET's kernel file $KERNEL not found in kernel directory $KERNELDIR."
	    fi
	fi
    fi

    if [ "$SERIES" ] ; then
        for series in $SERIES ; do
	    for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches" ; do
	        [ -r "$patchesdir/series/$series" ] && continue 2
	    done
	    fatal 1 "Target $TARGET's series $SERIES could not be found.\nSearched:\n\t$EXTERNAL_PATCHES/series\n\t$TOPDIR/lustre/lustre/kernel_patches/series."
	done
    fi

    CONFIG_FILE="$TOPDIR/lustre/lustre/kernel_patches/kernel_configs/$CONFIG"
    [ -r "$CONFIG_FILE" ] || \
	fatal 1 "Target $TARGET's config file $CONFIG missing from $TOPDIR/lustre/lustre/kernel_patches/kernel_configs/."

    if [ "$EXTRA_VERSION_save" ] ; then
	EXTRA_VERSION="$EXTRA_VERSION_save"
    elif ! (( $RELEASE )) ; then
	#remove the @VERSION@ (lustre version)
	EXTRA_VERSION=$(echo $EXTRA_VERSION | sed -e "s/\(.*_lustre\)\..*/\1/")
	EXTRA_VERSION="${EXTRA_VERSION}-${TAG}.${TIMESTAMP}"
    fi
    # EXTRA_VERSION=${EXTRA_VERSION//-/_}

    ALL_ARCHS="$BASE_ARCHS $BIGMEM_ARCHS $BOOT_ARCHS $JENSEN_ARCHS $SMP_ARCHS $BIGSMP_ARCHS $PSERIES64_ARCHS $UP_ARCHS"

    BUILD_ARCHS=
    for arch in $(uniqify "$ALL_ARCHS") ; do
	if [ -z "$TARGET_ARCHS" ] || echo "$TARGET_ARCHS" | grep "$arch" >/dev/null 2>/dev/null ; then
	    BUILD_ARCHS="$BUILD_ARCHS $arch"
	fi
    done
    [ "$BUILD_ARCHS" ] || usage 1 "No available target archs to build."
    echo "Building for: $BUILD_ARCHS"
}

tarflags()
{
    case "$1" in
	'')
	    fatal 1 "tarflags(): File name argument missing."
	    ;;
	*.tar.gz | *.tgz)
	    echo 'zxf'
	    ;;
	*.tar.bz2)
	    echo 'jxf'
	    ;;
	*.tar)
	    echo 'xf'
	    ;;
	*)
	    fatal 1 "tarflags(): Unrecognized tar extension in file: $1"
	    ;;
    esac
}

untar()
{
    echo "Untarring ${1##*/}..."
    tar $(tarflags "$1") "$1"
}

unpack_lustre()
{
    DIRNAME="lustre-$TAG-$TIMESTAMP"
    if [ "$LUSTRE" ] ; then
	untar "$LUSTRE"
	[ -d lustre ] || ln -sf lustre* lustre
    else
	if [ "$USE_DATESTAMP" ]; then
	    DATESTAMP="-D '$DATE'"
	else
	    DATESTAMP=""
	fi	    

	cvs -d "$CVSROOT" -qz3 co $DATESTAMP -d "$DIRNAME" lustre || \
	    fatal 1 "There was an error checking out toplevel Lustre from CVS."
	pushd "$DIRNAME" > /dev/null
	./lustrecvs "$TAG" || \
	    fatal 1 "There was an error checking out Lustre/Portals/Build from CVS."
	echo "Creating lustre tarball..."
	sh autogen.sh || fatal 1 "There was an error running autogen.sh."
	./configure --disable-{modules,utils,liblustre,tests,doc} || \
	    fatal 1 "There was an error running ./configure to create makefiles."
	make dist || fatal 1 "There was an error running 'make dist'."
	popd > /dev/null
	fname=`basename $DIRNAME/lustre-*.tar.gz`
	cp $DIRNAME/$fname . || fatal 1 "There was an error copying lustre tarball."
	LUSTRE="$PWD/$fname"
	ln -sf "$DIRNAME" lustre
    fi
}

unpack_linux()
{
    untar "$KERNEL_FILE"
    [ -d linux ] || ln -sf linux* linux
}

patch_linux()
{
    [ "$SERIES" ] || return 0
    FULL_PATCH="$PWD/lustre-kernel-${TARGET}-${EXTRA_VERSION}.patch"
    [ -f "$FULL_PATCH" ] && rm -f "$FULL_PATCH"
    pushd linux >/dev/null
    for series in $SERIES ; do
	echo -n "Applying series $series:"
	for patchesdir in "$EXTERNAL_PATCHES" "$TOPDIR/lustre/lustre/kernel_patches" ; do
	    [ -r "$patchesdir/series/$series" ] || continue
	    SERIES_FILE="$patchesdir/series/$series"
	    for patch in $(<"$SERIES_FILE") ; do
		echo -n " $patch"
		PATCH_FILE="$patchesdir/patches/$patch"
		[ -r "$PATCH_FILE" ] || \
		    fatal 1 "Patch $patch does not exist in Lustre tree."
		cat "$PATCH_FILE" >> "$FULL_PATCH" || \
		    fatal 1 "Error adding patch $patch to full patch."
		patch -s -p1 < "$PATCH_FILE" || fatal 1 "Error applying patch $patch."
	    done
	    break
	done
	echo
    done
    popd >/dev/null
    echo "Full patch has been saved in ${FULL_PATCH##*/}."
    echo "Replacing .config files..."
    [ -d linux/configs ] || mkdir linux/configs || \
        fatal 1 "Error creating configs directory."
    rm -f linux/configs/*
    copysuccess=0
    for patchesdir in "$EXTERNAL_PATCHES" "lustre/lustre/kernel_patches" ; do
	[ "$patchesdir" ] && \
	    cp -v $patchesdir/kernel_configs/kernel-${VERSION}-${TARGET}*.config linux/configs/ >/dev/null && copysuccess=1
    done
    [ "$copysuccess" = "1" ] || \
	fatal 1 "Error copying in kernel configs."
}

pack_linux()
{
    TARBALL="$(readlink linux)-$EXTRA_VERSION.tar.gz"
    echo "Creating patched linux tarball $TARBALL..."
    tar zcf "$TARBALL" "$(readlink linux)" \
	--exclude "CVS" --exclude ".cvsignore" || \
	--exclude "*.orig" --exclude "*~" --exclude "*.rej" || \
	fatal 1 "Error creating patched Linux tarball."
}

clean_linux()
{
    [ -d linux ] || return 0
    echo "Cleaning linux..."
    [ -L linux ] && rm -rf $(readlink linux)
    rm -rf linux
}

prep_kernel_build()
{
    # make .spec file
    ENABLE_INIT_SCRIPTS=""
    sed \
	-e "s^@BASE_ARCHS@^$BASE_ARCHS^g" \
	-e "s^@BIGMEM_ARCHS@^$BIGMEM_ARCHS^g" \
	-e "s^@BIGSMP_ARCHS@^$BIGSMP_ARCHS^g" \
	-e "s^@BOOT_ARCHS@^$BOOT_ARCHS^g" \
	-e "s^@CONFIGURE_FLAGS@^$CONFIGURE_FLAGS^g" \
	-e "s^@ENABLE_INIT_SCRIPTS@^$ENABLE_INIT_SCRIPTS^g" \
	-e "s^@JENSEN_ARCHS@^$BOOT_ARCHS^g" \
	-e "s^@KERNEL_EXTRA_VERSION@^$EXTRA_VERSION^g" \
	-e "s^@KERNEL_RELEASE@^${EXTRA_VERSION//-/_}^g" \
	-e "s^@KERNEL_SOURCE@^$KERNEL^g" \
	-e "s^@KERNEL_VERSION@^$VERSION^g" \
	-e "s^@LINUX26@^$LINUX26^g" \
	-e "s^@LUSTRE_SOURCE@^${LUSTRE##*/}^g" \
	-e "s^@LUSTRE_TARGET@^$TARGET^g" \
	-e "s^@PSERIES64_ARCHS@^$PSERIES64_ARCHS^g" \
	-e "s^@RHBUILD@^$RHBUILD^g" \
	-e "s^@SMP_ARCHS@^$SMP_ARCHS^g" \
	-e "s^@SUSEBUILD@^$SUSEBUILD^g" \
	-e "s^@SUSEBUILD@^$SUSEBUILD^g" \
	-e "s^@UP_ARCHS@^$UP_ARCHS^g" \
	< $TOPDIR/lustre/build/lustre-kernel-2.4.spec.in \
	> lustre-kernel-2.4.spec
    [ -d SRPMS ] || mkdir SRPMS
    [ -d RPMS ] || mkdir RPMS
    [ -d BUILD ] || mkdir BUILD
    [ -d SOURCES ] || mkdir SOURCES
    for script in linux-{rhconfig.h,merge-config.awk,merge-modules.awk} \
	suse-{functions.sh,post.sh,postun.sh,trigger-script.sh.in} \
	sles8-{pre,post,postun,update_{INITRD_MODULES,rcfile_setting}}.sh ; do
	cp $TOPDIR/lustre/build/$script SOURCES
    done
    cp "$LUSTRE" "$KERNEL_FILE" SOURCES
    if [ "$EXTERNAL_PATCHES" -a -d "$EXTERNAL_PATCHES" ] ; then
	tar zcf SOURCES/external-patches.tar.gz -C "$EXTERNAL_PATCHES" series targets patches kernel_configs
    else
	touch SOURCES/external-patches.tar.gz
    fi
}

clean_lustre()
{
    [ -d lustre ] || return 0
    echo "Cleaning Lustre..."
    [ -L lustre ] && rm -rf $(readlink lustre)
    rm -rf lustre
}

build_kernel()
{
    echo "Building kernel + Lustre RPMs for: $BUILD_ARCHS..."
    targets=
    for arch in $BUILD_ARCHS ; do
	targets="--target $arch $targets"
    done

    $RPMBUILD $targets -bb lustre-kernel-2.4.spec \
	--define "_topdir $TOPDIR" || \
	fatal 1 "Error building rpms for $BUILD_ARCHS."

    if (( $DO_SRC )) ; then
	$RPMBUILD -bs lustre-kernel-2.4.spec \
	    --define "_topdir $TOPDIR" || \
	    fatal 1 "Error building .src.rpm."
    fi
}

build_lustre()
{
    [ -d SRPMS ] || mkdir SRPMS
    [ -d RPMS ] || mkdir RPMS
    [ -d BUILD ] || mkdir BUILD
    [ -d SOURCES ] || mkdir SOURCES

    cp "$LUSTRE" SOURCES

    pushd lustre >/dev/null

    echo "Building Lustre RPMs for: $BUILD_ARCHS..."
    targets=
    for arch in $BUILD_ARCHS ; do
	targets="--target $arch $targets"
    done

    ./configure "--with-linux=${LINUX}" ${CONFIGURE_FLAGS}

    $RPMBUILD $targets -bb build/lustre.spec \
	--define "_topdir $TOPDIR" || \
	fatal 1 "Error building rpms for $BUILD_ARCHS."

    popd >/dev/null
}

publish()
{
    publishing || return 0
}

[ -r ~/.lbuildrc ] && . ~/.lbuildrc

options=$(getopt -o d:D:h -l disable-datestamp,external-patches:,extraversion:,kerneldir:,linux:,lustre:,nodownload,nosrc,publish,release,tag:,target:,target-archs:,with-linux: -- "$@")

if [ $? != 0 ] ; then
    usage 1
fi

eval set -- "$options"
    
while [ "$1" ] ; do
    case "$1" in
	'')
	    usage 1
	    ;;
	-d)
	    CVSROOT=$2
	    shift 2
	    ;;
	-D)
	    DATE=$2
	    shift 2
	    ;;
	--external-patches)
	    EXTERNAL_PATCHES=$2
	    shift 2
	    ;;
	--extraversion)
	    EXTRA_VERSION=$2
	    shift 2
	    ;;
	--help | -h)
	    usage 0
	    ;;
	--kerneldir)
	    KERNELDIR=$2
	    shift 2
	    ;;
	--linux | --with-linux)
	    LINUX=$2
	    shift 2
	    ;;
	--lustre)
	    LUSTRE=$2
	    shift 2
	    ;;
	--nodownload)
	    DOWNLOAD=0
	    shift 1
	    ;;
	--nosrc)
	    DO_SRC=0
	    shift 1
	    ;;
	--publish)
	    PUBLISH=1
	    shift
	    ;;
	--release)
	    RELEASE=1
	    shift
	    ;;
	--src)
	    DO_SRC=1
	    shift 1
	    ;;
	--tag)
	    TAG=$2
	    shift 2
	    ;;
	--target)
	    TARGET=$2
	    shift 2
	    ;;
	--target-archs)
	    TARGET_ARCHS=$2
	    shift 2
	    ;;
	--disable-datestamp)
	    USE_DATESTAMP=
	    shift
	    ;;
	--)
	    shift
	    CONFIGURE_FLAGS=$@
	    break
	    ;; 
	*)
	    usage 1 "Unrecognized option: $1"
	    ;;
    esac
done

check_options

unpack_lustre

# prep_build needs the .spec.in from the lustre source
if [ -z "$LINUX" ] ; then
    load_target

    if (( $DO_SRC )) ; then
	unpack_linux
	patch_linux
	pack_linux
	clean_linux
    fi

    prep_kernel_build
    clean_lustre

    build_kernel
else
    build_lustre
fi

publish