I updated the latest MVAPICH2 SRPM:
https://www.openfabrics.org/~rowland/ofed_1_2/
I am including a patch to the latest ofed_1_2_scripts git files. Since
these files are the same as those used in the OFED-1.2-20070208-1508.tgz
package, this patch can also be applied there. This patch is required to
use the new MVAPICH2 SRPM file and should not be used with the older
versions.
I've done the following:
- Updated some of the dependencies when mvapich2 is selected.
- Added new mvapich2 configuration prompts if mvapich2 is selected.
This is all contained within the mvapich2_config shell function. These
values are stored in the configuration file, etc. and prefixed with
MVAPICH2_CONF_.
There are two implementation choices for the MVAPICH2 build: OFA and
uDAPL. The OFA build should allow IB, IB + RDMA-CM, and iWARP to be
used. The mode is controlled by the following runtime environment variables:
IB
--
No additional environment variable required (default case).
IB + RDMA-CM
------------
MV2_USE_RDMA_CM=1
iWARP
-----
MV2_ENABLE_IWARP_MODE=1
--
Shaun Rowland [EMAIL PROTECTED]
http://www.cse.ohio-state.edu/~rowland/
diff --git a/build.sh b/build.sh
index 5eafb0d..c5f996c 100755
--- a/build.sh
+++ b/build.sh
@@ -448,18 +448,25 @@ mvapich()
mvapich2()
{
- local iwarp=0
-
- if [ "$MVAPICH2_IMPL" = "iwarp" ]; then
- iwarp=1
- fi
-
- echo
+ if [ $MVAPICH2_CONF_impl = "ofa" ]; then
+ echo "Building the MVAPICH2 RPM in the OFA configuration. Please
wait..."
+ elif [ $MVAPICH2_CONF_impl = "udapl" ]; then
+ echo "Building the MVAPICH2 RPM in the uDPAL configuration. Please
wait..."
+ if [ -d ${BUILD_ROOT}${STACK_PREFIX}/lib64 ]; then
+ MVAPICH2_DAT_LIB=${STACK_PREFIX}/lib64
+ elif [ -d ${BUILD_ROOT}${STACK_PREFIX}/lib ]; then
+ MVAPICH2_DAT_LIB=${STACK_PREFIX}/lib
+ else
+ echo "Could not find a proper uDAPL lib directory."
+ return 1
+ fi
- if [ $iwarp -eq 0 ]; then
- echo "Building the MVAPICH2 RPM with IB support. Please wait..."
- else
- echo "Building the MVAPICH2 RPM with iWARP support. Please wait..."
+ if [ -d ${BUILD_ROOT}${STACK_PREFIX}/include ]; then
+ MVAPICH2_DAT_INCLUDE=${STACK_PREFIX}/include
+ else
+ echo "Could not find a proper uDAPL include directory."
+ return 1
+ fi
fi
echo
@@ -484,7 +491,7 @@ mvapich2()
# On i686 the PathScale compiler requires -g optimization
# for MVAPICH2 in the shared library configuration.
- if [ "$ARCH" = "i686" ]; then
+ if [ "$ARCH" = "i686" ] && [ $MVAPICH2_CONF_shared_libs -eq 1
]; then
MVAPICH2_COMP_ENV="$MVAPICH2_COMP_ENV OPT_FLAG=-g"
fi
;;
@@ -492,25 +499,73 @@ mvapich2()
MVAPICH2_COMP_ENV="CC=pgcc CXX=pgCC F77=pgf77 F90=pgf90"
;;
intel)
- # The -i-dynamic flag is required for MVAPICH2 in the shared
- # library configuration.
- MVAPICH2_COMP_ENV='CC="icc -i-dynamic" CXX="icpc -i-dynamic"
F77="ifort -i-dynamic" F90="ifort -i-dynamic"'
+ if [ $MVAPICH2_CONF_shared_libs -eq 1 ]; then
+ # The -i-dynamic flag is required for MVAPICH2 in the
shared
+ # library configuration.
+ MVAPICH2_COMP_ENV='CC="icc -i-dynamic" CXX="icpc
-i-dynamic" F77="ifort -i-dynamic" F90="ifort -i-dynamic"'
+ else
+ MVAPICH2_COMP_ENV="CC=icc CXX=icpc F77=ifort F90=ifort"
+ fi
;;
esac
- ex rpmbuild --rebuild \
- --define \'_topdir ${RPM_DIR}\' \
- --define \'_name ${MVAPICH2_NAME}_${mpi_comp}\' \
- --define \'_prefix ${MVAPICH2_PREFIX}\' \
- --define \'build_root ${BUILD_ROOT}\' \
- --define \'open_ib_home ${STACK_PREFIX}\' \
- --define \'ofed_build_root ${BUILD_ROOT}\' \
- --define \'comp_env ${MVAPICH2_COMP_ENV}\' \
- --define \'iwarp ${iwarp}\' \
- --define \'romio 1\' \
- --define \'shared_libs 1\' \
- --define \'auto_req 1\' \
- $MVAPICH2_SRC_RPM
+ if [ $MVAPICH2_CONF_impl = "ofa" ] && [ $MVAPICH2_CONF_ckpt -eq 0 ];
then
+ ex rpmbuild --rebuild \
+ --define \'_topdir ${RPM_DIR}\' \
+ --define \'_prefix ${MVAPICH2_PREFIX}\' \
+ --define \'_name ${MVAPICH2_NAME}_${mpi_comp}\' \
+ --define \'build_root ${BUILD_ROOT}\' \
+ --define \'impl ofa\' \
+ --define \'multithread ${MVAPICH2_CONF_multithread}\' \
+ --define \'romio ${MVAPICH2_CONF_romio}\' \
+ --define \'shared_libs ${MVAPICH2_CONF_shared_libs}\' \
+ --define \'rdma_cm 1\' \
+ --define \'ckpt 0\' \
+ --define \'open_ib_home ${STACK_PREFIX}\' \
+ --define \'comp_env ${MVAPICH2_COMP_ENV}\' \
+ --define \'auto_req 0\' \
+ --define \'ofa_build 1\' \
+ $MVAPICH2_SRC_RPM
+ elif [ $MVAPICH2_CONF_impl = "ofa" ] && [ $MVAPICH2_CONF_ckpt -eq 1 ];
then
+ ex rpmbuild --rebuild \
+ --define \'_topdir ${RPM_DIR}\' \
+ --define \'_prefix ${MVAPICH2_PREFIX}\' \
+ --define \'_name ${MVAPICH2_NAME}_${mpi_comp}\' \
+ --define \'build_root ${BUILD_ROOT}\' \
+ --define \'impl ofa\' \
+ --define \'multithread 0\' \
+ --define \'romio ${MVAPICH2_CONF_romio}\' \
+ --define \'shared_libs ${MVAPICH2_CONF_shared_libs}\' \
+ --define \'rdma_cm 0\' \
+ --define \'ckpt 1\' \
+ --define \'blcr_home ${MVAPICH2_CONF_blcr_home}\' \
+ --define \'open_ib_home ${STACK_PREFIX}\' \
+ --define \'comp_env ${MVAPICH2_COMP_ENV}\' \
+ --define \'auto_req 0\' \
+ --define \'ofa_build 1\' \
+ $MVAPICH2_SRC_RPM
+ elif [ $MVAPICH2_CONF_impl = "udapl" ]; then
+ ex rpmbuild --rebuild \
+ --define \'_topdir ${RPM_DIR}\' \
+ --define \'_prefix ${MVAPICH2_PREFIX}\' \
+ --define \'_name ${MVAPICH2_NAME}_${mpi_comp}\' \
+ --define \'build_root ${BUILD_ROOT}\' \
+ --define \'impl udapl\' \
+ --define \'multithread ${MVAPICH2_CONF_multithread}\' \
+ --define \'romio ${MVAPICH2_CONF_romio}\' \
+ --define \'shared_libs ${MVAPICH2_CONF_shared_libs}\' \
+ --define \'vcluster ${MVAPICH2_CONF_vcluster}\' \
+ --define \'io_bus ${MVAPICH2_CONF_io_bus}\' \
+ --define \'link_speed ${MVAPICH2_CONF_link_speed}\' \
+ --define \'dapl_provider ${MVAPICH2_CONF_dapl_provider}\' \
+ --define \'dat_lib ${MVAPICH2_DAT_LIB}\' \
+ --define \'dat_include ${MVAPICH2_DAT_INCLUDE}\' \
+ --define \'comp_env ${MVAPICH2_COMP_ENV}\' \
+ --define \'auto_req 0\' \
+ --define \'ofa_build 1\' \
+ $MVAPICH2_SRC_RPM
+ fi
+
ex "$MV -f ${RPM_DIR}/RPMS/$build_arch/${MVAPICH2_RPM} $RPMS"
let BUILD_COUNTER++
diff --git a/build_env.sh b/build_env.sh
index 3128774..93891b3 100644
--- a/build_env.sh
+++ b/build_env.sh
@@ -971,6 +971,226 @@ is_compiler()
}
+# Prompt for MVAPICH2 build options.
+mvapich2_config() {
+ local choice=""
+ local blcr
+
+ if [ "$MVAPICH2_CONF_done" = 1 ]; then
+ return
+ fi
+
+ cat <<EOF
+
+Please choose an implementation of MVAPICH2:
+
+1) OFA (IB and iWARP)
+2) uDAPL
+
+EOF
+ while [ -z "$choice" ] || [[ $choice != [0-9] ]] || [ $choice -lt 1 ] || [
$choice -gt 2 ]; do
+ read -p "Implementation [1]: "
+ choice=${REPLY:-1}
+ done
+
+ if [ $choice -eq 1 ]; then
+ MVAPICH2_CONF_impl=ofa
+ elif [ $choice -eq 2 ]; then
+ MVAPICH2_CONF_impl=udapl
+ fi
+
+ if ! ( grep -w MVAPICH2_CONF_impl $CONFIG > $NULL 2>&1 ); then
+ echo "MVAPICH2_CONF_impl=\"${MVAPICH2_CONF_impl}\"" >> $CONFIG
+ fi
+
+ while [ -z "$MVAPICH2_CONF_romio" ]; do
+ read -p "Enable ROMIO support [Y/n]: " choice
+
+ if [ -z "$choice" ] || [[ $choice == [yY] ]] || [[ $choice ==
[yY][eE][sS] ]]; then
+ MVAPICH2_CONF_romio=1
+ elif [[ $choice == [nN] ]] || [[ $choice == [nN][oO] ]]; then
+ MVAPICH2_CONF_romio=0
+ fi
+ done
+
+ if ! ( grep -w MVAPICH2_CONF_romio $CONFIG > $NULL 2>&1 ); then
+ echo "MVAPICH2_CONF_romio=\"${MVAPICH2_CONF_romio}\"" >> $CONFIG
+ fi
+
+ while [ -z "$MVAPICH2_CONF_shared_libs" ]; do
+ read -p "Enable shared library support [Y/n]: " choice
+
+ if [ -z "$choice" ] || [[ $choice == [yY] ]] || [[ $choice ==
[yY][eE][sS] ]]; then
+ MVAPICH2_CONF_shared_libs=1
+ elif [[ $choice == [nN] ]] || [[ $choice == [nN][oO] ]]; then
+ MVAPICH2_CONF_shared_libs=0
+ fi
+ done
+
+ if ! ( grep -w MVAPICH2_CONF_shared_libs $CONFIG > $NULL 2>&1 ); then
+ echo "MVAPICH2_CONF_shared_libs=\"${MVAPICH2_CONF_shared_libs}\"" >>
$CONFIG
+ fi
+
+ cat <<EOF
+Multithread support should only be enabled only if thread safety is required.
+There may be a slight performance penalty for single threaded only use.
+EOF
+
+ while [ -z "$MVAPICH2_CONF_multithread" ]; do
+ read -p "Enable multithread support [y/N]: " choice
+
+ if [ -z "$choice" ] || [[ $choice == [nN] ]] || [[ $choice == [nN][oO]
]]; then
+ MVAPICH2_CONF_multithread=0
+ elif [[ $choice == [yY] ]] || [[ $choice == [yY][eE][sS] ]]; then
+ MVAPICH2_CONF_multithread=1
+ fi
+ done
+
+ if ! ( grep -w MVAPICH2_CONF_multithread $CONFIG > $NULL 2>&1 ); then
+ echo "MVAPICH2_CONF_multithread=\"${MVAPICH2_CONF_multithread}\"" >>
$CONFIG
+ fi
+
+ # OFA specific options.
+ if [ $MVAPICH2_CONF_impl = "ofa" ] && [ $MVAPICH2_CONF_multithread -eq 0
]; then
+ choice=0
+
+ while [ $choice = 0 ]; do
+ read -p "Enable Checkpoint-Restart support [y/N]: " choice
+
+ if [ -z "$choice" ] || [[ $choice == [nN] ]] || [[ $choice ==
[nN][oO] ]]; then
+ MVAPICH2_CONF_ckpt=0
+ choice=1
+ elif [[ $choice == [yY] ]] || [[ $choice == [yY][eE][sS] ]]; then
+ read -p "BLCR installation directory [or nothing if not
installed]: " blcr
+
+ if [ -d "$blcr" ]; then
+ MVAPICH2_CONF_ckpt=1
+ MVAPICH2_CONF_blcr_home="$blcr"
+ choice=1
+ else
+ echo "BLCR installation directory not found."
+ choice=0
+ fi
+ else
+ choice=0
+ fi
+ done
+ else
+ MVAPICH2_CONF_ckpt=0
+ fi
+
+ if [ $MVAPICH2_CONF_impl = "ofa" ]; then
+ if ! ( grep -w MVAPICH2_CONF_ckpt $CONFIG > $NULL 2>&1 ); then
+ echo "MVAPICH2_CONF_ckpt=\"${MVAPICH2_CONF_ckpt}\"" >> $CONFIG
+ fi
+
+ if [ $MVAPICH2_CONF_ckpt -eq 1 ]; then
+ if ! ( grep -w MVAPICH2_CONF_blcr_home $CONFIG > $NULL 2>&1 ); then
+ echo "MVAPICH2_CONF_blcr_home=\"${MVAPICH2_CONF_blcr_home}\""
>> $CONFIG
+ fi
+ fi
+ fi
+
+ # uDAPL specific options.
+ if [ $MVAPICH2_CONF_impl = "udapl" ]; then
+ cat <<EOF
+
+Cluster size:
+
+1) Small
+2) Medium
+3) Large
+
+EOF
+ choice=""
+
+ while [ -z "$choice" ] || [[ $choice != [0-9] ]] || [ $choice -lt 1 ]
|| [ $choice -gt 3 ]; do
+ read -p "Cluster size [1]: "
+ choice=${REPLY:-1}
+ done
+
+ if [ $choice -eq 1 ]; then
+ MVAPICH2_CONF_vcluster=small
+ elif [ $choice -eq 2 ]; then
+ MVAPICH2_CONF_vcluster=medium
+ elif [ $choice -eq 3 ]; then
+ MVAPICH2_CONF_vcluster=large
+ fi
+
+ if ! ( grep -w MVAPICH2_CONF_vcluster $CONFIG > $NULL 2>&1 ); then
+ echo "MVAPICH2_CONF_vcluster=\"${MVAPICH2_CONF_vcluster}\"" >>
$CONFIG
+ fi
+
+ cat <<EOF
+
+I/O Bus:
+
+1) PCI-Express
+2) PCI-X
+
+EOF
+ choice=""
+
+ while [ -z "$choice" ] || [[ $choice != [0-9] ]] || [ $choice -lt 1 ]
|| [ $choice -gt 2 ]; do
+ read -p "I/O Bus [1]: "
+ choice=${REPLY:-1}
+ done
+
+ if [ $choice -eq 1 ]; then
+ MVAPICH2_CONF_io_bus=pci-ex
+ elif [ $choice -eq 2 ]; then
+ MVAPICH2_CONF_io_bus=pci-x
+ fi
+
+ if ! ( grep -w MVAPICH2_CONF_io_bus $CONFIG > $NULL 2>&1 ); then
+ echo "MVAPICH2_CONF_io_bus=\"${MVAPICH2_CONF_io_bus}\"" >> $CONFIG
+ fi
+
+ if [ $MVAPICH2_CONF_io_bus = "pci-ex" ]; then
+ cat <<EOF
+
+Link Speed:
+
+1) SDR
+2) DDR
+
+EOF
+ choice=""
+
+ while [ -z "$choice" ] || [[ $choice != [0-9] ]] || [ $choice -lt
1 ] || [ $choice -gt 2 ]; do
+ read -p "Link Speed [1]: "
+ choice=${REPLY:-1}
+ done
+
+ if [ $choice -eq 1 ]; then
+ MVAPICH2_CONF_link_speed=sdr
+ elif [ $choice -eq 2 ]; then
+ MVAPICH2_CONF_link_speed=ddr
+ fi
+ else
+ MVAPICH2_CONF_link_speed=sdr
+ fi
+
+ if ! ( grep -w MVAPICH2_CONF_link_speed $CONFIG > $NULL 2>&1 ); then
+ echo "MVAPICH2_CONF_link_speed=\"${MVAPICH2_CONF_link_speed}\"" >>
$CONFIG
+ fi
+
+ read -p "Default DAPL provider [ib0]: "
+ MVAPICH2_CONF_dapl_provider=${REPLY:-ib0}
+
+ if ! ( grep -w MVAPICH2_CONF_dapl_provider $CONFIG > $NULL 2>&1 ); then
+ echo
"MVAPICH2_CONF_dapl_provider=\"${MVAPICH2_CONF_dapl_provider}\"" >> $CONFIG
+ fi
+ fi
+
+ MVAPICH2_CONF_done=1
+
+ if ! ( grep -w MVAPICH2_CONF_done $CONFIG > $NULL 2>&1 ); then
+ echo "MVAPICH2_CONF_done=\"${MVAPICH2_CONF_done}\"" >> $CONFIG
+ fi
+}
+
+
# Set Compilation environment for MPI
set_mpi_env()
{
@@ -998,6 +1218,7 @@ set_mpi_env()
echo
fi
+
printed_msg0=${printed_msg0:-0}
if [ $printed_msg0 -eq 0 ]; then
if [ $(echo -n ${COMPILERS_FOUND} | wc -w) -gt 1 ]; then
@@ -1014,24 +1235,8 @@ set_mpi_env()
read -p "Do you wish to create/install an ${mpipackage} RPM
with ${mpi_compiler}? [Y/n]:" ans
if [[ "$ans" == "" || "$ans" == "y" || "$ans" == "Y" || "$ans"
== "yes" ]]; then
MPI_COMPILER="$MPI_COMPILER ${mpi_compiler}"
-
- # MVAPICH2 can be built with iWARP support only if
- # librdmacm and librdmacm-devel are there.
- if [ "$mpipackage" = "mvapich2" ] &&
- (echo -n ${SELECTED_PACKAGES} | grep -w "librdmacm" >
$NULL) &&
- (echo -n ${SELECTED_PACKAGES} | grep -w
"librdmacm-devel" > $NULL); then
- read -p "Do you wish to build mvapich2 with iWARP
support only (default is IB) [y/N]:" ans
- if [[ "$ans" == "y" || "$ans" == "Y" || "$ans" ==
"yes" ]]; then
- MVAPICH2_IMPL=iwarp
- else
- MVAPICH2_IMPL=ib
- fi
- else
- MVAPICH2_IMPL=ib
- fi
fi
done
-
else # Unattended mode
case ${mpipackage} in
mvapich)
@@ -1095,18 +1300,17 @@ set_mpi_env()
warn_echo "No compilers for ${mpipackage} were found"
return 1
fi
-
MPI_COMPILER_mvapich2=${MPI_COMPILER}
if ! ( grep -w MPI_COMPILER_mvapich2 $CONFIG > $NULL 2>&1 );
then
echo "MPI_COMPILER_mvapich2=\"${MPI_COMPILER_mvapich2}\""
>> $CONFIG
fi
-
- if ! ( grep -w MVAPICH2_IMPL $CONFIG > $NULL 2>&1 ); then
- echo "MVAPICH2_IMPL=\"${MVAPICH2_IMPL}\"" >> $CONFIG
- fi
-
echo
echo "The following compiler(s) will be used to ${prog%*.*}
the ${mpipackage} RPM(s): $MPI_COMPILER_mvapich2"
+ # MVAPICH2 can be built with many options. The configuration
+ # function below asks the user how to build, and it only will
+ # do so if the configuration values have not already been
+ # read from the $CONFIG file.
+ mvapich2_config
;;
openmpi)
if [ ! -n "${COMPILERS_FOUND}" ]; then
@@ -1843,10 +2047,18 @@ set_package_deps()
export mvapich2=n
else
EXTRA_PACKAGES=$(echo "$EXTRA_PACKAGES
mvapich2" | tr -s ' ' '\n' | sort -rn | uniq)
- if [ "$MVAPICH2_IMPL" = "iwarp" ]; then
- OFA_PACKAGES=$(echo "$OFA_PACKAGES
libibverbs libibverbs-devel libibumad libibumad-devel librdmacm
librdmacm-devel" | tr -s ' ' '\n' | sort -n | uniq)
- else
- OFA_PACKAGES=$(echo "$OFA_PACKAGES
libibverbs libibverbs-devel libibumad libibumad-devel" | tr -s ' ' '\n' | sort
-n | uniq)
+ if [ "$MVAPICH2_CONF_impl" = "ofa" ] && [
"$MVAPICH2_CONF_ckpt" = 0 ]; then
+ # libibumad apparently needs libibcommon.
+ OFA_PACKAGES=$(echo "$OFA_PACKAGES
libibverbs libibverbs-devel libibumad libibumad-devel librdmacm librdmacm-devel
libibcommon libibcommon-devel" | tr -s ' ' '\n' | sort -n | uniq)
+ elif [ "$MVAPICH2_CONF_impl" = "ofa" ]; then
+ # Checkpoint-Restart does not support
+ # RDMA-CM, so it would not be required.
+ # libibumad apparently needs libibcommon.
+ OFA_PACKAGES=$(echo "$OFA_PACKAGES
libibverbs libibverbs-devel libibumad libibumad-devel libibcommon
libibcommon-devel" | tr -s ' ' '\n' | sort -n | uniq)
+ elif [ "$MVAPICH2_CONF_impl" = "udapl" ]; then
+ # dapl apparently needs libibverbs and
+ # librdmacm.
+ OFA_PACKAGES=$(echo "$OFA_PACKAGES dapl
dapl-devel libibverbs librdmacm" | tr -s ' ' '\n' | sort -n | uniq)
fi
fi
;;
diff --git a/install.sh b/install.sh
diff --git a/ofed-scripts.spec b/ofed-scripts.spec
diff --git a/propel.sh b/propel.sh
diff --git a/uninstall.sh b/uninstall.sh
_______________________________________________
openib-general mailing list
openib-general@openib.org
http://openib.org/mailman/listinfo/openib-general
To unsubscribe, please visit http://openib.org/mailman/listinfo/openib-general
