Author: bmahe
Date: Wed Nov 2 20:33:04 2011
New Revision: 1196791
URL: http://svn.apache.org/viewvc?rev=1196791&view=rev
Log:
Updates for hadoop 23
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/do-component-build
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/pig/SPECS/pig.spec
incubator/bigtop/branches/hadoop-0.23/bigtop.mk
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/do-component-build
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/do-component-build?rev=1196791&r1=1196790&r2=1196791&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/do-component-build
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/do-component-build
Wed Nov 2 20:33:04 2011
@@ -31,7 +31,7 @@ mkdir build
mkdir build/src
MAVEN_SKIP_TESTS="-DskipTests -DskipTest -DskipITs"
-mvn -Pdist -Pnative -Psrc -Pdocs -Dtar ${MAVEN_SKIP_TESTS} package "$@"
+mvn -Pdist -Pnative -Psrc -Dtar ${MAVEN_SKIP_TESTS} package "$@"
mvn install ${MAVEN_SKIP_TESTS} $@
mvn site site:stage ${MAVEN_SKIP_TESTS} $@
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh?rev=1196791&r1=1196790&r2=1196791&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hadoop/install_hadoop.sh
Wed Nov 2 20:33:04 2011
@@ -34,15 +34,17 @@ usage: $0 <options>
OPTS=$(getopt \
-n $0 \
-o '' \
- -l 'distro-dir:' \
-l 'prefix:' \
+ -l 'distro-dir:' \
-l 'build-dir:' \
-l 'native-build-string:' \
-l 'installed-lib-dir:' \
- -l 'lib-dir:' \
+ -l 'hadoop-dir:' \
+ -l 'system-include-dir:' \
-l 'system-lib-dir:' \
- -l 'src-dir:' \
- -l 'etc-dir:' \
+ -l 'system-libexec-dir:' \
+ -l 'hadoop-etc-dir:' \
+ -l 'yarn-etc-dir:' \
-l 'doc-dir:' \
-l 'man-dir:' \
-l 'example-dir:' \
@@ -65,9 +67,15 @@ while true ; do
--hadoop-dir)
HADOOP_DIR=$2 ; shift 2
;;
+ --system-include-dir)
+ SYSTEM_INCLUDE_DIR=$2 ; shift 2
+ ;;
--system-lib-dir)
SYSTEM_LIB_DIR=$2 ; shift 2
;;
+ --system-libexec-dir)
+ SYSTEM_LIBEXEC_DIR=$2 ; shift 2
+ ;;
--build-dir)
BUILD_DIR=$2 ; shift 2
;;
@@ -92,9 +100,6 @@ while true ; do
--example-dir)
EXAMPLE_DIR=$2 ; shift 2
;;
- --src-dir)
- SRC_DIR=$2 ; shift 2
- ;;
--)
shift ; break
;;
@@ -118,15 +123,17 @@ SYSTEM_LIB_DIR=${SYSTEM_LIB_DIR:-/usr/li
BIN_DIR=${BIN_DIR:-$PREFIX/usr/bin}
DOC_DIR=${DOC_DIR:-$PREFIX/usr/share/doc/hadoop}
MAN_DIR=${MAN_DIR:-$PREFIX/usr/man}
+SYSTEM_INCLUDE_DIR=${SYSTEM_INCLUDE_DIR:-$PREFIX/usr/include}
+SYSTEM_LIBEXEC_DIR=${SYSTEM_LIBEXEC_DIR:-$PREFIX/usr/libexec}
EXAMPLE_DIR=${EXAMPLE_DIR:-$DOC_DIR/examples}
-SRC_DIR=${SRC_DIR:-$PREFIX/usr/src/hadoop}
HADOOP_ETC_DIR=${HADOOP_ETC_DIR:-$PREFIX/etc/hadoop}
YARN_ETC_DIR=${YARN_ETC_DIR:-$PREFIX/etc/yarn}
INSTALLED_HADOOP_DIR=${INSTALLED_HADOOP_DIR:-/usr/lib/hadoop}
HADOOP_BIN_DIR=${HADOOP_DIR}/bin
-HADOOP_SBIN_DIR=${HADOOP_DIR}/bin
+HADOOP_SBIN_DIR=${HADOOP_DIR}/sbin
+HADOOP_LIB_DIR=${HADOOP_DIR}/lib
# Make bin wrappers
mkdir -p $BIN_DIR
@@ -142,14 +149,53 @@ EOF
chmod 755 $wrapper
done
-mkdir -p ${HADOOP_BIN_DIR}
+# bin
+install -d -m 0755 ${HADOOP_BIN_DIR}
cp -a ${BUILD_DIR}/bin/* ${HADOOP_BIN_DIR}/
-mkdir -p ${HADOOP_SBIN_DIR}
+# sbin
+install -d -m 0755 ${HADOOP_SBIN_DIR}
cp ${BUILD_DIR}/sbin/* ${HADOOP_SBIN_DIR}/
-install -d -m 0755 $PREFIX/$HADOOP_ETC_DIR/conf.empty
-install -d -m 0755 $PREFIX/$YARN_ETC_DIR/conf.empty
+# jars
+install -d -m 0755 ${HADOOP_LIB_DIR}
+cp ${BUILD_DIR}/lib/*.jar ${HADOOP_LIB_DIR}/
+
+# hadoop jar
+install -d -m 0755 ${HADOOP_DIR}
+cp ${BUILD_DIR}/modules/*.jar ${HADOOP_DIR}/
+
+# native libs
+install -d -m 0755 ${SYSTEM_LIB_DIR}
+cp ${BUILD_DIR}/lib/*.a ${SYSTEM_LIB_DIR}/
+for library in libhdfs.so.0.0.0 libhadoop.so.1.0.0; do
+ cp ${BUILD_DIR}/lib/${library} ${SYSTEM_LIB_DIR}/
+ ldconfig -vlN ${SYSTEM_LIB_DIR}/${library}
+done
+install -d -m 0755 ${SYSTEM_INCLUDE_DIR}
+cp ${BUILD_DIR}/../hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.h
${SYSTEM_INCLUDE_DIR}/
+
+#libexec
+install -d -m 0755 ${SYSTEM_LIBEXEC_DIR}
+rm -fv ${BUILD_DIR}/libexec/jsvc
+cp ${BUILD_DIR}/libexec/* ${SYSTEM_LIBEXEC_DIR}/
+
+# conf
+install -d -m 0755 $HADOOP_ETC_DIR/conf.empty
+install -d -m 0755 $YARN_ETC_DIR/conf.empty
+
+cp ${BUILD_DIR}/conf/* $YARN_ETC_DIR/conf.empty
+cp ${BUILD_DIR}/etc/hadoop/* $YARN_ETC_DIR/conf.empty
+
+# docs
+install -d -m 0755 ${DOC_DIR}
+cp -r ${BUILD_DIR}/../target/site/* ${DOC_DIR}/
+
+# source
+cp -r ${BUILD_DIR}/src ${DOC_DIR}/
+
+# man pages
+mkdir -p $MAN_DIR/man1
+gzip -c < $DISTRO_DIR/hadoop.1 > $MAN_DIR/man1/hadoop.1.gz
+
-cp ${BUILD_DIR}/conf/* $PREFIX/$YARN_ETC_DIR/conf.empty
-cp ${BUILD_DIR}/etc/hadoop/* $PREFIX/$YARN_ETC_DIR/conf.empty
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec?rev=1196791&r1=1196790&r2=1196791&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
Wed Nov 2 20:33:04 2011
@@ -31,7 +31,7 @@
%define log_hadoop %{log_hadoop_dirname}/%{name}
%define bin_hadoop %{_bindir}
%define man_hadoop %{_mandir}
-%define src_hadoop /usr/src/%{name}
+%define doc_hadoop %{_docdir}/%{name}-%{hadoop_version}
%define hadoop_username mapred
%define hadoop_services namenode secondarynamenode datanode jobtracker
tasktracker
# Hadoop outputs built binaries into %{hadoop_build}
@@ -219,14 +219,14 @@ assigns MapReduce work to the tasktracke
with an available work slot.
-%package conf-pseudo
-Summary: Hadoop installation in pseudo-distributed mode
-Group: System/Daemons
-Requires: %{name} = %{version}-%{release}, %{name}-namenode =
%{version}-%{release}, %{name}-datanode = %{version}-%{release},
%{name}-secondarynamenode = %{version}-%{release}, %{name}-tasktracker =
%{version}-%{release}, %{name}-jobtracker = %{version}-%{release}
-
-%description conf-pseudo
-Installation of this RPM will setup your machine to run in pseudo-distributed
mode
-where each Hadoop daemon runs in a separate Java process.
+#%package conf-pseudo
+#Summary: Hadoop installation in pseudo-distributed mode
+#Group: System/Daemons
+#Requires: %{name} = %{version}-%{release}, %{name}-namenode =
%{version}-%{release}, %{name}-datanode = %{version}-%{release},
%{name}-secondarynamenode = %{version}-%{release}, %{name}-tasktracker =
%{version}-%{release}, %{name}-jobtracker = %{version}-%{release}
+#
+#%description conf-pseudo
+#Installation of this RPM will setup your machine to run in pseudo-distributed
mode
+#where each Hadoop daemon runs in a separate Java process.
%package doc
Summary: Hadoop Documentation
@@ -235,40 +235,6 @@ Obsoletes: %{name}-docs
%description doc
Documentation for Hadoop
-%package source
-Summary: Source code for Hadoop
-Group: System/Daemons
-AutoReq: no
-
-%description source
-The Java source code for Hadoop and its contributed packages. This is handy
when
-trying to debug programs that depend on Hadoop.
-
-%package fuse
-Summary: Mountable HDFS
-Group: Development/Libraries
-Requires: %{name} = %{version}-%{release}, fuse
-AutoReq: no
-
-%if %{?suse_version:1}0
-Requires: libfuse2
-%else
-Requires: fuse-libs
-%endif
-
-
-%description fuse
-These projects (enumerated below) allow HDFS to be mounted (on most flavors of
Unix) as a standard file system using the mount command. Once mounted, the user
can operate on an instance of hdfs using standard Unix utilities such as 'ls',
'cd', 'cp', 'mkdir', 'find', 'grep', or use standard Posix libraries like open,
write, read, close from C, C++, Python, Ruby, Perl, Java, bash, etc.
-
-%package native
-Summary: Native libraries for Hadoop Compression
-Group: Development/Libraries
-Requires: %{name} = %{version}-%{release}
-AutoReq: no
-
-%description native
-Native libraries for Hadoop compression
-
%package libhdfs
Summary: Hadoop Filesystem Library
Group: Development/Libraries
@@ -279,14 +245,6 @@ AutoReq: no
%description libhdfs
Hadoop Filesystem Library
-%package pipes
-Summary: Hadoop Pipes Library
-Group: Development/Libraries
-Requires: %{name} = %{version}-%{release}
-
-%description pipes
-Hadoop Pipes Library
-
%package sbin
Summary: Binaries for secured Hadoop clusters
Group: System/Daemons
@@ -302,7 +260,7 @@ DataNodes to bind to a low (privileged)
before continuing operation.
%prep
-%setup -n apache-hadoop-common-ee19013
+%setup -n apache-hadoop-common-562cef1
%build
# This assumes that you installed Java JDK 6 and set JAVA_HOME
@@ -325,9 +283,9 @@ env HADOOP_VERSION=%{hadoop_version} HAD
bash %{SOURCE2} \
--distro-dir=$RPM_SOURCE_DIR \
--build-dir=$PWD/build \
- --src-dir=$RPM_BUILD_ROOT%{src_hadoop} \
- --lib-dir=$RPM_BUILD_ROOT%{lib_hadoop} \
- --system-lib-dir=%{_libdir} \
+ --system-include-dir=$RPM_BUILD_ROOT%{_includedir} \
+ --system-lib-dir=$RPM_BUILD_ROOT%{_libdir} \
+ --system-libexec-dir=$RPM_BUILD_ROOT%{_libexecdir} \
--hadoop-etc-dir=$RPM_BUILD_ROOT%{etc_hadoop} \
--yarn-etc-dir=$RPM_BUILD_ROOT%{etc_yarn} \
--prefix=$RPM_BUILD_ROOT \
@@ -337,8 +295,6 @@ bash %{SOURCE2} \
--installed-lib-dir=%{lib_hadoop} \
--man-dir=$RPM_BUILD_ROOT%{man_hadoop} \
-%__mv -f $RPM_BUILD_ROOT/usr/share/doc/libhdfs-devel
$RPM_BUILD_ROOT/%{_docdir}/libhdfs-%{hadoop_version}
-
# Init.d scripts
%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/
@@ -427,30 +383,16 @@ fi
%config(noreplace) /etc/default/hadoop
%config(noreplace) /etc/security/limits.d/hadoop.nofiles.conf
%{lib_hadoop}
+%{_libdir}/libhadoop*
%{bin_hadoop}/%{name}
-%{man_hadoop}/man1/hadoop.1.*z
%attr(0775,root,hadoop) /var/run/%{name}
%attr(0775,root,hadoop) %{log_hadoop}
-
-%exclude %{lib_hadoop}/lib/native
-%exclude %{lib_hadoop}/sbin/%{hadoop_arch}
-%exclude %{lib_hadoop}/bin/fuse_dfs
-%exclude %{lib_hadoop}/contrib/fuse-dfs
-%exclude %{lib_hadoop}/hdfs/contrib/fuse-dfs
-# FIXME: The following is a workaround for BIGTOP-139
-# %exclude %{lib_hadoop}/bin/task-controller
-%exclude %{lib_hadoop}/bin/jsvc*
-%exclude %{lib_hadoop}/hdfs/bin/jsvc*
+%{man_hadoop}/man1/hadoop.1.*
%files doc
%defattr(-,root,root)
%doc %{doc_hadoop}
-%files source
-%defattr(-,root,root)
-%{src_hadoop}
-
-
# Service file management RPMs
%define service_macro() \
@@ -483,54 +425,28 @@ fi
%service_macro tasktracker
# Pseudo-distributed Hadoop installation
-%post conf-pseudo
-%{alternatives_cmd} --install %{config_hadoop} %{name}-conf
%{etc_hadoop}/conf.pseudo 30
+#%post conf-pseudo
+#%{alternatives_cmd} --install %{config_hadoop} %{name}-conf
%{etc_hadoop}/conf.pseudo 30
-%files conf-pseudo
-%defattr(-,root,root)
-%config(noreplace) %attr(755,root,root) %{etc_hadoop}/conf.pseudo
-%dir %attr(0755,root,hadoop) /var/lib/%{name}
-%dir %attr(1777,root,hadoop) /var/lib/%{name}/cache
-
-%preun conf-pseudo
-if [ "$1" = 0 ]; then
- %{alternatives_cmd} --remove %{name}-conf %{etc_hadoop}/conf.pseudo
- rm -f %{etc_hadoop}/conf
-fi
-
-%files native
-%defattr(-,root,root)
-%{lib_hadoop}/lib/native
-
-%files fuse
-%defattr(-,root,root)
-%config(noreplace) /etc/default/hadoop-fuse
-%{lib_hadoop}/contrib/fuse-dfs
-%attr(0755,root,root) %{lib_hadoop}/bin/fuse_dfs
-%attr(0755,root,root) %{lib_hadoop}/bin/fuse_dfs_wrapper.sh
-%attr(0755,root,root) %{bin_hadoop}/hadoop-fuse-dfs
-%attr(0644,root,root) %{man_hadoop}/man1/hadoop-fuse-dfs.1.*
-
-%files pipes
-%defattr(-,root,root)
-%{_libdir}/libhadooppipes*
-%{_libdir}/libhadooputil*
-%{_includedir}/hadoop/*
+#%files conf-pseudo
+#%defattr(-,root,root)
+#%config(noreplace) %attr(755,root,root) %{etc_hadoop}/conf.pseudo
+#%dir %attr(0755,root,hadoop) /var/lib/%{name}
+#%dir %attr(1777,root,hadoop) /var/lib/%{name}/cache
+
+#%preun conf-pseudo
+#if [ "$1" = 0 ]; then
+# %{alternatives_cmd} --remove %{name}-conf %{etc_hadoop}/conf.pseudo
+# rm -f %{etc_hadoop}/conf
+#fi
%files libhdfs
%defattr(-,root,root)
%{_libdir}/libhdfs*
-%{_includedir}/hdfs.h
+#%{_includedir}/hdfs.h
# -devel should be its own package
-%doc %{_docdir}/libhdfs-%{hadoop_version}
+#%doc %{_docdir}/libhdfs-%{hadoop_version}
%files sbin
%defattr(-,root,root)
%dir %{lib_hadoop}/sbin
-%dir %{lib_hadoop}/sbin/%{hadoop_arch}
-# %attr(4750,root,mapred) %{lib_hadoop}/sbin/%{hadoop_arch}/task-controller
-%attr(0755,root,root) %{lib_hadoop}/sbin/%{hadoop_arch}/jsvc
-
-# FIXME: The following is a workaround for BIGTOP-139
-# %attr(4750,root,mapred) %{lib_hadoop}/bin/task-controller
-%attr(0755,root,root) %{lib_hadoop}/bin/jsvc*
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/pig/SPECS/pig.spec
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/pig/SPECS/pig.spec?rev=1196791&r1=1196790&r2=1196791&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/pig/SPECS/pig.spec
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/pig/SPECS/pig.spec
Wed Nov 2 20:33:04 2011
@@ -109,7 +109,7 @@ language called Pig Latin, which has the
%prep
%setup -n apache-pig-e42a1e6
-%patch0 -p0
+%patch0 -p1
%build
env PIG_BASE_VERSION=%{pig_base_version} bash %{SOURCE1}
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop.mk
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop.mk?rev=1196791&r1=1196790&r2=1196791&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop.mk (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop.mk Wed Nov 2 20:33:04 2011
@@ -18,13 +18,13 @@ BIGTOP_VERSION=0.1.0-incubating
# Hadoop 0.20.0-based hadoop package
HADOOP_NAME=hadoop
HADOOP_RELNOTES_NAME=Apache Hadoop
-HADOOP_BASE_VERSION=0.22.0
-HADOOP_PKG_VERSION=0.22.0
+HADOOP_BASE_VERSION=0.23.0
+HADOOP_PKG_VERSION=0.23.0
HADOOP_RELEASE_VERSION=1
HADOOP_TARBALL_DST=$(HADOOP_NAME)-$(HADOOP_BASE_VERSION).tar.gz
#HADOOP_TARBALL_SRC=$(HADOOP_TARBALL_DST)
#HADOOP_SITE=$(APACHE_MIRROR)/hadoop/common/$(HADOOP_NAME)-$(HADOOP_BASE_VERSION)/
-HADOOP_TARBALL_SRC=ee19013
+HADOOP_TARBALL_SRC=562cef1
HADOOP_SITE=https://github.com/apache/hadoop-common/tarball
$(eval $(call PACKAGE,hadoop,HADOOP))
