Author: rvs
Date: Mon Jul 9 21:07:22 2012
New Revision: 1359394
URL: http://svn.apache.org/viewvc?rev=1359394&view=rev
Log:
BIGTOP-669. Add DataFu to Bigtop distribution
Added:
incubator/bigtop/trunk/bigtop-packages/src/common/datafu/
incubator/bigtop/trunk/bigtop-packages/src/common/datafu/do-component-build
incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh
(with props)
incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/
incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/changelog
incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/compat
incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/control
incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/copyright
incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/pig-udf-datafu.install
incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules (with props)
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/BUILD/
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/BUILD/.gitignore
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/RPMS/
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/RPMS/.gitignore
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SOURCES/
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SOURCES/.gitignore
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/datafu.spec
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SRPMS/
incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SRPMS/.gitignore
Modified:
incubator/bigtop/trunk/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
incubator/bigtop/trunk/bigtop.mk
Added:
incubator/bigtop/trunk/bigtop-packages/src/common/datafu/do-component-build
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/datafu/do-component-build?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/datafu/do-component-build
(added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/datafu/do-component-build
Mon Jul 9 21:07:22 2012
@@ -0,0 +1,23 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+. `dirname $0`/bigtop.bom
+
+sed -i -e '/org.apache.pig/s#^.*$#<dependency org="org.apache.pig" name="pig"
rev="'$PIG_VERSION'"/>#' ivy.xml
+
+ant jar "$@"
Added:
incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh
(added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh
Mon Jul 9 21:07:22 2012
@@ -0,0 +1,100 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+ echo "
+usage: $0 <options>
+ Required not-so-options:
+ --build-dir=DIR path to pig dist.dir
+ --prefix=PREFIX path to install into
+
+ Optional options:
+ --doc-dir=DIR path to install docs into [/usr/share/doc/pig]
+ --lib-dir=DIR path to install pig home [/usr/lib/pig]
+ --installed-lib-dir=DIR path where lib-dir will end up on target
system
+ --bin-dir=DIR path to install bins [/usr/bin]
+ --examples-dir=DIR path to install examples [doc-dir/examples]
+ ... [ see source for more similar options ]
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'prefix:' \
+ -l 'doc-dir:' \
+ -l 'lib-dir:' \
+ -l 'installed-lib-dir:' \
+ -l 'bin-dir:' \
+ -l 'examples-dir:' \
+ -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+ case "$1" in
+ --prefix)
+ PREFIX=$2 ; shift 2
+ ;;
+ --build-dir)
+ BUILD_DIR=$2 ; shift 2
+ ;;
+ --doc-dir)
+ DOC_DIR=$2 ; shift 2
+ ;;
+ --lib-dir)
+ LIB_DIR=$2 ; shift 2
+ ;;
+ --installed-lib-dir)
+ INSTALLED_LIB_DIR=$2 ; shift 2
+ ;;
+ --bin-dir)
+ BIN_DIR=$2 ; shift 2
+ ;;
+ --examples-dir)
+ EXAMPLES_DIR=$2 ; shift 2
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+for var in PREFIX BUILD_DIR ; do
+ if [ -z "$(eval "echo \$$var")" ]; then
+ echo Missing param: $var
+ usage
+ fi
+done
+
+LIB_DIR=${LIB_DIR:-$PREFIX/usr/lib/pig}
+INSTALLED_LIB_DIR=${INSTALLED_LIB_DIR:-/usr/lib/pig}
+
+# First we'll move everything into lib
+install -d -m 0755 $LIB_DIR
+cp $BUILD_DIR/datafu-0.0.4.jar $LIB_DIR
Propchange:
incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/changelog
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/changelog?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/changelog (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/changelog Mon Jul 9
21:07:22 2012
@@ -0,0 +1 @@
+--- This is auto-generated
Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/compat
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/compat?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/compat (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/compat Mon Jul 9
21:07:22 2012
@@ -0,0 +1 @@
+6
Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/control
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/control?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/control (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/control Mon Jul 9
21:07:22 2012
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Source: pig-udf-datafu
+Section: misc
+Priority: extra
+Maintainer: Bigtop <[email protected]>
+Build-Depends: debhelper (>= 6)
+Standards-Version: 3.8.0
+Homepage: https://github.com/linkedin/datafu
+
+Package: pig-udf-datafu
+Architecture: all
+Depends: pig
+Description: A collection of user-defined functions for Hadoop and Pig.
+ DataFu is a collection of user-defined functions for working with large-scale
+ data in Hadoop and Pig. This library was born out of the need for a stable,
+ well-tested library of UDFs for data mining and statistics. It is used
+ at LinkedIn in many of our off-line workflows for data derived products like
+ "People You May Know" and "Skills".
+ .
+ It contains functions for: PageRank, Quantiles (median), variance,
Sessionization,
+ Convenience bag functions (e.g., set operations, enumerating bags, etc),
+ Convenience utility functions (e.g., assertions, easier writing of EvalFuncs)
+ and more...
+
Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/copyright
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/copyright?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/copyright (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/copyright Mon Jul 9
21:07:22 2012
@@ -0,0 +1,15 @@
+Format: http://dep.debian.net/deps/dep5
+Source: https://github.com/linkedin/datafu
+Upstream-Name: DataFu
+
+Files: *
+Copyright: 2010, LinkedIn, Inc
+License: Apache-2.0
+
+Files debian/*
+Copyright: 2011, The Apache Software Foundation
+License: Apache-2.0
+
+License: Apache-2.0
+ On Debian systems, the complete text of the Apache 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".
Added:
incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/pig-udf-datafu.install
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/pig-udf-datafu.install?rev=1359394&view=auto
==============================================================================
---
incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/pig-udf-datafu.install
(added)
+++
incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/pig-udf-datafu.install
Mon Jul 9 21:07:22 2012
@@ -0,0 +1 @@
+/usr/lib/pig
Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules Mon Jul 9
21:07:22 2012
@@ -0,0 +1,91 @@
+#!/usr/bin/make -f
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -*- makefile -*-
+
+# Uncomment this to turn on verbose mode.
+export DH_VERBOSE=1
+
+# This has to be exported to make some magic below work.
+export DH_OPTIONS
+
+patch: patch-stamp
+patch-stamp:
+ touch $@
+
+clean:
+ dh_testdir
+ dh_testroot
+ rm -f *-stamp
+ dh_clean
+ rm -Rf debian/tmp debian/mahout
+ find debian -name .\*swp -exec rm -f {} \;
+
+build-indep: build-indep-stamp
+build-indep-stamp: patch-stamp
+ # we'll just use the build from the tarball.
+ bash debian/do-component-build -Divy.home=${HOME}/.ivy2
+ touch $@
+
+install: install-indep
+install-indep:
+ dh_testdir
+ dh_testroot
+ sh -x debian/install_datafu.sh \
+ --build-dir=dist \
+ --prefix=debian/tmp/pig-udf-datafu
+ dh_install --sourcedir=debian/tmp/pig-udf-datafu -i
+
+ (dh_lintian) || /bin/true
+
+binary-common:
+ dh_testdir
+ dh_testroot
+ dh_installchangelogs
+ dh_installdocs
+# dh_installexamples
+# dh_installmenu
+# dh_installdebconf
+# dh_installlogrotate
+# dh_installemacsen
+# dh_installpam
+# dh_installmime
+# dh_python
+# dh_installinit
+# dh_installcron
+# dh_installinfo
+ dh_installman
+ dh_link
+ dh_strip
+ dh_compress
+ dh_fixperms
+# dh_perl
+ dh_makeshlibs
+ dh_installdeb
+ dh_shlibdeps
+ dh_gencontrol
+ dh_md5sums
+ dh_builddeb
+
+binary-indep: build-indep install-indep
+ $(MAKE) -f debian/rules DH_OPTIONS=-i binary-common
+
+binary-arch:
+
+
+binary: binary-indep
+.PHONY: build clean binary-indep binary install-indep binary-arch
Propchange: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules
------------------------------------------------------------------------------
svn:executable = *
Added: incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/BUILD/.gitignore
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/BUILD/.gitignore?rev=1359394&view=auto
==============================================================================
(empty)
Added: incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/RPMS/.gitignore
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/RPMS/.gitignore?rev=1359394&view=auto
==============================================================================
(empty)
Added: incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SOURCES/.gitignore
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SOURCES/.gitignore?rev=1359394&view=auto
==============================================================================
(empty)
Added: incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/datafu.spec
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/datafu.spec?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/datafu.spec
(added)
+++ incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/datafu.spec Mon
Jul 9 21:07:22 2012
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+%define datafu_name datafu
+%define lib_datafu /usr/lib/pig
+
+%if %{?suse_version:1}0
+%define doc_datafu %{_docdir}/datafu-doc
+%else
+%define doc_datafu %{_docdir}/datafu-doc-%{datafu_version}
+%endif
+
+# disable repacking jars
+%define __os_install_post %{nil}
+
+Name: pig-udf-datafu
+Version: %{datafu_version}
+Release: %{datafu_release}
+Summary: A collection of user-defined functions for Hadoop and Pig.
+URL: https://github.com/linkedin/datafu
+Group: Development/Libraries
+BuildArch: noarch
+Buildroot: %(mktemp -ud
%{_tmppath}/%{datafu_name}-%{version}-%{release}-XXXXXX)
+License: ASL 2.0
+Source0: %{datafu_name}-%{datafu_base_version}.tar.gz
+Source1: do-component-build
+Source2: install_%{datafu_name}.sh
+Requires: hadoop-client, bigtop-utils
+
+
+%description
+DataFu is a collection of user-defined functions for working with large-scale
+data in Hadoop and Pig. This library was born out of the need for a stable,
+well-tested library of UDFs for data mining and statistics. It is used
+at LinkedIn in many of our off-line workflows for data derived products like
+"People You May Know" and "Skills".
+
+It contains functions for: PageRank, Quantiles (median), variance,
Sessionization,
+Convenience bag functions (e.g., set operations, enumerating bags, etc),
+Convenience utility functions (e.g., assertions, easier writing of EvalFuncs)
+and more...
+
+%prep
+%setup -n %{datafu_name}-%{datafu_base_version}
+
+%build
+bash $RPM_SOURCE_DIR/do-component-build
+
+%install
+%__rm -rf $RPM_BUILD_ROOT
+sh $RPM_SOURCE_DIR/install_datafu.sh \
+ --build-dir=dist \
+ --prefix=$RPM_BUILD_ROOT
+
+#######################
+#### FILES SECTION ####
+#######################
+%files
+%defattr(-,root,root,755)
+%{lib_datafu}
Added: incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SRPMS/.gitignore
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SRPMS/.gitignore?rev=1359394&view=auto
==============================================================================
(empty)
Modified:
incubator/bigtop/trunk/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml?rev=1359394&r1=1359393&r2=1359394&view=diff
==============================================================================
---
incubator/bigtop/trunk/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
(original)
+++
incubator/bigtop/trunk/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
Mon Jul 9 21:07:22 2012
@@ -332,6 +332,25 @@ server.</description>
</pig-conf>
</alternatives>
</pig>
+ <pig-udf-datafu>
+ <metadata>
+ <summary>A collection of user-defined functions for Hadoop and
Pig.</summary>
+ <description> DataFu is a collection of user-defined functions for
working with large-scale
+ data in Hadoop and Pig. This library was born out of the need for a stable,
+ well-tested library of UDFs for data mining and statistics. It is used
+ at LinkedIn in many of our off-line workflows for data derived products like
+ "People You May Know" and "Skills".
+
+ It contains functions for: PageRank, Quantiles (median), variance,
Sessionization,
+ Convenience bag functions (e.g., set operations, enumerating bags, etc),
+ Convenience utility functions (e.g., assertions, easier writing of EvalFuncs)
+ and more...</description>
+ <url>https://github.com/linkedin/datafu</url>
+ </metadata>
+ <deps>
+ <pig/>
+ </deps>
+ </pig-udf-datafu>
<hive>
<metadata>
<summary>Hive is a data warehouse infrastructure built on top of
Hadoop</summary>
Modified: incubator/bigtop/trunk/bigtop.mk
URL:
http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop.mk?rev=1359394&r1=1359393&r2=1359394&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop.mk (original)
+++ incubator/bigtop/trunk/bigtop.mk Mon Jul 9 21:07:22 2012
@@ -187,6 +187,19 @@ HUE_SITE=https://github.com/downloads/cl
HUE_ARCHIVE=$(HUE_SITE)
$(eval $(call PACKAGE,hue,HUE))
+# DataFu
+DATAFU_NAME=datafu
+DATAFU_RELNOTES_NAME=Collection of user-defined functions
+DATAFU_PKG_NAME=pig-udf-datafu
+DATAFU_BASE_VERSION=0.0.4
+DATAFU_PKG_VERSION=0.0.4
+DATAFU_RELEASE_VERSION=1
+DATAFU_TARBALL_DST=datafu-$(DATAFU_BASE_VERSION).tar.gz
+DATAFU_TARBALL_SRC=$(DATAFU_TARBALL_DST)
+DATAFU_SITE=https://github.com/downloads/linkedin/datafu
+DATAFU_ARCHIVE=$(DATAFU_SITE)
+$(eval $(call PACKAGE,datafu,DATAFU))
+
# Bigtop-utils
BIGTOP_UTILS_NAME=bigtop-utils
BIGTOP_UTILS__RELNOTES_NAME=Bigtop-utils
