incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1359394 - in /incubator/bigtop/trunk: ./ bigtop-packages/src/common/datafu/ bigtop-packages/src/deb/datafu/ bigtop-packages/src/rpm/datafu/ bigtop-packages/src/rpm/datafu/BUILD/ bigtop-packages/src/rpm/datafu/RPMS/ bigtop-packages/src/rpm/...
Date Mon, 09 Jul 2012 21:07:22 GMT
Author: rvs
Date: Mon Jul  9 21:07:22 2012
New Revision: 1359394

URL: http://svn.apache.org/viewvc?rev=1359394&view=rev
Log:
BIGTOP-669. Add DataFu to Bigtop distribution

Added:
    incubator/bigtop/trunk/bigtop-packages/src/common/datafu/
    incubator/bigtop/trunk/bigtop-packages/src/common/datafu/do-component-build
    incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh   (with props)
    incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/
    incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/changelog
    incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/compat
    incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/control
    incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/copyright
    incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/pig-udf-datafu.install
    incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules   (with props)
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/BUILD/
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/BUILD/.gitignore
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/RPMS/
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/RPMS/.gitignore
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SOURCES/
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SOURCES/.gitignore
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/datafu.spec
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SRPMS/
    incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SRPMS/.gitignore
Modified:
    incubator/bigtop/trunk/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
    incubator/bigtop/trunk/bigtop.mk

Added: incubator/bigtop/trunk/bigtop-packages/src/common/datafu/do-component-build
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/datafu/do-component-build?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/datafu/do-component-build (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/datafu/do-component-build Mon Jul  9
21:07:22 2012
@@ -0,0 +1,23 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+. `dirname $0`/bigtop.bom
+
+sed -i -e '/org.apache.pig/s#^.*$#<dependency org="org.apache.pig" name="pig" rev="'$PIG_VERSION'"/>#'
ivy.xml
+
+ant jar "$@"

Added: incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh Mon Jul  9
21:07:22 2012
@@ -0,0 +1,100 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+  echo "
+usage: $0 <options>
+  Required not-so-options:
+     --build-dir=DIR             path to pig dist.dir
+     --prefix=PREFIX             path to install into
+
+  Optional options:
+     --doc-dir=DIR               path to install docs into [/usr/share/doc/pig]
+     --lib-dir=DIR               path to install pig home [/usr/lib/pig]
+     --installed-lib-dir=DIR     path where lib-dir will end up on target system
+     --bin-dir=DIR               path to install bins [/usr/bin]
+     --examples-dir=DIR          path to install examples [doc-dir/examples]
+     ... [ see source for more similar options ]
+  "
+  exit 1
+}
+
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'prefix:' \
+  -l 'doc-dir:' \
+  -l 'lib-dir:' \
+  -l 'installed-lib-dir:' \
+  -l 'bin-dir:' \
+  -l 'examples-dir:' \
+  -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+    usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+    case "$1" in
+        --prefix)
+        PREFIX=$2 ; shift 2
+        ;;
+        --build-dir)
+        BUILD_DIR=$2 ; shift 2
+        ;;
+        --doc-dir)
+        DOC_DIR=$2 ; shift 2
+        ;;
+        --lib-dir)
+        LIB_DIR=$2 ; shift 2
+        ;;
+        --installed-lib-dir)
+        INSTALLED_LIB_DIR=$2 ; shift 2
+        ;;
+        --bin-dir)
+        BIN_DIR=$2 ; shift 2
+        ;;
+        --examples-dir)
+        EXAMPLES_DIR=$2 ; shift 2
+        ;;
+        --)
+        shift ; break
+        ;;
+        *)
+        echo "Unknown option: $1"
+        usage
+        exit 1
+        ;;
+    esac
+done
+
+for var in PREFIX BUILD_DIR ; do
+  if [ -z "$(eval "echo \$$var")" ]; then
+    echo Missing param: $var
+    usage
+  fi
+done
+
+LIB_DIR=${LIB_DIR:-$PREFIX/usr/lib/pig}
+INSTALLED_LIB_DIR=${INSTALLED_LIB_DIR:-/usr/lib/pig}
+
+# First we'll move everything into lib
+install -d -m 0755 $LIB_DIR
+cp $BUILD_DIR/datafu-0.0.4.jar $LIB_DIR

Propchange: incubator/bigtop/trunk/bigtop-packages/src/common/datafu/install_datafu.sh
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/changelog
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/changelog?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/changelog (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/changelog Mon Jul  9 21:07:22 2012
@@ -0,0 +1 @@
+--- This is auto-generated 

Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/compat
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/compat?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/compat (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/compat Mon Jul  9 21:07:22 2012
@@ -0,0 +1 @@
+6

Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/control
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/control?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/control (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/control Mon Jul  9 21:07:22 2012
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Source: pig-udf-datafu 
+Section: misc
+Priority: extra
+Maintainer: Bigtop <bigtop-dev@incubator.apache.org>
+Build-Depends: debhelper (>= 6)
+Standards-Version: 3.8.0
+Homepage: https://github.com/linkedin/datafu
+
+Package: pig-udf-datafu
+Architecture: all
+Depends: pig
+Description: A collection of user-defined functions for Hadoop and Pig.
+ DataFu is a collection of user-defined functions for working with large-scale 
+ data in Hadoop and Pig. This library was born out of the need for a stable, 
+ well-tested library of UDFs for data mining and statistics. It is used 
+ at LinkedIn in many of our off-line workflows for data derived products like 
+ "People You May Know" and "Skills". 
+ .
+ It contains functions for: PageRank, Quantiles (median), variance, Sessionization, 
+ Convenience bag functions (e.g., set operations, enumerating bags, etc), 
+ Convenience utility functions (e.g., assertions, easier writing of EvalFuncs)
+ and more...
+

Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/copyright
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/copyright?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/copyright (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/copyright Mon Jul  9 21:07:22 2012
@@ -0,0 +1,15 @@
+Format: http://dep.debian.net/deps/dep5
+Source: https://github.com/linkedin/datafu
+Upstream-Name: DataFu
+
+Files: *
+Copyright: 2010, LinkedIn, Inc
+License: Apache-2.0
+
+Files debian/*
+Copyright: 2011, The Apache Software Foundation
+License: Apache-2.0
+
+License: Apache-2.0
+ On Debian systems, the complete text of the Apache 2.0 license
+ can be found in "/usr/share/common-licenses/Apache-2.0".

Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/pig-udf-datafu.install
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/pig-udf-datafu.install?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/pig-udf-datafu.install (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/pig-udf-datafu.install Mon Jul 
9 21:07:22 2012
@@ -0,0 +1 @@
+/usr/lib/pig

Added: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules Mon Jul  9 21:07:22 2012
@@ -0,0 +1,91 @@
+#!/usr/bin/make -f
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -*- makefile -*-
+
+# Uncomment this to turn on verbose mode.
+export DH_VERBOSE=1
+
+# This has to be exported to make some magic below work.
+export DH_OPTIONS
+
+patch: patch-stamp
+patch-stamp:	
+	touch $@
+
+clean:	
+	dh_testdir
+	dh_testroot
+	rm -f *-stamp
+	dh_clean
+	rm -Rf debian/tmp debian/mahout
+	find debian -name .\*swp -exec rm -f {} \;
+
+build-indep: build-indep-stamp
+build-indep-stamp: patch-stamp
+	# we'll just use the build from the tarball.
+	bash debian/do-component-build -Divy.home=${HOME}/.ivy2
+	touch $@
+
+install: install-indep
+install-indep:	
+	dh_testdir
+	dh_testroot
+	sh -x debian/install_datafu.sh \
+	  --build-dir=dist \
+	  --prefix=debian/tmp/pig-udf-datafu
+	dh_install --sourcedir=debian/tmp/pig-udf-datafu -i
+
+	(dh_lintian) || /bin/true
+
+binary-common:	
+	dh_testdir
+	dh_testroot
+	dh_installchangelogs
+	dh_installdocs
+#	dh_installexamples
+#	dh_installmenu
+#	dh_installdebconf
+#	dh_installlogrotate
+#	dh_installemacsen
+#	dh_installpam
+#	dh_installmime
+#	dh_python
+#	dh_installinit
+#	dh_installcron
+#	dh_installinfo
+	dh_installman
+	dh_link
+	dh_strip
+	dh_compress
+	dh_fixperms
+#	dh_perl
+	dh_makeshlibs
+	dh_installdeb
+	dh_shlibdeps
+	dh_gencontrol
+	dh_md5sums
+	dh_builddeb
+
+binary-indep: build-indep install-indep
+	$(MAKE) -f debian/rules DH_OPTIONS=-i binary-common
+
+binary-arch:	
+
+
+binary: binary-indep
+.PHONY: build clean binary-indep binary install-indep binary-arch

Propchange: incubator/bigtop/trunk/bigtop-packages/src/deb/datafu/rules
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/BUILD/.gitignore
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/BUILD/.gitignore?rev=1359394&view=auto
==============================================================================
    (empty)

Added: incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/RPMS/.gitignore
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/RPMS/.gitignore?rev=1359394&view=auto
==============================================================================
    (empty)

Added: incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SOURCES/.gitignore
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SOURCES/.gitignore?rev=1359394&view=auto
==============================================================================
    (empty)

Added: incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/datafu.spec
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/datafu.spec?rev=1359394&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/datafu.spec (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SPECS/datafu.spec Mon Jul  9 21:07:22
2012
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+%define datafu_name datafu
+%define lib_datafu /usr/lib/pig
+
+%if  %{?suse_version:1}0
+%define doc_datafu %{_docdir}/datafu-doc
+%else
+%define doc_datafu %{_docdir}/datafu-doc-%{datafu_version}
+%endif
+
+# disable repacking jars
+%define __os_install_post %{nil}
+
+Name: pig-udf-datafu
+Version: %{datafu_version}
+Release: %{datafu_release}
+Summary: A collection of user-defined functions for Hadoop and Pig.
+URL: https://github.com/linkedin/datafu
+Group: Development/Libraries
+BuildArch: noarch
+Buildroot: %(mktemp -ud %{_tmppath}/%{datafu_name}-%{version}-%{release}-XXXXXX)
+License: ASL 2.0 
+Source0: %{datafu_name}-%{datafu_base_version}.tar.gz
+Source1: do-component-build 
+Source2: install_%{datafu_name}.sh
+Requires: hadoop-client, bigtop-utils
+
+
+%description 
+DataFu is a collection of user-defined functions for working with large-scale
+data in Hadoop and Pig. This library was born out of the need for a stable,
+well-tested library of UDFs for data mining and statistics. It is used
+at LinkedIn in many of our off-line workflows for data derived products like
+"People You May Know" and "Skills".
+
+It contains functions for: PageRank, Quantiles (median), variance, Sessionization,
+Convenience bag functions (e.g., set operations, enumerating bags, etc),
+Convenience utility functions (e.g., assertions, easier writing of EvalFuncs)
+and more...
+
+%prep
+%setup -n %{datafu_name}-%{datafu_base_version}
+
+%build
+bash $RPM_SOURCE_DIR/do-component-build
+
+%install
+%__rm -rf $RPM_BUILD_ROOT
+sh $RPM_SOURCE_DIR/install_datafu.sh \
+          --build-dir=dist \
+          --prefix=$RPM_BUILD_ROOT
+
+#######################
+#### FILES SECTION ####
+#######################
+%files 
+%defattr(-,root,root,755)
+%{lib_datafu}

Added: incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SRPMS/.gitignore
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/rpm/datafu/SRPMS/.gitignore?rev=1359394&view=auto
==============================================================================
    (empty)

Modified: incubator/bigtop/trunk/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml?rev=1359394&r1=1359393&r2=1359394&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
(original)
+++ incubator/bigtop/trunk/bigtop-tests/test-artifacts/package/src/main/resources/package_data.xml
Mon Jul  9 21:07:22 2012
@@ -332,6 +332,25 @@ server.</description>
       </pig-conf>
     </alternatives>
   </pig>
+  <pig-udf-datafu>
+    <metadata>
+      <summary>A collection of user-defined functions for Hadoop and Pig.</summary>
+      <description> DataFu is a collection of user-defined functions for working with
large-scale
+ data in Hadoop and Pig. This library was born out of the need for a stable,
+ well-tested library of UDFs for data mining and statistics. It is used
+ at LinkedIn in many of our off-line workflows for data derived products like
+ "People You May Know" and "Skills".
+
+ It contains functions for: PageRank, Quantiles (median), variance, Sessionization,
+ Convenience bag functions (e.g., set operations, enumerating bags, etc),
+ Convenience utility functions (e.g., assertions, easier writing of EvalFuncs)
+ and more...</description>
+      <url>https://github.com/linkedin/datafu</url>
+    </metadata>
+    <deps>
+      <pig/>
+    </deps>
+  </pig-udf-datafu>
   <hive>
     <metadata>
       <summary>Hive is a data warehouse infrastructure built on top of Hadoop</summary>

Modified: incubator/bigtop/trunk/bigtop.mk
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop.mk?rev=1359394&r1=1359393&r2=1359394&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop.mk (original)
+++ incubator/bigtop/trunk/bigtop.mk Mon Jul  9 21:07:22 2012
@@ -187,6 +187,19 @@ HUE_SITE=https://github.com/downloads/cl
 HUE_ARCHIVE=$(HUE_SITE)
 $(eval $(call PACKAGE,hue,HUE))
 
+# DataFu 
+DATAFU_NAME=datafu
+DATAFU_RELNOTES_NAME=Collection of user-defined functions
+DATAFU_PKG_NAME=pig-udf-datafu
+DATAFU_BASE_VERSION=0.0.4
+DATAFU_PKG_VERSION=0.0.4
+DATAFU_RELEASE_VERSION=1
+DATAFU_TARBALL_DST=datafu-$(DATAFU_BASE_VERSION).tar.gz
+DATAFU_TARBALL_SRC=$(DATAFU_TARBALL_DST)
+DATAFU_SITE=https://github.com/downloads/linkedin/datafu
+DATAFU_ARCHIVE=$(DATAFU_SITE)
+$(eval $(call PACKAGE,datafu,DATAFU))
+
 # Bigtop-utils
 BIGTOP_UTILS_NAME=bigtop-utils
 BIGTOP_UTILS__RELNOTES_NAME=Bigtop-utils



Mime
View raw message