Mailing-List: contact commits-help@hawq.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@hawq.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: shivram@apache.org
To: commits@hawq.incubator.apache.org
Message-Id: <999eafad793a4cf3b546d991f0e77f6a@git.apache.org>
Subject: incubator-hawq git commit: HAWQ-1297. Make PXF install ready from
 source
Date: Mon, 30 Jan 2017 23:53:16 +0000 (UTC)
archived-at: Mon, 30 Jan 2017 23:53:36 -0000

Repository: incubator-hawq
Updated Branches:
  refs/heads/master d3983eb56 -> aac8868f8


HAWQ-1297. Make PXF install ready from source


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/aac8868f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/aac8868f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/aac8868f

Branch: refs/heads/master
Commit: aac8868f8e3d8c0158a1a58b8593a713056099e4
Parents: d3983eb
Author: Shivram Mani <shivram.mani@gmail.com>
Authored: Mon Jan 30 15:53:07 2017 -0800
Committer: Shivram Mani <shivram.mani@gmail.com>
Committed: Mon Jan 30 15:53:07 2017 -0800

----------------------------------------------------------------------
 pxf/Makefile                                    |  23 +++-
 pxf/build.gradle                                |  29 +++-
 .../src/main/resources/pxf-private.classpath    |  67 ++++++++++
 pxf/pxf-service/src/scripts/pxf-env.sh          |   6 +
 pxf/pxf-service/src/scripts/pxf-service         | 132 ++++++++++++++-----
 5 files changed, 211 insertions(+), 46 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aac8868f/pxf/Makefile
----------------------------------------------------------------------
diff --git a/pxf/Makefile b/pxf/Makefile
index 7669772..81298d0 100644
--- a/pxf/Makefile
+++ b/pxf/Makefile
@@ -19,26 +19,36 @@
 default: all
 
 ifneq "$(HD)" ""
-BUILD_PARAMS= -Dhd=$(HD)
+    BUILD_PARAMS= -Dhd=$(HD)
+else
+    ifneq "$(PXF_HOME)" ""
+        BUILD_PARAMS= -DdeployPath=$(PXF_HOME)
+    else ifneq "$(GPHOME)" ""
+        BUILD_PARAMS= -DdeployPath="$(GPHOME)/pxf"
+    else
+		@echo "Cannot invoke install without configuring either PXF_HOME or GPHOME"
+    endif
 endif
 
 ifneq "$(LICENSE)" ""
-BUILD_PARAMS+= -Plicense="$(LICENSE)"
+    BUILD_PARAMS+= -Plicense="$(LICENSE)"
 endif
+
 ifneq "$(VENDOR)" ""
-BUILD_PARAMS+= -Pvendor="$(VENDOR)"
+    BUILD_PARAMS+= -Pvendor="$(VENDOR)"
 endif
 
 help:
 	@echo 
-	@echo	"help it is then"
-	@echo	"Possible targets"
+	@echo"help it is then"
+	@echo   "Possible targets"
 	@echo	"  - all (clean, build, unittest, jar, tar, rpm)"
 	@echo	"  -  -  HD=<phd|hdp> - set classpath to match hadoop distribution. default phd"
 	@echo	"  -  -  LICENSE=<license info> - add license info to created RPMs"
 	@echo	"  -  -  VENDOR=<vendor name> - add vendor name to created RPMs"
 	@echo	"  - tomcat - builds tomcat rpm from downloaded tarball"
 	@echo	"  -  -  LICENSE and VENDOR parameters can be used as well"
+	@echo	"  - deploy - setup PXF along with tomcat in the configured deployPath"
 	@echo	"  - doc - creates aggregate javadoc under docs"
 
 all: 
@@ -65,3 +75,6 @@ doc:
 .PHONY: tomcat
 tomcat:
 	./gradlew tomcatRpm $(BUILD_PARAMS)
+
+install:
+	./gradlew install $(BUILD_PARAMS)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aac8868f/pxf/build.gradle
----------------------------------------------------------------------
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 385bf08..3c6d591 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -473,8 +473,7 @@ task rpm(type: Copy, dependsOn: [subprojects.build, distSubprojects.buildRpm]) {
 
 // tomcat 
 def tomcatName = "apache-tomcat-${tomcatVersion}"
-def tomcatTargetDir = "tomcat/build/"
-
+def tomcatTargetDir = "tomcat/build"
 
 task tomcatGet << {
 
@@ -506,12 +505,12 @@ task tomcatGet << {
 apply plugin: 'os-package'
 
 task tomcatRpm(type: Rpm) {
-    buildDir = 'tomcat/build/'
+    buildDir = "${tomcatTargetDir}"
 
     // clean should not delete the downloaded tarball
     // and RPM, so this is a bogus directory to delete instead.
     clean {
-        delete = 'tomcat/build/something'
+        delete = "${tomcatTargetDir}/something"
     }
 
     ospackage {
@@ -544,6 +543,26 @@ task tomcatRpm(type: Rpm) {
 
 tomcatRpm.dependsOn tomcatGet
 
+def pxfTargetDir = System.properties['deployPath'] ?: "build/"
+
+task install(type: Copy, dependsOn: [subprojects.build, tomcatGet]) {
+    into "${pxfTargetDir}"
+    subprojects { subProject ->
+        from("${project.name}/build/libs") { into 'lib' }
+    }
+    from("pxf-service/src/scripts/pxf-service") {
+        into 'bin'
+        fileMode 0755
+        rename('pxf-service', 'pxf')
+    }
+
+    from("${tomcatTargetDir}/${tomcatName}") { into 'apache-tomcat' }
+    from("pxf-service/src/main/resources") { into 'conf' }
+    from("pxf-service/src/configs/pxf-site.xml") { into 'conf' }
+    from("pxf-service/src/scripts/pxf-env.sh") { into 'conf' }
+    from("pxf-service/src/configs/tomcat") { into 'tomcat-templates' }
+}
+
 
 buildDir = '.'
-apply plugin: 'nebula-aggregate-javadocs'
\ No newline at end of file
+apply plugin: 'nebula-aggregate-javadocs'

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aac8868f/pxf/pxf-service/src/main/resources/pxf-private.classpath
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/resources/pxf-private.classpath b/pxf/pxf-service/src/main/resources/pxf-private.classpath
new file mode 100644
index 0000000..48ac2f9
--- /dev/null
+++ b/pxf/pxf-service/src/main/resources/pxf-private.classpath
@@ -0,0 +1,67 @@
+##################################################################
+# This file contains the internal classpaths required to run PXF.
+# Edit to set the base paths according to your specific package layout
+# Adding new resources should be done using pxf-public.classpath file.
+##################################################################
+
+# PXF Configuration
+pxf/conf
+
+# Hadoop Configuration
+hadoop/etc/hadoop
+
+# Hive Configuration
+hive/conf
+
+# Hbase Configuration
+hbase/conf
+
+# PXF Libraries
+pxf/pxf-hbase-*[0-9].jar
+pxf/pxf-hdfs-*[0-9].jar
+pxf/pxf-hive-*[0-9].jar
+pxf/pxf-json-*[0-9].jar
+
+# Hadoop Libraries
+hadoop/share/hadoop/hdfs/hadoop-hdfs-*[0-9].jar
+hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-core-*[0-9].jar
+hadoop/share/hadoop/common/lib/hadoop-auth-*[0-9].jar
+hadoop/share/hadoop/common/hadoop-common-*[0-9].jar
+hadoop/share/hadoop/common/lib/asm-*[0-9].jar
+hadoop/share/hadoop/common/lib/avro-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-cli-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-codec-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-collections-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-configuration-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-io-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-lang-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-logging-*[0-9].jar
+hadoop/share/hadoop/common/lib/guava-*[0-9].jar
+hadoop/share/hadoop/common/lib/htrace-core-*[0-9]*.jar
+hadoop/share/hadoop/common/lib/jetty-*.jar
+hadoop/share/hadoop/common/lib/jackson-core-asl-*[0-9].jar
+hadoop/share/hadoop/common/lib/jackson-mapper-asl-*[0-9].jar
+hadoop/share/hadoop/common/lib/jersey-core-*[0-9].jar
+hadoop/share/hadoop/common/lib/jersey-server-*[0-9].jar
+hadoop/share/hadoop/common/lib/log4j-*[0-9].jar
+hadoop/share/hadoop/common/lib/protobuf-java-*[0-9].jar
+hadoop/share/hadoop/common/lib/slf4j-api-*[0-9].jar
+
+# Hive Libraries
+hive/lib/antlr-runtime-*[0-9].jar
+hive/lib/datanucleus-api-jdo-*[0-9].jar
+hive/lib/datanucleus-core-*[0-9].jar
+hive/lib/hive-exec-*[0-9].jar
+hive/lib/hive-metastore-*[0-9].jar
+hive/lib/jdo-api-*[0-9].jar
+hive/lib/libfb303-*[0-9].jar
+# when running on OSx, 1.0.5 or higher version is required
+hive/lib/snappy-java-*[0-9].jar
+
+# HBase Libraries
+hbase/lib/hbase-client-*[0-9].jar
+hbase/lib/hbase-common-*[0-9].jar
+hbase/lib/hbase-protocol-*[0-9].jar
+hbase/lib/htrace-core-*[0-9]*.jar
+hbase/lib/netty-*[0-9].Final.jar
+hbase/lib/zookeeper-*[0-9].jar
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aac8868f/pxf/pxf-service/src/scripts/pxf-env.sh
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/scripts/pxf-env.sh b/pxf/pxf-service/src/scripts/pxf-env.sh
index 2ac80a9..ba9b368 100644
--- a/pxf/pxf-service/src/scripts/pxf-env.sh
+++ b/pxf/pxf-service/src/scripts/pxf-env.sh
@@ -28,3 +28,9 @@ export CATALINA_OUT=${PXF_LOGDIR}/catalina.out
 
 # Path to Run directory
 export PXF_RUNDIR=/var/run/pxf
+
+# Configured user
+export PXF_USER=pxf
+
+# Port
+export PXF_PORT=51200
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aac8868f/pxf/pxf-service/src/scripts/pxf-service
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/scripts/pxf-service b/pxf/pxf-service/src/scripts/pxf-service
index 7f62504..2d63310 100644
--- a/pxf/pxf-service/src/scripts/pxf-service
+++ b/pxf/pxf-service/src/scripts/pxf-service
@@ -20,18 +20,12 @@
 # pxf-service	start/stop/initialize/status the PXF instance
 #
 
-pxf_root=/usr/lib/pxf
-env_script=/etc/pxf/conf/pxf-env.sh
-tomcat_root=/opt/apache-tomcat
-tomcat_templates=/opt/pxf/tomcat-templates
-instance_root=/var/pxf
 
-pxf_user=pxf
-instance_name=pxf-service
-instance_port=51200
-instance_owner=pxf:pxf
-
-curl=`which curl`
+if [ -z $PXF_HOME ]; then
+    env_script=/etc/pxf/conf/pxf-env.sh
+else
+    env_script=$PXF_HOME/conf/pxf-env.sh
+fi
 
 # load pxf-env.sh script
 if [ ! -f $env_script ]; then
@@ -40,6 +34,29 @@ else
 	source $env_script
 fi
 
+pxf_user=$PXF_USER
+instance_port=$PXF_PORT
+instance_name=pxf-service
+
+if [ -z $PXF_HOME ]; then
+    # RPM based setup
+    pxf_root=/usr/lib/pxf
+    tomcat_root=/opt/apache-tomcat
+    tomcat_templates=/opt/pxf/tomcat-templates
+    instance_root=/var/pxf
+    instance_owner=$pxf_user:$pxf_user
+else
+    # OSS/Source code based setup
+    pxf_root=$PXF_HOME/lib
+    tomcat_root=$PXF_HOME/apache-tomcat
+    tomcat_templates=$PXF_HOME/tomcat-templates
+    instance_root=$PXF_HOME
+    instance_owner=$pxf_user
+fi
+
+curl=`which curl`
+
+
 # validate JAVA_HOME
 if [ ! -x $JAVA_HOME/bin/java ]; then
 	echo ERROR: \$JAVA_HOME is invalid
@@ -61,9 +78,8 @@ function createInstance()
 		return 1
 	fi
 
-	chown $instance_owner -R $instance_root
+	chown -R $instance_owner $instance_root
 	chmod 700 $instance_root/$instance_name
-
 	return 0
 }
 
@@ -97,7 +113,7 @@ function configureInstance()
 	# set pid
 	catalinaEnv=$instance_root/$instance_name/bin/setenv.sh
 	cat $catalinaEnv | \
-	sed "s|^[[:blank:]]*CATALINA_PID=.*$|CATALINA_PID=$PXF_RUNDIR/catalina.pid|g" \
+	sed -e "s|^[[:blank:]]*CATALINA_PID=.*$|CATALINA_PID=$PXF_RUNDIR/catalina.pid|g" \
 	> ${catalinaEnv}.tmp
 	rm $catalinaEnv
 	mv ${catalinaEnv}.tmp $catalinaEnv
@@ -176,7 +192,7 @@ function checkWebapp()
 # non zero otherwise
 function instanceExists()
 {
-	if [ ! -d $instance_root ]; then
+	if [ ! -d "$instance_root/$instance_name" ]; then
 		return 1
 	fi
 
@@ -193,39 +209,83 @@ function doInit()
 		return 0
 	fi
 
-	createInstance || return 1
+    createInstance || return 1
 	configureInstance || return 1
 	deployWebapp || return 1
 }
 
+#
+# patchWebapp patches the webapp config files
+# patch applied only if PXF_HOME is defined
+#
+function patchWebapp()
+{
+    if [ -z $PXF_HOME ]; then
+        # webapp doesn't require patch
+        return 0
+    fi
+    pushd $instance_root/$instance_name/webapps || return 1
+    rm -rf pxf
+    mkdir pxf
+    cd pxf
+    unzip -q ../pxf.war
+    popd
+
+    context_file=$instance_root/$instance_name/webapps/pxf/META-INF/context.xml
+    cat $context_file | \
+    sed  -e "s:classpathFiles=\"[a-zA-Z0-9\/\;.-]*\":classpathFiles=\"$PXF_HOME\/conf\/pxf-private.classpath\":" \
+    -e "s:secondaryClasspathFiles=\"[a-zA-Z0-9\/\;.-]*\":secondaryClasspathFiles=\"$PXF_HOME\/conf\/pxf-public.classpath\":" > context.xml.tmp
+    mv context.xml.tmp $context_file
+
+    web_file=$instance_root/$instance_name/webapps/pxf/WEB-INF/web.xml
+    cat $web_file | \
+    sed "s:<param-value>.*pxf-log4j.properties<\/param-value>:<param-value>$PXF_HOME\/conf\/pxf-log4j.properties<\/param-value>:" > web.xml.tmp
+    mv web.xml.tmp $web_file
+}
+
+function commandWebapp()
+{
+    command=$1
+    pushd $instance_root
+    su $pxf_user -c "$instance_root/$instance_name/bin/catalina.sh $command"
+    if [ $? -ne 0 ]; then
+        return 1
+    fi
+    popd
+}
+
 # 
-# doStartStop handles start/stop commands
-# commands are executed as the user $pxf_user
+# doStart handles start command
+# command is executed as the user $pxf_user
 #
 # after start, uses checkWebapp to verify the PXF webapp was loaded
 # successfully
 #
-function doStartStop()
+function doStart()
 {
-	command=$1
-
 	instanceExists
 	if [ $? -ne 0 ]; then
 		echo ERROR: cant find PXF instance, maybe call init?
 		return 1
 	fi
+	patchWebapp || return 1
+	commandWebapp start || return 1
+	checkWebapp 300 || return 1
+}
 
-	pushd $instance_root
-	su $pxf_user -c "$instance_root/$instance_name/bin/catalina.sh $command"
-	if [ $? -ne 0 ]; then
-		return 1
-	fi 
-	popd
-	
-	if [ "$command" = "start" ]; then
-		# try to connect for 5 minutes
-		checkWebapp 300 || return 1
-	fi
+#
+# doStart handles stop command
+# command is executed as the user $pxf_user
+#
+#
+function doStop()
+{
+    instanceExists
+    if [ $? -ne 0 ]; then
+        echo "ERROR: can't find PXF instance, maybe call init?"
+        return 1
+    fi
+    commandWebapp stop || return 1
 }
 
 function doStatus()
@@ -240,15 +300,15 @@ case "$command" in
 		doInit
 		;;
 	"start" )
-		doStartStop $command
+		doStart
 		;;
 	"stop" )
-		doStartStop $command
+		doStop
 		;;
 	"restart" )
-		doStartStop stop
+		doStop
 		sleep 1s
-		doStartStop start
+		doStart
 		;;
 	"status" )
 		doStatus