hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject svn commit: r1329381 - in /hive/trunk: ./ builtins/ cli/ common/ contrib/ hbase-handler/ hwi/ ivy/ jdbc/ metastore/ pdk/ pdk/scripts/ ql/ serde/ service/ shims/ testutils/
Date Mon, 23 Apr 2012 19:19:56 GMT
Author: cws
Date: Mon Apr 23 19:19:55 2012
New Revision: 1329381

URL: http://svn.apache.org/viewvc?rev=1329381&view=rev
Log:
HIVE-2646. Hive Ivy dependencies on Hadoop should depend on jars directly, not tarballs (Andrew Bayer and Thomas Weise)

Added:
    hive/trunk/testutils/hadoop   (with props)
Modified:
    hive/trunk/build-common.xml
    hive/trunk/build.properties
    hive/trunk/build.xml
    hive/trunk/builtins/build.xml
    hive/trunk/builtins/ivy.xml
    hive/trunk/cli/ivy.xml
    hive/trunk/common/ivy.xml
    hive/trunk/contrib/build.xml
    hive/trunk/contrib/ivy.xml
    hive/trunk/hbase-handler/build.xml
    hive/trunk/hbase-handler/ivy.xml
    hive/trunk/hwi/build.xml
    hive/trunk/hwi/ivy.xml
    hive/trunk/ivy/common-configurations.xml
    hive/trunk/ivy/ivysettings.xml
    hive/trunk/ivy/libraries.properties
    hive/trunk/jdbc/build.xml
    hive/trunk/jdbc/ivy.xml
    hive/trunk/metastore/ivy.xml
    hive/trunk/pdk/build.xml
    hive/trunk/pdk/ivy.xml
    hive/trunk/pdk/scripts/build-plugin.xml
    hive/trunk/ql/build.xml
    hive/trunk/ql/ivy.xml
    hive/trunk/serde/ivy.xml
    hive/trunk/service/build.xml
    hive/trunk/service/ivy.xml
    hive/trunk/shims/build.xml
    hive/trunk/shims/ivy.xml

Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Mon Apr 23 19:19:55 2012
@@ -66,20 +66,38 @@
   <property name="test.print.classpath" value="false"/>
   <property name="test.lang" value="en_US.UTF-8"/>
 
+  <property name="hadoop.opts.23" value="-D mapreduce.framework.name=local" />
+  <property name="hadoop.opts.20" value="" />
+  
   <path id="test.classpath">
     <pathelement location="${test.build.classes}" />
     <pathelement location="${test.build.resources}" />
     <pathelement location="" />
     <pathelement location="${test.src.data.dir}/conf"/>
     <pathelement location="${hive.conf.dir}"/>
-    <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
-    <path refid="classpath"/>
-    <fileset dir="${hadoop.root}">
-      <!-- below is for 0.23 onwards -->
-      <include name="share/hadoop/common/lib/*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-mapreduce-*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-yarn-*.jar" />
-    </fileset>
+    <pathelement location="${build.dir.hive}/cli/test/classes"/>
+    <pathelement location="${build.dir.hive}/common/test/classes"/>
+    <pathelement location="${build.dir.hive}/hbase-handler/test/classes"/>
+    <pathelement location="${build.dir.hive}/hwi/test/classes"/>
+    <pathelement location="${build.dir.hive}/jdbc/test/classes"/>
+    <pathelement location="${build.dir.hive}/metastore/test/classes"/>
+    <pathelement location="${build.dir.hive}/ql/test/classes"/>
+    <pathelement location="${build.dir.hive}/serde/test/classes"/>
+    <pathelement location="${build.dir.hive}/service/test/classes"/>
+    <pathelement location="${build.dir.hive}/shims/test/classes"/>
+
+    <fileset dir="${hive.root}/build/ivy/lib/test" includes="*.jar" erroronmissingdir="false" excludes="**/hive_contrib*.jar,**/hive-contrib*.jar"/>
+    <fileset dir="${hive.root}/build/ivy/lib/default" includes="*.jar" erroronmissingdir="false" excludes="**/hive_contrib*.jar,**/hive-contrib*.jar" />
+    <fileset dir="${hive.root}/testlibs" includes="*.jar"/>
+    <fileset dir="${hive.root}/build/ivy/lib/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" />
+    <pathelement location="${build.classes}" />
+  </path>
+
+  <!-- include contrib on local classpath, but not on cluster -->
+  <!-- https://reviews.facebook.net/D2133#comment-47 -->
+  <path id="test.local.classpath">
+    <path refid="${test.classpath.id}"/>
+    <fileset dir="${hive.root}/build/ivy/lib/test" includes="hive-contrib*.jar" erroronmissingdir="false"/>
   </path>
 
 
@@ -125,47 +143,41 @@
   	<ivy:retrieve settingsRef="${ant.project.name}-${hadoop.version.ant-internal}.ivy.settings"
       pattern="${build.dir.hadoop}/[artifact]-[revision].[ext]"/>
   </target>
-  
-  <available property="hadoopcore.${hadoop.version.ant-internal}.install.done"
-    file="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.installed"/>
 
-  <target name="install-hadoopcore-internal" depends="ivy-retrieve-hadoop-source"
-    unless="hadoopcore.${hadoop.version.ant-internal}.install.done">
+  <target name="ivy-resolve-test" depends="ivy-init-settings" unless="offline">
     <echo message="Project: ${ant.project.name}"/>
-    <untar src="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.tar.gz" dest="${build.dir.hadoop}" compression="gzip"/>
-    <chmod file="${hadoop.root}/bin/hadoop" perm="+x"/>
-    <touch file="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.installed"/>
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings"
+      conf="test" log="${ivyresolvelog}"/>
   </target>
 
-  <target name="install-hadoopcore-default" unless="hadoop.root.nondefault">
+  <target name="ivy-retrieve-test" depends="ivy-resolve-test"
+    description="Retrieve Ivy-managed artifacts">
     <echo message="Project: ${ant.project.name}"/>
-    <antcall target="install-hadoopcore-internal"/>
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+      log="${ivyresolvelog}" conf="test"/>
   </target>
 
-  <target name="install-hadoopcore">
+  <target name="ivy-resolve-hadoop-shim" depends="ivy-init-settings" unless="offline">
     <echo message="Project: ${ant.project.name}"/>
-    <condition property="hadoop.root.nondefault">
-      <not>
-        <equals arg1="${hadoop.root}" arg2="${hadoop.root.default}"/>
-      </not>
-    </condition>
-    <antcall target="install-hadoopcore-default"/>
+    <ivy:resolve settingsRef="${ant.project.name}.ivy.settings"
+      conf="${ivy.hadoop.shim.conf}" log="${ivyresolvelog}"/>
+  </target>
+
+  <target name="ivy-retrieve-hadoop-shim" depends="ivy-resolve-hadoop-shim"
+    description="Retrieve Ivy-managed artifacts">
+    <echo message="Project: ${ant.project.name}"/>
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+      pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+      log="${ivyresolvelog}" conf="${ivy.hadoop.shim.conf}"/>
   </target>
 
   <!-- the normal classpath -->
   <path id="common-classpath">
-    <pathelement location="${hadoop.oldstyle-name.jar}"/>
-    <pathelement location="${hadoop.oldstyle-name.tools.jar}"/>
-    <pathelement location="${hadoop.newstyle-name.jar}"/>
-    <pathelement location="${hadoop.newstyle-name.tools.jar}"/>
-    <pathelement location="${hadoop.common.jar}"/>
-    <pathelement location="${hadoop.hdfs.jar}"/>
-    <pathelement location="${hadoop.mapreduce.jar}"/>
-    <pathelement location="${hadoop.mapreduce.tools.jar}"/>
     <pathelement location="${build.dir.hive}/classes"/>
     <fileset dir="${build.dir.hive}" includes="*/*.jar"/>
     <fileset dir="${hive.root}/lib" includes="*.jar"/>
-    <fileset dir="${build.ivy.lib.dir}/default" includes="*.jar" excludes="*hadoop*.jar"
+    <fileset dir="${build.ivy.lib.dir}/default" includes="*.jar" 
              erroronmissingdir="false"/>
   </path>
 
@@ -178,10 +190,10 @@
     <pathelement location="${build.dir.hive}/cli/classes"/>
     <pathelement location="${build.dir.hive}/shims/classes"/>
     <pathelement location="${build.dir.hive}/hwi/classes"/>
-    <fileset erroronmissingdir="false" dir="${build.dir.hadoop}/hadoop-${hadoop.security.version}"
-             includes="lib/commons-codec-*.jar"/>
-    <path refid="common-classpath"/>
+    <pathelement location="${build.dir.hive}/jdbc/classes"/>
+    <pathelement location="${build.dir.hive}/hbase-handler/classes"/>
     <fileset dir="${basedir}" includes="lib/*.jar"/>
+    <path refid="common-classpath"/>
   </path>
 
   <target name="create-dirs">
@@ -251,10 +263,13 @@
       </manifest>
       <metainf dir="${hive.root}" includes="LICENSE,NOTICE"/>
     </jar>
+    <ivy:publish settingsRef="${ant.project.name}.ivy.settings"
+                 resolver="local" pubrevision="${version}" overwrite="true"
+                 artifactspattern="${build.dir}/${ivy.publish.pattern}"/>
   </target>
 
   <!-- target to compile tests -->
-  <target name="compile-test" depends="compile">
+  <target name="compile-test" depends="compile,ivy-retrieve-test">
     <echo message="Project: ${ant.project.name}"/>
     <javac
      encoding="${build.encoding}"
@@ -356,17 +371,26 @@
   <target name="test"
   	depends="test-conditions,gen-test,compile-test,test-jar,test-init">
     <echo message="Project: ${ant.project.name}"/>
+    <property name="hadoop.testcp" refid="test.classpath"/>
+    <if>
+      <equals arg1="${hadoop.mr.rev}" arg2="23" />
+      <then>
+        <property name="hadoop.opts" value="${hadoop.opts.23}" />
+      </then>
+      <else>
+        <property name="hadoop.opts" value="${hadoop.opts.20}" />
+      </else>
+    </if>
     <if>
       <equals arg1="${test.print.classpath}" arg2="true" />
       <then>        
-        <property name="testcp" refid="test.classpath"/>
-        <echo message="Test Classpath: ${testcp}"/>
+        <echo message="Test Classpath: ${hadoop.testcp}"/>
       </then>
     </if>
     <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no"
            fork="yes" maxmemory="512m" dir="${basedir}" timeout="${test.timeout}"
            errorProperty="tests.failed" failureProperty="tests.failed" filtertrace="off">
-      <env key="LANG" value="${test.lang}"/> 
+      <env key="HIVE_HADOOP_TEST_CLASSPATH" value="${hadoop.testcp}"/>
       <env key="HADOOP_HOME" value="${hadoop.root}"/>
       <env key="HADOOP_CLASSPATH" value="${test.src.data.dir}/conf:${build.dir.hive}/dist/lib/derby-${derby.version}.jar:${build.dir.hive}/dist/lib/JavaEWAH-${javaewah.version}.jar:${hadoop.root}/modules/*"/> <!-- Modules needed for Hadoop 0.23 -->
       <env key="TZ" value="US/Pacific"/>
@@ -391,8 +415,9 @@
       <sysproperty key="build.ivy.lib.dir" value="${build.ivy.lib.dir}"/>
       <sysproperty key="derby.version" value="${derby.version}"/>
       <sysproperty key="hive.version" value="${version}"/>
+      <sysproperty key="hadoop.bin.path" value="${test.hadoop.bin.path}"/>
 
-      <classpath refid="${test.classpath.id}"/>
+      <classpath refid="test.local.classpath"/>
       <formatter type="${test.junit.output.format}" usefile="${test.junit.output.usefile}" />
       <batchtest todir="${test.build.dir}" unless="testcase">
         <fileset dir="${test.build.classes}"

Modified: hive/trunk/build.properties
URL: http://svn.apache.org/viewvc/hive/trunk/build.properties?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/build.properties (original)
+++ hive/trunk/build.properties Mon Apr 23 19:19:55 2012
@@ -26,13 +26,17 @@ javac.deprecation=off
 javac.args=
 javac.args.warnings=
 
-hadoop-0.20.version=0.20.1
+hadoop-0.20.version=0.20.2
 hadoop-0.20S.version=1.0.0
-hadoop-0.23.version=0.23.0
+hadoop-0.23.version=0.23.1
 hadoop.version=${hadoop-0.20.version}
 hadoop.security.version=${hadoop-0.20S.version}
 hadoop.mirror=http://mirror.facebook.net/facebook/hive-deps
 hadoop.mirror2=http://archive.cloudera.com/hive-deps
+# Used to determine which set of Hadoop artifacts we depend on.
+# - 20: hadoop-core, hadoop-test
+# - 23: hadoop-common, hadoop-mapreduce-*, etc
+hadoop.mr.rev=20
 
 build.dir.hive=${hive.root}/build
 build.dir.hadoop=${build.dir.hive}/hadoopcore
@@ -41,6 +45,7 @@ build.dir.hadoop=${build.dir.hive}/hadoo
 hadoop.version.ant-internal=${hadoop.version}
 hadoop.root.default=${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}
 hadoop.root=${hadoop.root.default}
+test.hadoop.bin.path=${hive.root}/testutils/hadoop
 # Newer versions of Hadoop name the jar as hadoop-{core,test}-VERSION instead of hadoop-VERSION-{core,test}
 # We will add both styles to the classpath and it will pick up whichever is there
 hadoop.oldstyle-name.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar
@@ -76,6 +81,9 @@ build.ivy.maven.dir=${build.ivy.dir}/mav
 ivy.conf.dir=${hive.root}/ivy
 ivy.version=2.1.0
 ivy.jar=${build.ivy.lib.dir}/ivy-${ivy.version}.jar
+ivy.changingPattern=.*SNAPSHOT
+ivy.publish.pattern=[artifact]-[revision].[ext]
+ivy.artifact.retrieve.pattern=[conf]/[artifact]-[revision](-[classifier]).[ext]
 ivysettings.xml=${ivy.conf.dir}/ivysettings.xml
 ivyresolvelog=download-only
 ivy.mvn.repo=http://repo2.maven.org/maven2

Modified: hive/trunk/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/build.xml (original)
+++ hive/trunk/build.xml Mon Apr 23 19:19:55 2012
@@ -508,6 +508,7 @@
       <fileset dir="${build.dir.hive}/ivy/lib/default">
         <include name="*.jar"/>
         <exclude name="*.tar.gz"/>
+        <exclude name="hadoop-*.jar" />
 	<exclude name="**/*high-scale-lib-*"/>
         <exclude name="**/hamcrest-core-*jar"/>
         <exclude name="**/junit*.jar"/>
@@ -701,13 +702,6 @@
       <link href="${javadoc.link.java}"/>
 
       <classpath >
-        <fileset dir="${hadoop.root}/lib">
-          <include name="**/*.jar" />
-          <exclude name="**/excluded/" />
-	  <exclude name="**/commons-logging-*.jar" />
-	  <exclude name="**/commons-codec-*.jar" />
-        </fileset>
-	<pathelement path="${hadoop.oldstyle-name.test.jar}"/>
         <path refid="classpath" />
         <pathelement path="${java.class.path}"/>
       </classpath>

Modified: hive/trunk/builtins/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/build.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/builtins/build.xml (original)
+++ hive/trunk/builtins/build.xml Mon Apr 23 19:19:55 2012
@@ -17,12 +17,12 @@
    limitations under the License.
 -->
 
-<project name="builtins" default="jar">
+<project xmlns:ivy="antlib:org.apache.ivy.ant" name="builtins" default="jar">
 
   <property name="src.dir"  location="${basedir}/src"/>
   <import file="../build-common.xml"/>
 
-  <target name="compile" depends="init, setup">
+  <target name="compile" depends="init, setup, ivy-retrieve">
     <echo message="Project: ${ant.project.name}"/>
     <!-- defer compilation until package phase -->
   </target>
@@ -32,15 +32,18 @@
     <!-- defer compilation until package phase -->
   </target>
 
-  <target name="package">
+  <target name="package" depends="init, setup, ivy-retrieve">
     <echo message="Project: ${ant.project.name}"/>
     <ant antfile="build-plugin.xml" target="package" inheritAll="false">
       <property name="hive.install.dir" value="${build.dir.hive}/dist"/>
       <property name="hadoop.home" value="${hadoop.root}"/>
     </ant>
+    <ivy:publish settingsRef="${ant.project.name}.ivy.settings"
+                 resolver="local" pubrevision="${version}" overwrite="true"
+                 artifactspattern="${build.dir}/${ivy.publish.pattern}"/>
   </target>
 
-  <target name="test" unless="testcase">
+  <target name="test" unless="testcase" depends="init, setup, ivy-retrieve">
     <echo message="Project: ${ant.project.name}"/>
     <ant antfile="build-plugin.xml" target="test" inheritAll="false">
       <property name="hive.install.dir" value="${build.dir.hive}/dist"/>

Modified: hive/trunk/builtins/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/builtins/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/builtins/ivy.xml (original)
+++ hive/trunk/builtins/ivy.xml Mon Apr 23 19:19:55 2012
@@ -23,6 +23,14 @@
         https://cwiki.apache.org/confluence/display/Hive/Home
       </description>
     </info>
-    <dependencies>
-    </dependencies>
+  <configurations>
+    <include file="${ivy.conf.dir}/common-configurations.xml"/>
+  </configurations>
+  <dependencies>
+    <dependency org="org.apache.hive" name="hive-exec" rev="${version}"
+                conf="compile->default" />
+    <dependency org="org.apache.hive" name="hive-pdk" rev="${version}"
+                conf="compile->default" transitive="false" />
+  </dependencies>
+
 </ivy-module>

Modified: hive/trunk/cli/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/cli/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/cli/ivy.xml (original)
+++ hive/trunk/cli/ivy.xml Mon Apr 23 19:19:55 2012
@@ -28,9 +28,7 @@
   </configurations>
   <dependencies>
     <!-- Runtime Dependencies -->
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
-      <artifact name="hadoop" type="source" ext="tar.gz"/>
-    </dependency>
+
     <dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}"/>
     <dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}"/>
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}"
@@ -42,16 +40,8 @@
     <dependency org="org.apache.thrift" name="libthrift" rev="${libthrift.version}"
                 transitive="false"/>
 
-    <dependency org="org.apache.hive" name="hive-common" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-metastore" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-exec" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-serde" rev="${version}"
-                conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-service" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" />
     <dependency org="org.apache.hive" name="hive-shims" rev="${version}"
                 conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-builtins" rev="${version}"

Modified: hive/trunk/common/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/common/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/common/ivy.xml (original)
+++ hive/trunk/common/ivy.xml Mon Apr 23 19:19:55 2012
@@ -27,11 +27,62 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>
   <dependencies>
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
-      <artifact name="hadoop" type="source" ext="tar.gz"/>
-    </dependency> 
+    <dependency org="org.apache.hadoop" name="hadoop-common"
+                rev="${hadoop.version}"
+                conf="hadoop23.compile->default" transitive="false">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-auth"
+                rev="${hadoop.version}"
+                conf="hadoop23.compile->default" transitive="false">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core"
+                rev="${hadoop.version}"
+                conf="hadoop23.compile->default" transitive="false">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-archives"
+                rev="${hadoop.version}"
+                conf="hadoop23.compile->default" transitive="false">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.avro" name="avro" rev="${avro.version}"
+                conf="hadoop23.compile->default" transitive="false" />
+
+    <dependency org="org.apache.hadoop" name="hadoop-core"
+                rev="${hadoop.version}"
+                conf="hadoop20.compile->default" transitive="false">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+
+    <dependency org="org.apache.hadoop" name="hadoop-test"
+                rev="${hadoop.version}"
+                conf="hadoop20.compile->default" transitive="false">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-tools"
+                rev="${hadoop.version}"
+                conf="hadoop20.compile->default" transitive="false">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+
     <dependency org="org.apache.hive" name="hive-shims" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" transitive="false" />
     <dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}"/>
     <dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}"/>
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}"

Modified: hive/trunk/contrib/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/build.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/contrib/build.xml (original)
+++ hive/trunk/contrib/build.xml Mon Apr 23 19:19:55 2012
@@ -28,32 +28,6 @@
 
   <import file="../build-common.xml"/>
 
-  <path id="test.classpath">
-    <pathelement location="${build.dir.hive}/ql/test/classes" />
-    <pathelement location="${test.build.classes}" />
-    <pathelement location="" />
-    <pathelement location="${hadoop.conf.dir}"/>
-    <pathelement location="${test.src.data.dir}/conf"/>
-    <pathelement location="${hive.conf.dir}"/>
-    <pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
-    <pathelement location="${jetty.test.jar}"/>
-    <pathelement location="${servlet.test.jar}"/>
-    <pathelement location="${jasper.test.jar}"/>
-    <pathelement location="${jasperc.test.jar}"/>
-    <pathelement location="${jsp.test.jar}"/>
-    <pathelement location="${common.jar}"/>
-    <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
-    <fileset dir="${hadoop.root}">
-      <include name="lib/**/*.jar" />
-      <exclude name="lib/**/excluded/" />
-      <!-- below is for 0.23 onwards -->
-      <include name="share/hadoop/common/lib/*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-mapreduce-*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-yarn-*.jar" />
-    </fileset>
-    <path refid="classpath"/>
-  </path>
-
   <target name="test-jar" depends="compile-test, jar">
     <echo message="Project: ${ant.project.name}"/>
     <delete file="${test.build.dir}/test-udfs.jar"/>

Modified: hive/trunk/contrib/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/contrib/ivy.xml (original)
+++ hive/trunk/contrib/ivy.xml Mon Apr 23 19:19:55 2012
@@ -26,19 +26,10 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>  
   <dependencies>
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
-      <artifact name="hadoop" type="source" ext="tar.gz"/>
-    </dependency>
-    <dependency org="org.apache.hive" name="hive-common" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-metastore" rev="${version}"
-                conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-exec" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-serde" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" />
     <dependency org="org.apache.hive" name="hive-shims" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" transitive="false" />
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}"
                 transitive="false"/>
     <dependency org="commons-logging" name="commons-logging-api" rev="${commons-logging-api.version}"

Modified: hive/trunk/hbase-handler/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/build.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/hbase-handler/build.xml (original)
+++ hive/trunk/hbase-handler/build.xml Mon Apr 23 19:19:55 2012
@@ -29,31 +29,6 @@
 
   <import file="../build-common.xml"/>
 
-  <path id="test.classpath">
-    <pathelement location="${build.dir.hive}/ql/test/classes" />
-    <pathelement location="${test.build.classes}" />
-    <pathelement location="" />
-    <pathelement location="${hadoop.conf.dir}"/>
-    <pathelement location="${test.src.data.dir}/conf"/>
-    <pathelement location="${hive.conf.dir}"/>
-    <pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
-    <pathelement location="${hadoop.oldstyle-name.test.jar}"/>
-    <pathelement location="${hadoop.newstyle-name.test.jar}"/>
-    <pathelement location="${hadoop.common.test.jar}"/>
-    <pathelement location="${hadoop.hdfs.test.jar}"/>
-    <pathelement location="${hadoop.mapreduce.test.jar}"/>
-    <pathelement location="${jetty.test.jar}"/>
-    <pathelement location="${servlet.test.jar}"/>
-    <pathelement location="${jasper.test.jar}"/>
-    <pathelement location="${jasperc.test.jar}"/>
-    <pathelement location="${jsp.test.jar}"/>
-    <pathelement location="${common.jar}"/>
-    <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
-    <fileset dir="${hadoop.root}/lib" includes="*.jar"/>
-    <fileset dir="${hadoop.root}/lib" includes="jsp-2.1/*.jar"/>
-    <path refid="classpath"/>
-  </path>
-
   <target name="test-jar" depends="compile-test, jar">
     <echo message="Project: ${ant.project.name}"/>
     <delete file="${test.build.dir}/test-udfs.jar"/>

Modified: hive/trunk/hbase-handler/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/hbase-handler/ivy.xml (original)
+++ hive/trunk/hbase-handler/ivy.xml Mon Apr 23 19:19:55 2012
@@ -26,17 +26,8 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>  
   <dependencies>
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
-      <artifact name="hadoop" type="source" ext="tar.gz"/>
-    </dependency>
-    <dependency org="org.apache.hive" name="hive-common" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-metastore" rev="${version}"
-                conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-exec" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-serde" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" />
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}"
                 transitive="false"/>
     <dependency org="commons-logging" name="commons-logging-api" rev="${commons-logging-api.version}"

Modified: hive/trunk/hwi/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hwi/build.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/hwi/build.xml (original)
+++ hive/trunk/hwi/build.xml Mon Apr 23 19:19:55 2012
@@ -37,18 +37,6 @@
     <path refid="classpath"/>
   </path>
 
-  <!-- We also need the Jetty jar as HWI is a stand-alone Jetty instance. -->
-  <path id="test.classpath">
-    <pathelement location="${test.build.classes}" />
-    <pathelement location="" />
-    <pathelement location="${test.src.data.dir}/conf"/>
-    <pathelement location="${hive.conf.dir}"/>
-    <!-- We are running unit tests like the one inside ql -->
-    <pathelement location="${build.dir.hive}/ql/test/classes"/> 
-    <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
-    <path refid="classpath-hwi"/>
-  </path>
-
   <!--hive_hwi.war file contains only the JSP sources -->
   <target name="war">
     <echo message="Project: ${ant.project.name}"/>
@@ -78,7 +66,7 @@
      debug="${javac.debug}"
      deprecation="${javac.deprecation}">
       <compilerarg line="${javac.args} ${javac.args.warnings}" />
-      <classpath refid="classpath-hwi"/>
+      <classpath refid="classpath"/>
     </javac>
   </target>
 

Modified: hive/trunk/hwi/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hwi/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/hwi/ivy.xml (original)
+++ hive/trunk/hwi/ivy.xml Mon Apr 23 19:19:55 2012
@@ -26,20 +26,17 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>  
   <dependencies>
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
-      <artifact name="hadoop" type="source" ext="tar.gz"/>
-    </dependency>
-    <dependency org="org.apache.hive" name="hive-common" rev="${version}"
-                conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-cli" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-exec" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-shims" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" />
+    <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}" />
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}"
                 transitive="false"/>
     <dependency org="commons-logging" name="commons-logging-api" rev="${commons-logging-api.version}"
                 transitive="false"/>
+    <dependency org="org.apache.hive" name="hive-builtins" rev="${version}"
+                conf="test->default" transitive="false"/>
+    <dependency org="commons-httpclient" name="commons-httpclient" rev="${commons-httpclient.version}"
+                conf="test->default"
+                transitive="false"/>
   </dependencies>
 </ivy-module>

Modified: hive/trunk/ivy/common-configurations.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/common-configurations.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/ivy/common-configurations.xml (original)
+++ hive/trunk/ivy/common-configurations.xml Mon Apr 23 19:19:55 2012
@@ -18,6 +18,14 @@
   <!--these match the Maven configurations-->
   <conf name="default" extends="master,compile"/>
   <conf name="master" description="contains the artifact but no dependencies"/>
-  <conf name="compile" description="contains the artifact but no dependencies"/>
+  <conf name="compile" extends="hadoop${hadoop.mr.rev}.compile" description="contains the artifact but no dependencies" visibility="private"/>
   <conf name="runtime" description="runtime but not the artifact"/>
+  <conf name="test" extends="hadoop${hadoop.mr.rev}.test,compile" visibility="private" />
+  <conf name="hadoop20.compile" visibility="private"/>
+  <conf name="hadoop23.compile" visibility="private"/>
+  <conf name="hadoop20.test" visibility="private"/>
+  <conf name="hadoop23.test" visibility="private"/>
+  <conf name="hadoop0.20.shim" visibility="private"/>
+  <conf name="hadoop0.20S.shim" visibility="private"/>
+  <conf name="hadoop0.23.shim" visibility="private"/>
 </configurations>
\ No newline at end of file

Modified: hive/trunk/ivy/ivysettings.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/ivysettings.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/ivy/ivysettings.xml (original)
+++ hive/trunk/ivy/ivysettings.xml Mon Apr 23 19:19:55 2012
@@ -36,6 +36,8 @@
   <property name="repo.dir" value="${user.home}/.m2/repository"/>
   <property name="maven2.pattern.ext"  value="${maven2.pattern}.[ext]"/>
   <property name="resolvers" value="default" override="false"/>
+  <!-- pull in the local repository -->
+  <include url="${ivy.default.conf.dir}/ivyconf-local.xml"/>
   <settings defaultResolver="${resolvers}"/>
 
   <resolvers>
@@ -62,8 +64,9 @@
     </filesystem>
 
     <chain name="default" dual="true" returnFirst="true" 
-           checkmodified="${ivy.checkmodified}" 
+           checkmodified="true" 
            changingPattern="${ivy.changingPattern}">
+      <resolver ref="local" />
       <resolver ref="apache-snapshot"/>
       <resolver ref="maven2"/>
       <resolver ref="datanucleus-repo"/>
@@ -72,6 +75,7 @@
     </chain>
 
     <chain name="internal" dual="true">
+      <resolver ref="local" />
       <resolver ref="fs"/>
       <resolver ref="apache-snapshot"/>
       <resolver ref="maven2"/>
@@ -86,7 +90,8 @@
   </resolvers>
 
   <modules>
-     <module organisation="org.apache.hadoop" name="hadoop-*" resolver="${resolvers}"/>
+    <module organisation="org.apache.hadoop" name="hadoop-*" resolver="${resolvers}"/>
+    <module organisation="org.apache.hive" name=".*" resolver="${resolvers}"/>
   </modules>
 
   <caches default="${ivy.cache.name}" 

Modified: hive/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/libraries.properties?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/ivy/libraries.properties (original)
+++ hive/trunk/ivy/libraries.properties Mon Apr 23 19:19:55 2012
@@ -23,15 +23,18 @@ ant-task.version=2.0.10
 antlr.version=3.0.1
 antlr-runtime.version=3.0.1
 asm.version=3.1
+avro.version=1.5.3
 datanucleus-connectionpool.version=2.0.3
 datanucleus-core.version=2.0.3
 datanucleus-enhancer.version=2.0.3
 datanucleus-rdbms.version=2.0.3
 checkstyle.version=5.0
 commons-cli.version=1.2
-commons-codec.version=1.3
+commons-codec.version=1.4
 commons-collections.version=3.2.1
+commons-configuration.version=1.6
 commons-dbcp.version=1.4
+commons-httpclient.version=3.0.1
 commons-lang.version=2.4
 commons-logging.version=1.0.4
 commons-logging-api.version=1.0.4
@@ -43,6 +46,7 @@ jackson.version=1.8.8
 javaewah.version=0.3.2
 jdo-api.version=2.3-ec
 jdom.version=1.1
+jetty.version=6.1.26
 jline.version=0.9.94
 json.version=20090211
 junit.version=4.10

Modified: hive/trunk/jdbc/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/jdbc/build.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/jdbc/build.xml (original)
+++ hive/trunk/jdbc/build.xml Mon Apr 23 19:19:55 2012
@@ -19,13 +19,6 @@
 
 <project name="jdbc" default="jar">
   <property name="src.dir" value="${basedir}/src"/>
-    
-  <!--
-    Override properties defined in ../build-common.xml.
-  <property name="test.junit.output.format" value="brief"/>
-  <property name="test.output" value="false"/>
-  <property name="test.junit.output.usefile" value="false"/>
-  -->
 
   <import file="../build-common.xml"/>
 
@@ -37,15 +30,8 @@
     <fileset dir="${test.src.data.dir}" includes="files/*.jar"/>
     <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
     <pathelement location="${build.dir.hive}/ql/test/classes"/>
-    <fileset dir="${hadoop.root}">
-      <include name="lib/**/*.jar" />
-      <exclude name="lib/**/excluded/" />
-      <!-- below is for 0.23 onwards -->
-      <include name="share/hadoop/common/lib/*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-mapreduce-*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-yarn-*.jar" />
-    </fileset>
-    <path refid="classpath"/>
+    <fileset dir="${hive.root}/build/ivy/lib/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" />
+    <path refid="classpath" />
   </path>
 
   <target name="compile" depends="init,ivy-retrieve">

Modified: hive/trunk/jdbc/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/jdbc/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/jdbc/ivy.xml (original)
+++ hive/trunk/jdbc/ivy.xml Mon Apr 23 19:19:55 2012
@@ -27,14 +27,8 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>
   <dependencies>
-    <dependency org="org.apache.hive" name="hive-common" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-metastore" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-serde" rev="${version}"
-                conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-service" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" />
     <dependency org="org.apache.thrift" name="libthrift" rev="${libthrift.version}"
                 transitive="false"/>
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}"

Modified: hive/trunk/metastore/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/metastore/ivy.xml (original)
+++ hive/trunk/metastore/ivy.xml Mon Apr 23 19:19:55 2012
@@ -26,15 +26,10 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>
   <dependencies>
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
-      <artifact name="hadoop" type="source" ext="tar.gz"/>
-    </dependency>
-    <dependency org="org.apache.hive" name="hive-common" rev="${version}"
-                conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-serde" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" />
     <dependency org="org.apache.hive" name="hive-shims" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" transitive="false" />
     <dependency org="org.antlr" name="antlr" rev="${antlr.version}"/>
     <dependency org="org.antlr" name="antlr-runtime" rev="${antlr-runtime.version}"/>
     <dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}"/>
@@ -62,10 +57,10 @@
     <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j-api.version}"/>
     <dependency org="org.slf4j" name="slf4j-log4j12" rev="${slf4j-log4j12.version}"
                 transitive="false"/>
-    <dependency org="org.apache.thrift" name="libthrift" rev="${libthrift.version}"
-                transitive="false"/>
-    <dependency org="org.apache.thrift" name="libfb303" rev="${libfb303.version}"
-                transitive="false"/>
+    <dependency org="org.apache.thrift" name="libthrift" rev="${libthrift.version}" transitive="false" />
+
+    <dependency org="org.apache.thrift" name="libfb303" rev="${libfb303.version}" transitive="false" />
+
     <dependency org="asm" name="asm" rev="${asm.version}"/>
   </dependencies>
 </ivy-module>

Modified: hive/trunk/pdk/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/pdk/build.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/pdk/build.xml (original)
+++ hive/trunk/pdk/build.xml Mon Apr 23 19:19:55 2012
@@ -52,11 +52,14 @@
          target="package" inheritAll="false" inheritRefs="false">
       <property name="hive.install.dir" value="${build.dir.hive}/dist"/>
       <property name="hadoop.home" value="${hadoop.root}"/>
+      <property name="build.ivy.lib.dir" value="${build.ivy.lib.dir}"/>
     </ant>
     <ant antfile="${build.dir}/test-plugin/build.xml"
          target="test" inheritAll="false" inheritRefs="false">
       <property name="hive.install.dir" value="${build.dir.hive}/dist"/>
       <property name="hadoop.home" value="${hadoop.root}"/>
+      <property name="build.ivy.lib.dir" value="${build.ivy.lib.dir}"/>
+      <property name="test.hadoop.bin.path" value="${test.hadoop.bin.path}"/>
     </ant>
   </target>
 

Modified: hive/trunk/pdk/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/pdk/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/pdk/ivy.xml (original)
+++ hive/trunk/pdk/ivy.xml Mon Apr 23 19:19:55 2012
@@ -27,5 +27,7 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>
   <dependencies>
+    <dependency org="org.apache.hive" name="hive-cli" rev="${version}"
+                conf="compile->default" transitive="false" />
   </dependencies>
 </ivy-module>

Modified: hive/trunk/pdk/scripts/build-plugin.xml
URL: http://svn.apache.org/viewvc/hive/trunk/pdk/scripts/build-plugin.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/pdk/scripts/build-plugin.xml (original)
+++ hive/trunk/pdk/scripts/build-plugin.xml Mon Apr 23 19:19:55 2012
@@ -33,19 +33,21 @@
   <property name="hadoop.home" value="${env.HADOOP_HOME}"/>
 
   <path id="plugin.classpath">
-    <fileset dir="${install.dir}" includes="lib/hive-exec-*.jar"/>
-    <fileset dir="${install.dir}" includes="lib/hive-pdk-*.jar"/>
-    <fileset dir="${install.dir}" includes="lib/commons-lang-*.jar"/>
-    <fileset dir="${install.dir}" includes="lib/commons-logging-*.jar"/>
-    <fileset dir="${hadoop.home}" includes="hadoop-*-core.jar"/>
+    <fileset dir="${build.ivy.lib.dir}/default" includes="hive-exec-*.jar"/>
+    <fileset dir="${build.ivy.lib.dir}/default" includes="hive-pdk-*.jar"/>
+    <fileset dir="${build.ivy.lib.dir}/default" includes="commons-lang-*.jar"/>
+    <fileset dir="${build.ivy.lib.dir}/default" includes="commons-logging-*.jar"/>
+    <fileset dir="${build.ivy.lib.dir}/default" includes="hadoop-*.jar" />
     <pathelement location="${build.classes}"/>
   </path>
 
   <path id="plugin.test.classpath">
     <pathelement path="${install.dir}/conf"/>
-    <fileset dir="${install.dir}" includes="lib/*.jar"/>
-    <fileset dir="${hadoop.home}" includes="hadoop-*-core.jar"/>
+    <fileset dir="${build.ivy.lib.dir}/default" includes="*.jar"/>
+    <fileset dir="${build.ivy.lib.dir}/default" includes="hadoop-*.jar" />
+
     <fileset dir="${build.dir}" includes="${plugin.jar.basename}"/>
+    <fileset dir="${install.dir}/lib" includes="hive-builtins-*.jar"/>
   </path>
 
   <target name="compile">
@@ -118,12 +120,18 @@
   <target name="package" depends="jar"/>
 
   <target name="test" depends="package" unless="testcase">
+    <path id="hive.hadoop.mock.classpath">
+       <fileset dir="${build.ivy.lib.dir}/test" includes="*.jar" excludes="hive*.jar"/>
+    </path>
+    <property name="hive.hadoop.mock.classpath" refid="hive.hadoop.mock.classpath"/>
     <junit fork="true" printsummary="on" haltonfailure="true"
            dir="${build.dir}">
       <classpath refid="plugin.test.classpath"/>
+      <sysproperty key="hadoop.bin.path" value="${test.hadoop.bin.path}"/>
       <sysproperty key="hive.plugin.root.dir" value="${plugin.dir}"/>
       <sysproperty key="hive.plugin.class.list" value="${class.list}"/>
       <sysproperty key="hive.metastore.warehouse.dir" value="${build.dir}/warehouse"/>
+      <env key="HIVE_HADOOP_TEST_CLASSPATH" value="${hive.hadoop.mock.classpath}"/>
       <env key="HADOOP_HOME" value="${hadoop.home}"/>
       <env key="HIVE_CONF_DIR" value="${install.dir}/conf"/>
       <env key="HIVE_HOME" value="${install.dir}"/>

Modified: hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/build.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/ql/build.xml (original)
+++ hive/trunk/ql/build.xml Mon Apr 23 19:19:55 2012
@@ -18,7 +18,7 @@
 -->
 
 
-<project name="ql" default="jar">
+<project xmlns:ivy="antlib:org.apache.ivy.ant" name="ql" default="jar">
 
   <property name="ql.lib.dir" value="${basedir}/lib"/>
   <property name="src.dir"  location="${basedir}/src/java"/>
@@ -32,37 +32,6 @@
   <import file="../build-common.xml"/>
   <property name="ql.test.clientpositive.exclude" value="${minimr.query.files}"/>
 
-  <path id="test.classpath">
-    <pathelement location="${test.build.classes}" />
-    <pathelement location="" />
-    <pathelement location="${hadoop.conf.dir}"/>
-    <pathelement location="${test.src.data.dir}/conf"/>
-    <pathelement location="${hive.conf.dir}"/>
-    <pathelement location="${hadoop.oldstyle-name.test.jar}"/>
-    <pathelement location="${hadoop.newstyle-name.test.jar}"/>
-    <pathelement location="${hadoop.common.test.jar}"/>
-    <pathelement location="${hadoop.hdfs.test.jar}"/>
-    <pathelement location="${hadoop.mapreduce.test.jar}"/>
-    <pathelement location="${jetty.test.jar}"/>
-    <pathelement location="${servlet.test.jar}"/>
-    <pathelement location="${jasper.test.jar}"/>
-    <pathelement location="${jasperc.test.jar}"/>
-    <pathelement location="${jsp.test.jar}"/>
-    <pathelement location="${common.jar}"/>
-    <pathelement location="${hive.root}/build/ivy/lib/default/derby-${derby.version}.jar"/>
-    <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
-    <fileset dir="${hadoop.root}">
-      <include name="lib/**/*.jar" />
-      <exclude name="lib/**/excluded/" />
-      <!-- below is for 0.23 onwards -->
-      <include name="share/hadoop/common/lib/*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-mapreduce-*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-yarn-*.jar" />
-    </fileset>
-
-    <path refid="classpath"/>
-  </path>
-
   <target name="thriftif" depends="check-thrift-home">
     <echo message="${ant.project.name}"/>
     <fail unless="thrift.home">You must set the 'thrift.home' property!</fail>
@@ -245,6 +214,9 @@
       </manifest>
       <metainf dir="${hive.root}" includes="LICENSE,NOTICE"/>
     </jar>
+    <ivy:publish settingsRef="${ant.project.name}.ivy.settings"
+                 resolver="local" pubrevision="${version}" overwrite="true"
+                 artifactspattern="${build.dir}/${ivy.publish.pattern}"/>
   </target>
 
   <!-- Override deploy since we are deploying hive_exec and not hive_ql -->

Modified: hive/trunk/ql/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ql/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/ql/ivy.xml (original)
+++ hive/trunk/ql/ivy.xml Mon Apr 23 19:19:55 2012
@@ -26,21 +26,19 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>
   <dependencies>
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
-      <artifact name="hadoop" type="source" ext="tar.gz"/>
-    </dependency>
-    <dependency org="org.apache.hive" name="hive-common" rev="${version}"
-                conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-metastore" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-serde" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" />
     <dependency org="org.apache.hive" name="hive-shims" rev="${version}"
                 conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-builtins" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="test->default,runtime" transitive="false"/>
+    <dependency org="org.apache.hive" name="hive-hbase-handler" rev="${version}"
+                conf="test->default" transitive="false"/>
+    <dependency org="org.apache.hive" name="hive-contrib" rev="${version}"
+                conf="test->default" transitive="false"/>
     <dependency org="org.antlr" name="antlr" rev="${antlr.version}"
                 transitive="false"/>
+
     <dependency org="org.antlr" name="antlr-runtime" rev="${antlr-runtime.version}"/>
     <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j-api.version}"/>
     <dependency org="org.slf4j" name="slf4j-log4j12" rev="${slf4j-log4j12.version}"
@@ -61,8 +59,28 @@
                 transitive="false"/>
     <dependency org="commons-logging" name="commons-logging-api" rev="${commons-logging-api.version}"
                 transitive="false"/>
+    <dependency org="commons-configuration" name="commons-configuration" rev="${commons-configuration.version}"
+                transitive="false"/>
     <dependency org="org.apache.derby" name="derby" rev="${derby.version}"/>
     <dependency org="com.googlecode.javaewah" name="JavaEWAH" rev="${javaewah.version}"/>
     <dependency org="junit" name="junit" rev="${junit.version}"/>
+
+    <dependency org="org.datanucleus" name="datanucleus-connectionpool" rev="${datanucleus-connectionpool.version}"
+                transitive="false"/>
+    <dependency org="org.datanucleus" name="datanucleus-core" rev="${datanucleus-core.version}"
+                transitive="false"/>
+    <dependency org="org.datanucleus" name="datanucleus-enhancer" rev="${datanucleus-enhancer.version}"
+                transitive="false"/>
+    <dependency org="org.datanucleus" name="datanucleus-rdbms" rev="${datanucleus-rdbms.version}"
+                transitive="false"/>
+
+    <!-- Hack to get jobclient tests dependency in. -->
+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-jobclient" rev="${hadoop.version.ant-internal}"
+                conf="hadoop23.test->default" transitive="false">
+      <artifact name="hadoop-mapreduce-client-jobclient" ext="jar" />
+      <artifact name="hadoop-mapreduce-client-jobclient" type="tests" ext="jar" m:classifier="tests"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
   </dependencies>
 </ivy-module>

Modified: hive/trunk/serde/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/serde/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/serde/ivy.xml (original)
+++ hive/trunk/serde/ivy.xml Mon Apr 23 19:19:55 2012
@@ -26,13 +26,10 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>
   <dependencies>
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
-      <artifact name="hadoop" type="source" ext="tar.gz"/>
-    </dependency>
     <dependency org="org.apache.hive" name="hive-common" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" />
     <dependency org="org.apache.hive" name="hive-shims" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" transitive="false"  />
     <dependency org="org.slf4j" name="slf4j-api" rev="${slf4j-api.version}"/>
     <dependency org="org.slf4j" name="slf4j-log4j12" rev="${slf4j-log4j12.version}"
                 transitive="false"/>

Modified: hive/trunk/service/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/service/build.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/service/build.xml (original)
+++ hive/trunk/service/build.xml Mon Apr 23 19:19:55 2012
@@ -22,23 +22,6 @@
 
   <import file="../build-common.xml"/>
 
-  <path id="test.classpath">
-    <pathelement location="${test.build.classes}" />
-    <pathelement location="" />
-    <pathelement location="${test.src.data.dir}/conf"/>
-    <pathelement location="${hive.conf.dir}"/>
-    <fileset dir="${test.src.data.dir}" includes="files/*.jar"/>
-    <fileset dir="${hive.root}" includes="testlibs/*.jar"/>
-    <pathelement location="${build.dir.hive}/ql/test/classes"/>
-    <path refid="classpath"/>
-    <fileset dir="${hadoop.root}">
-      <!-- below is for 0.23 onwards -->
-      <include name="share/hadoop/common/lib/*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-mapreduce-*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-yarn-*.jar" />
-    </fileset>
-  </path>
-
   <target name="thriftif" depends="check-thrift-home">
     <echo message="Project: ${ant.project.name}"/>
     <fail unless="thrift.home">You must set the 'thrift.home' property!</fail>

Modified: hive/trunk/service/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/service/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/service/ivy.xml (original)
+++ hive/trunk/service/ivy.xml Mon Apr 23 19:19:55 2012
@@ -27,12 +27,8 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>
   <dependencies>
-    <dependency org="org.apache.hive" name="hive-common" rev="${version}"
-                conf="runtime" transitive="false"/>
-    <dependency org="org.apache.hive" name="hive-metastore" rev="${version}"
-                conf="runtime" transitive="false"/>
     <dependency org="org.apache.hive" name="hive-exec" rev="${version}"
-                conf="runtime" transitive="false"/>
+                conf="compile->default" />
     <dependency org="org.apache.thrift" name="libthrift" rev="${libthrift.version}"
                 transitive="false"/>
     <dependency org="org.apache.thrift" name="libfb303" rev="${libfb303.version}"

Modified: hive/trunk/shims/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/shims/build.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/shims/build.xml (original)
+++ hive/trunk/shims/build.xml Mon Apr 23 19:19:55 2012
@@ -22,35 +22,9 @@
 Before you can run these subtargets directly, you need
 to call at top-level: ant deploy-contrib compile-core-test
 -->
-<project name="shims" default="jar">
+<project xmlns:ivy="antlib:org.apache.ivy.ant" name="shims" default="jar">
   <import file="../build-common.xml"/>
 
-  <path id="classpath">
-    <fileset dir="${hadoop.root}">
-      <include name="lib/**/*.jar" />
-      <exclude name="lib/**/excluded/" />
-      <!-- below is for 0.23 onwards -->
-      <include name="share/hadoop/common/lib/*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-mapreduce-*.jar" />
-      <exclude name="share/hadoop/common/lib/hadoop-yarn-*.jar" />
-    </fileset>
-    <pathelement location="${hadoop.oldstyle-name.jar}"/>
-    <pathelement location="${hadoop.oldstyle-name.tools.jar}"/>
-    <pathelement location="${hadoop.oldstyle-name.test.jar}"/>
-    <pathelement location="${hadoop.newstyle-name.jar}"/>
-    <pathelement location="${hadoop.newstyle-name.test.jar}"/>
-    <pathelement location="${hadoop.newstyle-name.tools.jar}"/>
-    <pathelement location="${hadoop.common.jar}"/>
-    <pathelement location="${hadoop.common.test.jar}"/>
-    <pathelement location="${hadoop.hdfs.jar}"/>
-    <pathelement location="${hadoop.hdfs.test.jar}"/>
-    <pathelement location="${hadoop.mapreduce.jar}"/>
-    <pathelement location="${hadoop.mapreduce.test.jar}"/>
-    <pathelement location="${hadoop.mapreduce.tools.jar}"/>
-  	<fileset dir="../lib" includes="*.jar" />
-    <path refid="common-classpath"/>
-  </path>
-
   <!-- default list of shims to build -->
   <property name="shims.include" value="0.20,0.20S,0.23"/>
   <property name="shims.common.sources" value="${basedir}/src/common/java"/>	
@@ -62,11 +36,37 @@ to call at top-level: ant deploy-contrib
   <property name="shims.0.20S.version" value="${hadoop-0.20S.version}" />	
   <property name="shims.0.23.sources" value="${shims.common.secure.sources};${basedir}/src/0.23/java" />	
   <property name="shims.0.23.version" value="${hadoop-0.23.version}" />	
-	
-  <target name="build_shims" depends="install-hadoopcore-internal"
+  <property name="shims.0.20.hadoop.ivy.dir" value="${hive.root}/build/ivy/lib/hadoop0.20.shim" />
+  <property name="shims.0.20S.hadoop.ivy.dir" value="${hive.root}/build/ivy/lib/hadoop0.20S.shim" />
+  <property name="shims.0.23.hadoop.ivy.dir" value="${hive.root}/build/ivy/lib/hadoop0.23.shim" />
+
+  <path id="test.classpath">
+    <pathelement location="${test.build.classes}" />
+    <pathelement location="" />
+    <pathelement location="${test.src.data.dir}/conf"/>
+    <pathelement location="${hive.conf.dir}"/>
+    <!-- We are running unit tests like the one inside ql -->
+    <fileset dir="${hive.root}/testlibs" includes="*.jar" />
+    <fileset dir="${shims.0.20S.hadoop.ivy.dir}" includes="*.jar" excludes="slf4j-*.jar" />
+    <fileset dir="../lib" includes="*.jar" />
+    <path refid="classpath" />
+  </path>
+
+  <target name="build_shims"
           description="Build shims against a particular hadoop version">
     <echo message="Project: ${ant.project.name}"/>
     <echo message="Compiling ${sources} against hadoop ${hadoop.version.ant-internal} (${hadoop.root})"/>
+
+    <antcall target="ivy-retrieve-hadoop-shim" inheritRefs="false" inheritAll="false">
+      <param name="ivy.hadoop.shim.conf" value="hadoop${shim.name}.shim" />
+    </antcall>
+    
+    <path id="shims.classpath">
+      <fileset dir="${hadoop.ivy.dir}" includes="*.jar" />
+      <fileset dir="../lib" includes="*.jar" />
+      <fileset dir="${hive.root}/build/ivy/lib/default" includes="*.jar" />
+    </path>
+
     <javac
      encoding="${build.encoding}"
      includes="**/*.java"
@@ -76,24 +76,26 @@ to call at top-level: ant deploy-contrib
      srcdir="${sources}"
      includeantruntime="false">
       <compilerarg line="${javac.args} ${javac.args.warnings}" />
-      <classpath refid="classpath"/>
+      <classpath refid="shims.classpath"/>
     </javac>
   </target>
 	
   <target name="compile" depends="init,ivy-retrieve">
-  		<echo message="Project: ${ant.project.name}"/>
-		<for param="shimName" list="${shims.include}">
-		  <sequential>
-		    <echo>Building shims @{shimName}</echo>
-		  	<antcall target="build_shims" inheritRefs="false" inheritAll="false">
-		      <param name="hadoop.version.ant-internal" value="${shims.@{shimName}.version}" />
-		      <param name="sources" value="${shims.@{shimName}.sources}" />
-		    </antcall>
-		  </sequential>
-	  	</for>  	
+    <echo message="Project: ${ant.project.name}"/>
+    <for param="shimName" list="${shims.include}">
+      <sequential>
+        <echo>Building shims @{shimName}</echo>
+        <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+          <param name="hadoop.version.ant-internal" value="${shims.@{shimName}.version}" />
+          <param name="sources" value="${shims.@{shimName}.sources}" />
+          <param name="hadoop.ivy.dir" value="${shims.@{shimName}.hadoop.ivy.dir}" />
+          <param name="shim.name" value="@{shimName}" />
+        </antcall>
+      </sequential>
+    </for>  	
   </target>
   
-  <target name="compile_secure_test" depends="install-hadoopcore-internal" 
+  <target name="compile_secure_test" 
           description="Test shims against a particular hadoop version">
     <echo message="Project: ${ant.project.name}"/>
     <getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version.ant-internal}" />
@@ -111,7 +113,7 @@ to call at top-level: ant deploy-contrib
      deprecation="${javac.deprecation}"
      includeantruntime="false">
       <compilerarg line="${javac.args} ${javac.args.warnings}" />
-      <classpath refid="classpath"/>
+      <classpath refid="test.classpath"/>
     </javac>
     <javac
      encoding="${build.encoding}"
@@ -125,11 +127,11 @@ to call at top-level: ant deploy-contrib
      deprecation="${javac.deprecation}"
      includeantruntime="false">
       <compilerarg line="${javac.args} ${javac.args.warnings}" />
-      <classpath refid="classpath"/>
+      <classpath refid="test.classpath"/>
     </javac>
   </target>
   
-  <target name="compile-test" depends="compile">
+  <target name="compile-test" depends="compile,ivy-retrieve-test">
     <echo message="Project: ${ant.project.name}"/>
     <!-- TODO: move tests to version directory -->
     <antcall target="compile_secure_test" inheritRefs="false" inheritAll="false">

Modified: hive/trunk/shims/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/shims/ivy.xml?rev=1329381&r1=1329380&r2=1329381&view=diff
==============================================================================
--- hive/trunk/shims/ivy.xml (original)
+++ hive/trunk/shims/ivy.xml Mon Apr 23 19:19:55 2012
@@ -15,7 +15,7 @@
    See the License for the specific language governing permissions and
    limitations under the License.
 -->
-<ivy-module version="2.0">
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
   <info organisation="${hive.ivy.org}" module="hive-shims" revision="${version}">
     <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt" />
     <description homepage="http://hive.apache.org">
@@ -27,10 +27,7 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>
   <dependencies>
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
-      <artifact name="hadoop" type="source" ext="tar.gz"/>
-      <exclude org="org.codehaus.jackson" name="*"/>
-    </dependency>
+    <!-- General dependencies. -->
     <dependency org="org.apache.zookeeper" name="zookeeper"
                 rev="${zookeeper.version}" transitive="false">
       <include type="jar"/>
@@ -45,6 +42,84 @@
       transitive="false"/>
     <dependency org="org.codehaus.jackson" name="jackson-core-asl" rev="${jackson.version}"/>
     <dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="${jackson.version}"/>
+
+    <!-- Hadoop 0.23 dependencies. Used both for shims and for building against Hadoop 0.23. -->
+    <dependency org="org.apache.hadoop" name="hadoop-common"
+                rev="${hadoop-0.23.version}"
+                conf="hadoop0.23.shim->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core"
+                rev="${hadoop-0.23.version}"
+                conf="hadoop0.23.shim->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-archives"
+                rev="${hadoop-0.23.version}"
+                conf="hadoop0.23.shim->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-hdfs"
+                rev="${hadoop-0.23.version}"
+                conf="hadoop0.23.shim->default">
+      <artifact name="hadoop-hdfs" ext="jar" />
+      <artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+
+    <!-- Hadoop 0.20 shim dependencies. Used for building 0.20 shims. -->
+    <dependency org="org.apache.hadoop" name="hadoop-core"
+                rev="${hadoop-0.20.version}"
+                conf="hadoop0.20.shim->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-tools"
+                rev="${hadoop-0.20.version}"
+                conf="hadoop0.20.shim->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-test"
+                rev="${hadoop-0.20.version}"
+                conf="hadoop0.20.shim->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+
+    <!-- Hadoop 0.20S (or 1.0.0) shim dependencies. Used for building 0.20S shims. -->
+    <dependency org="org.apache.hadoop" name="hadoop-core"
+                rev="${hadoop-0.20S.version}"
+                conf="hadoop0.20S.shim->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-tools"
+                rev="${hadoop-0.20S.version}"
+                conf="hadoop0.20S.shim->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-test"
+                rev="${hadoop-0.20S.version}"
+                conf="hadoop0.20S.shim->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+
     <conflict manager="all" />
   </dependencies>
 </ivy-module>

Added: hive/trunk/testutils/hadoop
URL: http://svn.apache.org/viewvc/hive/trunk/testutils/hadoop?rev=1329381&view=auto
==============================================================================
--- hive/trunk/testutils/hadoop (added)
+++ hive/trunk/testutils/hadoop Mon Apr 23 19:19:55 2012
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script satisfies Hive's bin/hadoop dependency when running tests.
+# It's a quick and dirty hack, and should eventually be made obsolete by
+# completely eliminating Hive's dependency on bin/hadoop.
+
+bin=`which $0`
+bin=`dirname ${bin}`
+bin=`cd "$bin"; pwd`
+
+# Newer versions of glibc use an arena memory allocator that causes virtual
+# memory usage to explode. This interacts badly with the many threads that
+# we use in Hadoop. Tune the variable down to prevent vmem explosion.
+export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m 
+
+# check envvars which might override default args
+if [ "$HADOOP_HEAPSIZE" != "" ]; then
+  JAVA_HEAP_MAX="-Xmx""$HADOOP_HEAPSIZE""m"
+fi
+
+# CLASSPATH initially contains $HADOOP_CONF_DIR
+CLASSPATH="${HADOOP_CONF_DIR}"
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+BUILD_ROOT="$bin"/../build
+
+if [ "$HIVE_HADOOP_TEST_CLASSPATH" != "" ]; then
+  CLASSPATH=${CLASSPATH}:${HIVE_HADOOP_TEST_CLASSPATH}
+else
+  echo "Error: HIVE_HADOOP_TEST_CLASSPATH not defined."
+  exit 1
+fi
+
+# restore ordinary behaviour
+unset IFS
+
+mkdir -p $BUILD_ROOT/test/hadoop/logs
+HADOOP_LOG_DIR=$BUILD_ROOT/test/hadoop/logs
+HADOOP_LOGFILE=hadoop.log
+
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+  HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+fi  
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.policy.file=$HADOOP_POLICYFILE"
+
+# Disable ipv6 as it can cause issues
+HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
+
+COMMAND=$1
+# the core commands
+if [ "$COMMAND" = "fs" ] ; then
+  CLASS=org.apache.hadoop.fs.FsShell
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "version" ] ; then
+  CLASS=org.apache.hadoop.util.VersionInfo
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "jar" ] ; then
+  CLASS=org.apache.hadoop.util.RunJar
+elif [ "$COMMAND" = "distcp" ] ; then
+  CLASS=org.apache.hadoop.tools.DistCp
+  CLASSPATH=${CLASSPATH}:${TOOL_PATH}
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "daemonlog" ] ; then
+  CLASS=org.apache.hadoop.log.LogLevel
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [ "$COMMAND" = "archive" ] ; then
+  CLASS=org.apache.hadoop.tools.HadoopArchives
+  CLASSPATH=${CLASSPATH}:${TOOL_PATH}
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+elif [[ "$COMMAND" = -*  ]] ; then
+  # class and package names cannot begin with a -
+  echo "Error: No command named \`$COMMAND' was found. Perhaps you meant \`hadoop ${COMMAND#-}'"
+  exit 1
+else
+  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
+  CLASS=$COMMAND
+fi
+shift
+
+#make sure security appender is turned off
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}"
+
+export CLASSPATH=$CLASSPATH
+exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"

Propchange: hive/trunk/testutils/hadoop
------------------------------------------------------------------------------
    svn:executable = *



Mime
View raw message