hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amareshw...@apache.org
Subject svn commit: r1229584 - in /hive/trunk: ./ contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/ ql/src/java/org/apache/hadoop/hive/ql/io/ ql/src/java/org/apache/hadoop/hive/ql/lib/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/java/org/ap...
Date Tue, 10 Jan 2012 15:08:48 GMT
Author: amareshwari
Date: Tue Jan 10 15:08:45 2012
New Revision: 1229584

URL: http://svn.apache.org/viewvc?rev=1229584&view=rev
Log:
HIVE-2629. Reverting previous commit 

Added:
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/DelegationTokenIdentifier.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/DelegationTokenIdentifier.java
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/DelegationTokenSecretManager.java
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/DelegationTokenSelector.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/DelegationTokenSelector.java
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/TokenStoreDelegationTokenSecretManager.java
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/client/TUGIAssumingTransport.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/client/TUGIAssumingTransport.java
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/security/token/delegation/HiveDelegationTokenSupport.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.20S/java/org/apache/hadoop/security/token/delegation/HiveDelegationTokenSupport.java
    hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/thrift/DelegationTokenIdentifier23.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/thrift/DelegationTokenIdentifier23.java
    hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/thrift/DelegationTokenSelector23.java
      - copied unchanged from r1229509, hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/thrift/DelegationTokenSelector23.java
Removed:
    hive/trunk/shims/src/common-secure/
Modified:
    hive/trunk/README.txt   (props changed)
    hive/trunk/build-common.xml
    hive/trunk/build.properties
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMax.java   (props changed)
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMin.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataInputBuffer.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataOutputBuffer.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/JoinCond.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/JoinType.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AggregationDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CollectDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateFunctionDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DescFunctionDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DescTableDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropFunctionDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplosionDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeColumnDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeFieldDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeNullDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExtractDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ForwardDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SchemaDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowFunctionsDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowPartitionsDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowTableStatusDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowTablesDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java   (props changed)
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/UnionDesc.java   (props changed)
    hive/trunk/serde/README.txt   (props changed)
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java   (props changed)
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/   (props changed)
    hive/trunk/shims/build.xml
    hive/trunk/shims/ivy.xml
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java

Propchange: hive/trunk/README.txt
            ('svn:mergeinfo' removed)

Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1229584&r1=1229583&r2=1229584&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Tue Jan 10 15:08:45 2012
@@ -115,14 +115,14 @@
       log="${ivyresolvelog}"/>
   </target>
 
-  <target name="ivy-retrieve-hadoop-source"
+
+  <target name="ivy-retrieve-hadoop-source" depends="ivy-init-settings"
     description="Retrieve Ivy-managed Hadoop source artifacts" unless="ivy.skip">
     <echo message="Project: ${ant.project.name}"/>
-  	<echo message="hadoop.version.ant-internal: ${hadoop.version.ant-internal}"/>
-  	<ivy:settings id="${ant.project.name}-${hadoop.version.ant-internal}.ivy.settings" file="${ivysettings.xml}"/>
-  	<ivy:retrieve settingsRef="${ant.project.name}-${hadoop.version.ant-internal}.ivy.settings"
+    <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
       pattern="${build.dir.hadoop}/[artifact]-[revision].[ext]"/>
   </target>
+
   
   <available property="hadoopcore.${hadoop.version.ant-internal}.install.done"
     file="${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}.installed"/>

Modified: hive/trunk/build.properties
URL: http://svn.apache.org/viewvc/hive/trunk/build.properties?rev=1229584&r1=1229583&r2=1229584&view=diff
==============================================================================
--- hive/trunk/build.properties (original)
+++ hive/trunk/build.properties Tue Jan 10 15:08:45 2012
@@ -10,11 +10,9 @@ javac.deprecation=off
 javac.args=
 javac.args.warnings=
 
-hadoop-0.20.version=0.20.1
-hadoop-0.20S.version=0.20.3-CDH3-SNAPSHOT
-hadoop-0.23.version=0.23.0
-hadoop.version=${hadoop-0.20.version}
-hadoop.security.version=${hadoop-0.20S.version}
+hadoop.version=0.20.1
+hadoop.security.version=0.20.3-CDH3-SNAPSHOT
+hadoop.security.version.prefix=0.20S
 hadoop.mirror=http://mirror.facebook.net/facebook/hive-deps
 hadoop.mirror2=http://archive.cloudera.com/hive-deps
 

Propchange: hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMax.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMin.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataInputBuffer.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/NonSyncDataOutputBuffer.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/JoinCond.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/JoinType.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AggregationDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CollectDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CopyWork.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateFunctionDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateViewDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DescFunctionDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DescTableDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropFunctionDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplosionDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeColumnDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeFieldDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeGenericFuncDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeNullDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ExtractDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ForwardDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/GroupByDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LimitDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredLocalWork.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SchemaDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ScriptDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowFunctionsDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowPartitionsDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowTableStatusDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowTablesDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/UDTFDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/UnionDesc.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/serde/README.txt
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
            ('svn:mergeinfo' removed)

Propchange: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/
            ('svn:mergeinfo' removed)

Modified: hive/trunk/shims/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/shims/build.xml?rev=1229584&r1=1229583&r2=1229584&view=diff
==============================================================================
--- hive/trunk/shims/build.xml (original)
+++ hive/trunk/shims/build.xml Tue Jan 10 15:08:45 2012
@@ -51,49 +51,50 @@ to call at top-level: ant deploy-contrib
     <path refid="common-classpath"/>
   </path>
 
-  <!-- default list of shims to build -->
-  <property name="shims.include" value="0.20,0.20S,0.23"/>
-  <property name="shims.common.sources" value="${basedir}/src/common/java"/>	
-  <property name="shims.common.secure.sources" value="${basedir}/src/common/java;${basedir}/src/common-secure/java"/>
-  <!-- sources and hadoop version for each shim -->
-  <property name="shims.0.20.sources" value="${shims.common.sources};${basedir}/src/0.20/java" />	
-  <property name="shims.0.20.version" value="${hadoop-0.20.version}" />	
-  <property name="shims.0.20S.sources" value="${shims.common.secure.sources};${basedir}/src/0.20S/java" />	
-  <property name="shims.0.20S.version" value="${hadoop-0.20S.version}" />	
-  <property name="shims.0.23.sources" value="${shims.common.secure.sources};${basedir}/src/0.23/java" />	
-  <property name="shims.0.23.version" value="${hadoop-0.23.version}" />	
-	
-  <target name="build_shims" depends="install-hadoopcore-internal"
+  <target name="build_shims" depends="install-hadoopcore-internal, ivy-retrieve-hadoop-source"
           description="Build shims against a particular hadoop version">
     <echo message="Project: ${ant.project.name}"/>
-    <echo message="Compiling ${sources} against hadoop ${hadoop.version.ant-internal} (${hadoop.root})"/>
+    <getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version.ant-internal}" />
+    <echo message="Compiling shims against hadoop ${hadoop.version.ant-internal} (${hadoop.root})"/>
     <javac
      encoding="${build.encoding}"
      includes="**/*.java"
+     excludes="**/Proxy*.java"
      destdir="${build.classes}"
      debug="${javac.debug}"
      deprecation="${javac.deprecation}"
-     srcdir="${sources}"
      includeantruntime="false">
       <compilerarg line="${javac.args} ${javac.args.warnings}" />
       <classpath refid="classpath"/>
+      <src path="${basedir}/src/${hadoop.version.ant-internal.prefix}/java" />
+      <src path="${basedir}/src/common/java" />
     </javac>
   </target>
-	
+
   <target name="compile" depends="init,ivy-retrieve">
-  		<echo message="Project: ${ant.project.name}"/>
-		<for param="shimName" list="${shims.include}">
-		  <sequential>
-		    <echo>Building shims @{shimName}</echo>
-		  	<antcall target="build_shims" inheritRefs="false" inheritAll="false">
-		      <param name="hadoop.version.ant-internal" value="${shims.@{shimName}.version}" />
-		      <param name="sources" value="${shims.@{shimName}.sources}" />
-		    </antcall>
-		  </sequential>
-	  	</for>  	
+    <echo message="Project: ${ant.project.name}"/>
+    <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+      <param name="hadoop.version.ant-internal" value="${hadoop.version}" />
+    </antcall>
+    <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+      <param name="hadoop.version.ant-internal" value="${hadoop.security.version}" />
+      <param name="hadoop.version.ant-internal.prefix" value="${hadoop.security.version.prefix}" />
+    </antcall>
+    <getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version}" />
+    <javac
+     encoding="${build.encoding}"
+     includes="**/Proxy*.java"
+     destdir="${build.classes}"
+     debug="${javac.debug}"
+     deprecation="${javac.deprecation}"
+     includeantruntime="false">
+      <compilerarg line="${javac.args} ${javac.args.warnings}" />
+      <classpath refid="classpath"/>
+      <src path="${basedir}/src/common/java" />
+    </javac>
   </target>
   
-  <target name="compile_secure_test" depends="install-hadoopcore-internal" 
+  <target name="compile_secure_test" depends="install-hadoopcore-internal, ivy-retrieve-hadoop-source" 
           description="Test shims against a particular hadoop version">
     <echo message="Project: ${ant.project.name}"/>
     <getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version.ant-internal}" />
@@ -132,8 +133,9 @@ to call at top-level: ant deploy-contrib
   <target name="compile-test" depends="compile">
     <echo message="Project: ${ant.project.name}"/>
     <!-- TODO: move tests to version directory -->
-    <antcall target="compile_secure_test" inheritRefs="false" inheritAll="false">
+    <!--antcall target="compile_secure_test" inheritRefs="false" inheritAll="false">
       <param name="hadoop.version.ant-internal" value="${hadoop.security.version}" />
-    </antcall>
+      <param name="hadoop.version.ant-internal.prefix" value="${hadoop.security.version.prefix}" />
+    </antcall-->
   </target>
 </project>

Modified: hive/trunk/shims/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/shims/ivy.xml?rev=1229584&r1=1229583&r2=1229584&view=diff
==============================================================================
--- hive/trunk/shims/ivy.xml (original)
+++ hive/trunk/shims/ivy.xml Tue Jan 10 15:08:45 2012
@@ -27,7 +27,10 @@
     <include file="${ivy.conf.dir}/common-configurations.xml"/>
   </configurations>
   <dependencies>
-    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
+    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version}">
+      <artifact name="hadoop" type="source" ext="tar.gz"/>
+    </dependency> 
+    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.security.version}">
       <artifact name="hadoop" type="source" ext="tar.gz"/>
     </dependency>
     <dependency org="org.apache.zookeeper" name="zookeeper"

Modified: hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1229584&r1=1229583&r2=1229584&view=diff
==============================================================================
--- hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Tue Jan 10 15:08:45 2012
@@ -17,17 +17,510 @@
  */
 package org.apache.hadoop.hive.shims;
 
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.shims.HadoopShimsSecure;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
+import org.apache.hadoop.hive.thrift.DelegationTokenSelector;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.OutputCommitter;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+import org.apache.hadoop.mapred.TaskCompletionEvent;
+import org.apache.hadoop.mapred.TaskID;
+import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
+import org.apache.hadoop.mapred.lib.CombineFileSplit;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.apache.hadoop.tools.HadoopArchives;
 import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
- * Implemention of shims against Hadoop 0.20 with Security.
+ * Implemention of shims against Hadoop 0.20.0.
  */
-public class Hadoop20SShims extends HadoopShimsSecure {
+public class Hadoop20SShims implements HadoopShims {
+  public boolean usesJobShell() {
+    return false;
+  }
+
+  public boolean fileSystemDeleteOnExit(FileSystem fs, Path path)
+      throws IOException {
+
+    return fs.deleteOnExit(path);
+  }
+
+  public void inputFormatValidateInput(InputFormat fmt, JobConf conf)
+      throws IOException {
+    // gone in 0.18+
+  }
+
+  public boolean isJobPreparing(RunningJob job) throws IOException {
+    return job.getJobState() == JobStatus.PREP;
+  }
+  /**
+   * Workaround for hadoop-17 - jobclient only looks at commandlineconfig.
+   */
+  public void setTmpFiles(String prop, String files) {
+    // gone in 20+
+  }
+
+  public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
+      int numDataNodes,
+      boolean format,
+      String[] racks) throws IOException {
+    return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
+  }
+
+  /**
+   * MiniDFSShim.
+   *
+   */
+  public class MiniDFSShim implements HadoopShims.MiniDFSShim {
+    private final MiniDFSCluster cluster;
+
+    public MiniDFSShim(MiniDFSCluster cluster) {
+      this.cluster = cluster;
+    }
+
+    public FileSystem getFileSystem() throws IOException {
+      return cluster.getFileSystem();
+    }
+
+    public void shutdown() {
+      cluster.shutdown();
+    }
+  }
+
+  /**
+   * We define this function here to make the code compatible between
+   * hadoop 0.17 and hadoop 0.20.
+   *
+   * Hive binary that compiled Text.compareTo(Text) with hadoop 0.20 won't
+   * work with hadoop 0.17 because in hadoop 0.20, Text.compareTo(Text) is
+   * implemented in org.apache.hadoop.io.BinaryComparable, and Java compiler
+   * references that class, which is not available in hadoop 0.17.
+   */
+  public int compareText(Text a, Text b) {
+    return a.compareTo(b);
+  }
+
+  @Override
+  public long getAccessTime(FileStatus file) {
+    return file.getAccessTime();
+  }
+
+  public HadoopShims.CombineFileInputFormatShim getCombineFileInputFormat() {
+    return new CombineFileInputFormatShim() {
+      @Override
+      public RecordReader getRecordReader(InputSplit split,
+          JobConf job, Reporter reporter) throws IOException {
+        throw new IOException("CombineFileInputFormat.getRecordReader not needed.");
+      }
+    };
+  }
+
+  public static class InputSplitShim extends CombineFileSplit implements HadoopShims.InputSplitShim {
+    long shrinkedLength;
+    boolean _isShrinked;
+    public InputSplitShim() {
+      super();
+      _isShrinked = false;
+    }
+
+    public InputSplitShim(CombineFileSplit old) throws IOException {
+      super(old);
+      _isShrinked = false;
+    }
+
+    @Override
+    public void shrinkSplit(long length) {
+      _isShrinked = true;
+      shrinkedLength = length;
+    }
+
+    public boolean isShrinked() {
+      return _isShrinked;
+    }
+
+    public long getShrinkedLength() {
+      return shrinkedLength;
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+      super.readFields(in);
+      _isShrinked = in.readBoolean();
+      if (_isShrinked) {
+        shrinkedLength = in.readLong();
+      }
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+      super.write(out);
+      out.writeBoolean(_isShrinked);
+      if (_isShrinked) {
+        out.writeLong(shrinkedLength);
+      }
+    }
+  }
+
+  /* This class should be replaced with org.apache.hadoop.mapred.lib.CombineFileRecordReader class, once
+   * https://issues.apache.org/jira/browse/MAPREDUCE-955 is fixed. This code should be removed - it is a copy
+   * of org.apache.hadoop.mapred.lib.CombineFileRecordReader
+   */
+  public static class CombineFileRecordReader<K, V> implements RecordReader<K, V> {
+
+    static final Class[] constructorSignature = new Class[] {
+        InputSplit.class,
+        Configuration.class,
+        Reporter.class,
+        Integer.class
+        };
+
+    protected CombineFileSplit split;
+    protected JobConf jc;
+    protected Reporter reporter;
+    protected Class<RecordReader<K, V>> rrClass;
+    protected Constructor<RecordReader<K, V>> rrConstructor;
+    protected FileSystem fs;
+
+    protected int idx;
+    protected long progress;
+    protected RecordReader<K, V> curReader;
+    protected boolean isShrinked;
+    protected long shrinkedLength;
+
+    public boolean next(K key, V value) throws IOException {
+
+      while ((curReader == null)
+          || !doNextWithExceptionHandler((K) ((CombineHiveKey) key).getKey(),
+              value)) {
+        if (!initNextRecordReader(key)) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    public K createKey() {
+      K newKey = curReader.createKey();
+      return (K)(new CombineHiveKey(newKey));
+    }
+
+    public V createValue() {
+      return curReader.createValue();
+    }
+
+    /**
+     * Return the amount of data processed.
+     */
+    public long getPos() throws IOException {
+      return progress;
+    }
+
+    public void close() throws IOException {
+      if (curReader != null) {
+        curReader.close();
+        curReader = null;
+      }
+    }
+
+    /**
+     * Return progress based on the amount of data processed so far.
+     */
+    public float getProgress() throws IOException {
+      return Math.min(1.0f, progress / (float) (split.getLength()));
+    }
+
+    /**
+     * A generic RecordReader that can hand out different recordReaders
+     * for each chunk in the CombineFileSplit.
+     */
+    public CombineFileRecordReader(JobConf job, CombineFileSplit split,
+        Reporter reporter,
+        Class<RecordReader<K, V>> rrClass)
+        throws IOException {
+      this.split = split;
+      this.jc = job;
+      this.rrClass = rrClass;
+      this.reporter = reporter;
+      this.idx = 0;
+      this.curReader = null;
+      this.progress = 0;
+
+      isShrinked = false;
+
+      assert (split instanceof Hadoop20Shims.InputSplitShim);
+      if (((InputSplitShim) split).isShrinked()) {
+        isShrinked = true;
+        shrinkedLength = ((InputSplitShim) split).getShrinkedLength();
+      }      
+      
+      try {
+        rrConstructor = rrClass.getDeclaredConstructor(constructorSignature);
+        rrConstructor.setAccessible(true);
+      } catch (Exception e) {
+        throw new RuntimeException(rrClass.getName() +
+            " does not have valid constructor", e);
+      }
+      initNextRecordReader(null);
+    }
+    
+    /**
+     * do next and handle exception inside it. 
+     * @param key
+     * @param value
+     * @return
+     * @throws IOException
+     */
+    private boolean doNextWithExceptionHandler(K key, V value) throws IOException {
+      try {
+        return curReader.next(key, value);
+      } catch (Exception e) {
+        return HiveIOExceptionHandlerUtil
+            .handleRecordReaderNextException(e, jc);
+      }
+    }
+
+    /**
+     * Get the record reader for the next chunk in this CombineFileSplit.
+     */
+    protected boolean initNextRecordReader(K key) throws IOException {
+
+      if (curReader != null) {
+        curReader.close();
+        curReader = null;
+        if (idx > 0) {
+          progress += split.getLength(idx - 1); // done processing so far
+        }
+      }
+
+      // if all chunks have been processed, nothing more to do.
+      if (idx == split.getNumPaths() || (isShrinked && progress > shrinkedLength)) {
+        return false;
+      }
+
+      // get a record reader for the idx-th chunk
+      try {
+        curReader = rrConstructor.newInstance(new Object[]
+            {split, jc, reporter, Integer.valueOf(idx)});
+
+        // change the key if need be
+        if (key != null) {
+          K newKey = curReader.createKey();
+          ((CombineHiveKey)key).setKey(newKey);
+        }
+
+        // setup some helper config variables.
+        jc.set("map.input.file", split.getPath(idx).toString());
+        jc.setLong("map.input.start", split.getOffset(idx));
+        jc.setLong("map.input.length", split.getLength(idx));
+      } catch (Exception e) {
+        curReader = HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(
+            e, jc);
+      }
+      idx++;
+      return true;
+    }
+  }
+
+  public abstract static class CombineFileInputFormatShim<K, V> extends
+      CombineFileInputFormat<K, V>
+      implements HadoopShims.CombineFileInputFormatShim<K, V> {
+
+    public Path[] getInputPathsShim(JobConf conf) {
+      try {
+        return FileInputFormat.getInputPaths(conf);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    @Override
+    public void createPool(JobConf conf, PathFilter... filters) {
+      super.createPool(conf, filters);
+    }
+
+    @Override
+    public InputSplitShim[] getSplits(JobConf job, int numSplits) throws IOException {
+      long minSize = job.getLong("mapred.min.split.size", 0);
+
+      // For backward compatibility, let the above parameter be used
+      if (job.getLong("mapred.min.split.size.per.node", 0) == 0) {
+        super.setMinSplitSizeNode(minSize);
+      }
+
+      if (job.getLong("mapred.min.split.size.per.rack", 0) == 0) {
+        super.setMinSplitSizeRack(minSize);
+      }
+
+      if (job.getLong("mapred.max.split.size", 0) == 0) {
+        super.setMaxSplitSize(minSize);
+      }
+
+      CombineFileSplit[] splits = (CombineFileSplit[]) super.getSplits(job, numSplits);
+
+      InputSplitShim[] isplits = new InputSplitShim[splits.length];
+      for (int pos = 0; pos < splits.length; pos++) {
+        isplits[pos] = new InputSplitShim(splits[pos]);
+      }
+
+      return isplits;
+    }
+
+    public InputSplitShim getInputSplitShim() throws IOException {
+      return new InputSplitShim();
+    }
+
+    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
+        Reporter reporter,
+        Class<RecordReader<K, V>> rrClass)
+        throws IOException {
+      CombineFileSplit cfSplit = (CombineFileSplit) split;
+      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
+    }
+
+  }
+
+  public String getInputFormatClassName() {
+    return "org.apache.hadoop.hive.ql.io.CombineHiveInputFormat";
+  }
+
+  String[] ret = new String[2];
+
+  @Override
+  public String[] getTaskJobIDs(TaskCompletionEvent t) {
+    TaskID tid = t.getTaskAttemptId().getTaskID();
+    ret[0] = tid.toString();
+    ret[1] = tid.getJobID().toString();
+    return ret;
+  }
+
+  public void setFloatConf(Configuration conf, String varName, float val) {
+    conf.setFloat(varName, val);
+  }
+
+  @Override
+  public int createHadoopArchive(Configuration conf, Path sourceDir, Path destDir,
+      String archiveName) throws Exception {
+
+    HadoopArchives har = new HadoopArchives(conf);
+    List<String> args = new ArrayList<String>();
+
+    if (conf.get("hive.archive.har.parentdir.settable") == null) {
+      throw new RuntimeException("hive.archive.har.parentdir.settable is not set");
+    }
+    boolean parentSettable =
+      conf.getBoolean("hive.archive.har.parentdir.settable", false);
+
+    if (parentSettable) {
+      args.add("-archiveName");
+      args.add(archiveName);
+      args.add("-p");
+      args.add(sourceDir.toString());
+      args.add(destDir.toString());
+    } else {
+      args.add("-archiveName");
+      args.add(archiveName);
+      args.add(sourceDir.toString());
+      args.add(destDir.toString());
+    }
+
+    return ToolRunner.run(har, args.toArray(new String[0]));
+  }
+
+  public static class NullOutputCommitter extends OutputCommitter {
+    @Override
+    public void setupJob(JobContext jobContext) { }
+    @Override
+    public void cleanupJob(JobContext jobContext) { }
+
+    @Override
+    public void setupTask(TaskAttemptContext taskContext) { }
+    @Override
+    public boolean needsTaskCommit(TaskAttemptContext taskContext) {
+      return false;
+    }
+    @Override
+    public void commitTask(TaskAttemptContext taskContext) { }
+    @Override
+    public void abortTask(TaskAttemptContext taskContext) { }
+  }
+
+  public void setNullOutputFormat(JobConf conf) {
+    conf.setOutputFormat(NullOutputFormat.class);
+    conf.setOutputCommitter(Hadoop20Shims.NullOutputCommitter.class);
+
+    // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463)
+    // but can be backported. So we disable setup/cleanup in all versions >= 0.19
+    conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
+
+    // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206)
+    // but can be backported. So we disable setup/cleanup in all versions >= 0.19
+    conf.setBoolean("mapreduce.job.committer.task.cleanup.needed", false);
+  }
+
+  @Override
+  public UserGroupInformation getUGIForConf(Configuration conf) throws IOException {
+    return UserGroupInformation.getCurrentUser();
+  }
+  
+  @Override
+  public boolean isSecureShimImpl() {
+    return true;
+  }
+  
+  @Override
+  public String getShortUserName(UserGroupInformation ugi) {
+    return ugi.getShortUserName();
+  }
+
+  @Override
+  public String getTokenStrForm(String tokenSignature) throws IOException {
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    TokenSelector<? extends TokenIdentifier> tokenSelector = new DelegationTokenSelector();
+
+    Token<? extends TokenIdentifier> token = tokenSelector.selectToken(
+        tokenSignature == null ? new Text() : new Text(tokenSignature), ugi.getTokens());
+    return token != null ? token.encodeToUrlString() : null;
+  }
+  
+  @Override
+  public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction<Void> pvea) throws IOException, InterruptedException {
+    ugi.doAs(pvea);
+  }
+
+  @Override
+  public UserGroupInformation createRemoteUser(String userName, List<String> groupNames) {
+    return UserGroupInformation.createRemoteUser(userName);
+  }
 
   @Override
   public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception {
@@ -42,7 +535,7 @@ public class Hadoop20SShims extends Hado
       throw new Exception(errorMsg);
     }
   }
-
+  
   @Override
   public org.apache.hadoop.mapreduce.TaskAttemptContext newTaskAttemptContext(Configuration conf, final Progressable progressable) {
     return new org.apache.hadoop.mapreduce.TaskAttemptContext(conf, new TaskAttemptID()) {

Modified: hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1229584&r1=1229583&r2=1229584&view=diff
==============================================================================
--- hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Tue Jan 10 15:08:45 2012
@@ -17,22 +17,505 @@
  */
 package org.apache.hadoop.hive.shims;
 
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.security.auth.login.LoginException;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain;
+import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
 import org.apache.hadoop.hive.shims.HadoopShims.JobTrackerState;
-import org.apache.hadoop.hive.shims.HadoopShimsSecure;
+import org.apache.hadoop.hive.thrift.DelegationTokenSelector23;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.OutputCommitter;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+import org.apache.hadoop.mapred.TaskCompletionEvent;
+import org.apache.hadoop.mapred.TaskID;
+import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
+import org.apache.hadoop.mapred.lib.CombineFileSplit;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.apache.hadoop.tools.HadoopArchives;
 import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * Implemention of shims against Hadoop 0.23.0.
  */
-public class Hadoop23Shims extends HadoopShimsSecure {
+public class Hadoop23Shims implements HadoopShims {
+  public boolean usesJobShell() {
+    return false;
+  }
+
+  public boolean fileSystemDeleteOnExit(FileSystem fs, Path path)
+      throws IOException {
+
+    return fs.deleteOnExit(path);
+  }
+
+  public void inputFormatValidateInput(InputFormat fmt, JobConf conf)
+      throws IOException {
+    // gone in 0.18+
+  }
+
+  public boolean isJobPreparing(RunningJob job) throws IOException {
+    return job.getJobState() == JobStatus.PREP;
+  }
+  /**
+   * Workaround for hadoop-17 - jobclient only looks at commandlineconfig.
+   */
+  public void setTmpFiles(String prop, String files) {
+    // gone in 20+
+  }
+
+  public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
+      int numDataNodes,
+      boolean format,
+      String[] racks) throws IOException {
+    return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
+  }
+
+  /**
+   * MiniDFSShim.
+   *
+   */
+  public class MiniDFSShim implements HadoopShims.MiniDFSShim {
+    private final MiniDFSCluster cluster;
+
+    public MiniDFSShim(MiniDFSCluster cluster) {
+      this.cluster = cluster;
+    }
+
+    public FileSystem getFileSystem() throws IOException {
+      return cluster.getFileSystem();
+    }
+
+    public void shutdown() {
+      cluster.shutdown();
+    }
+  }
+
+  /**
+   * We define this function here to make the code compatible between
+   * hadoop 0.17 and hadoop 0.20.
+   *
+   * Hive binary that compiled Text.compareTo(Text) with hadoop 0.20 won't
+   * work with hadoop 0.17 because in hadoop 0.20, Text.compareTo(Text) is
+   * implemented in org.apache.hadoop.io.BinaryComparable, and Java compiler
+   * references that class, which is not available in hadoop 0.17.
+   */
+  public int compareText(Text a, Text b) {
+    return a.compareTo(b);
+  }
+
+  @Override
+  public long getAccessTime(FileStatus file) {
+    return file.getAccessTime();
+  }
+
+  public HadoopShims.CombineFileInputFormatShim getCombineFileInputFormat() {
+    return new CombineFileInputFormatShim() {
+      @Override
+      public RecordReader getRecordReader(InputSplit split,
+          JobConf job, Reporter reporter) throws IOException {
+        throw new IOException("CombineFileInputFormat.getRecordReader not needed.");
+      }
+    };
+  }
+
+  public static class InputSplitShim extends CombineFileSplit implements HadoopShims.InputSplitShim {
+    long shrinkedLength;
+    boolean _isShrinked;
+    public InputSplitShim() {
+      super();
+      _isShrinked = false;
+    }
+
+    public InputSplitShim(CombineFileSplit old) throws IOException {
+      super(old);
+      _isShrinked = false;
+    }
+
+    @Override
+    public void shrinkSplit(long length) {
+      _isShrinked = true;
+      shrinkedLength = length;
+    }
+
+    public boolean isShrinked() {
+      return _isShrinked;
+    }
+
+    public long getShrinkedLength() {
+      return shrinkedLength;
+    }
+
+    @Override
+    public void readFields(DataInput in) throws IOException {
+      super.readFields(in);
+      _isShrinked = in.readBoolean();
+      if (_isShrinked) {
+        shrinkedLength = in.readLong();
+      }
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+      super.write(out);
+      out.writeBoolean(_isShrinked);
+      if (_isShrinked) {
+        out.writeLong(shrinkedLength);
+      }
+    }
+  }
+
+  /* This class should be replaced with org.apache.hadoop.mapred.lib.CombineFileRecordReader class, once
+   * https://issues.apache.org/jira/browse/MAPREDUCE-955 is fixed. This code should be removed - it is a copy
+   * of org.apache.hadoop.mapred.lib.CombineFileRecordReader
+   */
+  public static class CombineFileRecordReader<K, V> implements RecordReader<K, V> {
+
+    static final Class[] constructorSignature = new Class[] {
+        InputSplit.class,
+        Configuration.class,
+        Reporter.class,
+        Integer.class
+        };
+
+    protected CombineFileSplit split;
+    protected JobConf jc;
+    protected Reporter reporter;
+    protected Class<RecordReader<K, V>> rrClass;
+    protected Constructor<RecordReader<K, V>> rrConstructor;
+    protected FileSystem fs;
+
+    protected int idx;
+    protected long progress;
+    protected RecordReader<K, V> curReader;
+    protected boolean isShrinked;
+    protected long shrinkedLength;
+    
+    public boolean next(K key, V value) throws IOException {
+
+      while ((curReader == null)
+          || !doNextWithExceptionHandler((K) ((CombineHiveKey) key).getKey(),
+              value)) {
+        if (!initNextRecordReader(key)) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    public K createKey() {
+      K newKey = curReader.createKey();
+      return (K)(new CombineHiveKey(newKey));
+    }
+
+    public V createValue() {
+      return curReader.createValue();
+    }
+
+    /**
+     * Return the amount of data processed.
+     */
+    public long getPos() throws IOException {
+      return progress;
+    }
+
+    public void close() throws IOException {
+      if (curReader != null) {
+        curReader.close();
+        curReader = null;
+      }
+    }
+
+    /**
+     * Return progress based on the amount of data processed so far.
+     */
+    public float getProgress() throws IOException {
+      return Math.min(1.0f, progress / (float) (split.getLength()));
+    }
+
+    /**
+     * A generic RecordReader that can hand out different recordReaders
+     * for each chunk in the CombineFileSplit.
+     */
+    public CombineFileRecordReader(JobConf job, CombineFileSplit split,
+        Reporter reporter,
+        Class<RecordReader<K, V>> rrClass)
+        throws IOException {
+      this.split = split;
+      this.jc = job;
+      this.rrClass = rrClass;
+      this.reporter = reporter;
+      this.idx = 0;
+      this.curReader = null;
+      this.progress = 0;
+
+      isShrinked = false;
+
+      assert (split instanceof InputSplitShim);
+      if (((InputSplitShim) split).isShrinked()) {
+        isShrinked = true;
+        shrinkedLength = ((InputSplitShim) split).getShrinkedLength();
+      }
+
+      try {
+        rrConstructor = rrClass.getDeclaredConstructor(constructorSignature);
+        rrConstructor.setAccessible(true);
+      } catch (Exception e) {
+        throw new RuntimeException(rrClass.getName() +
+            " does not have valid constructor", e);
+      }
+      initNextRecordReader(null);
+    }
+    
+    /**
+     * do next and handle exception inside it. 
+     * @param key
+     * @param value
+     * @return
+     * @throws IOException
+     */
+    private boolean doNextWithExceptionHandler(K key, V value) throws IOException {
+      try {
+        return curReader.next(key, value);
+      } catch (Exception e) {
+        return HiveIOExceptionHandlerUtil.handleRecordReaderNextException(e, jc);
+      }
+    }
+
+    /**
+     * Get the record reader for the next chunk in this CombineFileSplit.
+     */
+    protected boolean initNextRecordReader(K key) throws IOException {
+
+      if (curReader != null) {
+        curReader.close();
+        curReader = null;
+        if (idx > 0) {
+          progress += split.getLength(idx - 1); // done processing so far
+        }
+      }
+
+      // if all chunks have been processed or reached the length, nothing more to do.
+      if (idx == split.getNumPaths() || (isShrinked && progress > shrinkedLength)) {
+        return false;
+      }
+
+      // get a record reader for the idx-th chunk
+      try {
+        curReader = rrConstructor.newInstance(new Object[]
+            {split, jc, reporter, Integer.valueOf(idx)});
+
+        // change the key if need be
+        if (key != null) {
+          K newKey = curReader.createKey();
+          ((CombineHiveKey)key).setKey(newKey);
+        }
+
+        // setup some helper config variables.
+        jc.set("map.input.file", split.getPath(idx).toString());
+        jc.setLong("map.input.start", split.getOffset(idx));
+        jc.setLong("map.input.length", split.getLength(idx));
+      } catch (Exception e) {
+        curReader=HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(e, jc);
+      }
+      idx++;
+      return true;
+    }
+  }
+
+  public abstract static class CombineFileInputFormatShim<K, V> extends
+      CombineFileInputFormat<K, V>
+      implements HadoopShims.CombineFileInputFormatShim<K, V> {
+
+    public Path[] getInputPathsShim(JobConf conf) {
+      try {
+        return FileInputFormat.getInputPaths(conf);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    @Override
+    public void createPool(JobConf conf, PathFilter... filters) {
+      super.createPool(conf, filters);
+    }
+
+    @Override
+    public InputSplitShim[] getSplits(JobConf job, int numSplits) throws IOException {
+      long minSize = job.getLong("mapred.min.split.size", 0);
+
+      // For backward compatibility, let the above parameter be used
+      if (job.getLong("mapred.min.split.size.per.node", 0) == 0) {
+        super.setMinSplitSizeNode(minSize);
+      }
+
+      if (job.getLong("mapred.min.split.size.per.rack", 0) == 0) {
+        super.setMinSplitSizeRack(minSize);
+      }
+
+      if (job.getLong("mapred.max.split.size", 0) == 0) {
+        super.setMaxSplitSize(minSize);
+      }
+
+      InputSplit[] splits = super.getSplits(job, numSplits);
+
+      InputSplitShim[] isplits = new InputSplitShim[splits.length];
+      for (int pos = 0; pos < splits.length; pos++) {
+        isplits[pos] = new InputSplitShim((CombineFileSplit) splits[pos]);
+      }
+
+      return isplits;
+    }
+
+    public InputSplitShim getInputSplitShim() throws IOException {
+      return new InputSplitShim();
+    }
+
+    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
+        Reporter reporter,
+        Class<RecordReader<K, V>> rrClass)
+        throws IOException {
+      CombineFileSplit cfSplit = (CombineFileSplit) split;
+      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
+    }
+
+  }
+
+  public String getInputFormatClassName() {
+    return "org.apache.hadoop.hive.ql.io.CombineHiveInputFormat";
+  }
+
+  String[] ret = new String[2];
+
+  @Override
+  public String[] getTaskJobIDs(TaskCompletionEvent t) {
+    TaskID tid = t.getTaskAttemptId().getTaskID();
+    ret[0] = tid.toString();
+    ret[1] = tid.getJobID().toString();
+    return ret;
+  }
+
+  public void setFloatConf(Configuration conf, String varName, float val) {
+    conf.setFloat(varName, val);
+  }
 
   @Override
+  public int createHadoopArchive(Configuration conf, Path sourceDir, Path destDir,
+      String archiveName) throws Exception {
+
+    HadoopArchives har = new HadoopArchives(conf);
+    List<String> args = new ArrayList<String>();
+
+    if (conf.get("hive.archive.har.parentdir.settable") == null) {
+      throw new RuntimeException("hive.archive.har.parentdir.settable is not set");
+    }
+    boolean parentSettable =
+      conf.getBoolean("hive.archive.har.parentdir.settable", false);
+
+    if (parentSettable) {
+      args.add("-archiveName");
+      args.add(archiveName);
+      args.add("-p");
+      args.add(sourceDir.toString());
+      args.add(destDir.toString());
+    } else {
+      args.add("-archiveName");
+      args.add(archiveName);
+      args.add(sourceDir.toString());
+      args.add(destDir.toString());
+    }
+
+    return ToolRunner.run(har, args.toArray(new String[0]));
+  }
+
+  public static class NullOutputCommitter extends OutputCommitter {
+    @Override
+    public void setupJob(JobContext jobContext) { }
+    @Override
+    public void cleanupJob(JobContext jobContext) { }
+
+    @Override
+    public void setupTask(TaskAttemptContext taskContext) { }
+    @Override
+    public boolean needsTaskCommit(TaskAttemptContext taskContext) {
+      return false;
+    }
+    @Override
+    public void commitTask(TaskAttemptContext taskContext) { }
+    @Override
+    public void abortTask(TaskAttemptContext taskContext) { }
+  }
+
+  public void setNullOutputFormat(JobConf conf) {
+    conf.setOutputFormat(NullOutputFormat.class);
+    conf.setOutputCommitter(Hadoop23Shims.NullOutputCommitter.class);
+
+    // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463)
+    // but can be backported. So we disable setup/cleanup in all versions >= 0.19
+    conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
+
+    // option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206)
+    // but can be backported. So we disable setup/cleanup in all versions >= 0.19
+    conf.setBoolean("mapreduce.job.committer.task.cleanup.needed", false);
+  }
+
+  @Override
+  public UserGroupInformation getUGIForConf(Configuration conf) throws IOException {
+    return UserGroupInformation.getCurrentUser();
+  }
+
+  @Override
+  public boolean isSecureShimImpl() {
+    return true;
+  }
+
+  @Override
+  public String getShortUserName(UserGroupInformation ugi) {
+    return ugi.getShortUserName();
+  }
+
+  @Override
+  public String getTokenStrForm(String tokenSignature) throws IOException {
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    TokenSelector<? extends TokenIdentifier> tokenSelector = new DelegationTokenSelector23();
+
+    Token<? extends TokenIdentifier> token = tokenSelector.selectToken(
+        tokenSignature == null ? new Text() : new Text(tokenSignature), ugi.getTokens());
+    return token != null ? token.encodeToUrlString() : null;
+  }
+  
+  @Override
   public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception {
     JobTrackerState state;
     switch (clusterStatus.getJobTrackerStatus()) {
@@ -45,7 +528,7 @@ public class Hadoop23Shims extends Hadoo
       throw new Exception(errorMsg);
     }
   }
-
+  
   @Override
   public org.apache.hadoop.mapreduce.TaskAttemptContext newTaskAttemptContext(Configuration conf, final Progressable progressable) {
     return new TaskAttemptContextImpl(conf, new TaskAttemptID()) {

Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java?rev=1229584&r1=1229583&r2=1229584&view=diff
==============================================================================
--- hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java (original)
+++ hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java Tue Jan 10 15:08:45 2012
@@ -79,7 +79,7 @@ public abstract class ShimLoader {
   }
 
   public static synchronized HadoopThriftAuthBridge getHadoopThriftAuthBridge() {
-      if (getHadoopShims().isSecureShimImpl()) {
+        if ("0.20S".equals(getMajorVersion())) {
           return createShim("org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge20S",
                             HadoopThriftAuthBridge.class);
         } else {
@@ -87,6 +87,7 @@ public abstract class ShimLoader {
         }
       }
 
+  @SuppressWarnings("unchecked")
   private static <T> T loadShims(Map<String, String> classMap, Class<T> xface) {
     String vers = getMajorVersion();
     String className = classMap.get(vers);
@@ -95,7 +96,7 @@ public abstract class ShimLoader {
 
     private static <T> T createShim(String className, Class<T> xface) {
     try {
-      Class<?> clazz = Class.forName(className);
+      Class clazz = Class.forName(className);
       return xface.cast(clazz.newInstance());
     } catch (Exception e) {
       throw new RuntimeException("Could not load shims in class " +



Mime
View raw message