hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject svn commit: r1651972 - in /hive/trunk: ./ hbase-handler/ itests/ itests/qtest-spark/ itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/exec/spark/ ql/src/java/org/apache/hadoop/hive/ql/optimize...
Date Thu, 15 Jan 2015 08:45:17 GMT
Author: szehon
Date: Thu Jan 15 08:45:16 2015
New Revision: 1651972

URL: http://svn.apache.org/r1651972
Log:
HIVE-9352 : Merge from spark to trunk (follow-up of HIVE-9257) (Szehon, reviewed by Brock)

Removed:
    hive/trunk/ql/src/test/queries/clientpositive/spark_test.q
    hive/trunk/ql/src/test/results/clientpositive/spark/spark_test.q.out
    hive/trunk/ql/src/test/results/clientpositive/spark_test.q.out
Modified:
    hive/trunk/   (props changed)
    hive/trunk/hbase-handler/pom.xml   (props changed)
    hive/trunk/itests/pom.xml
    hive/trunk/itests/qtest-spark/pom.xml
    hive/trunk/itests/src/test/resources/testconfiguration.properties
    hive/trunk/pom.xml
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/KryoSerializer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java
    hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    hive/trunk/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java

Propchange: hive/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 15 08:45:16 2015
@@ -1,5 +1,5 @@
 /hive/branches/branch-0.11:1480385,1480458,1481120,1481344,1481346,1481348,1481352,1483872,1505184
 /hive/branches/cbo:1605012-1627125
-/hive/branches/spark:1608589-1650662
+/hive/branches/spark:1608589-1651242
 /hive/branches/tez:1494760-1622766
 /hive/branches/vectorization:1466908-1527856

Propchange: hive/trunk/hbase-handler/pom.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jan 15 08:45:16 2015
@@ -1,6 +1,6 @@
 /hive/branches/branch-0.11/hbase-handler/pom.xml:1480385,1480458,1481120,1481344,1481346,1481348,1481352,1483872,1505184
 /hive/branches/cbo/hbase-handler/pom.xml:1605012-1627125
-/hive/branches/spark/hbase-handler/pom.xml:1608589-1650662
+/hive/branches/spark/hbase-handler/pom.xml:1608589-1651242
 /hive/branches/tez/hbase-handler/pom.xml:1494760-1622766
 /hive/branches/vectorization/hbase-handler/pom.xml:1466908-1527856
 /hive/trunk/hbase-handler/pom.xml:1494760-1537575

Modified: hive/trunk/itests/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/pom.xml?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/itests/pom.xml (original)
+++ hive/trunk/itests/pom.xml Thu Jan 15 08:45:16 2015
@@ -90,7 +90,7 @@
                     mv $BASE_DIR/${finalName}* $BASE_DIR/$finalName
                   }
                   mkdir -p $DOWNLOAD_DIR
-                  download "http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark-1.2.1-SNAPSHOT-bin-hadoop2-without-hive.tgz"
"spark"
+                  download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz"
"spark"
                   cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/
                 </echo>
               </target>

Modified: hive/trunk/itests/qtest-spark/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/itests/qtest-spark/pom.xml?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/itests/qtest-spark/pom.xml (original)
+++ hive/trunk/itests/qtest-spark/pom.xml Thu Jan 15 08:45:16 2015
@@ -54,38 +54,6 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.apache.tez</groupId>
-      <artifactId>tez-api</artifactId>
-      <version>${tez.version}</version>
-      <optional>true</optional>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-mapreduce-client-core</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-mapreduce-client-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-hdfs</artifactId>
-        </exclusion>
-       <exclusion>
-         <groupId>org.apache.hadoop</groupId>
-         <artifactId>hadoop-yarn-client</artifactId>
-       </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-util</artifactId>
       <version>${spark.jetty.version}</version>

Modified: hive/trunk/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/trunk/itests/src/test/resources/testconfiguration.properties?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/trunk/itests/src/test/resources/testconfiguration.properties Thu Jan 15 08:45:16
2015
@@ -842,7 +842,6 @@ spark.query.files=add_part_multiple.q, \
   smb_mapjoin_8.q, \
   smb_mapjoin_9.q, \
   sort.q, \
-  spark_test.q, \
   stats0.q, \
   stats1.q, \
   stats10.q, \

Modified: hive/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/pom.xml?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/pom.xml (original)
+++ hive/trunk/pom.xml Thu Jan 15 08:45:16 2015
@@ -154,7 +154,7 @@
     <ST4.version>4.0.4</ST4.version>
     <tez.version>0.5.2</tez.version>
     <super-csv.version>2.2.0</super-csv.version>
-    <spark.version>1.2.1-SNAPSHOT</spark.version>
+    <spark.version>1.2.0</spark.version>
     <scala.binary.version>2.10</scala.binary.version>
     <scala.version>2.10.4</scala.version>
     <tempus-fugit.version>1.1</tempus-fugit.version>
@@ -212,17 +212,7 @@
        <snapshots>
          <enabled>false</enabled>
        </snapshots>
-     </repository>
-     <repository>
-       <id>spark-snapshot</id>
-       <url>http://ec2-50-18-79-139.us-west-1.compute.amazonaws.com/data/spark_2.10-1.2-SNAPSHOT/</url>
-       <releases>
-         <enabled>false</enabled>
-       </releases>
-       <snapshots>
-         <enabled>true</enabled>
-       </snapshots>
-     </repository>
+    </repository>
   </repositories>
 
   <!-- Hadoop dependency management is done at the bottom under profiles -->
@@ -833,6 +823,7 @@
             <LANG>en_US.UTF-8</LANG>
             <HADOOP_CLASSPATH>${test.tmp.dir}/conf:${basedir}/${hive.path.to.root}/conf</HADOOP_CLASSPATH>
             <HIVE_HADOOP_TEST_CLASSPATH>${test.hive.hadoop.classpath}</HIVE_HADOOP_TEST_CLASSPATH>
+            <SPARK_SUBMIT_CLASSPATH>${spark.home}/lib/spark-assembly-${spark.version}-hadoop2.4.0.jar:${test.hive.hadoop.classpath}</SPARK_SUBMIT_CLASSPATH>
             <PATH>${env.PATH}${test.extra.path}</PATH>
           </environmentVariables>
           <systemPropertyVariables>

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java Thu Jan 15 08:45:16
2015
@@ -32,13 +32,13 @@ import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.LinkedHashMap;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -48,7 +48,12 @@ import org.apache.hadoop.hive.ql.hooks.R
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
-import org.apache.hadoop.hive.ql.plan.*;
+import org.apache.hadoop.hive.ql.plan.Explain;
+import org.apache.hadoop.hive.ql.plan.ExplainWork;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.SparkWork;
+import org.apache.hadoop.hive.ql.plan.TezWork;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
 import org.apache.hadoop.hive.ql.security.authorization.AuthorizationFactory;
 import org.apache.hadoop.hive.ql.session.SessionState;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Thu Jan
15 08:45:16 2015
@@ -96,8 +96,8 @@ public class HashTableSinkOperator exten
   protected transient MapJoinPersistableTableContainer[] mapJoinTables;
   protected transient MapJoinTableContainerSerDe[] mapJoinTableSerdes;
 
-  private final Object[] EMPTY_OBJECT_ARRAY = new Object[0];
-  private final MapJoinEagerRowContainer EMPTY_ROW_CONTAINER = new MapJoinEagerRowContainer();
+  private final Object[] emptyObjectArray = new Object[0];
+  private final MapJoinEagerRowContainer emptyRowContainer = new MapJoinEagerRowContainer();
 
   private long rowNumber = 0;
   protected transient LogHelper console;
@@ -118,7 +118,7 @@ public class HashTableSinkOperator exten
     boolean isSilent = HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVESESSIONSILENT);
     console = new LogHelper(LOG, isSilent);
     memoryExhaustionHandler = new MapJoinMemoryExhaustionHandler(console, conf.getHashtableMemoryUsage());
-    EMPTY_ROW_CONTAINER.addRow(EMPTY_OBJECT_ARRAY);
+    emptyRowContainer.addRow(emptyObjectArray);
 
     // for small tables only; so get the big table position first
     posBigTableAlias = conf.getPosBigTable();
@@ -229,7 +229,7 @@ public class HashTableSinkOperator exten
     MapJoinKeyObject key = new MapJoinKeyObject();
     key.readFromRow(currentKey, joinKeysObjectInspectors[alias]);
 
-    Object[] value = EMPTY_OBJECT_ARRAY;
+    Object[] value = emptyObjectArray;
     if((hasFilter(alias) && filterMaps[alias].length > 0) || joinValues[alias].size()
> 0) {
       value = JoinUtil.computeMapJoinValues(row, joinValues[alias],
         joinValuesObjectInspectors[alias], joinFilters[alias], joinFilterObjectInspectors[alias],
@@ -242,14 +242,14 @@ public class HashTableSinkOperator exten
         rowContainer = new MapJoinEagerRowContainer();
         rowContainer.addRow(value);
       } else {
-        rowContainer = EMPTY_ROW_CONTAINER;
+        rowContainer = emptyRowContainer;
       }
       rowNumber++;
       if (rowNumber > hashTableScale && rowNumber % hashTableScale == 0) {
         memoryExhaustionHandler.checkMemoryStatus(tableContainer.size(), rowNumber);
       }
       tableContainer.put(key, rowContainer);
-    } else if (rowContainer == EMPTY_ROW_CONTAINER) {
+    } else if (rowContainer == emptyRowContainer) {
       rowContainer = rowContainer.copy();
       rowContainer.addRow(value);
       tableContainer.put(key, rowContainer);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
Thu Jan 15 08:45:16 2015
@@ -26,13 +26,13 @@ import org.apache.spark.api.java.functio
 
 
 public abstract class HivePairFlatMapFunction<T, K, V> implements PairFlatMapFunction<T,
K, V> {
-  private static final NumberFormat TASK_ID_FORMAT = NumberFormat.getInstance();
-  private static final NumberFormat STAGE_ID_FORMAT = NumberFormat.getInstance();
-  static {
-    TASK_ID_FORMAT.setGroupingUsed(false);
-    TASK_ID_FORMAT.setMinimumIntegerDigits(6);
-    STAGE_ID_FORMAT.setGroupingUsed(false);
-    STAGE_ID_FORMAT.setMinimumIntegerDigits(4);
+  private final NumberFormat taskIdFormat = NumberFormat.getInstance();
+  private final NumberFormat stageIdFormat = NumberFormat.getInstance();
+  {
+    taskIdFormat.setGroupingUsed(false);
+    taskIdFormat.setMinimumIntegerDigits(6);
+    stageIdFormat.setGroupingUsed(false);
+    stageIdFormat.setMinimumIntegerDigits(4);
   }
 
   protected transient JobConf jobConf;
@@ -60,7 +60,7 @@ public abstract class HivePairFlatMapFun
     StringBuilder taskAttemptIdBuilder = new StringBuilder("attempt_");
     taskAttemptIdBuilder.append(System.currentTimeMillis())
       .append("_")
-      .append(STAGE_ID_FORMAT.format(TaskContext.get().stageId()))
+      .append(stageIdFormat.format(TaskContext.get().stageId()))
       .append("_");
 
     if (isMap()) {
@@ -71,7 +71,7 @@ public abstract class HivePairFlatMapFun
 
     // Spark task attempt id is increased by Spark context instead of task, which may introduce
     // unstable qtest output, since non Hive features depends on this, we always set it to
0 here.
-    taskAttemptIdBuilder.append(TASK_ID_FORMAT.format(TaskContext.get().partitionId()))
+    taskAttemptIdBuilder.append(taskIdFormat.format(TaskContext.get().partitionId()))
       .append("_0");
 
     String taskAttemptIdStr = taskAttemptIdBuilder.toString();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/KryoSerializer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/KryoSerializer.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/KryoSerializer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/KryoSerializer.java Thu Jan
15 08:45:16 2015
@@ -55,13 +55,13 @@ public class KryoSerializer {
     try {
       jobConf.write(new DataOutputStream(out));
     } catch (IOException e) {
-      LOG.error("Error serializing job configuration", e);
+      LOG.error("Error serializing job configuration: " + e, e);
       return null;
     } finally {
       try {
         out.close();
       } catch (IOException e) {
-        LOG.error("Error closing output stream", e);
+        LOG.error("Error closing output stream: " + e, e);
       }
     }
 
@@ -74,8 +74,8 @@ public class KryoSerializer {
     try {
       conf.readFields(new DataInputStream(new ByteArrayInputStream(buffer)));
     } catch (IOException e) {
-      LOG.error("Error de-serializing job configuration");
-      return null;
+      String msg = "Error de-serializing job configuration: " + e;
+      throw new IllegalStateException(msg, e);
     }
     return conf;
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
Thu Jan 15 08:45:16 2015
@@ -39,7 +39,6 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.StringUtils;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -57,9 +56,9 @@ import java.util.List;
  *
  */
 public class SparkMapRecordHandler extends SparkRecordHandler {
+  private static final Log LOG = LogFactory.getLog(SparkMapRecordHandler.class);
   private static final String PLAN_KEY = "__MAP_PLAN__";
   private MapOperator mo;
-  public static final Log LOG = LogFactory.getLog(SparkMapRecordHandler.class);
   private MapredLocalWork localWork = null;
   private boolean isLogInfoEnabled = false;
   private ExecMapperContext execContext;
@@ -125,7 +124,7 @@ public class SparkMapRecordHandler exten
         // Don't create a new object if we are already out of memory
         throw (OutOfMemoryError) e;
       } else {
-        throw new RuntimeException("Map operator initialization failed", e);
+        throw new RuntimeException("Map operator initialization failed: " + e, e);
       }
     }
     perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_INIT_OPERATORS);
@@ -149,8 +148,9 @@ public class SparkMapRecordHandler exten
         // Don't create a new object if we are already out of memory
         throw (OutOfMemoryError) e;
       } else {
-        LOG.fatal(StringUtils.stringifyException(e));
-        throw new RuntimeException(e);
+        String msg = "Error processing row: " + e;
+        LOG.fatal(msg, e);
+        throw new RuntimeException(msg, e);
       }
     }
   }
@@ -196,8 +196,9 @@ public class SparkMapRecordHandler exten
     } catch (Exception e) {
       if (!abort) {
         // signal new failure to map-reduce
-        LOG.error("Hit error while closing operators - failing tree");
-        throw new IllegalStateException("Error while closing operators", e);
+        String msg = "Hit error while closing operators - failing tree: " + e;
+        LOG.error(msg, e);
+        throw new IllegalStateException(msg, e);
       }
     } finally {
       MapredContext.close();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenSparkSkewJoinProcessor.java
Thu Jan 15 08:45:16 2015
@@ -322,10 +322,6 @@ public class GenSparkSkewJoinProcessor {
         }
       }
 
-      // keep it as reference in case we need fetch work
-//        localPlan.getAliasToFetchWork().put(small_alias.toString(),
-//            new FetchWork(tblDir, tableDescList.get(small_alias)));
-
       listWorks.add(skewJoinMapJoinTask.getWork());
       listTasks.add(skewJoinMapJoinTask);
     }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWorkWalker.java Thu
Jan 15 08:45:16 2015
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.ql.plan.Op
 
 /**
  * Walks the operator tree in DFS fashion.
- * 
+ *
  * Cloned from GenTezWorkWarlker.
  */
 public class GenSparkWorkWalker extends DefaultGraphWalker {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java Thu Jan
15 08:45:16 2015
@@ -98,11 +98,6 @@ public class SparkCompiler extends TaskC
   }
 
   @Override
-  public void init(HiveConf conf, LogHelper console, Hive db) {
-    super.init(conf, console, db);
-  }
-
-  @Override
   protected void optimizeOperatorPlan(ParseContext pCtx, Set<ReadEntity> inputs,
       Set<WriteEntity> outputs) throws SemanticException {
     PERF_LOGGER.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/SparkEdgeProperty.java Thu Jan 15
08:45:16 2015
@@ -42,7 +42,7 @@ public class SparkEdgeProperty {
   public boolean isShuffleNone() {
     return edgeType == SHUFFLE_NONE;
   }
-  
+
   public void setShuffleNone() {
     edgeType = SHUFFLE_NONE;
   }
@@ -80,7 +80,7 @@ public class SparkEdgeProperty {
     if (isShuffleNone()) {
       return "NONE";
     }
-    
+
     StringBuilder sb = new StringBuilder();
     if (isShuffleGroup()) {
       sb.append("GROUP");

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/CounterStatsPublisher.java Thu
Jan 15 08:45:16 2015
@@ -53,7 +53,8 @@ public class CounterStatsPublisher imple
       try {
         reporter.incrCounter(fileID, entry.getKey(), Long.valueOf(entry.getValue()));
       } catch (Exception e) {
-        LOG.error("Failed to increment counter value " + entry.getValue() + " for " + entry.getKey());
+        LOG.error("Failed to increment counter value " + entry.getValue() + " for " + entry.getKey()
+          + ": " + e, e);
         return false;
       }
     }

Modified: hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Thu
Jan 15 08:45:16 2015
@@ -71,7 +71,6 @@ import org.apache.hadoop.mapreduce.TaskI
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-import org.apache.hadoop.mapreduce.util.ResourceBundles;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.authentication.util.KerberosName;
 import org.apache.hadoop.security.UserGroupInformation;

Modified: hive/trunk/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java?rev=1651972&r1=1651971&r2=1651972&view=diff
==============================================================================
--- hive/trunk/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
(original)
+++ hive/trunk/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
Thu Jan 15 08:45:16 2015
@@ -49,6 +49,7 @@ import org.slf4j.LoggerFactory;
 
 import com.google.common.base.Charsets;
 import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Lists;
@@ -63,6 +64,7 @@ class SparkClientImpl implements SparkCl
 
   private static final String DRIVER_OPTS_KEY = "spark.driver.extraJavaOptions";
   private static final String EXECUTOR_OPTS_KEY = "spark.executor.extraJavaOptions";
+  private static final String DRIVER_EXTRA_CLASSPATH = "spark.driver.extraClassPath";
 
   private final Map<String, String> conf;
   private final AtomicInteger childIdGenerator;
@@ -231,6 +233,17 @@ class SparkClientImpl implements SparkCl
       allProps.put(DRIVER_OPTS_KEY, driverJavaOpts);
       allProps.put(EXECUTOR_OPTS_KEY, executorJavaOpts);
 
+      String hiveHadoopTestClasspath = Strings.nullToEmpty(System.getenv("HIVE_HADOOP_TEST_CLASSPATH"));
+      if (!hiveHadoopTestClasspath.isEmpty()) {
+        String extraClasspath = Strings.nullToEmpty((String)allProps.get(DRIVER_EXTRA_CLASSPATH));
+        if (extraClasspath.isEmpty()) {
+          allProps.put(DRIVER_EXTRA_CLASSPATH, hiveHadoopTestClasspath);
+        } else {
+          extraClasspath = extraClasspath.endsWith(File.pathSeparator) ? extraClasspath :
extraClasspath + File.pathSeparator;
+          allProps.put(DRIVER_EXTRA_CLASSPATH, extraClasspath + hiveHadoopTestClasspath);
+        }
+      }
+
       Writer writer = new OutputStreamWriter(new FileOutputStream(properties), Charsets.UTF_8);
       try {
         allProps.store(writer, "Spark Context configuration");



Mime
View raw message