hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject svn commit: r1650429 - in /hive/branches/spark: ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/ ql/src/java/org/apache/hadoop/hive/ql/io/ ql/src/java/org/apache/...
Date Fri, 09 Jan 2015 01:53:14 GMT
Author: szehon
Date: Fri Jan  9 01:53:13 2015
New Revision: 1650429

URL: http://svn.apache.org/r1650429
Log:
HIVE-9319 : Cleanup Modified Files [Spark Branch] (Szehon, reviewed by Xuefu)

Modified:
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Fri Jan  9 01:53:13
2015
@@ -1342,9 +1342,9 @@ public class Driver implements CommandPr
         }
       }
 
-      int jobs = Utilities.getMRTasks(plan.getRootTasks()).size() +
-                 Utilities.getTezTasks(plan.getRootTasks()).size() +
-                 Utilities.getSparkTasks(plan.getRootTasks()).size();
+      int jobs = Utilities.getMRTasks(plan.getRootTasks()).size()
+        + Utilities.getTezTasks(plan.getRootTasks()).size()
+        + Utilities.getSparkTasks(plan.getRootTasks()).size();
       if (jobs > 0) {
         console.printInfo("Query ID = " + plan.getQueryId());
         console.printInfo("Total jobs = " + jobs);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java Fri Jan
 9 01:53:13 2015
@@ -424,12 +424,12 @@ public class ExplainTask extends Task<Ex
               json.accumulate(ent.getKey().toString(), jsonDep);
             }
           }
-        } else if (ent.getValue() != null && !((List<?>)ent.getValue()).isEmpty()
-            && ((List<?>)ent.getValue()).get(0) != null &&
-            ((List<?>)ent.getValue()).get(0) instanceof SparkWork.Dependency) {
+        } else if (ent.getValue() != null && !((List<?>) ent.getValue()).isEmpty()
+            && ((List<?>) ent.getValue()).get(0) != null &&
+            ((List<?>) ent.getValue()).get(0) instanceof SparkWork.Dependency) {
           if (out != null) {
             boolean isFirst = true;
-            for (SparkWork.Dependency dep: (List<SparkWork.Dependency>)ent.getValue())
{
+            for (SparkWork.Dependency dep: (List<SparkWork.Dependency>) ent.getValue())
{
               if (!isFirst) {
                 out.print(", ");
               } else {
@@ -446,7 +446,7 @@ public class ExplainTask extends Task<Ex
             out.println();
           }
           if (jsonOutput) {
-            for (SparkWork.Dependency dep: (List<SparkWork.Dependency>)ent.getValue())
{
+            for (SparkWork.Dependency dep: (List<SparkWork.Dependency>) ent.getValue())
{
               JSONObject jsonDep = new JSONObject();
               jsonDep.put("parent", dep.getName());
               jsonDep.put("type", dep.getShuffleType());

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Fri Jan
 9 01:53:13 2015
@@ -363,8 +363,8 @@ public final class Utilities {
       path = getPlanPath(conf, name);
       LOG.info("PLAN PATH = " + path);
       assert path != null;
-      if (!gWorkMap.containsKey(path) ||
-          HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark"))
{
+      if (!gWorkMap.containsKey(path)
+        || HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark"))
{
         Path localPath;
         if (conf.getBoolean("mapreduce.task.uberized", false) && name.equals(REDUCE_PLAN_NAME))
{
           localPath = new Path(name);
@@ -2673,7 +2673,8 @@ public final class Utilities {
     return sparkTasks;
   }
 
-  private static void getSparkTasks(List<Task<? extends Serializable>> tasks,
List<SparkTask> sparkTasks) {
+  private static void getSparkTasks(List<Task<? extends Serializable>> tasks,
+    List<SparkTask> sparkTasks) {
     for (Task<? extends Serializable> task : tasks) {
       if (task instanceof SparkTask && !sparkTasks.contains(task)) {
         sparkTasks.add((SparkTask) task);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
Fri Jan  9 01:53:13 2015
@@ -154,7 +154,7 @@ public class MapJoinTableContainerSerDe
       return tableContainer;
     } catch (IOException e) {
       throw new HiveException("IO error while trying to create table container", e);
-    } catch(Exception e) {
+    } catch (Exception e) {
       throw new HiveException("Error while trying to create table container", e);
     }
   }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveKey.java Fri Jan  9 01:53:13
2015
@@ -24,7 +24,6 @@ import org.apache.hadoop.io.WritableComp
 /**
  * HiveKey is a simple wrapper on Text which allows us to set the hashCode
  * easily. hashCode is used for hadoop partitioner.
- * 
  */
 public class HiveKey extends BytesWritable {
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
Fri Jan  9 01:53:13 2015
@@ -861,10 +861,10 @@ public final class GenMapRedUtils {
         }
       }
     } else if (task instanceof SparkTask) {
-      SparkWork work = (SparkWork)task.getWork();
+      SparkWork work = (SparkWork) task.getWork();
       for (BaseWork w : work.getAllWorkUnsorted()) {
         if (w instanceof MapWork) {
-          ((MapWork)w).deriveExplainAttributes();
+          ((MapWork) w).deriveExplainAttributes();
         }
       }
     }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java Fri
Jan  9 01:53:13 2015
@@ -102,10 +102,12 @@ public class Optimizer {
     }
     transformations.add(new SamplePruner());
 
-    MapJoinProcessor mapJoinProcessor = isSparkExecEngine ? new SparkMapJoinProcessor() :
new MapJoinProcessor();
+    MapJoinProcessor mapJoinProcessor = isSparkExecEngine ? new SparkMapJoinProcessor()
+      : new MapJoinProcessor();
     transformations.add(mapJoinProcessor);
 
-    if ((HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTBUCKETMAPJOIN)) &&
!isTezExecEngine && !isSparkExecEngine) {
+    if ((HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTBUCKETMAPJOIN))
+      && !isTezExecEngine && !isSparkExecEngine) {
       transformations.add(new BucketMapJoinOptimizer());
       bucketMapJoinOptimizer = true;
     }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/unionproc/UnionProcessor.java
Fri Jan  9 01:53:13 2015
@@ -96,8 +96,8 @@ public class UnionProcessor implements T
     // Walk the tree again to see if the union can be removed completely
     HiveConf conf = pCtx.getConf();
     opRules.clear();
-    if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE) &&
-        !conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
+    if (conf.getBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_UNION_REMOVE)
+      && !conf.getVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
 
       if (!conf.getBoolVar(HiveConf.ConfVars.HIVE_HADOOP_SUPPORTS_SUBDIRECTORIES)) {
         throw new

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
Fri Jan  9 01:53:13 2015
@@ -147,9 +147,7 @@ public class MapReduceCompiler extends T
   }
 
   // loop over all the operators recursively
-  // TODO: changed from private to protected for SparkCompiler to use. It will be changed
back onece SparkCompiler
-  //       stands alone.
-  protected void breakOperatorTree(Operator<? extends OperatorDesc> topOp) {
+  private void breakOperatorTree(Operator<? extends OperatorDesc> topOp) {
     if (topOp instanceof ReduceSinkOperator) {
       topOp.setChildOperators(null);
     }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java Fri Jan 
9 01:53:13 2015
@@ -129,7 +129,7 @@ public abstract class BaseWork extends A
     // add all children
     opStack.addAll(opSet);
 
-    while(!opStack.empty()) {
+    while (!opStack.empty()) {
       Operator<?> op = opStack.pop();
       if (op.getNumChild() == 0) {
         returnSet.add(op);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
Fri Jan  9 01:53:13 2015
@@ -149,7 +149,7 @@ public class ConditionalResolverMergeFil
           work = ((MapredWork) mrTask.getWork()).getMapWork();
         } else if (mrTask.getWork() instanceof TezWork){
           work = (MapWork) ((TezWork) mrTask.getWork()).getAllWork().get(0);
-        } else if (mrTask.getWork() instanceof SparkWork){
+        } else if (mrTask.getWork() instanceof SparkWork) {
           work = (MapWork) ((SparkWork) mrTask.getWork()).getAllWork().get(0);
         } else {
           work = (MapWork) mrTask.getWork();

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Fri
Jan  9 01:53:13 2015
@@ -1271,7 +1271,7 @@ public class SessionState {
     if (sparkSession != null) {
       try {
         SparkSessionManagerImpl.getInstance().closeSession(sparkSession);
-      } catch(Exception ex) {
+      } catch (Exception ex) {
         LOG.error("Error closing spark session.", ex);
       } finally {
         sparkSession = null;

Modified: hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1650429&r1=1650428&r2=1650429&view=diff
==============================================================================
--- hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
(original)
+++ hive/branches/spark/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
Fri Jan  9 01:53:13 2015
@@ -518,14 +518,14 @@ public interface HadoopShims {
   Path getCurrentTrashPath(Configuration conf, FileSystem fs);
 
   /**
-   * Returns a shim to wrap KerberosName
+   * Check whether file is directory.
    */
-  public KerberosNameShim getKerberosNameShim(String name) throws IOException;
+  boolean isDirectory(FileStatus fileStatus);
 
   /**
-   * Check whether file is directory.
+   * Returns a shim to wrap KerberosName
    */
-  boolean isDirectory(FileStatus fileStatus);
+  public KerberosNameShim getKerberosNameShim(String name) throws IOException;
 
   /**
    * Shim for KerberosName



Mime
View raw message