hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1489279 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql: exec/ index/ io/rcfile/merge/ io/rcfile/stats/ io/rcfile/truncate/ parse/
Date Tue, 04 Jun 2013 03:14:18 GMT
Author: hashutosh
Date: Tue Jun  4 03:14:17 2013
New Revision: 1489279

URL: http://svn.apache.org/r1489279
Log:
HIVE-4585 : Remove unused MR Temp file localization from Tasks (Gunther Hagleitner via Ashutosh
Chauhan)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java Tue Jun  4
03:14:17 2013
@@ -390,17 +390,4 @@ public class ColumnStatsTask extends Tas
   public String getName() {
     return "COLUMNSTATS TASK";
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    FetchWork fWork = work.getfWork();
-    String s = fWork.getTblDir();
-    if ((s != null) && ctx.isMRTmpFileURI(s)) {
-      fWork.setTblDir(ctx.localizeMRTmpFileURI(s));
-    }
-    ArrayList<String> ls = fWork.getPartDir();
-    if (ls != null) {
-      ctx.localizePaths(ls);
-    }
-  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ConditionalTask.java Tue Jun  4
03:14:17 2013
@@ -205,15 +205,6 @@ public class ConditionalTask extends Tas
   }
 
   @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    if (getListTasks() != null) {
-      for (Task<? extends Serializable> t : getListTasks()) {
-        t.localizeMRTmpFiles(ctx);
-      }
-    }
-  }
-
-  @Override
   public List<Task<? extends Serializable>> getDependentTasks() {
     return listTasks;
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java Tue Jun  4 03:14:17
2013
@@ -106,12 +106,4 @@ public class CopyTask extends Task<CopyW
   public String getName() {
     return "COPY";
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // copy task is only used by the load command and
-    // does not use any map-reduce tmp files
-    // we don't expect to enter this code path at all
-    throw new RuntimeException ("Unexpected call");
-  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Tue Jun  4 03:14:17
2013
@@ -4059,9 +4059,4 @@ public class DDLTask extends Task<DDLWor
   public String getName() {
     return "DDL";
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // no-op
-  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DependencyCollectionTask.java Tue
Jun  4 03:14:17 2013
@@ -53,12 +53,4 @@ public class DependencyCollectionTask ex
   public String getName() {
     return "DEPENDENCY_COLLECTION";
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // copy task doesn't have any execution and so
-    // does not use any map-reduce tmp files
-    // we don't expect to enter this code path at all
-    throw new RuntimeException ("Unexpected call");
-  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Tue Jun  4 03:14:17
2013
@@ -915,57 +915,6 @@ public class ExecDriver extends Task<Map
   }
 
   @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-
-    // localize any map-reduce input paths
-    ctx.localizeKeys((Map<String, Object>) ((Object) work.getPathToAliases()));
-    ctx.localizeKeys((Map<String, Object>) ((Object) work.getPathToPartitionInfo()));
-
-    // localize any input paths for maplocal work
-    MapredLocalWork l = work.getMapLocalWork();
-    if (l != null) {
-      Map<String, FetchWork> m = l.getAliasToFetchWork();
-      if (m != null) {
-        for (FetchWork fw : m.values()) {
-          String s = fw.getTblDir();
-          if ((s != null) && ctx.isMRTmpFileURI(s)) {
-            fw.setTblDir(ctx.localizeMRTmpFileURI(s));
-          }
-        }
-      }
-    }
-
-    // fix up outputs
-    Map<String, ArrayList<String>> pa = work.getPathToAliases();
-    if (pa != null) {
-      for (List<String> ls : pa.values()) {
-        for (String a : ls) {
-          ArrayList<Operator<? extends OperatorDesc>> opList =
-            new ArrayList<Operator<? extends OperatorDesc>>();
-          opList.add(work.getAliasToWork().get(a));
-
-          while (!opList.isEmpty()) {
-            Operator<? extends OperatorDesc> op = opList.remove(0);
-
-            if (op instanceof FileSinkOperator) {
-              FileSinkDesc fdesc = ((FileSinkOperator) op).getConf();
-              String s = fdesc.getDirName();
-              if ((s != null) && ctx.isMRTmpFileURI(s)) {
-                fdesc.setDirName(ctx.localizeMRTmpFileURI(s));
-              }
-              ((FileSinkOperator) op).setConf(fdesc);
-            }
-
-            if (op.getChildOperators() != null) {
-              opList.addAll(op.getChildOperators());
-            }
-          }
-        }
-      }
-    }
-  }
-
-  @Override
   public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
     for (Operator<? extends OperatorDesc> op : work.getAliasToWork().values()) {
       op.updateCounters(ctrs);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java Tue Jun  4 03:14:17
2013
@@ -717,13 +717,6 @@ public class ExplainTask extends Task<Ex
   }
 
   @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // explain task has nothing to localize
-    // we don't expect to enter this code path at all
-    throw new RuntimeException("Unexpected call");
-  }
-
-  @Override
   public List<FieldSchema> getResultSchema() {
     FieldSchema tmpFieldSchema = new FieldSchema();
     List<FieldSchema> colList = new ArrayList<FieldSchema>();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java Tue Jun  4 03:14:17
2013
@@ -163,19 +163,6 @@ public class FetchTask extends Task<Fetc
     return "FETCH";
   }
 
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    String s = work.getTblDir();
-    if ((s != null) && ctx.isMRTmpFileURI(s)) {
-      work.setTblDir(ctx.localizeMRTmpFileURI(s));
-    }
-
-    ArrayList<String> ls = work.getPartDir();
-    if (ls != null) {
-      ctx.localizePaths(ls);
-    }
-  }
-
   /**
    * Clear the Fetch Operator.
    *

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java Tue Jun  4 03:14:17
2013
@@ -114,9 +114,4 @@ public class FunctionTask extends Task<F
   public String getName() {
     return "FUNCTION";
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    throw new RuntimeException ("Unexpected call");
-  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapredLocalTask.java Tue Jun  4
03:14:17 2013
@@ -448,11 +448,6 @@ public class MapredLocalTask extends Tas
   }
 
   @Override
-  public void localizeMRTmpFilesImpl(Context ctx) {
-
-  }
-
-  @Override
   public boolean isMapRedLocalTask() {
     return true;
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Tue Jun  4 03:14:17
2013
@@ -537,10 +537,4 @@ public class MoveTask extends Task<MoveW
   public String getName() {
     return "MOVE";
   }
-
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // no-op
-  }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java Tue Jun  4 03:14:17
2013
@@ -263,11 +263,6 @@ public class StatsTask extends Task<Stat
     return "STATS";
   }
 
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // Nothing to do for StatsTask here.
-  }
-
   private int aggregateStats() {
 
     StatsAggregator statsAggregator = null;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Tue Jun  4 03:14:17 2013
@@ -384,37 +384,6 @@ public abstract class Task<T extends Ser
   public abstract StageType getType();
 
   /**
-   * If this task uses any map-reduce intermediate data (either for reading or for writing),
-   * localize them (using the supplied Context). Map-Reduce intermediate directories are
allocated
-   * using Context.getMRTmpFileURI() and can be localized using localizeMRTmpFileURI().
-   *
-   * This method is declared abstract to force any task code to explicitly deal with this
aspect of
-   * execution.
-   *
-   * @param ctx
-   *          context object with which to localize
-   */
-  abstract protected void localizeMRTmpFilesImpl(Context ctx);
-
-  /**
-   * Localize a task tree
-   *
-   * @param ctx
-   *          context object with which to localize
-   */
-  public final void localizeMRTmpFiles(Context ctx) {
-    localizeMRTmpFilesImpl(ctx);
-
-    if (childTasks == null) {
-      return;
-    }
-
-    for (Task<? extends Serializable> t : childTasks) {
-      t.localizeMRTmpFiles(ctx);
-    }
-  }
-
-  /**
    * Subscribe the feed of publisher. To prevent cycles, a task can only subscribe to its
ancestor.
    * Feed is a generic form of execution-time feedback (type, value) pair from one task to
another
    * task. Examples include dynamic partitions (which are only available at execution time).
The

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java Tue
Jun  4 03:14:17 2013
@@ -98,9 +98,4 @@ public class IndexMetadataChangeTask ext
   public StageType getType() {
     return StageType.DDL;
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-  }
-
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java Tue
Jun  4 03:14:17 2013
@@ -378,10 +378,4 @@ public class BlockMergeTask extends Task
   public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
     // no op
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // no op
-  }
-
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
Tue Jun  4 03:14:17 2013
@@ -379,11 +379,4 @@ public class PartialScanTask extends Tas
   public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
     // no op
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // no op
-  }
-
-
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/truncate/ColumnTruncateTask.java
Tue Jun  4 03:14:17 2013
@@ -254,10 +254,4 @@ public class ColumnTruncateTask extends 
   public void updateCounters(Counters ctrs, RunningJob rj) throws IOException {
     // no op
   }
-
-  @Override
-  protected void localizeMRTmpFilesImpl(Context ctx) {
-    // no op
-  }
-
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1489279&r1=1489278&r2=1489279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Jun 
4 03:14:17 2013
@@ -9637,15 +9637,6 @@ public class SemanticAnalyzer extends Ba
       ctx.setOriginalTracker(ShimLoader.getHadoopShims().getJobLauncherRpcAddress(conf));
       ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf, "local");
       console.printInfo("Automatically selecting local only mode for query");
-
-      // If all the tasks can be run locally, we can use local disk for
-      // storing intermediate data.
-
-      /**
-       * This code is commented out pending further testing/development
-       * for (Task<? extends OperatorDesc> t: rootTasks)
-       * t.localizeMRTmpFiles(ctx);
-       */
     }
   }
 



Mime
View raw message