hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1033988 - in /hive/trunk: CHANGES.txt common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
Date Thu, 11 Nov 2010 16:36:54 GMT
Author: namit
Date: Thu Nov 11 16:36:54 2010
New Revision: 1033988

URL: http://svn.apache.org/viewvc?rev=1033988&view=rev
Log:
HIVE-1776 parallel execution and auto-local mode combine to place
plan file in wrong file system (Joydeep Sen Sarma via namit)


Modified:
    hive/trunk/CHANGES.txt
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java

Modified: hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hive/trunk/CHANGES.txt?rev=1033988&r1=1033987&r2=1033988&view=diff
==============================================================================
--- hive/trunk/CHANGES.txt (original)
+++ hive/trunk/CHANGES.txt Thu Nov 11 16:36:54 2010
@@ -483,6 +483,9 @@ Trunk -  Unreleased
     HIVE-1780 Typo in hive-default.xml
     (Youngwoo Kim via namit)
 
+    HIVE-1776 parallel execution and auto-local mode combine to place
+    plan file in wrong file system (Joydeep Sen Sarma via namit)
+
   TESTS
 
     HIVE-1464. improve  test query performance

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1033988&r1=1033987&r2=1033988&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Nov 11 16:36:54
2010
@@ -483,6 +483,16 @@ public class HiveConf extends Configurat
     initialize(cls);
   }
 
+  /**
+   * Copy constructor
+   */
+  public HiveConf(HiveConf other) {
+    super(other);
+    hiveJar = other.hiveJar;
+    auxJars = other.auxJars;
+    origProp = (Properties)other.origProp.clone();
+  }
+
   private Properties getUnderlyingProps() {
     Iterator<Map.Entry<String, String>> iter = this.iterator();
     Properties p = new Properties();

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java?rev=1033988&r1=1033987&r2=1033988&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapRedTask.java Thu Nov 11 16:36:54
2010
@@ -102,8 +102,8 @@ public class MapRedTask extends ExecDriv
 
         String reason = MapRedTask.isEligibleForLocalMode(conf, inputSummary, numReducers);
         if (reason == null) {
-          // set the JT to local for the duration of this job
-          ctx.setOriginalTracker(conf.getVar(HiveConf.ConfVars.HADOOPJT));
+          // clone configuration before modifying it on per-task basis
+          cloneConf();
           conf.setVar(HiveConf.ConfVars.HADOOPJT, "local");
           console.printInfo("Selecting local mode for task: " + getId());
         } else {
@@ -121,6 +121,9 @@ public class MapRedTask extends ExecDriv
         return super.execute(driverContext);
       }
 
+      // we need to edit the configuration to setup cmdline. clone it first
+      cloneConf();
+
       // enable assertion
       String hadoopExec = conf.getVar(HiveConf.ConfVars.HADOOPBIN);
       String hiveJar = conf.getJar();
@@ -263,10 +266,6 @@ public class MapRedTask extends ExecDriv
       return (1);
     } finally {
       try {
-        // in case we decided to run everything in local mode, restore the
-        // the jobtracker setting to its initial value
-        ctx.restoreOriginalTracker();
-
         // creating the context can create a bunch of files. So make
         // sure to clear it out
         if(ctxCreated) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=1033988&r1=1033987&r2=1033988&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Thu Nov 11 16:36:54 2010
@@ -58,6 +58,7 @@ public abstract class Task<T extends Ser
   protected transient TaskHandle taskHandle;
   protected transient HashMap<String, Long> taskCounters;
   protected transient DriverContext driverContext;
+  protected transient boolean clonedConf = false;
 
   // Descendants tasks who subscribe feeds from this task
   protected transient List<Task<? extends Serializable>> feedSubscribers;
@@ -381,4 +382,11 @@ public abstract class Task<T extends Ser
   // a subscriber accept the feed and do something depending on the Task type
   protected void receiveFeed(FeedType feedType, Object feedValue) {
   }
+
+  protected void cloneConf () {
+    if (!clonedConf) {
+      clonedConf = true;
+      conf = new HiveConf(conf);
+    }
+  }
 }
\ No newline at end of file



Mime
View raw message