hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1629344 - in /hive/branches/spark/ql/src: java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/exec/mr/ java/org/apache/hadoop/hive/ql/exec/spark/ java/org/apache/hadoop/hive/ql/io/ test/org/apache/hadoop/hive/ql/exec/ test/...
Date Sat, 04 Oct 2014 01:03:32 GMT
Author: xuefu
Date: Sat Oct  4 01:03:32 2014
New Revision: 1629344

URL: http://svn.apache.org/r1629344
Log:
HIVE-8331: HIVE-8303 followup, investigate result diff [Spark Branch] (Chao via Xuefu)

Modified:
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
    hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
    hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java
    hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_18.q.out

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java?rev=1629344&r1=1629343&r2=1629344&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java Sat
Oct  4 01:03:32 2014
@@ -76,7 +76,7 @@ public class FilterOperator extends Oper
       statsMap.put(Counter.FILTERED, filtered_count);
       statsMap.put(Counter.PASSED, passed_count);
       conditionInspector = null;
-      ioContext = IOContext.get(hconf.get(Utilities.INPUT_NAME));
+      ioContext = IOContext.get(hconf);
     } catch (Throwable e) {
       throw new HiveException(e);
     }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1629344&r1=1629343&r2=1629344&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Sat Oct
 4 01:03:32 2014
@@ -339,7 +339,7 @@ public class MapOperator extends Operato
   }
 
   public void setChildren(Configuration hconf) throws HiveException {
-    Path fpath = IOContext.get(hconf.get(Utilities.INPUT_NAME)).getInputPath();
+    Path fpath = IOContext.get(hconf).getInputPath();
 
     boolean schemeless = fpath.toUri().getScheme() == null;
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java?rev=1629344&r1=1629343&r2=1629344&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java
Sat Oct  4 01:03:32 2014
@@ -63,7 +63,7 @@ public class ExecMapperContext {
 
   public ExecMapperContext(JobConf jc) {
     this.jc = jc;
-    ioCxt = IOContext.get(jc.get(Utilities.INPUT_NAME));
+    ioCxt = IOContext.get(jc);
   }
 
   public void clear() {

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java?rev=1629344&r1=1629343&r2=1629344&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkMapRecordHandler.java
Sat Oct  4 01:03:32 2014
@@ -88,7 +88,6 @@ public class SparkMapRecordHandler exten
       }
       mo.setConf(mrwork);
       l4j.info("Main input name is " + mrwork.getName());
-      jc.set(Utilities.INPUT_NAME, mrwork.getName());
       // initialize map operator
       mo.setChildren(job);
       l4j.info(mo.dump(0));

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java?rev=1629344&r1=1629343&r2=1629344&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
Sat Oct  4 01:03:32 2014
@@ -197,7 +197,6 @@ public class SparkPlanGenerator {
     }
     if (work instanceof MapWork) {
       List<Path> inputPaths = Utilities.getInputPaths(cloned, (MapWork) work, scratchDir,
context, false);
-      cloned.set(Utilities.INPUT_NAME, work.getName());
       Utilities.setInputPaths(cloned, inputPaths);
       Utilities.setMapWork(cloned, (MapWork) work, scratchDir, false);
       Utilities.createTmpDirs(cloned, (MapWork) work);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java?rev=1629344&r1=1629343&r2=1629344&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
Sat Oct  4 01:03:32 2014
@@ -161,7 +161,7 @@ public abstract class HiveContextAwareRe
   }
 
   public IOContext getIOContext() {
-    return IOContext.get(jobConf.get(Utilities.INPUT_NAME));
+    return IOContext.get(jobConf);
   }
 
   private void initIOContext(long startPos, boolean isBlockPointer,

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java?rev=1629344&r1=1629343&r2=1629344&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java Sat Oct  4
01:03:32 2014
@@ -21,7 +21,11 @@ package org.apache.hadoop.hive.ql.io;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.mapred.JobConf;
 
 
 /**
@@ -49,13 +53,17 @@ public class IOContext {
     return inputNameIOContextMap;
   }
 
-  public static IOContext get(String inputName) {
-    if (inputNameIOContextMap.containsKey(inputName) == false) {
-      IOContext ioContext = new IOContext();
-      inputNameIOContextMap.put(inputName, ioContext);
+  public static IOContext get(Configuration conf) {
+    if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
+      return get();
+    } else {
+      String inputName = conf.get(Utilities.INPUT_NAME);
+      if (inputNameIOContextMap.containsKey(inputName) == false) {
+        IOContext ioContext = new IOContext();
+        inputNameIOContextMap.put(inputName, ioContext);
+      }
+      return inputNameIOContextMap.get(inputName);
     }
-
-    return inputNameIOContextMap.get(inputName);
   }
 
   public static void clear() {

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java?rev=1629344&r1=1629343&r2=1629344&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java Sat
Oct  4 01:03:32 2014
@@ -331,7 +331,7 @@ public class TestOperators extends TestC
       Configuration hconf = new JobConf(TestOperators.class);
       HiveConf.setVar(hconf, HiveConf.ConfVars.HADOOPMAPFILENAME,
           "hdfs:///testDir/testFile");
-      IOContext.get(hconf.get(Utilities.INPUT_NAME)).setInputPath(
+      IOContext.get(hconf).setInputPath(
           new Path("hdfs:///testDir/testFile"));
 
       // initialize pathToAliases

Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java?rev=1629344&r1=1629343&r2=1629344&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java
(original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java
Sat Oct  4 01:03:32 2014
@@ -116,7 +116,7 @@ public class TestHiveBinarySearchRecordR
 
   private void resetIOContext() {
     conf.set(Utilities.INPUT_NAME, "TestHiveBinarySearchRecordReader");
-    ioContext = IOContext.get(conf.get(Utilities.INPUT_NAME));
+    ioContext = IOContext.get(conf);
     ioContext.setUseSorted(false);
     ioContext.setIsBinarySearching(false);
     ioContext.setEndBinarySearch(false);

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_18.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_18.q.out?rev=1629344&r1=1629343&r2=1629344&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_18.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_remove_18.q.out Sat
Oct  4 01:03:32 2014
@@ -306,15 +306,15 @@ POSTHOOK: Input: default@outputtbl1@ds=1
 POSTHOOK: Input: default@outputtbl1@ds=18
 POSTHOOK: Input: default@outputtbl1@ds=28
 #### A masked pattern was here ####
-1	1	13
-1	1	13
-2	1	13
-2	1	13
+1	1	11
+1	1	11
+2	1	12
+2	1	12
 3	1	13
 3	1	13
-7	1	13
-7	1	13
-8	1	28
-8	1	28
+7	1	17
+7	1	17
+8	1	18
+8	1	18
 8	1	28
 8	1	28



Mime
View raw message