hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1648397 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io: CombineHiveInputFormat.java CombineHiveRecordReader.java HiveInputFormat.java
Date Mon, 29 Dec 2014 17:42:12 GMT
Author: xuefu
Date: Mon Dec 29 17:42:11 2014
New Revision: 1648397

URL: http://svn.apache.org/r1648397
Log:
HIVE-9153: Perf enhancement on CombineHiveInputFormat and HiveInputFormat (Rui via Xuefu)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java?rev=1648397&r1=1648396&r2=1648397&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java Mon Dec
29 17:42:11 2014
@@ -275,7 +275,8 @@ public class CombineHiveInputFormat<K ex
   /**
    * Create Hive splits based on CombineFileSplit.
    */
-  private InputSplit[] getCombineSplits(JobConf job, int numSplits, Map<String, PartitionDesc>
pathToPartitionInfo)
+  private InputSplit[] getCombineSplits(JobConf job, int numSplits,
+      Map<String, PartitionDesc> pathToPartitionInfo)
       throws IOException {
     PerfLogger perfLogger = PerfLogger.getPerfLogger();
     perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.GET_SPLITS);
@@ -512,7 +513,8 @@ public class CombineHiveInputFormat<K ex
     if (combinablePaths.size() > 0) {
       FileInputFormat.setInputPaths(job, combinablePaths.toArray
           (new Path[combinablePaths.size()]));
-      Map<String, PartitionDesc> pathToPartitionInfo = Utilities.getMapWork(job).getPathToPartitionInfo();
+      Map<String, PartitionDesc> pathToPartitionInfo = this.pathToPartitionInfo !=
null ?
+          this.pathToPartitionInfo : Utilities.getMapWork(job).getPathToPartitionInfo();
       InputSplit[] splits = getCombineSplits(job, numSplits, pathToPartitionInfo);
       for (InputSplit split : splits) {
         result.add(split);
@@ -658,7 +660,7 @@ public class CombineHiveInputFormat<K ex
 
     return ShimLoader.getHadoopShims().getCombineFileInputFormat()
         .getRecordReader(job,
-        ((CombineHiveInputSplit) split).getInputSplitShim(), reporter,
+        (CombineFileSplit) split, reporter,
         CombineHiveRecordReader.class);
   }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java?rev=1648397&r1=1648396&r2=1648397&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java Mon Dec
29 17:42:11 2014
@@ -44,8 +44,9 @@ public class CombineHiveRecordReader<K e
   public CombineHiveRecordReader(InputSplit split, Configuration conf,
       Reporter reporter, Integer partition) throws IOException {
     super((JobConf)conf);
-    CombineHiveInputSplit hsplit = new CombineHiveInputSplit(jobConf,
-        (CombineFileSplit) split);
+    CombineHiveInputSplit hsplit = split instanceof CombineHiveInputSplit ?
+        (CombineHiveInputSplit) split :
+        new CombineHiveInputSplit(jobConf, (CombineFileSplit) split);
     String inputFormatClassName = hsplit.inputFormatClassName();
     Class inputFormatClass = null;
     try {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=1648397&r1=1648396&r2=1648397&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java Mon Dec 29 17:42:11
2014
@@ -258,15 +258,17 @@ public class HiveInputFormat<K extends W
   }
 
   protected void init(JobConf job) {
-    if (HiveConf.getVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
-      mrwork = (MapWork) Utilities.getMergeWork(job);
-      if (mrwork == null) {
+    if (mrwork == null || pathToPartitionInfo == null) {
+      if (HiveConf.getVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
+        mrwork = (MapWork) Utilities.getMergeWork(job);
+        if (mrwork == null) {
+          mrwork = Utilities.getMapWork(job);
+        }
+      } else {
         mrwork = Utilities.getMapWork(job);
       }
-    } else {
-      mrwork = Utilities.getMapWork(job);
+      pathToPartitionInfo = mrwork.getPathToPartitionInfo();
     }
-    pathToPartitionInfo = mrwork.getPathToPartitionInfo();
   }
 
   /*



Mime
View raw message