hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1576978 - in /hive/branches/branch-0.13: hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/ hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/ hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/ ql/src/java/org...
Date Wed, 12 Mar 2014 22:54:19 GMT
Author: hashutosh
Date: Wed Mar 12 22:54:18 2014
New Revision: 1576978

URL: http://svn.apache.org/r1576978
Log:
HIVE-6572 : Use shimmed version of hadoop conf names for mapred.{min,max}.split.size{.*} (Sushanth
Sowmyan via Ashutosh Chauhan)

Modified:
    hive/branches/branch-0.13/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java
    hive/branches/branch-0.13/hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java
    hive/branches/branch-0.13/hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
    hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
    hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
    hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java

Modified: hive/branches/branch-0.13/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java?rev=1576978&r1=1576977&r2=1576978&view=diff
==============================================================================
--- hive/branches/branch-0.13/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java
(original)
+++ hive/branches/branch-0.13/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatConstants.java
Wed Mar 12 22:54:18 2014
@@ -88,7 +88,9 @@ public final class HCatConstants {
    * The desired number of input splits produced for each partition. When the
    * input files are large and few, we want to split them into many splits,
    * so as to increase the parallelizm of loading the splits. Try also two
-   * other parameters, mapred.min.split.size and mapred.max.split.size, to
+   * other parameters, mapred.min.split.size and mapred.max.split.size for
+   * hadoop 1.x, or mapreduce.input.fileinputformat.split.minsize and
+   * mapreduce.input.fileinputformat.split.maxsize in hadoop 2.x to
    * control the number of input splits.
    */
   public static final String HCAT_DESIRED_PARTITION_NUM_SPLITS =

Modified: hive/branches/branch-0.13/hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java?rev=1576978&r1=1576977&r2=1576978&view=diff
==============================================================================
--- hive/branches/branch-0.13/hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java
(original)
+++ hive/branches/branch-0.13/hcatalog/core/src/main/java/org/apache/hive/hcatalog/rcfile/RCFileMapReduceInputFormat.java
Wed Mar 12 22:54:18 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.List;
 
 import org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable;
+import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.mapreduce.InputSplit;
@@ -44,7 +45,9 @@ public class RCFileMapReduceInputFormat<
   @Override
   public List<InputSplit> getSplits(JobContext job) throws IOException {
 
-    job.getConfiguration().setLong("mapred.min.split.size", SequenceFile.SYNC_INTERVAL);
+    job.getConfiguration().setLong(
+        ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
+        SequenceFile.SYNC_INTERVAL);
     return super.getSplits(job);
   }
 }

Modified: hive/branches/branch-0.13/hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java?rev=1576978&r1=1576977&r2=1576978&view=diff
==============================================================================
--- hive/branches/branch-0.13/hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java
(original)
+++ hive/branches/branch-0.13/hcatalog/core/src/test/java/org/apache/hive/hcatalog/rcfile/TestRCFileMapReduceInputFormat.java
Wed Mar 12 22:54:18 2014
@@ -228,7 +228,8 @@ public class TestRCFileMapReduceInputFor
     Configuration jonconf = new Configuration(cloneConf);
     jonconf.set("mapred.input.dir", testDir.toString());
     JobContext context = new Job(jonconf);
-    context.getConfiguration().setLong("mapred.max.split.size", maxSplitSize);
+    context.getConfiguration().setLong(
+        ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), maxSplitSize);
     List<InputSplit> splits = inputFormat.getSplits(context);
     assertEquals("splits length should be " + splitNumber, splits.size(), splitNumber);
     int readCount = 0;

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java?rev=1576978&r1=1576977&r2=1576978&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
Wed Mar 12 22:54:18 2014
@@ -284,7 +284,9 @@ public class HiveInputFormat<K extends W
       if (headerCount != 0 || footerCount != 0) {
         
         // Input file has header or footer, cannot be splitted.
-        conf.setLong("mapred.min.split.size", Long.MAX_VALUE);
+        conf.setLong(
+            ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
+            Long.MAX_VALUE);
       }
     }
 

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1576978&r1=1576977&r2=1576978&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
Wed Mar 12 22:54:18 2014
@@ -77,8 +77,10 @@ public class OrcInputFormat  implements 
   VectorizedOrcInputFormat voif = new VectorizedOrcInputFormat();
 
   private static final Log LOG = LogFactory.getLog(OrcInputFormat.class);
-  static final String MIN_SPLIT_SIZE = "mapred.min.split.size";
-  static final String MAX_SPLIT_SIZE = "mapred.max.split.size";
+  static final String MIN_SPLIT_SIZE =
+      ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE");
+  static final String MAX_SPLIT_SIZE =
+      ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE");
 
   private static final long DEFAULT_MIN_SPLIT_SIZE = 16 * 1024 * 1024;
   private static final long DEFAULT_MAX_SPLIT_SIZE = 256 * 1024 * 1024;

Modified: hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java?rev=1576978&r1=1576977&r2=1576978&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java (original)
+++ hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java Wed
Mar 12 22:54:18 2014
@@ -58,6 +58,7 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
@@ -640,7 +641,9 @@ public class TestRCFile {
     RCFileInputFormat inputFormat = new RCFileInputFormat();
     JobConf jobconf = new JobConf(cloneConf);
     jobconf.set("mapred.input.dir", testDir.toString());
-    jobconf.setLong("mapred.min.split.size", fileLen);
+    jobconf.setLong(
+        ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
+        fileLen);
     InputSplit[] splits = inputFormat.getSplits(jobconf, 1);
     RCFileRecordReader rr = new RCFileRecordReader(jobconf, (FileSplit)splits[0]);
     long lastSync = 0;
@@ -707,7 +710,9 @@ public class TestRCFile {
     RCFileInputFormat inputFormat = new RCFileInputFormat();
     JobConf jonconf = new JobConf(cloneConf);
     jonconf.set("mapred.input.dir", testDir.toString());
-    jonconf.setLong("mapred.min.split.size", minSplitSize);
+    jonconf.setLong(
+        ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
+        minSplitSize);
     InputSplit[] splits = inputFormat.getSplits(jonconf, splitNumber);
     assertEquals("splits length should be " + splitNumber, splits.length, splitNumber);
     int readCount = 0;

Modified: hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java?rev=1576978&r1=1576977&r2=1576978&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
(original)
+++ hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java
Wed Mar 12 22:54:18 2014
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.udf.gen
 import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.InputFormat;
@@ -106,8 +107,8 @@ public class TestOrcSplitElimination {
         100000, CompressionKind.NONE, 10000, 10000);
     writeData(writer);
     writer.close();
-    conf.set("mapred.min.split.size", "1000");
-    conf.set("mapred.max.split.size", "5000");
+    conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
"1000");
+    conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"),
"5000");
     InputFormat<?, ?> in = new OrcInputFormat();
     FileInputFormat.setInputPaths(conf, testFilePath.toString());
 
@@ -184,8 +185,8 @@ public class TestOrcSplitElimination {
         100000, CompressionKind.NONE, 10000, 10000);
     writeData(writer);
     writer.close();
-    conf.set("mapred.min.split.size", "1000");
-    conf.set("mapred.max.split.size", "150000");
+    conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
"1000");
+    conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"),
"150000");
     InputFormat<?, ?> in = new OrcInputFormat();
     FileInputFormat.setInputPaths(conf, testFilePath.toString());
 
@@ -273,8 +274,8 @@ public class TestOrcSplitElimination {
         100000, CompressionKind.NONE, 10000, 10000);
     writeData(writer);
     writer.close();
-    conf.set("mapred.min.split.size", "1000");
-    conf.set("mapred.max.split.size", "150000");
+    conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"),
"1000");
+    conf.set(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"),
"150000");
     InputFormat<?, ?> in = new OrcInputFormat();
     FileInputFormat.setInputPaths(conf, testFilePath.toString());
 

Modified: hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1576978&r1=1576977&r2=1576978&view=diff
==============================================================================
--- hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
(original)
+++ hive/branches/branch-0.13/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
Wed Mar 12 22:54:18 2014
@@ -778,8 +778,8 @@ public class Hadoop20Shims implements Ha
     ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapred.input.dir.recursive");
     ret.put("MAPREDMAXSPLITSIZE", "mapred.max.split.size");
     ret.put("MAPREDMINSPLITSIZE", "mapred.min.split.size");
-    ret.put("MAPREDMINSPLITSIZEPERNODE", "mapred.min.split.size.per.rack");
-    ret.put("MAPREDMINSPLITSIZEPERRACK", "mapred.min.split.size.per.node");
+    ret.put("MAPREDMINSPLITSIZEPERRACK", "mapred.min.split.size.per.rack");
+    ret.put("MAPREDMINSPLITSIZEPERNODE", "mapred.min.split.size.per.node");
     ret.put("HADOOPNUMREDUCERS", "mapred.reduce.tasks");
     ret.put("HADOOPJOBNAME", "mapred.job.name");
     ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapred.reduce.tasks.speculative.execution");

Modified: hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1576978&r1=1576977&r2=1576978&view=diff
==============================================================================
--- hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
(original)
+++ hive/branches/branch-0.13/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
Wed Mar 12 22:54:18 2014
@@ -404,8 +404,8 @@ public class Hadoop20SShims extends Hado
     ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapred.input.dir.recursive");
     ret.put("MAPREDMAXSPLITSIZE", "mapred.max.split.size");
     ret.put("MAPREDMINSPLITSIZE", "mapred.min.split.size");
-    ret.put("MAPREDMINSPLITSIZEPERNODE", "mapred.min.split.size.per.rack");
-    ret.put("MAPREDMINSPLITSIZEPERRACK", "mapred.min.split.size.per.node");
+    ret.put("MAPREDMINSPLITSIZEPERNODE", "mapred.min.split.size.per.node");
+    ret.put("MAPREDMINSPLITSIZEPERRACK", "mapred.min.split.size.per.rack");
     ret.put("HADOOPNUMREDUCERS", "mapred.reduce.tasks");
     ret.put("HADOOPJOBNAME", "mapred.job.name");
     ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapred.reduce.tasks.speculative.execution");

Modified: hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1576978&r1=1576977&r2=1576978&view=diff
==============================================================================
--- hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
(original)
+++ hive/branches/branch-0.13/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
Wed Mar 12 22:54:18 2014
@@ -568,8 +568,8 @@ public class Hadoop23Shims extends Hadoo
     ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapreduce.input.fileinputformat.input.dir.recursive");
     ret.put("MAPREDMAXSPLITSIZE", "mapreduce.input.fileinputformat.split.maxsize");
     ret.put("MAPREDMINSPLITSIZE", "mapreduce.input.fileinputformat.split.minsize");
-    ret.put("MAPREDMINSPLITSIZEPERNODE", "mapreduce.input.fileinputformat.split.minsize.per.rack");
-    ret.put("MAPREDMINSPLITSIZEPERRACK", "mapreduce.input.fileinputformat.split.minsize.per.node");
+    ret.put("MAPREDMINSPLITSIZEPERNODE", "mapreduce.input.fileinputformat.split.minsize.per.node");
+    ret.put("MAPREDMINSPLITSIZEPERRACK", "mapreduce.input.fileinputformat.split.minsize.per.rack");
     ret.put("HADOOPNUMREDUCERS", "mapreduce.job.reduces");
     ret.put("HADOOPJOBNAME", "mapreduce.job.name");
     ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapreduce.reduce.speculative");



Mime
View raw message