hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [48/53] [abbrv] hive git commit: HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun Radhakrishnan, reviewed by Daniel Dai)
Date Thu, 06 Aug 2015 00:50:54 GMT
HIVE-11456: HCatStorer should honor mapreduce.output.basename (Mithun Radhakrishnan, reviewed
by Daniel Dai)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0f934d29
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0f934d29
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0f934d29

Branch: refs/heads/llap
Commit: 0f934d29bb6bfc992eca050337b8f48fb47aaed8
Parents: 96f5914
Author: Daniel Dai <daijy@hortonworks.com>
Authored: Wed Aug 5 13:50:19 2015 -0700
Committer: Daniel Dai <daijy@hortonworks.com>
Committed: Wed Aug 5 13:50:19 2015 -0700

----------------------------------------------------------------------
 .../hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java | 7 ++++---
 .../mapreduce/DynamicPartitionFileRecordWriterContainer.java  | 3 ++-
 .../hive/hcatalog/mapreduce/FileOutputFormatContainer.java    | 3 ++-
 3 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
index 3a07b0c..f620b83 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DefaultOutputFormatContainer.java
@@ -51,8 +51,9 @@ class DefaultOutputFormatContainer extends OutputFormatContainer {
     super(of);
   }
 
-  static synchronized String getOutputName(int partition) {
-    return "part-" + NUMBER_FORMAT.format(partition);
+  static synchronized String getOutputName(TaskAttemptContext context) {
+    return context.getConfiguration().get("mapreduce.output.basename", "part")
+        + "-" + NUMBER_FORMAT.format(context.getTaskAttemptID().getTaskID().getId());
   }
 
   /**
@@ -65,7 +66,7 @@ class DefaultOutputFormatContainer extends OutputFormatContainer {
   @Override
   public RecordWriter<WritableComparable<?>, HCatRecord>
   getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {
-    String name = getOutputName(context.getTaskAttemptID().getTaskID().getId());
+    String name = getOutputName(context);
     return new DefaultRecordWriterContainer(context,
       getBaseOutputFormat().getRecordWriter(null, new JobConf(context.getConfiguration()),
name, InternalUtil.createReporter(context)));
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
index 60f1b60..320ace4 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/DynamicPartitionFileRecordWriterContainer.java
@@ -205,7 +205,8 @@ class DynamicPartitionFileRecordWriterContainer extends FileRecordWriterContaine
 
       Path parentDir = new Path(currTaskContext.getConfiguration().get("mapred.work.output.dir"));
       Path childPath =
-          new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext, "part", ""));
+          new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext,
+              currTaskContext.getConfiguration().get("mapreduce.output.basename", "part"),
""));
 
       RecordWriter baseRecordWriter =
           baseOF.getRecordWriter(parentDir.getFileSystem(currTaskContext.getConfiguration()),

http://git-wip-us.apache.org/repos/asf/hive/blob/0f934d29/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
index 001b59b..95ee3b4 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputFormatContainer.java
@@ -97,7 +97,8 @@ class FileOutputFormatContainer extends OutputFormatContainer {
           (org.apache.hadoop.mapred.RecordWriter)null, context);
     } else {
       Path parentDir = new Path(context.getConfiguration().get("mapred.work.output.dir"));
-      Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()),
"part"));
+      Path childPath = new Path(parentDir,FileOutputFormat.getUniqueName(new JobConf(context.getConfiguration()),
+               context.getConfiguration().get("mapreduce.output.basename", "part")));
 
       rw = new StaticPartitionFileRecordWriterContainer(
           getBaseOutputFormat().getRecordWriter(


Mime
View raw message