hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pxi...@apache.org
Subject [2/2] hive git commit: HIVE-14014: zero length file is being created for empty bucket in tez mode (II) (Pengcheng Xiong, reviewed by Ashutosh Chauhan)
Date Mon, 20 Jun 2016 17:33:17 GMT
HIVE-14014: zero length file is being created for empty bucket in tez mode (II) (Pengcheng
Xiong, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8ada8eaf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8ada8eaf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8ada8eaf

Branch: refs/heads/branch-2.1
Commit: 8ada8eafaa2b84b9daefdbb7284ec837983ceee7
Parents: a38745a
Author: Pengcheng Xiong <pxiong@apache.org>
Authored: Mon Jun 20 10:30:36 2016 -0700
Committer: Pengcheng Xiong <pxiong@apache.org>
Committed: Mon Jun 20 10:33:03 2016 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/ql/exec/FileSinkOperator.java    | 9 ++++++---
 ql/src/test/results/clientpositive/tez/orc_merge5.q.out     | 4 ++--
 ql/src/test/results/clientpositive/tez/orc_merge6.q.out     | 8 ++++----
 .../test/results/clientpositive/tez/union_fast_stats.q.out  | 6 +++---
 4 files changed, 15 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/8ada8eaf/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 0ac9109..1f95885 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.common.StatsSetupConst;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.CompilationOpContext;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -228,12 +229,14 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc>
implements
             // then skip the rename.  If it does try it.  We could just blindly try the rename
             // and avoid the extra stat, but that would mask other errors.
             try {
-              FileStatus stat = fs.getFileStatus(outPaths[idx]);
+              if (outPaths[idx] != null) {
+                FileStatus stat = fs.getFileStatus(outPaths[idx]);
+              }
             } catch (FileNotFoundException fnfe) {
               needToRename = false;
             }
           }
-          if (needToRename && !fs.rename(outPaths[idx], finalPaths[idx])) {
+          if (needToRename && outPaths[idx] != null && !fs.rename(outPaths[idx],
finalPaths[idx])) {
             throw new HiveException("Unable to rename output from: " +
                 outPaths[idx] + " to: " + finalPaths[idx]);
           }
@@ -1009,7 +1012,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc>
implements
     row_count.set(numRows);
     LOG.info(toString() + ": records written - " + numRows);
 
-    if (!bDynParts && !filesCreated) {
+    if (!bDynParts && !filesCreated && !"tez".equalsIgnoreCase(hconf.get(ConfVars.HIVE_EXECUTION_ENGINE.varname)))
{
       createBucketFiles(fsp);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/8ada8eaf/ql/src/test/results/clientpositive/tez/orc_merge5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/orc_merge5.q.out b/ql/src/test/results/clientpositive/tez/orc_merge5.q.out
index c2d841e..3126c2a 100644
--- a/ql/src/test/results/clientpositive/tez/orc_merge5.q.out
+++ b/ql/src/test/results/clientpositive/tez/orc_merge5.q.out
@@ -104,7 +104,7 @@ analyze table orc_merge5b compute statistics noscan
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_merge5b
 POSTHOOK: Output: default@orc_merge5b
-Found 3 items
+Found 2 items
 #### A masked pattern was here ####
 PREHOOK: query: select * from orc_merge5b
 PREHOOK: type: QUERY
@@ -271,7 +271,7 @@ POSTHOOK: query: analyze table orc_merge5b compute statistics noscan
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_merge5b
 POSTHOOK: Output: default@orc_merge5b
-Found 3 items
+Found 2 items
 #### A masked pattern was here ####
 PREHOOK: query: select * from orc_merge5b
 PREHOOK: type: QUERY

http://git-wip-us.apache.org/repos/asf/hive/blob/8ada8eaf/ql/src/test/results/clientpositive/tez/orc_merge6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/orc_merge6.q.out b/ql/src/test/results/clientpositive/tez/orc_merge6.q.out
index 73bb4a7..52c6d95 100644
--- a/ql/src/test/results/clientpositive/tez/orc_merge6.q.out
+++ b/ql/src/test/results/clientpositive/tez/orc_merge6.q.out
@@ -134,9 +134,9 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_merge5a
 POSTHOOK: Output: default@orc_merge5a
 POSTHOOK: Output: default@orc_merge5a@year=2001/hour=24
-Found 3 items
+Found 2 items
 #### A masked pattern was here ####
-Found 3 items
+Found 2 items
 #### A masked pattern was here ####
 PREHOOK: query: show partitions orc_merge5a
 PREHOOK: type: SHOWPARTITIONS
@@ -388,9 +388,9 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: default@orc_merge5a
 POSTHOOK: Output: default@orc_merge5a
 POSTHOOK: Output: default@orc_merge5a@year=2001/hour=24
-Found 3 items
+Found 2 items
 #### A masked pattern was here ####
-Found 3 items
+Found 2 items
 #### A masked pattern was here ####
 PREHOOK: query: show partitions orc_merge5a
 PREHOOK: type: SHOWPARTITIONS

http://git-wip-us.apache.org/repos/asf/hive/blob/8ada8eaf/ql/src/test/results/clientpositive/tez/union_fast_stats.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/union_fast_stats.q.out b/ql/src/test/results/clientpositive/tez/union_fast_stats.q.out
index 0cc0b9c..efe9336 100644
--- a/ql/src/test/results/clientpositive/tez/union_fast_stats.q.out
+++ b/ql/src/test/results/clientpositive/tez/union_fast_stats.q.out
@@ -177,7 +177,7 @@ Retention:          	0
 Table Type:         	MANAGED_TABLE       	 
 Table Parameters:	 	 
 	COLUMN_STATS_ACCURATE	{\"BASIC_STATS\":\"true\"}
-	numFiles            	4                   
+	numFiles            	3                   
 	numRows             	0                   
 	rawDataSize         	0                   
 	totalSize           	4003                
@@ -230,7 +230,7 @@ Retention:          	0
 Table Type:         	MANAGED_TABLE       	 
 Table Parameters:	 	 
 	COLUMN_STATS_ACCURATE	{\"BASIC_STATS\":\"true\"}
-	numFiles            	4                   
+	numFiles            	3                   
 	numRows             	15                  
 	rawDataSize         	3483                
 	totalSize           	4003                
@@ -295,7 +295,7 @@ Retention:          	0
 Table Type:         	MANAGED_TABLE       	 
 Table Parameters:	 	 
 	COLUMN_STATS_ACCURATE	{\"BASIC_STATS\":\"true\"}
-	numFiles            	5                   
+	numFiles            	4                   
 	numRows             	20                  
 	rawDataSize         	4552                
 	totalSize           	5360                


Mime
View raw message