Mailing-List: contact commits-help@hive.apache.org; run by ezmlm
Precedence: bulk
Reply-To: hive-dev@hive.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: sershe@apache.org
To: commits@hive.apache.org
Date: Wed, 19 Apr 2017 19:57:24 -0000
Message-Id: <840a4becab5147cfb604b411279ff1f5@git.apache.org>
In-Reply-To: <63f526604e6144588e0de1be1eb258ba@git.apache.org>
References: <63f526604e6144588e0de1be1eb258ba@git.apache.org>
Subject: [2/5] hive git commit: HIVE-16473 : Hive-on-Tez may fail to write to
 an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)
archived-at: Wed, 19 Apr 2017 19:57:25 -0000

HIVE-16473 : Hive-on-Tez may fail to write to an HBase table (Sergey Shelukhin, reviewed by Siddharth Seth)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1d5374a9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1d5374a9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1d5374a9

Branch: refs/heads/branch-2.3
Commit: 1d5374a9a64edef01c3628858078475a39f78883
Parents: d9927aa
Author: sergey <sershe@apache.org>
Authored: Wed Apr 19 12:27:04 2017 -0700
Committer: sergey <sershe@apache.org>
Committed: Wed Apr 19 12:31:50 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/exec/tez/TezTask.java   | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/1d5374a9/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 6c8bf29..1c84c6a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -18,6 +18,10 @@
 
 package org.apache.hadoop.hive.ql.exec.tez;
 
+import java.io.Serializable;
+import org.apache.hadoop.hive.ql.exec.ConditionalTask;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
@@ -324,6 +328,14 @@ public class TezTask extends Task<TezWork> {
     }
   }
 
+  void checkOutputSpec(BaseWork work, JobConf jc) throws IOException {
+    for (Operator<?> op : work.getAllOperators()) {
+      if (op instanceof FileSinkOperator) {
+        ((FileSinkOperator) op).checkOutputSpecs(null, jc);
+      }
+    }
+  }
+
   DAG build(JobConf conf, TezWork work, Path scratchDir,
       LocalResource appJarLr, List<LocalResource> additionalLr, Context ctx)
       throws Exception {
@@ -357,7 +369,6 @@ public class TezTask extends Task<TezWork> {
     setAccessControlsForCurrentUser(dag, queryPlan.getQueryId(), conf);
 
     for (BaseWork w: ws) {
-
       boolean isFinal = work.getLeaves().contains(w);
 
       // translate work to vertex
@@ -379,6 +390,8 @@ public class TezTask extends Task<TezWork> {
             children.add(v);
           }
         }
+        JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+        checkOutputSpec(w, parentConf);
 
         // create VertexGroup
         Vertex[] vertexArray = new Vertex[unionWorkItems.size()];
@@ -391,7 +404,7 @@ public class TezTask extends Task<TezWork> {
 
         // For a vertex group, all Outputs use the same Key-class, Val-class and partitioner.
         // Pick any one source vertex to figure out the Edge configuration.
-        JobConf parentConf = workToConf.get(unionWorkItems.get(0));
+       
 
         // now hook up the children
         for (BaseWork v: children) {
@@ -404,6 +417,7 @@ public class TezTask extends Task<TezWork> {
       } else {
         // Regular vertices
         JobConf wxConf = utils.initializeVertexConf(conf, ctx, w);
+        checkOutputSpec(w, wxConf);
         Vertex wx =
             utils.createVertex(wxConf, w, scratchDir, appJarLr, additionalLr, fs, ctx, !isFinal,
                 work, work.getVertexType(w));