hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject hive git commit: HIVE-17125. Lineage: Generate lineage information on need basis when atlas hook is enabled (Rajesh Balamohan, reviewed by Ashutosh Chauhan)
Date Wed, 19 Jul 2017 22:28:26 GMT
Repository: hive
Updated Branches:
  refs/heads/master fb6b023dc -> 9f86d14af


HIVE-17125. Lineage: Generate lineage information on need basis when atlas hook is enabled
(Rajesh Balamohan, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f86d14a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f86d14a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f86d14a

Branch: refs/heads/master
Commit: 9f86d14af1cbc381050cefedf006c691cc7ece70
Parents: fb6b023
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Thu Jul 20 03:58:06 2017 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Thu Jul 20 03:58:06 2017 +0530

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/QueryProperties.java  |  1 +
 .../hadoop/hive/ql/optimizer/Optimizer.java     |  2 +-
 .../hive/ql/optimizer/lineage/Generator.java    | 21 ++++++++++++++++++++
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  2 +-
 4 files changed, 24 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/9f86d14a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
index 650792b..154fec4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
@@ -283,6 +283,7 @@ public class QueryProperties {
     analyzeRewrite = false;
     ctas = false;
     outerQueryLimit = -1;
+    isMaterializedView = false;
 
     hasJoin = false;
     hasGroupBy = false;

http://git-wip-us.apache.org/repos/asf/hive/blob/9f86d14a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
index 3e4584d..e1340c7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
@@ -79,7 +79,7 @@ public class Optimizer {
     if (postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.PostExecutePrinter")
         || postExecHooks.contains("org.apache.hadoop.hive.ql.hooks.LineageLogger")
         || postExecHooks.contains("org.apache.atlas.hive.hook.HiveHook")) {
-      transformations.add(new Generator());
+      transformations.add(new Generator(postExecHooks));
     }
 
     // Try to transform OR predicates in Filter into simpler IN clauses first

http://git-wip-us.apache.org/repos/asf/hive/blob/9f86d14a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
index 747aae0..e6c0771 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.optimizer.lineage;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
 import org.apache.hadoop.hive.ql.exec.FilterOperator;
@@ -59,12 +60,32 @@ public class Generator extends Transform {
 
   private static final Logger LOG = LoggerFactory.getLogger(Generator.class);
 
+  private final Set<String> hooks;
+  private static final String ATLAS_HOOK_CLASSNAME = "org.apache.atlas.hive.hook.HiveHook";
+
+  public Generator(Set<String> hooks) {
+    this.hooks = hooks;
+  }
+
   /* (non-Javadoc)
    * @see org.apache.hadoop.hive.ql.optimizer.Transform#transform(org.apache.hadoop.hive.ql.parse.ParseContext)
    */
   @Override
   public ParseContext transform(ParseContext pctx) throws SemanticException {
 
+    if (hooks != null && hooks.contains(ATLAS_HOOK_CLASSNAME)) {
+      // Atlas would be interested in lineage information for insert,load,create etc.
+      if (!pctx.getQueryProperties().isCTAS()
+          && !pctx.getQueryProperties().isMaterializedView()
+          && pctx.getQueryProperties().isQuery()
+          && pctx.getCreateTable() == null
+          && pctx.getCreateViewDesc() == null
+          && (pctx.getLoadTableWork() == null || pctx.getLoadTableWork().isEmpty()))
{
+        LOG.debug("Not evaluating lineage");
+        return pctx;
+      }
+    }
+
     Index index = SessionState.get() != null ?
       SessionState.get().getLineageState().getIndex() : new Index();
 

http://git-wip-us.apache.org/repos/asf/hive/blob/9f86d14a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 56e785e..4faec05 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -11412,7 +11412,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
             || postExecHooks.contains("org.apache.atlas.hive.hook.HiveHook")) {
           ArrayList<Transform> transformations = new ArrayList<Transform>();
           transformations.add(new HiveOpConverterPostProc());
-          transformations.add(new Generator());
+          transformations.add(new Generator(postExecHooks));
           for (Transform t : transformations) {
             pCtx = t.transform(pCtx);
           }


Mime
View raw message