Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 44C5618034 for ; Mon, 17 Aug 2015 22:00:36 +0000 (UTC) Received: (qmail 85865 invoked by uid 500); 17 Aug 2015 22:00:33 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 85758 invoked by uid 500); 17 Aug 2015 22:00:33 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 83254 invoked by uid 99); 17 Aug 2015 22:00:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Aug 2015 22:00:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4A97BE0435; Mon, 17 Aug 2015 22:00:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sershe@apache.org To: commits@hive.apache.org Date: Mon, 17 Aug 2015 22:00:57 -0000 Message-Id: <1f412fb6d7e44f4497c3db95c646a27f@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [27/50] [abbrv] hive git commit: HIVE-11464: lineage info missing if there are multiple outputs (Jimmy) HIVE-11464: lineage info missing if there are multiple outputs (Jimmy) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1a75644d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1a75644d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1a75644d Branch: refs/heads/hbase-metastore Commit: 1a75644d68c8c61fbafb4058fe45b7823492491c Parents: f26b256 Author: Jimmy Xiang Authored: Wed Aug 5 08:02:50 2015 -0700 Committer: Jimmy Xiang Committed: Thu Aug 13 13:44:03 2015 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/ql/Driver.java | 8 ++-- .../hadoop/hive/ql/hooks/LineageInfo.java | 9 ++-- .../hadoop/hive/ql/hooks/LineageLogger.java | 44 +++++++++++++------- .../ql/optimizer/lineage/ExprProcFactory.java | 9 ++-- .../hive/ql/optimizer/lineage/LineageCtx.java | 34 +++++++++++---- .../ql/optimizer/lineage/OpProcFactory.java | 10 ++--- ql/src/test/queries/clientpositive/lineage3.q | 15 +++++++ .../test/results/clientpositive/lineage3.q.out | 32 +++++++++++++- 8 files changed, 118 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index e7b7b55..c0c1b2e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -441,8 +441,11 @@ public class Driver implements CommandProcessor { // to avoid returning sensitive data String queryStr = HookUtils.redactLogString(conf, command); + // get the output schema + schema = getSchema(sem, conf); + plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId, - SessionState.get().getHiveOperation(), getSchema(sem, conf)); + SessionState.get().getHiveOperation(), schema); conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr); @@ -454,9 +457,6 @@ public class Driver implements CommandProcessor { plan.getFetchTask().initialize(conf, plan, null); } - // get the output schema - schema = getSchema(sem, conf); - //do the authorization check if (!sem.skipAuthorization() && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) { http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java index fe0841e..2806c54 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java @@ -22,7 +22,6 @@ import java.io.Serializable; import java.util.Collections; import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.List; import java.util.Map; import java.util.Set; @@ -375,9 +374,9 @@ public class LineageInfo implements Serializable { private String expr; /** - * The list of base columns that the particular column depends on. + * The set of base columns that the particular column depends on. */ - private List baseCols; + private Set baseCols; /** * @return the type @@ -410,14 +409,14 @@ public class LineageInfo implements Serializable { /** * @return the baseCols */ - public List getBaseCols() { + public Set getBaseCols() { return baseCols; } /** * @param baseCols the baseCols to set */ - public void setBaseCols(List baseCols) { + public void setBaseCols(Set baseCols) { this.baseCols = baseCols; } http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java index d615372..3c6ce94 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java @@ -33,6 +33,7 @@ import org.apache.commons.io.output.StringBuilderWriter; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Table; @@ -147,6 +148,7 @@ public class LineageLogger implements ExecuteWithHookContext { // Don't emit user/timestamp info in test mode, // so that the test golden output file is fixed. long queryTime = plan.getQueryStartTime().longValue(); + if (queryTime == 0) queryTime = System.currentTimeMillis(); writer.name("user").value(hookContext.getUgi().getUserName()); writer.name("timestamp").value(queryTime/1000); writer.name("jobIds"); @@ -209,23 +211,28 @@ public class LineageLogger implements ExecuteWithHookContext { * For each target column, find out its sources based on the dependency index. */ private List getEdges(QueryPlan plan, Index index) { - List fieldSchemas = plan.getResultSchema().getFieldSchemas(); - int fields = fieldSchemas == null ? 0 : fieldSchemas.size(); - SelectOperator finalSelOp = index.getFinalSelectOp(); + LinkedHashMap> finalSelOps = index.getFinalSelectOps(); + Set allTargets = new LinkedHashSet(); + Map allSources = new LinkedHashMap(); List edges = new ArrayList(); - if (finalSelOp != null && fields > 0) { - Map colMap = index.getDependencies(finalSelOp); - List dependencies = colMap != null ? Lists.newArrayList(colMap.values()) : null; - if (dependencies == null || dependencies.size() != fields) { - log("Result schema has " + fields - + " fields, but we don't get as many dependencies"); + for (ObjectPair pair: finalSelOps.values()) { + List fieldSchemas = plan.getResultSchema().getFieldSchemas(); + SelectOperator finalSelOp = pair.getFirst(); + org.apache.hadoop.hive.ql.metadata.Table t = pair.getSecond(); + String destTableName = null; + List colNames = null; + if (t != null) { + destTableName = t.getDbName() + "." + t.getTableName(); + fieldSchemas = t.getCols(); } else { - String destTableName = null; - List colNames = null; // Based on the plan outputs, find out the target table name and column names. for (WriteEntity output : plan.getOutputs()) { - if (output.getType() == Entity.Type.TABLE) { - org.apache.hadoop.hive.ql.metadata.Table t = output.getTable(); + Entity.Type entityType = output.getType(); + if (entityType == Entity.Type.TABLE + || entityType == Entity.Type.PARTITION) { + t = output.getTable(); destTableName = t.getDbName() + "." + t.getTableName(); List cols = t.getCols(); if (cols != null && !cols.isEmpty()) { @@ -234,10 +241,15 @@ public class LineageLogger implements ExecuteWithHookContext { break; } } - + } + int fields = fieldSchemas.size(); + Map colMap = index.getDependencies(finalSelOp); + List dependencies = colMap != null ? Lists.newArrayList(colMap.values()) : null; + if (dependencies == null || dependencies.size() != fields) { + log("Result schema has " + fields + + " fields, but we don't get as many dependencies"); + } else { // Go through each target column, generate the lineage edges. - Set allTargets = new LinkedHashSet(); - Map allSources = new LinkedHashMap(); for (int i = 0; i < fields; i++) { Vertex target = new Vertex( getTargetFieldName(i, destTableName, colNames, fieldSchemas)); http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java index 455a525..38040e3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Stack; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -124,7 +125,7 @@ public class ExprProcFactory { bci_set.addAll(child_dep.getBaseCols()); } - dep.setBaseCols(new ArrayList(bci_set)); + dep.setBaseCols(bci_set); dep.setType(new_type); return dep; @@ -146,7 +147,7 @@ public class ExprProcFactory { // Create a dependency that has no basecols Dependency dep = new Dependency(); dep.setType(LineageInfo.DependencyType.SIMPLE); - dep.setBaseCols(new ArrayList()); + dep.setBaseCols(new LinkedHashSet()); return dep; } @@ -218,9 +219,9 @@ public class ExprProcFactory { Dependency dep = lctx.getIndex().getDependency(inpOp, internalName); if ((tabAlias == null || tabAlias.startsWith("_") || tabAlias.startsWith("$")) && (dep != null && dep.getType() == DependencyType.SIMPLE)) { - List baseCols = dep.getBaseCols(); + Set baseCols = dep.getBaseCols(); if (baseCols != null && !baseCols.isEmpty()) { - BaseColumnInfo baseCol = baseCols.get(0); + BaseColumnInfo baseCol = baseCols.iterator().next(); tabAlias = baseCol.getTabAlias().getAlias(); alias = baseCol.getColumn().getName(); } http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java index d26d8da..c33d775 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java @@ -25,7 +25,9 @@ import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; +import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.ql.exec.ColumnInfo; +import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.hooks.LineageInfo; @@ -33,6 +35,7 @@ import org.apache.hadoop.hive.ql.hooks.LineageInfo.BaseColumnInfo; import org.apache.hadoop.hive.ql.hooks.LineageInfo.Dependency; import org.apache.hadoop.hive.ql.hooks.LineageInfo.Predicate; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.plan.OperatorDesc; @@ -59,7 +62,11 @@ public class LineageCtx implements NodeProcessorCtx { */ private final Map, Set> condMap; - private SelectOperator finalSelectOp; + /** + * A map from a final select operator id to the select operator + * and the corresponding target table in case an insert into query. + */ + private LinkedHashMap> finalSelectOps; /** * Constructor. @@ -69,6 +76,8 @@ public class LineageCtx implements NodeProcessorCtx { new LinkedHashMap, LinkedHashMap>(); condMap = new HashMap, Set>(); + finalSelectOps = + new LinkedHashMap>(); } /** @@ -146,7 +155,7 @@ public class LineageCtx implements NodeProcessorCtx { old_dep.setType(new_type); Set bci_set = new LinkedHashSet(old_dep.getBaseCols()); bci_set.addAll(dep.getBaseCols()); - old_dep.setBaseCols(new ArrayList(bci_set)); + old_dep.setBaseCols(bci_set); // TODO: Fix the expressions later. old_dep.setExpr(null); } @@ -179,16 +188,27 @@ public class LineageCtx implements NodeProcessorCtx { return condMap.get(op); } - public void setFinalSelectOp(SelectOperator sop) { - finalSelectOp = sop; + public void addFinalSelectOp( + SelectOperator sop, Operator sinkOp) { + String operatorId = sop.getOperatorId(); + if (!finalSelectOps.containsKey(operatorId)) { + Table table = null; + if (sinkOp instanceof FileSinkOperator) { + FileSinkOperator fso = (FileSinkOperator) sinkOp; + table = fso.getConf().getTable(); + } + finalSelectOps.put(operatorId, + new ObjectPair(sop, table)); + } } - public SelectOperator getFinalSelectOp() { - return finalSelectOp; + public LinkedHashMap> getFinalSelectOps() { + return finalSelectOps; } public void clear() { - finalSelectOp = null; + finalSelectOps.clear(); depMap.clear(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java index f670db8..5c5d0d6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java @@ -120,7 +120,7 @@ public class OpProcFactory { } dep.setType(new_type); - dep.setBaseCols(new ArrayList(col_set)); + dep.setBaseCols(col_set); boolean isScript = op instanceof ScriptOperator; @@ -186,7 +186,7 @@ public class OpProcFactory { // Populate the dependency dep.setType(LineageInfo.DependencyType.SIMPLE); - dep.setBaseCols(new ArrayList()); + dep.setBaseCols(new LinkedHashSet()); dep.getBaseCols().add(bci); // Put the dependency in the map @@ -396,7 +396,7 @@ public class OpProcFactory { } if (op == null || (op.getChildOperators().isEmpty() && op instanceof FileSinkOperator)) { - lctx.getIndex().setFinalSelectOp(sop); + lctx.getIndex().addFinalSelectOp(sop, op); } return null; @@ -450,7 +450,7 @@ public class OpProcFactory { new_type = LineageCtx.getNewDependencyType(expr_dep.getType(), new_type); bci_set.addAll(expr_dep.getBaseCols()); if (expr_dep.getType() == LineageInfo.DependencyType.SIMPLE) { - BaseColumnInfo col = expr_dep.getBaseCols().get(0); + BaseColumnInfo col = expr_dep.getBaseCols().iterator().next(); Table t = col.getTabAlias().getTable(); if (t != null) { sb.append(t.getDbName()).append(".").append(t.getTableName()).append("."); @@ -514,7 +514,7 @@ public class OpProcFactory { } } - dep.setBaseCols(new ArrayList(bci_set)); + dep.setBaseCols(bci_set); dep.setType(new_type); lctx.getIndex().putDependency(gop, col_infos.get(cnt++), dep); } http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/test/queries/clientpositive/lineage3.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/lineage3.q b/ql/src/test/queries/clientpositive/lineage3.q index 53fff0f..c24ff7d 100644 --- a/ql/src/test/queries/clientpositive/lineage3.q +++ b/ql/src/test/queries/clientpositive/lineage3.q @@ -1,5 +1,20 @@ set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.LineageLogger; +drop table if exists d1; +create table d1(a int); + +from (select a.ctinyint x, b.cstring1 y +from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t +insert into table d1 select x + length(y); + +drop table if exists d2; +create table d2(b varchar(128)); + +from (select a.ctinyint x, b.cstring1 y +from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t +insert into table d1 select x where y is null +insert into table d2 select y where x > 0; + drop table if exists t; create table t as select * from http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/test/results/clientpositive/lineage3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/lineage3.q.out b/ql/src/test/results/clientpositive/lineage3.q.out index 75d88f8..b6b4e0b 100644 --- a/ql/src/test/results/clientpositive/lineage3.q.out +++ b/ql/src/test/results/clientpositive/lineage3.q.out @@ -1,3 +1,31 @@ +PREHOOK: query: drop table if exists d1 +PREHOOK: type: DROPTABLE +PREHOOK: query: create table d1(a int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@d1 +PREHOOK: query: from (select a.ctinyint x, b.cstring1 y +from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t +insert into table d1 select x + length(y) +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@d1 +{"version":"1.0","engine":"mr","hash":"4c9b7b8d89403cef78668f15d393e542","queryText":"from (select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on a.cint = b.cbigint) t\ninsert into table d1 select x + length(y)","edges":[{"sources":[1,2],"targets":[0],"expression":"(UDFToInteger(a.ctinyint) + length(a.cstring1))","edgeType":"PROJECTION"},{"sources":[3,4],"targets":[0],"expression":"(UDFToLong(a.cint) = a.cbigint)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]} +PREHOOK: query: drop table if exists d2 +PREHOOK: type: DROPTABLE +PREHOOK: query: create table d2(b varchar(128)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@d2 +PREHOOK: query: from (select a.ctinyint x, b.cstring1 y +from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t +insert into table d1 select x where y is null +insert into table d2 select y where x > 0 +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@d1 +PREHOOK: Output: default@d2 +{"version":"1.0","engine":"mr","hash":"8703e4091ebd4c96afd3cac83e3a2957","queryText":"from (select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on a.cint = b.cbigint) t\ninsert into table d1 select x where y is null\ninsert into table d2 select y where x > 0","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(x)","edgeType":"PROJECTION"},{"sources":[3],"targets":[0,1],"expression":"t.y is null","edgeType":"PREDICATE"},{"sources":[4,5],"targets":[0,1],"expression":"(UDFToLong(a.cint) = b.cbigint)","edgeType":"PREDICATE"},{"sources":[3],"targets":[1],"expression":"CAST( y AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(t.x > 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.d2.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc. cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]} PREHOOK: query: drop table if exists t PREHOOK: type: DROPTABLE PREHOOK: query: create table t as @@ -23,7 +51,7 @@ where cint is not null and cint < 0 order by cint, cs limit 5 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Output: default@dest_l1@ds=today -{"version":"1.0","engine":"mr","hash":"2b5891d094ff74e23ec6acf5b4990f45","queryText":"insert into table dest_l1 partition (ds='today')\nselect cint, cast(cstring1 as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cint < 0 order by cint, cs limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( alltypesorc.cstring1 AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cint < 0))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"cint"},{"id":1,"vertexType":"COLUMN","vertexId":"cs"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]} +{"version":"1.0","engine":"mr","hash":"2b5891d094ff74e23ec6acf5b4990f45","queryText":"insert into table dest_l1 partition (ds='today')\nselect cint, cast(cstring1 as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cint < 0 order by cint, cs limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( alltypesorc.cstring1 AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cint < 0))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]} PREHOOK: query: insert into table dest_l1 partition (ds='tomorrow') select min(cint), cast(min(cstring1) as varchar(128)) as cs from alltypesorc @@ -33,7 +61,7 @@ having min(cbigint) > 10 PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Output: default@dest_l1@ds=tomorrow -{"version":"1.0","engine":"mr","hash":"4ad6338a8abfe3fe0342198fcbd1f11d","queryText":"insert into table dest_l1 partition (ds='tomorrow')\nselect min(cint), cast(min(cstring1) as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cboolean1 = true\ngroup by csmallint\nhaving min(cbigint) > 10","edges":[{"sources":[2],"targets":[0],"expression":"min(default.alltypesorc.cint)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( min(default.alltypesorc.cstring1) AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2,4],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cboolean1 = true))","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1],"expression":"(min(default.alltypesorc.cbigint) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c0"},{"id":1,"vertexType":"COLUMN","vertexId":"cs"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN", "vertexId":"default.alltypesorc.cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]} +{"version":"1.0","engine":"mr","hash":"4ad6338a8abfe3fe0342198fcbd1f11d","queryText":"insert into table dest_l1 partition (ds='tomorrow')\nselect min(cint), cast(min(cstring1) as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cboolean1 = true\ngroup by csmallint\nhaving min(cbigint) > 10","edges":[{"sources":[2],"targets":[0],"expression":"min(default.alltypesorc.cint)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( min(default.alltypesorc.cstring1) AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2,4],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cboolean1 = true))","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1],"expression":"(min(default.alltypesorc.cbigint) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"}, {"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]} PREHOOK: query: select cint, rank() over(order by cint) from alltypesorc where cint > 10 and cint < 10000 limit 10 PREHOOK: type: QUERY