hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject [38/50] [abbrv] hive git commit: HIVE-11311: Avoid dumping AST tree String in Explain unless necessary (Jesus Camacho Rodriguez, reviewed by Hari Sankar Sivarama Subramaniyan)
Date Wed, 22 Jul 2015 20:48:06 GMT
HIVE-11311: Avoid dumping AST tree String in Explain unless necessary (Jesus Camacho Rodriguez,
reviewed by Hari Sankar Sivarama Subramaniyan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dfdc6700
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dfdc6700
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dfdc6700

Branch: refs/heads/hbase-metastore
Commit: dfdc6700c7a81320a3e0909305645db0248bf01a
Parents: 55853b4
Author: Jesus Camacho Rodriguez <jcamacho@apache.org>
Authored: Tue Jul 21 17:38:39 2015 +0100
Committer: Jesus Camacho Rodriguez <jcamacho@apache.org>
Committed: Tue Jul 21 17:38:39 2015 +0100

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/ql/Driver.java  |  6 ++--
 .../apache/hadoop/hive/ql/exec/ExplainTask.java |  7 ++--
 .../hive/ql/parse/ExplainSemanticAnalyzer.java  |  2 +-
 .../apache/hadoop/hive/ql/plan/ExplainWork.java | 19 +++++++----
 .../parse/TestUpdateDeleteSemanticAnalyzer.java | 34 ++++++++++----------
 5 files changed, 37 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index b74e5fa..6ad3f49 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -482,7 +482,7 @@ public class Driver implements CommandProcessor {
       }
 
       if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) {
-        String explainOutput = getExplainOutput(sem, plan, tree.dump());
+        String explainOutput = getExplainOutput(sem, plan, tree);
         if (explainOutput != null) {
           LOG.info("EXPLAIN output for queryid " + queryId + " : "
               + explainOutput);
@@ -533,7 +533,7 @@ public class Driver implements CommandProcessor {
    * @throws java.io.IOException
    */
   private String getExplainOutput(BaseSemanticAnalyzer sem, QueryPlan plan,
-      String astStringTree) throws IOException {
+      ASTNode astTree) throws IOException {
     String ret = null;
     ExplainTask task = new ExplainTask();
     task.initialize(conf, plan, null);
@@ -541,7 +541,7 @@ public class Driver implements CommandProcessor {
     PrintStream ps = new PrintStream(baos);
     try {
       List<Task<?>> rootTasks = sem.getRootTasks();
-      task.getJSONPlan(ps, astStringTree, rootTasks, sem.getFetchTask(), false, true, true);
+      task.getJSONPlan(ps, astTree, rootTasks, sem.getFetchTask(), false, true, true);
       ret = baos.toString();
     } catch (Exception e) {
       LOG.warn("Exception generating explain output: " + e, e);

http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index c6b49bf..21de3cf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.ql.DriverContext;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
 import org.apache.hadoop.hive.ql.plan.Explain;
 import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -171,11 +172,11 @@ public class ExplainTask extends Task<ExplainWork> implements
Serializable {
 
   public JSONObject getJSONPlan(PrintStream out, ExplainWork work)
       throws Exception {
-    return getJSONPlan(out, work.getAstStringTree(), work.getRootTasks(), work.getFetchTask(),
+    return getJSONPlan(out, work.getAstTree(), work.getRootTasks(), work.getFetchTask(),
                        work.isFormatted(), work.getExtended(), work.isAppendTaskType());
   }
 
-  public JSONObject getJSONPlan(PrintStream out, String ast, List<Task<?>> tasks,
Task<?> fetchTask,
+  public JSONObject getJSONPlan(PrintStream out, ASTNode ast, List<Task<?>> tasks,
Task<?> fetchTask,
       boolean jsonOutput, boolean isExtended, boolean appendTaskType) throws Exception {
 
     // If the user asked for a formatted output, dump the json output
@@ -188,7 +189,7 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable
{
 
     // Print out the parse AST
     if (ast != null && isExtended) {
-      String jsonAST = outputAST(ast, out, jsonOutput, 0);
+      String jsonAST = outputAST(ast.dump(), out, jsonOutput, 0);
       if (out != null) {
         out.println();
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
index 66d1546..2d365a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
@@ -102,7 +102,7 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
         pCtx,
         tasks,
         fetchTask,
-        input.dump(),
+        input,
         sem,
         extended,
         formatted,

http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
index aa208a5..7a2f883 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
@@ -26,8 +26,9 @@ import java.util.List;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
-import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
 
 /**
  * ExplainWork.
@@ -39,6 +40,7 @@ public class ExplainWork implements Serializable {
   private Path resFile;
   private ArrayList<Task<? extends Serializable>> rootTasks;
   private Task<? extends Serializable> fetchTask;
+  private ASTNode astTree;
   private String astStringTree;
   private HashSet<ReadEntity> inputs;
   private ParseContext pCtx;
@@ -63,7 +65,7 @@ public class ExplainWork implements Serializable {
       ParseContext pCtx,
       List<Task<? extends Serializable>> rootTasks,
       Task<? extends Serializable> fetchTask,
-      String astStringTree,
+      ASTNode astTree,
       BaseSemanticAnalyzer analyzer,
       boolean extended,
       boolean formatted,
@@ -75,7 +77,7 @@ public class ExplainWork implements Serializable {
     this.resFile = resFile;
     this.rootTasks = new ArrayList<Task<? extends Serializable>>(rootTasks);
     this.fetchTask = fetchTask;
-    this.astStringTree = astStringTree;
+    this.astTree = astTree;
     this.analyzer = analyzer;
     this.inputs = analyzer.getInputs();
     this.extended = extended;
@@ -112,12 +114,15 @@ public class ExplainWork implements Serializable {
     this.fetchTask = fetchTask;
   }
 
-  public String getAstStringTree() {
-    return astStringTree;
+  public ASTNode getAstTree() {
+    return astTree;
   }
 
-  public void setAstStringTree(String astStringTree) {
-    this.astStringTree = astStringTree;
+  public String getAstStringTree() {
+    if (astStringTree == null) {
+      astStringTree = astTree.dump();
+    }
+    return astStringTree;
   }
 
   public HashSet<ReadEntity> getInputs() {

http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
index f0435cb..f78f226 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
@@ -59,7 +59,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
     try {
       ReturnInfo rc = parseAndAnalyze("insert into table T select a, b from U", "testInsertSelect");
 
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
 
     } finally {
       cleanupTables();
@@ -70,7 +70,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
   public void testDeleteAllNonPartitioned() throws Exception {
     try {
       ReturnInfo rc = parseAndAnalyze("delete from T", "testDeleteAllNonPartitioned");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -80,7 +80,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
   public void testDeleteWhereNoPartition() throws Exception {
     try {
       ReturnInfo rc = parseAndAnalyze("delete from T where a > 5", "testDeleteWhereNoPartition");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -90,7 +90,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
   public void testDeleteAllPartitioned() throws Exception {
     try {
       ReturnInfo rc = parseAndAnalyze("delete from U", "testDeleteAllPartitioned");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -100,7 +100,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
   public void testDeleteAllWherePartitioned() throws Exception {
     try {
       ReturnInfo rc = parseAndAnalyze("delete from U where a > 5", "testDeleteAllWherePartitioned");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -111,7 +111,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
     try {
       ReturnInfo rc = parseAndAnalyze("delete from U where ds = 'today'",
           "testDeleteFromPartitionOnly");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -122,7 +122,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
     try {
       ReturnInfo rc = parseAndAnalyze("delete from U where ds = 'today' and a > 5",
           "testDeletePartitionWhere");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -132,7 +132,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
   public void testUpdateAllNonPartitioned() throws Exception {
     try {
       ReturnInfo rc = parseAndAnalyze("update T set b = 5", "testUpdateAllNonPartitioned");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -143,7 +143,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
     try {
       ReturnInfo rc = parseAndAnalyze("update T set b = 5 where b > 5",
           "testUpdateAllNonPartitionedWhere");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -153,7 +153,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
   public void testUpdateAllPartitioned() throws Exception {
     try {
       ReturnInfo rc = parseAndAnalyze("update U set b = 5", "testUpdateAllPartitioned");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -164,7 +164,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
     try {
       ReturnInfo rc = parseAndAnalyze("update U set b = 5 where b > 5",
           "testUpdateAllPartitionedWhere");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -175,7 +175,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
     try {
       ReturnInfo rc = parseAndAnalyze("update U set b = 5 where ds = 'today'",
           "testUpdateOnePartition");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -186,7 +186,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
     try {
       ReturnInfo rc = parseAndAnalyze("update U set b = 5 where ds = 'today' and b > 5",
           "testUpdateOnePartitionWhere");
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
     } finally {
       cleanupTables();
     }
@@ -198,7 +198,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
       ReturnInfo rc = parseAndAnalyze("insert into table T values ('abc', 3), ('ghi', null)",
           "testInsertValues");
 
-      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
 
     } finally {
       cleanupTables();
@@ -212,7 +212,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
               "('abc', 3, 'today'), ('ghi', 5, 'tomorrow')",
           "testInsertValuesPartitioned");
 
-      LOG.info(explain((SemanticAnalyzer) rc.sem, rc.plan, rc.ast.dump()));
+      LOG.info(explain((SemanticAnalyzer) rc.sem, rc.plan, rc.ast));
 
     } finally {
       cleanupTables();
@@ -285,7 +285,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
     return new ReturnInfo(tree, sem, plan);
   }
 
-  private String explain(SemanticAnalyzer sem, QueryPlan plan, String astStringTree) throws
+  private String explain(SemanticAnalyzer sem, QueryPlan plan, ASTNode astTree) throws
       IOException {
     FileSystem fs = FileSystem.get(conf);
     File f = File.createTempFile("TestSemanticAnalyzer", "explain");
@@ -293,7 +293,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
     fs.create(tmp);
     fs.deleteOnExit(tmp);
     ExplainWork work = new ExplainWork(tmp, sem.getParseContext(), sem.getRootTasks(),
-        sem.getFetchTask(), astStringTree, sem, true, false, false, false, false, false,
null);
+        sem.getFetchTask(), astTree, sem, true, false, false, false, false, false, null);
     ExplainTask task = new ExplainTask();
     task.setWork(work);
     task.initialize(conf, plan, null);


Mime
View raw message