quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zu...@apache.org
Subject [05/38] incubator-quickstep git commit: Visualized Execution DAG in the distributed version.
Date Mon, 06 Mar 2017 20:18:02 GMT
Visualized Execution DAG in the distributed version.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f5c063a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f5c063a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f5c063a1

Branch: refs/heads/reorder-partitioned-hash-join
Commit: f5c063a19d0b9ff4327041f707a1dc38c343f727
Parents: 2b2d7ba
Author: Zuyu Zhang <zuyuz@apache.org>
Authored: Fri Feb 10 22:01:48 2017 -0800
Committer: Zuyu Zhang <zuyuz@apache.org>
Committed: Thu Feb 23 00:39:05 2017 -0800

----------------------------------------------------------------------
 query_execution/CMakeLists.txt                |  5 ++++-
 query_execution/PolicyEnforcerBase.cpp        |  7 ++----
 query_execution/PolicyEnforcerBase.hpp        | 16 +++++++++++--
 query_execution/PolicyEnforcerDistributed.cpp | 26 +++++++++++++++++++++-
 query_execution/QueryManagerBase.cpp          | 11 +++++++++
 query_execution/QueryManagerBase.hpp          | 12 ++++++++++
 query_optimizer/QueryHandle.hpp               |  7 ++++++
 7 files changed, 75 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 5ad6999..50bf694 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -166,6 +166,7 @@ if (ENABLE_DISTRIBUTED)
                         quickstep_queryexecution_ShiftbossDirectory
                         quickstep_queryoptimizer_QueryHandle
                         quickstep_storage_StorageBlockInfo
+                        quickstep_utility_ExecutionDAGVisualizer
                         quickstep_utility_Macros
                         tmb
                         ${GFLAGS_LIB_NAME})
@@ -246,7 +247,9 @@ target_link_libraries(quickstep_queryexecution_QueryManagerBase
                       quickstep_relationaloperators_WorkOrder
                       quickstep_storage_StorageBlockInfo
                       quickstep_utility_DAG
-                      quickstep_utility_Macros)
+                      quickstep_utility_ExecutionDAGVisualizer
+                      quickstep_utility_Macros
+                      ${GFLAGS_LIB_NAME})
 if (ENABLE_DISTRIBUTED)
   target_link_libraries(quickstep_queryexecution_QueryManagerDistributed
                         quickstep_catalog_CatalogTypedefs

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/PolicyEnforcerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp
index 082f6e9..1ffde4d 100644
--- a/query_execution/PolicyEnforcerBase.cpp
+++ b/query_execution/PolicyEnforcerBase.cpp
@@ -40,15 +40,12 @@
 
 namespace quickstep {
 
+DECLARE_bool(visualize_execution_dag);
+
 DEFINE_bool(profile_and_report_workorder_perf, false,
     "If true, Quickstep will record the exceution time of all the individual "
     "normal work orders and report it at the end of query execution.");
 
-DEFINE_bool(visualize_execution_dag, false,
-            "If true, visualize the execution plan DAG into a graph in DOT "
-            "format (DOT is a plain text graph description language) which is "
-            "then printed via stderr.");
-
 PolicyEnforcerBase::PolicyEnforcerBase(CatalogDatabaseLite *catalog_database)
     : catalog_database_(catalog_database),
       profile_individual_workorders_(FLAGS_profile_and_report_workorder_perf || FLAGS_visualize_execution_dag)
{

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/PolicyEnforcerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp
index 4107817..f66134b 100644
--- a/query_execution/PolicyEnforcerBase.hpp
+++ b/query_execution/PolicyEnforcerBase.hpp
@@ -113,6 +113,19 @@ class PolicyEnforcerBase {
   }
 
   /**
+   * @brief Check if the given query has profiling results.
+   *
+   * @note Even enabled profiling, not every query has profiling results.
+   *       For example, CreateTable and CreateIndex do not produce work orders,
+   *       so they do not have profiling results.
+   *
+   * @return True if it has profiling results, otherwise false.
+   **/
+  bool hasProfilingResults(const std::size_t query_id) const {
+    return workorder_time_recorder_.find(query_id) != workorder_time_recorder_.end();
+  }
+
+  /**
    * @brief Get the profiling results for individual work order execution for a
    *        given query.
    *
@@ -127,8 +140,7 @@ class PolicyEnforcerBase {
   inline const std::vector<WorkOrderTimeEntry>& getProfilingResults(
       const std::size_t query_id) const {
     DCHECK(profile_individual_workorders_);
-    DCHECK(workorder_time_recorder_.find(query_id) !=
-           workorder_time_recorder_.end());
+    DCHECK(hasProfilingResults(query_id));
     return workorder_time_recorder_.at(query_id);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index 8f0332d..6ee58a8 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -15,11 +15,14 @@
 #include "query_execution/PolicyEnforcerDistributed.hpp"
 
 #include <cstddef>
+#include <cstdio>
 #include <cstdlib>
 #include <memory>
 #include <queue>
-#include <utility>
+#include <sstream>
+#include <string>
 #include <unordered_map>
+#include <utility>
 #include <vector>
 
 #include "catalog/Catalog.pb.h"
@@ -33,6 +36,7 @@
 #include "query_execution/QueryManagerDistributed.hpp"
 #include "query_optimizer/QueryHandle.hpp"
 #include "storage/StorageBlockInfo.hpp"
+#include "utility/ExecutionDAGVisualizer.hpp"
 
 #include "gflags/gflags.h"
 #include "glog/logging.h"
@@ -53,6 +57,8 @@ using tmb::TaggedMessage;
 
 namespace quickstep {
 
+DECLARE_bool(visualize_execution_dag);
+
 namespace S = serialization;
 
 DEFINE_uint64(max_msgs_per_dispatch_round, 20, "Maximum number of messages that"
@@ -228,6 +234,24 @@ void PolicyEnforcerDistributed::onQueryCompletion(QueryManagerBase *query_manage
   const tmb::client_id cli_id = query_handle->getClientId();
   const std::size_t query_id = query_handle->query_id();
 
+  if (FLAGS_visualize_execution_dag && hasProfilingResults(query_id)) {
+    ExecutionDAGVisualizer* dag_visualizer = query_manager->dag_visualizer();
+    dag_visualizer->bindProfilingStats(getProfilingResults(query_id));
+
+    std::ostringstream dot_filename;
+    dot_filename << query_id << ".dot";
+
+    FILE *fp = std::fopen(dot_filename.str().c_str(), "w");
+    CHECK_NOTNULL(fp);
+    const std::string dot_file_content(dag_visualizer->toDOT());
+    const std::size_t dot_file_length = dot_file_content.length();
+
+    CHECK_EQ(dot_file_length,
+             std::fwrite(dot_file_content.c_str(), sizeof(char), dot_file_length, fp));
+
+    std::fclose(fp);
+  }
+
   // TODO(quickstep-team): Dynamically scale-up/down Shiftbosses.
   tmb::Address shiftboss_addresses;
   for (std::size_t i = 0; i < shiftboss_directory_->size(); ++i) {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/QueryManagerBase.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp
index 5f8c6a3..f84ad4e 100644
--- a/query_execution/QueryManagerBase.cpp
+++ b/query_execution/QueryManagerBase.cpp
@@ -30,12 +30,18 @@
 #include "relational_operators/WorkOrder.hpp"
 #include "storage/StorageBlockInfo.hpp"
 
+#include "gflags/gflags.h"
 #include "glog/logging.h"
 
 using std::pair;
 
 namespace quickstep {
 
+DEFINE_bool(visualize_execution_dag, false,
+            "If true, visualize the execution plan DAG into a graph in DOT "
+            "format (DOT is a plain text graph description language) which is "
+            "then printed via stderr.");
+
 QueryManagerBase::QueryManagerBase(QueryHandle *query_handle)
     : query_handle_(DCHECK_NOTNULL(query_handle)),
       query_id_(query_handle->query_id()),
@@ -45,6 +51,11 @@ QueryManagerBase::QueryManagerBase(QueryHandle *query_handle)
       output_consumers_(num_operators_in_dag_),
       blocking_dependencies_(num_operators_in_dag_),
       query_exec_state_(new QueryExecutionState(num_operators_in_dag_)) {
+  if (FLAGS_visualize_execution_dag) {
+    dag_visualizer_ =
+        std::make_unique<quickstep::ExecutionDAGVisualizer>(query_handle_->getQueryPlan());
+  }
+
   for (dag_node_index node_index = 0;
        node_index < num_operators_in_dag_;
        ++node_index) {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/QueryManagerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp
index d0bb0ea..27fa6dc 100644
--- a/query_execution/QueryManagerBase.hpp
+++ b/query_execution/QueryManagerBase.hpp
@@ -31,6 +31,7 @@
 #include "relational_operators/WorkOrder.hpp"
 #include "storage/StorageBlockInfo.hpp"
 #include "utility/DAG.hpp"
+#include "utility/ExecutionDAGVisualizer.hpp"
 #include "utility/Macros.hpp"
 
 namespace quickstep {
@@ -149,6 +150,15 @@ class QueryManagerBase {
    **/
   QueryStatusCode queryStatus(const dag_node_index op_index);
 
+  /**
+   * @brief Get the execution DAG visualizer.
+   *
+   * @return the execution DAG visualizer.
+   **/
+  ExecutionDAGVisualizer* dag_visualizer() {
+    return dag_visualizer_.get();
+  }
+
  protected:
   /**
    * @brief Process a current relational operator: Get its workorders and store
@@ -276,6 +286,8 @@ class QueryManagerBase {
 
   std::unique_ptr<QueryExecutionState> query_exec_state_;
 
+  std::unique_ptr<ExecutionDAGVisualizer> dag_visualizer_;
+
  private:
   /**
    * @brief Check if the given operator's normal execution is over.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_optimizer/QueryHandle.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryHandle.hpp b/query_optimizer/QueryHandle.hpp
index cbd1cd9..7cb4f68 100644
--- a/query_optimizer/QueryHandle.hpp
+++ b/query_optimizer/QueryHandle.hpp
@@ -87,6 +87,13 @@ class QueryHandle {
   }
 
   /**
+   * @return The const query plan.
+   */
+  const QueryPlan& getQueryPlan() const {
+    return *query_plan_;
+  }
+
+  /**
    * @return The mutable query plan.
    */
   QueryPlan* getQueryPlanMutable() {


Mime
View raw message