quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hbdeshm...@apache.org
Subject [01/18] incubator-quickstep git commit: Added ExecutionStats class [Forced Update!]
Date Fri, 01 Jul 2016 04:41:31 GMT
Repository: incubator-quickstep
Updated Branches:
  refs/heads/scheduler++ ad8f0d3c2 -> efd88a768 (forced update)


Added ExecutionStats class

- To keep track of query execution statistics for a given query.
- The stats class organizes execution time on a per-operator basis.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/13383a1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/13383a1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/13383a1b

Branch: refs/heads/scheduler++
Commit: 13383a1b41ec0c7d84e128d45bf5e1e2ab0d72f6
Parents: b258821
Author: Harshad Deshmukh <hbdeshmukh@apache.org>
Authored: Sun Jun 19 10:06:02 2016 -0500
Committer: Harshad Deshmukh <hbdeshmukh@apache.org>
Committed: Thu Jun 30 23:41:04 2016 -0500

----------------------------------------------------------------------
 query_execution/CMakeLists.txt     |   6 ++
 query_execution/ExecutionStats.hpp | 177 ++++++++++++++++++++++++++++++++
 query_execution/PolicyEnforcer.hpp |   2 +-
 3 files changed, 184 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13383a1b/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index b031a44..fcd4f48 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -32,6 +32,7 @@ if (ENABLE_DISTRIBUTED)
   add_library(quickstep_queryexecution_BlockLocator BlockLocator.cpp BlockLocator.hpp)
 endif()
 add_library(quickstep_queryexecution_AdmitRequestMessage ../empty_src.cpp AdmitRequestMessage.hpp)
+add_library(quickstep_queryexecution_ExecutionStats ../empty_src.cpp ExecutionStats.hpp)
 add_library(quickstep_queryexecution_Foreman Foreman.cpp Foreman.hpp)
 add_library(quickstep_queryexecution_ForemanLite ../empty_src.cpp ForemanLite.hpp)
 add_library(quickstep_queryexecution_PolicyEnforcer PolicyEnforcer.cpp PolicyEnforcer.hpp)
@@ -69,6 +70,9 @@ if (ENABLE_DISTRIBUTED)
                         quickstep_utility_Macros
                         tmb)
 endif()
+target_link_libraries(quickstep_queryexecution_ExecutionStats
+                      glog
+                      quickstep_utility_Macros)
 target_link_libraries(quickstep_queryexecution_Foreman
                       ${GFLAGS_LIB_NAME} 
                       glog
@@ -91,6 +95,7 @@ target_link_libraries(quickstep_queryexecution_ForemanLite
 target_link_libraries(quickstep_queryexecution_PolicyEnforcer
                       ${GFLAGS_LIB_NAME}
                       glog
+                      quickstep_queryexecution_ExecutionStats
                       quickstep_catalog_CatalogTypedefs
                       quickstep_queryexecution_QueryExecutionMessages_proto
                       quickstep_queryexecution_QueryExecutionTypedefs
@@ -199,6 +204,7 @@ target_link_libraries(quickstep_queryexecution_WorkerSelectionPolicy
 add_library(quickstep_queryexecution ../empty_src.cpp QueryExecutionModule.hpp)
 target_link_libraries(quickstep_queryexecution
                       quickstep_queryexecution_AdmitRequestMessage
+                      quickstep_queryexecution_ExecutionStats
                       quickstep_queryexecution_Foreman
                       quickstep_queryexecution_ForemanLite
                       quickstep_queryexecution_PolicyEnforcer

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13383a1b/query_execution/ExecutionStats.hpp
----------------------------------------------------------------------
diff --git a/query_execution/ExecutionStats.hpp b/query_execution/ExecutionStats.hpp
new file mode 100644
index 0000000..f28f367
--- /dev/null
+++ b/query_execution/ExecutionStats.hpp
@@ -0,0 +1,177 @@
+/**
+ *   Copyright 2016, Quickstep Research Group, Computer Sciences Department,
+ *     University of Wisconsin—Madison.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ **/
+
+#ifndef QUICKSTEP_QUERY_EXECUTION_EXECUTION_STATS_HPP_
+#define QUICKSTEP_QUERY_EXECUTION_EXECUTION_STATS_HPP_
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <deque>
+#include <memory>
+#include <unordered_map>
+#include <utility>
+
+#include "utility/Macros.hpp"
+
+#include "glog/logging.h"
+
+namespace quickstep {
+
+/** \addtogroup QueryExecution
+ *  @{
+ */
+
+/**
+ * @brief Record the execution stats of a query.
+ **/
+class ExecutionStats {
+ public:
+  /**
+   * @brief Constructor
+   *
+   * @param max_entries The maximum number of entries we remember for each
+   *        operator.
+   **/
+  explicit ExecutionStats(const std::size_t max_entries)
+      : max_entries_(max_entries), cached_stats_(std::make_pair(0, 0)) {}
+
+  /**
+   * @brief Get the number of active operators in stats.
+   **/
+  const std::size_t getNumActiveOperators() const {
+    return active_operators_.size();
+  }
+
+  /**
+   * @brief Get the current stats.
+   *
+   * @note This function updates the cache, hence it can't be const. We are lazy
+   *       in updating the cache, instead of eagerly updating the cache upon
+   *       each update.
+   *
+   * @return A pair - 1st element is total time, 2nd element is total number of
+   *         WorkOrders for the whole query.
+   **/
+  std::pair<std::uint64_t, std::uint64_t> getCurrentStats() {
+    if (active_operators_.empty()) {
+      return cached_stats_;
+    } else {
+      std::pair<std::uint64_t, std::uint64_t> result = std::make_pair(0, 0);
+      for (auto it = active_operators_.begin(); it != active_operators_.end(); ++it) {
+        DCHECK(it->second.get() != nullptr);
+        std::pair<std::uint64_t, std::size_t> op_stats = it->second->getStats();
+        result.first += op_stats.first;
+        result.second += op_stats.second;
+      }
+      if (result.first == 0 || result.second == 0) {
+        // If one of the element in the pair is 0, use old result.
+        return cached_stats_;
+      } else if (result.first != 0 && result.second != 0) {
+        cached_stats_ = result;
+      }
+      return result;
+    }
+  }
+
+  /**
+   * @brief Add a new entry to stats.
+   *
+   * @param value The value to be added.
+   * @param operator_index The operator index which the value belongs to.
+   **/
+  void addEntry(std::size_t value, std::size_t operator_index) {
+    if (hasOperator(operator_index)) {
+      // This is not the first entry for the given operator.
+      active_operators_[operator_index]->addEntry(value);
+    } else {
+      // Create the OperatorStats object for this operator.
+      active_operators_[operator_index] =
+          std::unique_ptr<OperatorStats>(new OperatorStats(max_entries_));
+    }
+  }
+
+  /**
+   * @brief Remove the operator with given index. This should be called only
+   *        when the given operator finishes its execution.
+   **/
+  void removeOperator(std::size_t operator_index) {
+    DCHECK(hasOperator(operator_index));
+    active_operators_.erase(operator_index);
+  }
+
+ private:
+  /**
+   * @brief Stats for an operator within the query.
+   *
+   * @note We remember only the last N entries for the operator.
+   **/
+  class OperatorStats {
+   public:
+    /**
+     * @brief Constructor.
+     *
+     * @param max_entries The maximum number of entries we remember. Typically
+     *        these are the last N (=max_entries) entries.
+     **/
+    explicit OperatorStats(const std::size_t max_entries) : max_entries_(max_entries) {
+      DCHECK_GE(max_entries, 0);
+    }
+
+    inline std::pair<std::uint64_t, std::size_t> getStats() const {
+      return std::make_pair(std::accumulate(times_.begin(), times_.end(), 0),
+                            times_.size());
+    }
+
+    inline void addEntry(std::uint64_t time_value) {
+      if (times_.size() == max_entries_) {
+        times_.pop_front();
+      }
+      times_.push_back(time_value);
+      DCHECK_LE(times_.size(), max_entries_);
+    }
+
+   private:
+    const std::size_t max_entries_;
+    std::deque<std::uint64_t> times_;
+
+    DISALLOW_COPY_AND_ASSIGN(OperatorStats);
+  };
+
+  /**
+   * @brief Check if the operator with given index is present in the stats.
+   **/
+  inline bool hasOperator(const std::size_t operator_index) const {
+    return active_operators_.find(operator_index) != active_operators_.end();
+  }
+
+  const std::size_t max_entries_;
+
+  std::unordered_map<std::size_t, std::unique_ptr<OperatorStats>>
+      active_operators_;
+
+  // Cached stats for the whole query.
+  std::pair<std::uint64_t, std::uint64_t> cached_stats_;
+
+  DISALLOW_COPY_AND_ASSIGN(ExecutionStats);
+};
+
+/** @} */
+
+}  // namespace quickstep
+
+#endif  // QUICKSTEP_QUERY_EXECUTION_EXECUTION_STATS_HPP_

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/13383a1b/query_execution/PolicyEnforcer.hpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcer.hpp b/query_execution/PolicyEnforcer.hpp
index 470ff2a..b7f5735 100644
--- a/query_execution/PolicyEnforcer.hpp
+++ b/query_execution/PolicyEnforcer.hpp
@@ -214,4 +214,4 @@ class PolicyEnforcer {
 
 }  // namespace quickstep
 
-#endif  // QUICKSTEP_QUERY_EXECUTION_QUERY_MANAGER_HPP_
+#endif  // QUICKSTEP_QUERY_EXECUTION_POLICY_ENFORCER_HPP_


Mime
View raw message