quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hbdeshm...@apache.org
Subject incubator-quickstep git commit: Function to log work order execution statistics. [Forced Update!]
Date Fri, 01 Jul 2016 18:12:44 GMT
Repository: incubator-quickstep
Updated Branches:
  refs/heads/scheduler++ 4c3169e45 -> ad2f2b489 (forced update)


Function to log work order execution statistics.

- Change log severity levels.
- Minor style fixes.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/ad2f2b48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/ad2f2b48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/ad2f2b48

Branch: refs/heads/scheduler++
Commit: ad2f2b489562180723ab12b09569a5ca32f10bcb
Parents: efd88a7
Author: Harshad Deshmukh <hbdeshmukh@apache.org>
Authored: Fri Jul 1 11:38:18 2016 -0500
Committer: Harshad Deshmukh <hbdeshmukh@apache.org>
Committed: Fri Jul 1 13:12:09 2016 -0500

----------------------------------------------------------------------
 cli/InputParserUtil.cpp                         |  7 +++--
 query_execution/CMakeLists.txt                  |  2 --
 query_execution/Learner.cpp                     | 27 +++++++++++++++++---
 query_execution/Learner.hpp                     |  3 +++
 query_execution/PriorityPolicyEnforcer.cpp      | 13 +++++-----
 query_execution/QueryExecutionMessages.proto    |  6 +++++
 query_execution/QueryExecutionUtil.hpp          |  1 +
 query_execution/Worker.cpp                      |  5 ++++
 .../tests/ExecutionGeneratorTestRunner.cpp      |  1 +
 9 files changed, 48 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ad2f2b48/cli/InputParserUtil.cpp
----------------------------------------------------------------------
diff --git a/cli/InputParserUtil.cpp b/cli/InputParserUtil.cpp
index ffc997c..74a2188 100644
--- a/cli/InputParserUtil.cpp
+++ b/cli/InputParserUtil.cpp
@@ -40,9 +40,9 @@
 using std::string;
 
 namespace quickstep {
-  class CatalogRelation;
-  class CatalogDatabase;
-  class StorageManager;
+class CatalogRelation;
+class CatalogDatabase;
+class StorageManager;
 }
 
 namespace quickstep {
@@ -114,7 +114,6 @@ void InputParserUtil::PrintAndDropOutputRelation(
                        query_processor->getDefaultDatabase(),
                        query_processor->getStorageManager());
   }
-
 }
 
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ad2f2b48/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index 13b74e3..9ab86b2 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -107,7 +107,6 @@ target_link_libraries(quickstep_queryexecution_Learner
 target_link_libraries(quickstep_queryexecution_PolicyEnforcer
                       ${GFLAGS_LIB_NAME}
                       glog
-                      quickstep_queryexecution_ExecutionStats
                       quickstep_catalog_CatalogTypedefs
                       quickstep_queryexecution_ProbabilityStore
                       quickstep_queryexecution_QueryExecutionMessages_proto
@@ -122,7 +121,6 @@ target_link_libraries(quickstep_queryexecution_PolicyEnforcer
 target_link_libraries(quickstep_queryexecution_PriorityPolicyEnforcer
                       ${GFLAGS_LIB_NAME}
                       glog
-                      quickstep_queryexecution_ExecutionStats
                       quickstep_catalog_CatalogTypedefs
                       quickstep_queryexecution_Learner
                       quickstep_queryexecution_ProbabilityStore

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ad2f2b48/query_execution/Learner.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Learner.cpp b/query_execution/Learner.cpp
index 3c6f42b..193f33e 100644
--- a/query_execution/Learner.cpp
+++ b/query_execution/Learner.cpp
@@ -19,6 +19,7 @@
 
 #include <algorithm>
 #include <cstddef>
+#include <string>
 #include <unordered_map>
 #include <utility>
 #include <vector>
@@ -61,16 +62,17 @@ void Learner::addCompletionFeedback(
   }
   updateProbabilitiesForQueriesInPriorityLevel(priority_level, query_id);
   updateProbabilitiesOfAllPriorityLevels();
-  printProbabilitiesForPriorityLevel(priority_level);
+  // printProbabilitiesForPriorityLevel(priority_level);
+  printWorkOrderDetails(workorder_completion_proto);
 }
 
 void Learner::updateProbabilitiesForQueriesInPriorityLevel(
     const std::size_t priority_level, const std::size_t query_id) {
   DCHECK(isPriorityLevelPresent(priority_level));
   if (execution_stats_[priority_level].empty()) {
-    LOG(INFO) << "Updating probabilities for query ID: " << query_id
-              << " and priority level: " << priority_level
-              << " that has no queries";
+    LOG(WARNING) << "Updating probabilities for query ID: " << query_id
+                 << " and priority level: " << priority_level
+                 << " that has no queries";
     return;
   } else if (execution_stats_[priority_level].size() == 1u) {
     DCHECK(current_probabilities_[priority_level] != nullptr);
@@ -286,4 +288,21 @@ void Learner::printProbabilitiesForPriorityLevel(const std::size_t priority_leve
   }
 }
 
+void Learner::printWorkOrderDetails(
+    const serialization::NormalWorkOrderCompletionMessage &proto) const {
+  // Format: Query ID, Operator ID, Worker ID, Time in micros, WO execution end timestamp.
+  std::string result = "";
+  result.reserve(30);
+  result += std::to_string(proto.query_id());  // 2 chars
+  result += "|";
+  result += std::to_string(proto.operator_index());  // 2 chars
+  result += "|";
+  result += std::to_string(proto.worker_thread_index());  // 2 chars
+  result += "|";
+  result += std::to_string(proto.execution_time_in_microseconds());  // 5 chars
+  result += "|";
+  result += std::to_string(proto.execution_end_timestamp());  // 12 chars
+  LOG(INFO) << result;
+}
+
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ad2f2b48/query_execution/Learner.hpp
----------------------------------------------------------------------
diff --git a/query_execution/Learner.hpp b/query_execution/Learner.hpp
index 6cea325..8654544 100644
--- a/query_execution/Learner.hpp
+++ b/query_execution/Learner.hpp
@@ -472,6 +472,9 @@ class Learner {
 
   void printProbabilitiesForPriorityLevel(const std::size_t priority_level);
 
+  void printWorkOrderDetails(
+      const serialization::NormalWorkOrderCompletionMessage &proto) const;
+
   // Key = Priority level, value = A vector of pairs.
   // Each pair:
   // 1st element: Query ID.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ad2f2b48/query_execution/PriorityPolicyEnforcer.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PriorityPolicyEnforcer.cpp b/query_execution/PriorityPolicyEnforcer.cpp
index 93908a9..84082ed 100644
--- a/query_execution/PriorityPolicyEnforcer.cpp
+++ b/query_execution/PriorityPolicyEnforcer.cpp
@@ -51,7 +51,8 @@ bool PriorityPolicyEnforcer::admitQuery(QueryHandle *query_handle) {
       admitted_queries_[query_id].reset(
           new QueryManager(foreman_client_id_, num_numa_nodes_, query_handle,
                            catalog_database_, storage_manager_, bus_));
-      LOG(INFO) << "Admitted query with ID: " << query_handle->query_id()
<< " priority: " << query_handle->query_priority();
+      DLOG(INFO) << "Admitted query with ID: " << query_handle->query_id()
+                 << " priority: " << query_handle->query_priority();
       priority_query_ids_[query_handle->query_priority()].emplace_back(query_id);
       learner_->addQuery(*query_handle);
       query_handle->setAdmissionTime();
@@ -63,7 +64,7 @@ bool PriorityPolicyEnforcer::admitQuery(QueryHandle *query_handle) {
     }
   } else {
     // This query will have to wait.
-    LOG(INFO) << "Query " << query_handle->query_id() << " waitlisted";
+    DLOG(INFO) << "Query " << query_handle->query_id() << " waitlisted";
     query_id_to_handle_[query_handle->query_id()] = query_handle;
     waiting_queries_.push(query_handle);
     return false;
@@ -139,7 +140,7 @@ void PriorityPolicyEnforcer::processMessage(const TaggedMessage &tagged_message)
   DCHECK(admitted_queries_.find(query_id) != admitted_queries_.end());
   const QueryManager::QueryStatusCode return_code =
       admitted_queries_[query_id]->processMessage(tagged_message);
-  //NOTE: kQueryExecuted takes precedence over kOperatorExecuted.
+  // NOTE: kQueryExecuted takes precedence over kOperatorExecuted.
   if (return_code == QueryManager::QueryStatusCode::kQueryExecuted) {
     removeQuery(query_id);
     if (!waiting_queries_.empty()) {
@@ -178,11 +179,10 @@ void PriorityPolicyEnforcer::getWorkerMessages(
     while (checked_priority_levels.size() < priority_query_ids_.size()) {
       const int chosen_priority_level = learner_->pickRandomPriorityLevel();
       if (chosen_priority_level == kInvalidPriorityLevel) {
-        LOG(INFO) << "No valid priority level chosen";
+        DLOG(INFO) << "No valid priority level chosen";
         break;
       } else if (checked_priority_levels.find(static_cast<std::size_t>(
                      chosen_priority_level)) != checked_priority_levels.end()) {
-        DLOG(INFO) << "The chosen priority level " << chosen_priority_level <<
" was checked already";
         continue;
       } else {
         WorkerMessage *next_worker_message =
@@ -230,7 +230,7 @@ void PriorityPolicyEnforcer::removeQuery(const std::size_t query_id) {
   // Remove the query from the learner.
   learner_->removeQuery(query_id);
   // TODO(harshad) - Admit waiting queries, if any.
-  LOG(INFO) << "Removed query: " << query_id << " with priority: " <<
query_priority;
+  DLOG(INFO) << "Removed query: " << query_id << " with priority: " <<
query_priority;
 }
 
 bool PriorityPolicyEnforcer::admitQueries(
@@ -279,7 +279,6 @@ WorkerMessage* PriorityPolicyEnforcer::getNextWorkerMessageFromPriorityLevel(
           std::unique_ptr<WorkerMessage> next_worker_message(
               chosen_query_manager->getNextWorkerMessage(0, kAnyNUMANodeID));
           if (next_worker_message != nullptr) {
-            // LOG(INFO) << "Selecting a work order from query " << qid <<
" instead";
             return next_worker_message.release();
           } else {
             // This query doesn't have any WorkerMessage right now. Mark as checked.

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ad2f2b48/query_execution/QueryExecutionMessages.proto
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionMessages.proto b/query_execution/QueryExecutionMessages.proto
index 65a8946..db35ecc 100644
--- a/query_execution/QueryExecutionMessages.proto
+++ b/query_execution/QueryExecutionMessages.proto
@@ -39,6 +39,8 @@ message NormalWorkOrderCompletionMessage {
   required uint64 worker_thread_index = 2;
   required uint64 query_id = 3;
   optional uint64 execution_time_in_microseconds = 4;
+  // Indicates time elapsed in microseconds since epoch.  
+  optional uint64 execution_end_timestamp = 5;  
 }
 
 // A message sent upon completion of a rebuild WorkOrder execution.
@@ -47,6 +49,10 @@ message RebuildWorkOrderCompletionMessage {
   required uint64 worker_thread_index = 2;
   required uint64 query_id = 3;
   optional uint64 execution_time_in_microseconds = 4;
+  optional uint64 execution_end_timestamp = 5;  // Indicates time elapsed in 
+                                                // microseconds since epoch, 
+                                                // when the work order execution
+                                                // got over. 
 }
 
 message CatalogRelationNewBlockMessage {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ad2f2b48/query_execution/QueryExecutionUtil.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryExecutionUtil.hpp b/query_execution/QueryExecutionUtil.hpp
index 2b95f1f..95ebc8c 100644
--- a/query_execution/QueryExecutionUtil.hpp
+++ b/query_execution/QueryExecutionUtil.hpp
@@ -19,6 +19,7 @@
 
 #include <memory>
 #include <utility>
+#include <vector>
 
 #include "query_execution/AdmitRequestMessage.hpp"
 #include "query_execution/QueryExecutionTypedefs.hpp"

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ad2f2b48/query_execution/Worker.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Worker.cpp b/query_execution/Worker.cpp
index ae889c7..d05caed 100644
--- a/query_execution/Worker.cpp
+++ b/query_execution/Worker.cpp
@@ -123,11 +123,16 @@ void Worker::executeWorkOrderHelper(const TaggedMessage &tagged_message,
   const uint64_t execution_time_microseconds =
       std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
           .count();
+  const uint64_t execution_end_timestamp =
+      std::chrono::duration_cast<std::chrono::microseconds>(
+          end.time_since_epoch())
+          .count();
   // Construct the proto message.
   proto->set_operator_index(worker_message.getRelationalOpIndex());
   proto->set_query_id(query_id_for_workorder);
   proto->set_worker_thread_index(worker_thread_index_);
   proto->set_execution_time_in_microseconds(execution_time_microseconds);
+  proto->set_execution_end_timestamp(execution_end_timestamp);
 }
 
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/ad2f2b48/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
index 92baf19..0fef398 100644
--- a/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
+++ b/query_optimizer/tests/ExecutionGeneratorTestRunner.cpp
@@ -20,6 +20,7 @@
 #include <cstdio>
 #include <set>
 #include <string>
+#include <vector>
 
 #include "cli/DropRelation.hpp"
 #include "cli/PrintToScreen.hpp"


Mime
View raw message