quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zu...@apache.org
Subject incubator-quickstep git commit: Moved some Cli flags into a common file.
Date Fri, 10 Mar 2017 09:09:17 GMT
Repository: incubator-quickstep
Updated Branches:
  refs/heads/master 64316f858 -> 0528c779b


Moved some Cli flags into a common file.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/0528c779
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/0528c779
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/0528c779

Branch: refs/heads/master
Commit: 0528c779be168f01b65e9bd88233d29cb16df037
Parents: 64316f8
Author: Zuyu Zhang <zuyuz@apache.org>
Authored: Fri Mar 10 01:09:11 2017 -0800
Committer: Zuyu Zhang <zuyuz@apache.org>
Committed: Fri Mar 10 01:09:11 2017 -0800

----------------------------------------------------------------------
 cli/Flags.cpp                                 | 12 ++++++++++++
 cli/Flags.hpp                                 |  4 ++++
 cli/InputParserUtil.cpp                       |  6 +++---
 cli/QuickstepCli.cpp                          | 10 ----------
 cli/distributed/CMakeLists.txt                |  1 +
 cli/distributed/Executor.cpp                  | 11 +++++++++--
 query_execution/PolicyEnforcerDistributed.cpp |  1 +
 storage/PreloaderThread.hpp                   |  2 +-
 8 files changed, 31 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0528c779/cli/Flags.cpp
----------------------------------------------------------------------
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
index 1e3ce1d..74915ae 100644
--- a/cli/Flags.cpp
+++ b/cli/Flags.cpp
@@ -58,6 +58,13 @@ DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is
"
 static const volatile bool num_workers_dummy
     = gflags::RegisterFlagValidator(&FLAGS_num_workers, &ValidateNumWorkers);
 
+DEFINE_string(worker_affinities, "",
+              "A comma-separated list of CPU IDs to pin worker threads to "
+              "(leaving this empty will cause all worker threads to inherit "
+              "the affinity mask of the Quickstep process, which typically "
+              "means that they will all be runable on any CPU according to "
+              "the kernel's own scheduling policy).");
+
 static bool ValidateStoragePath(const char *flagname,
                                 const std::string &value) {
   if (!value.empty() && value.back() != kPathSeparator) {
@@ -71,4 +78,9 @@ DEFINE_string(storage_path, kDefaultStoragePath,
 static const volatile bool storage_path_dummy
     = gflags::RegisterFlagValidator(&FLAGS_storage_path, &ValidateStoragePath);
 
+DEFINE_bool(preload_buffer_pool, false,
+            "If true, pre-load all known blocks into buffer pool before "
+            "accepting queries (should also set --buffer_pool_slots to be "
+            "large enough to accomodate the entire database).");
+
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0528c779/cli/Flags.hpp
----------------------------------------------------------------------
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
index 70aee98..a268e39 100644
--- a/cli/Flags.hpp
+++ b/cli/Flags.hpp
@@ -37,8 +37,12 @@ DECLARE_bool(initialize_db);
 
 DECLARE_int32(num_workers);
 
+DECLARE_string(worker_affinities);
+
 DECLARE_string(storage_path);
 
+DECLARE_bool(preload_buffer_pool);
+
 /** @} */
 
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0528c779/cli/InputParserUtil.cpp
----------------------------------------------------------------------
diff --git a/cli/InputParserUtil.cpp b/cli/InputParserUtil.cpp
index e45605c..b092dfa 100644
--- a/cli/InputParserUtil.cpp
+++ b/cli/InputParserUtil.cpp
@@ -51,9 +51,9 @@ std::vector<int> InputParserUtil::ParseWorkerAffinities(
     LOG(INFO) << "Empty worker affinities provided, switching to default "
                  "worker affinities";
   } else if (!ParseIntString(affinity_string, ',', &affinities)) {
-      switch_to_default_affinities = true;
-      LOG(INFO) << "Invalid worker affinities provided, switching to default "
-                   "affinities";
+    switch_to_default_affinities = true;
+    LOG(INFO) << "Invalid worker affinities provided, switching to default "
+                 "affinities";
   }
 
   for (const int affinity : affinities) {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0528c779/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index eddee8c..26cb154 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -119,16 +119,6 @@ using tmb::client_id;
 
 namespace quickstep {
 
-DEFINE_bool(preload_buffer_pool, false,
-            "If true, pre-load all known blocks into buffer pool before "
-            "accepting queries (should also set --buffer_pool_slots to be "
-            "large enough to accomodate the entire database).");
-DEFINE_string(worker_affinities, "",
-              "A comma-separated list of CPU IDs to pin worker threads to "
-              "(leaving this empty will cause all worker threads to inherit "
-              "the affinity mask of the Quickstep process, which typically "
-              "means that they will all be runable on any CPU according to "
-              "the kernel's own scheduling policy).");
 DEFINE_bool(print_query, false,
             "Print each input query statement. This is useful when running a "
             "large number of queries in a batch.");

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0528c779/cli/distributed/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/distributed/CMakeLists.txt b/cli/distributed/CMakeLists.txt
index 2db27e5..6a31895 100644
--- a/cli/distributed/CMakeLists.txt
+++ b/cli/distributed/CMakeLists.txt
@@ -56,6 +56,7 @@ target_link_libraries(quickstep_cli_distributed_Executor
                       glog
                       quickstep_catalog_CatalogTypedefs
                       quickstep_cli_Flags
+                      quickstep_cli_InputParserUtil
                       quickstep_cli_distributed_Role
                       quickstep_queryexecution_BlockLocatorUtil
                       quickstep_queryexecution_QueryExecutionTypedefs

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0528c779/cli/distributed/Executor.cpp
----------------------------------------------------------------------
diff --git a/cli/distributed/Executor.cpp b/cli/distributed/Executor.cpp
index a95ed41..1415e99 100644
--- a/cli/distributed/Executor.cpp
+++ b/cli/distributed/Executor.cpp
@@ -25,6 +25,7 @@
 
 #include "catalog/CatalogTypedefs.hpp"
 #include "cli/Flags.hpp"
+#include "cli/InputParserUtil.hpp"
 #include "query_execution/BlockLocatorUtil.hpp"
 #include "query_execution/QueryExecutionTypedefs.hpp"
 #include "query_execution/Shiftboss.hpp"
@@ -56,13 +57,19 @@ void Executor::init() {
   bus_.RegisterClientAsSender(executor_client_id_, kBlockDomainRegistrationMessage);
   bus_.RegisterClientAsReceiver(executor_client_id_, kBlockDomainRegistrationResponseMessage);
 
+  // Parse the CPU affinities for workers and the preloader thread, if enabled
+  // to warm up the buffer pool.
+  const vector<int> worker_cpu_affinities =
+      InputParserUtil::ParseWorkerAffinities(FLAGS_num_workers, FLAGS_worker_affinities);
+
+  const vector<numa_node_id> worker_numa_nodes(FLAGS_num_workers, kAnyNUMANodeID);
   vector<client_id> worker_client_ids;
-  vector<numa_node_id> worker_numa_nodes(FLAGS_num_workers, kAnyNUMANodeID);
 
   for (std::size_t worker_thread_index = 0;
        worker_thread_index < FLAGS_num_workers;
        ++worker_thread_index) {
-    workers_.push_back(make_unique<Worker>(worker_thread_index, &bus_local_));
+    workers_.push_back(make_unique<Worker>(worker_thread_index, &bus_local_,
+                                           worker_cpu_affinities[worker_thread_index]));
     worker_client_ids.push_back(workers_.back()->getBusClientID());
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0528c779/query_execution/PolicyEnforcerDistributed.cpp
----------------------------------------------------------------------
diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp
index 857d9aa..9f246ed 100644
--- a/query_execution/PolicyEnforcerDistributed.cpp
+++ b/query_execution/PolicyEnforcerDistributed.cpp
@@ -88,6 +88,7 @@ void PolicyEnforcerDistributed::getWorkOrderProtoMessages(
   if (admitted_queries_.empty()) {
     if (waiting_queries_.empty()) {
       return;
+    }
 
     // Admit the earliest waiting query.
     QueryHandle *new_query = waiting_queries_.front();

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/0528c779/storage/PreloaderThread.hpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp
index aaea790..d9853b8 100644
--- a/storage/PreloaderThread.hpp
+++ b/storage/PreloaderThread.hpp
@@ -58,7 +58,7 @@ class PreloaderThread : public Thread {
    **/
   PreloaderThread(const CatalogDatabase &database,
                   StorageManager *storage_manager,
-                  const int cpu_id)
+                  const int cpu_id = -1)
       : database_(database),
         storage_manager_(storage_manager),
         cpu_id_(cpu_id) {


Mime
View raw message