quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zu...@apache.org
Subject incubator-quickstep git commit: Moved FLAGS_num_workers into a file for shared flags.
Date Mon, 21 Nov 2016 06:43:01 GMT
Repository: incubator-quickstep
Updated Branches:
  refs/heads/FLAGS_num_workers [created] e666a81f4


Moved FLAGS_num_workers into a file for shared flags.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/e666a81f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/e666a81f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/e666a81f

Branch: refs/heads/FLAGS_num_workers
Commit: e666a81f472e4c8ed81e98995c1914ce655e247a
Parents: 212e6cd
Author: Zuyu Zhang <zuyuz@apache.org>
Authored: Sun Nov 20 22:42:49 2016 -0800
Committer: Zuyu Zhang <zuyuz@apache.org>
Committed: Sun Nov 20 22:42:49 2016 -0800

----------------------------------------------------------------------
 cli/CMakeLists.txt   |  1 +
 cli/Flags.cpp        | 29 +++++++++++++++++++++++++++++
 cli/Flags.hpp        |  2 ++
 cli/QuickstepCli.cpp | 31 +++++--------------------------
 4 files changed, 37 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 9928f3f..9847787 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -112,6 +112,7 @@ if(QUICKSTEP_HAVE_LIBNUMA)
                       ${LIBNUMA_LIBRARY})
 endif()
 target_link_libraries(quickstep_cli_Flags
+                      quickstep_cli_DefaultsConfigurator
                       quickstep_storage_StorageConstants
                       ${GFLAGS_LIB_NAME})
 target_link_libraries(quickstep_cli_InputParserUtil

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/Flags.cpp
----------------------------------------------------------------------
diff --git a/cli/Flags.cpp b/cli/Flags.cpp
index 384ef71..87f9f73 100644
--- a/cli/Flags.cpp
+++ b/cli/Flags.cpp
@@ -19,14 +19,43 @@
 
 #include "cli/Flags.hpp"
 
+#include <cstddef>
+#include <cstdio>
 #include <string>
 
+#include "cli/DefaultsConfigurator.hpp"
 #include "storage/StorageConstants.hpp"
 
 #include "gflags/gflags.h"
 
+using std::fprintf;
+
 namespace quickstep {
 
+static bool ValidateNumWorkers(const char *flagname, int value) {
+  if (value > 0) {
+    return true;
+  }
+
+  // Detect the hardware concurrency level.
+  const std::size_t num_hw_threads =
+      DefaultsConfigurator::GetNumHardwareThreads();
+
+  // Use the command-line value if that was supplied, else use the value
+  // that we computed above, provided it did return a valid value.
+  // TODO(jmp): May need to change this at some point to keep one thread
+  //            available for the OS if the hardware concurrency level is high.
+  FLAGS_num_workers = num_hw_threads != 0 ? num_hw_threads : 1;
+
+  return FLAGS_num_workers > 0;
+}
+DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is "
+             "specified and is greater than 0, then this user-supplied value is "
+             "used. Else (i.e. the default case), we examine the reported "
+             "hardware concurrency level, and use that.");
+static const volatile bool num_workers_dummy
+    = gflags::RegisterFlagValidator(&FLAGS_num_workers, &ValidateNumWorkers);
+
 static bool ValidateStoragePath(const char *flagname,
                                 const std::string &value) {
   if (!value.empty() && value.back() != kPathSeparator) {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/Flags.hpp
----------------------------------------------------------------------
diff --git a/cli/Flags.hpp b/cli/Flags.hpp
index a623448..b020a3e 100644
--- a/cli/Flags.hpp
+++ b/cli/Flags.hpp
@@ -32,6 +32,8 @@ namespace quickstep {
  * @brief A collection of common flags shared by Quickstep CLIs in both the
  * single-node and the distributed version.
  **/
+DECLARE_int32(num_workers);
+
 DECLARE_string(storage_path);
 
 /** @} */

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/e666a81f/cli/QuickstepCli.cpp
----------------------------------------------------------------------
diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp
index 33a9ff4..6a7c500 100644
--- a/cli/QuickstepCli.cpp
+++ b/cli/QuickstepCli.cpp
@@ -109,6 +109,7 @@ using quickstep::AdmitRequestMessage;
 using quickstep::CatalogRelation;
 using quickstep::DefaultsConfigurator;
 using quickstep::DropRelation;
+using quickstep::FLAGS_num_workers;
 using quickstep::FLAGS_storage_path;
 using quickstep::ForemanSingleNode;
 using quickstep::InputParserUtil;
@@ -136,11 +137,6 @@ using tmb::client_id;
 
 namespace quickstep {
 
-DEFINE_int32(num_workers, 0, "Number of worker threads. If this value is "
-                             "specified and is greater than 0, then this "
-                             "user-supplied value is used. Else (i.e. the"
-                             "default case), we examine the reported "
-                             "hardware concurrency level, and use that.");
 DEFINE_bool(preload_buffer_pool, false,
             "If true, pre-load all known blocks into buffer pool before "
             "accepting queries (should also set --buffer_pool_slots to be "
@@ -189,25 +185,8 @@ int main(int argc, char* argv[]) {
   google::InitGoogleLogging(argv[0]);
   gflags::ParseCommandLineFlags(&argc, &argv, true);
 
-  // Detect the hardware concurrency level.
-  const std::size_t num_hw_threads =
-      DefaultsConfigurator::GetNumHardwareThreads();
-
-  // Use the command-line value if that was supplied, else use the value
-  // that we computed above, provided it did return a valid value.
-  // TODO(jmp): May need to change this at some point to keep one thread
-  //            available for the OS if the hardware concurrency level is high.
-  if (quickstep::FLAGS_num_workers <= 0) {
-    LOG(INFO) << "Quickstep expects at least one worker thread, switching to "
-                 "the default number of worker threads";
-  }
-  const int real_num_workers = quickstep::FLAGS_num_workers > 0
-                                   ? quickstep::FLAGS_num_workers
-                                   : (num_hw_threads != 0 ? num_hw_threads : 1);
-
-  DCHECK_GT(real_num_workers, 0);
-  printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool\n",
-         real_num_workers,
+  printf("Starting Quickstep with %d worker thread(s) and a %.2f GB buffer pool.\n",
+         FLAGS_num_workers,
          (static_cast<double>(quickstep::FLAGS_buffer_pool_slots) * quickstep::kSlotSizeBytes)/quickstep::kAGigaByte);
 
 #ifdef QUICKSTEP_HAVE_FILE_MANAGER_HDFS
@@ -290,7 +269,7 @@ int main(int argc, char* argv[]) {
   // Parse the CPU affinities for workers and the preloader thread, if enabled
   // to warm up the buffer pool.
   const vector<int> worker_cpu_affinities =
-      InputParserUtil::ParseWorkerAffinities(real_num_workers,
+      InputParserUtil::ParseWorkerAffinities(FLAGS_num_workers,
                                              quickstep::FLAGS_worker_affinities);
 
   const std::size_t num_numa_nodes_system = DefaultsConfigurator::GetNumNUMANodes();
@@ -323,7 +302,7 @@ int main(int argc, char* argv[]) {
   vector<client_id> worker_client_ids;
 
   // Initialize the worker threads.
-  DCHECK_EQ(static_cast<std::size_t>(real_num_workers),
+  DCHECK_EQ(static_cast<std::size_t>(FLAGS_num_workers),
             worker_cpu_affinities.size());
   for (std::size_t worker_thread_index = 0;
        worker_thread_index < worker_cpu_affinities.size();


Mime
View raw message