quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hbdeshm...@apache.org
Subject [09/48] incubator-quickstep git commit: Cleanup in Preloader class (#209)
Date Thu, 09 Jun 2016 01:09:38 GMT
Cleanup in Preloader class (#209)

Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/cb763bab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/cb763bab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/cb763bab

Branch: refs/heads/query-manager-used-in-foreman
Commit: cb763bab42fdff5ba407490c60094b7679b04c8e
Parents: 1a685be
Author: Harshad Deshmukh <d.harshad17@gmail.com>
Authored: Fri May 6 22:29:41 2016 -0500
Committer: Zuyu Zhang <zzhang@pivotal.io>
Committed: Wed Jun 8 11:57:41 2016 -0700

----------------------------------------------------------------------
 storage/PreloaderThread.cpp | 44 ++++++++++++++++++++++++----------------
 storage/PreloaderThread.hpp |  7 +++++++
 2 files changed, 34 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cb763bab/storage/PreloaderThread.cpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.cpp b/storage/PreloaderThread.cpp
index 8f600b8..4bfa9c5 100644
--- a/storage/PreloaderThread.cpp
+++ b/storage/PreloaderThread.cpp
@@ -49,37 +49,46 @@ void PreloaderThread::run() {
   std::size_t blocks_loaded = 0;
 
   for (const CatalogRelation &relation : database_) {
-    if (relation.hasPartitionScheme()) {
+    if (relation.hasPartitionScheme() && relation.hasNUMAPlacementScheme()) {
+#ifdef QUICKSTEP_HAVE_LIBNUMA
       blocks_loaded += preloadNUMAAware(relation, blocks_loaded, num_slots);
+#endif
     } else {
+      // NUMA agnostic preloading of relation.
       std::vector<block_id> blocks = relation.getBlocksSnapshot();
       for (block_id current_block_id : blocks) {
         try {
-          BlockReference current_block = storage_manager_->getBlock(current_block_id,
relation);
+          BlockReference current_block =
+              storage_manager_->getBlock(current_block_id, relation);
         } catch (...) {
-          LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks\n";
+          LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks";
           throw;
         }
         ++blocks_loaded;
         if (blocks_loaded == num_slots) {
-          // The buffer pool has filled up. But, some database blocks are not loaded.
-          printf(" The database is larger than the buffer pool. Only %lu blocks were loaded
",
-                 blocks_loaded);
+          // The buffer pool has filled up. But, some database blocks are not
+          // loaded.
+          printf(
+              " The database is larger than the buffer pool. Only %lu blocks "
+              "were loaded ", blocks_loaded);
           return;
         }
       }
+      LOG(INFO) << "Relation " << relation.getName()
+                << " completely preloaded in buffer pool";
     }
   }
   printf(" Loaded %lu blocks ", blocks_loaded);
 }
 
+#ifdef QUICKSTEP_HAVE_LIBNUMA
 std::size_t PreloaderThread::preloadNUMAAware(
     const CatalogRelation &relation,
     const std::size_t num_previously_loaded_blocks,
     const std::size_t num_slots) {
-#ifdef QUICKSTEP_HAVE_LIBNUMA
   std::size_t blocks_loaded = 0;
-  const NUMAPlacementScheme *placement_scheme = relation.getNUMAPlacementSchemePtr();
+  const NUMAPlacementScheme *placement_scheme =
+      relation.getNUMAPlacementSchemePtr();
   DCHECK(placement_scheme != nullptr);
   DCHECK(relation.hasPartitionScheme());
   const PartitionScheme &part_scheme = relation.getPartitionScheme();
@@ -96,15 +105,17 @@ std::size_t PreloaderThread::preloadNUMAAware(
         BlockReference current_block = storage_manager_->getBlock(
             curr_block_id, relation, partition_numa_node_id);
       } catch (...) {
-        LOG(ERROR) << "Error after loading "
+        LOG(ERROR) << "Error while preloading: After loading total "
                    << blocks_loaded + num_previously_loaded_blocks
-                   << " blocks\n";
+                   << " blocks and " << blocks_loaded
+                   << " blocks of relation " << relation.getName();
         throw;
       }
       ++blocks_loaded;
       num_blocks_loaded[partition_numa_node_id]++;
       if ((blocks_loaded + num_previously_loaded_blocks) == num_slots) {
-        // The buffer pool has filled up. But, some database blocks are not loaded.
+        // The buffer pool has filled up. But, some database blocks are not
+        // loaded.
         printf(
             " The database is larger than the buffer pool. Only %lu blocks "
             "were loaded ",
@@ -116,14 +127,13 @@ std::size_t PreloaderThread::preloadNUMAAware(
   LOG(INFO) << "For relation: " << relation.getName();
   for (auto numa_block_loaded_info : num_blocks_loaded) {
     LOG(INFO) << "NUMA node: " << numa_block_loaded_info.first
-              << " Number of loaded blocks: " << numa_block_loaded_info.second;
+              << " Number of loaded blocks: "
+              << numa_block_loaded_info.second;
   }
+  LOG(INFO) << "Relation " << relation.getName()
+            << " completely preloaded in buffer pool in a NUMA aware fashion";
   return blocks_loaded;
-#else
-  LOG(INFO) << "Relation: " << relation.getName()
-            << " has partition scheme but the system doesn't support NUMA";
-  return 0;
-#endif
 }
+#endif
 
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/cb763bab/storage/PreloaderThread.hpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp
index f16fd50..16df34a 100644
--- a/storage/PreloaderThread.hpp
+++ b/storage/PreloaderThread.hpp
@@ -20,6 +20,7 @@
 
 #include <cstddef>
 
+#include "storage/StorageConfig.h"
 #include "threading/Thread.hpp"
 #include "utility/Macros.hpp"
 
@@ -68,6 +69,7 @@ class PreloaderThread : public Thread {
   void run() override;
 
  private:
+#ifdef QUICKSTEP_HAVE_LIBNUMA
   /**
    * @brief Preload a relation which has a partition and a NUMA placement scheme.
    *
@@ -82,11 +84,16 @@ class PreloaderThread : public Thread {
    *          allocated sufficient amount of memory so as not to exceed that
    *          socket's memory limit.
    *
+   * TODO(harshad) - Allow multiple preloader threads, each pinned to a NUMA
+   *          socket in the system. Each thread should preload only its share of
+   *          storage blocks.
+   *
    * @return The number of blocks loaded during this function call.
    **/
   std::size_t preloadNUMAAware(const CatalogRelation &relation,
                                const std::size_t num_previously_loaded_blocks,
                                const std::size_t num_slots);
+#endif
 
   const CatalogDatabase &database_;
   StorageManager *storage_manager_;


Mime
View raw message