quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zu...@apache.org
Subject [1/2] incubator-quickstep git commit: Fixed the catalog relation schema reference bug in the preloading in the distributed version. [Forced Update!]
Date Sun, 12 Mar 2017 02:41:54 GMT
Repository: incubator-quickstep
Updated Branches:
  refs/heads/fix-admit-queries-bug 59c2184bd -> 9a991bb00 (forced update)


Fixed the catalog relation schema reference bug in the preloading in the distributed version.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4a95738d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4a95738d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4a95738d

Branch: refs/heads/fix-admit-queries-bug
Commit: 4a95738de730b03ccfb2026843f5a4e6838cfe2f
Parents: a5c68dd
Author: Zuyu Zhang <zuyuz@apache.org>
Authored: Fri Mar 10 14:58:39 2017 -0800
Committer: Zuyu Zhang <zuyuz@apache.org>
Committed: Fri Mar 10 14:58:39 2017 -0800

----------------------------------------------------------------------
 catalog/tests/Catalog_unittest.cpp |  5 ++++-
 query_execution/Shiftboss.cpp      |  4 +++-
 storage/CMakeLists.txt             |  1 +
 storage/PreloaderThread.cpp        | 17 +++++++++++++----
 storage/PreloaderThread.hpp        | 12 ++++++++++--
 5 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/catalog/tests/Catalog_unittest.cpp
----------------------------------------------------------------------
diff --git a/catalog/tests/Catalog_unittest.cpp b/catalog/tests/Catalog_unittest.cpp
index 276bfa9..f761026 100644
--- a/catalog/tests/Catalog_unittest.cpp
+++ b/catalog/tests/Catalog_unittest.cpp
@@ -571,7 +571,10 @@ TEST_F(CatalogTest, CatalogIndexTest) {
 }
 
 TEST_F(CatalogTest, CatalogDatabaseCacheTest) {
-  CatalogRelationSchema* const rel = createCatalogRelation("rel");
+  CatalogRelation * const rel = createCatalogRelation("rel");
+
+  rel->addBlock(BlockIdUtil::GetBlockId(0 /* domain */, 7));
+  rel->addBlock(BlockIdUtil::GetBlockId(0 /* domain */, 14));
 
   rel->addAttribute(new CatalogAttribute(nullptr, "attr_int", TypeFactory::GetType(kInt),
-1 /* id */, "int"));
   rel->addAttribute(new CatalogAttribute(nullptr, "attr_long", TypeFactory::GetType(kLong)));

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/query_execution/Shiftboss.cpp
----------------------------------------------------------------------
diff --git a/query_execution/Shiftboss.cpp b/query_execution/Shiftboss.cpp
index 905464f..72d8913 100644
--- a/query_execution/Shiftboss.cpp
+++ b/query_execution/Shiftboss.cpp
@@ -340,9 +340,11 @@ void Shiftboss::processShiftbossRegistrationResponseMessage() {
   storage_manager_->sendBlockDomainToShiftbossIndexMessage(shiftboss_index_);
 
   if (FLAGS_preload_buffer_pool) {
+    database_cache_.update(proto.catalog_database());
+
     const CatalogDatabase catalog_database(proto.catalog_database());
 
-    PreloaderThread preloader(catalog_database, storage_manager_, cpu_id_);
+    PreloaderThread preloader(catalog_database, storage_manager_, cpu_id_, &database_cache_);
 
     printf("Preloading the buffer pool ... \n");
     const std::chrono::time_point<std::chrono::steady_clock> preload_start = std::chrono::steady_clock::now();

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/storage/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index 8b68150..6fe6436 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -827,6 +827,7 @@ target_link_libraries(quickstep_storage_PartitionedHashTablePool
 target_link_libraries(quickstep_storage_PreloaderThread
                       glog
                       quickstep_catalog_CatalogDatabase
+                      quickstep_catalog_CatalogDatabaseLite
                       quickstep_catalog_CatalogRelation
                       quickstep_catalog_CatalogTypedefs
                       quickstep_storage_StorageBlock

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/storage/PreloaderThread.cpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.cpp b/storage/PreloaderThread.cpp
index edaa143..354e468 100644
--- a/storage/PreloaderThread.cpp
+++ b/storage/PreloaderThread.cpp
@@ -27,6 +27,7 @@
 #endif
 
 #include "catalog/CatalogDatabase.hpp"
+#include "catalog/CatalogDatabaseLite.hpp"
 #include "catalog/CatalogRelation.hpp"
 #include "catalog/CatalogTypedefs.hpp"
 #include "storage/StorageBlock.hpp"
@@ -60,8 +61,12 @@ void PreloaderThread::run() {
       std::vector<block_id> blocks = relation.getBlocksSnapshot();
       for (block_id current_block_id : blocks) {
         try {
-          BlockReference current_block =
-              storage_manager_->getBlock(current_block_id, relation);
+          const CatalogRelationSchema &relation_schema =
+              database_for_schemas_ ? database_for_schemas_->getRelationSchemaById(relation.getID())
+                                    : relation;
+
+          const BlockReference current_block =
+              storage_manager_->getBlock(current_block_id, relation_schema);
         } catch (...) {
           LOG(ERROR) << "Error after loading " << blocks_loaded << "blocks";
           throw;
@@ -104,8 +109,12 @@ std::size_t PreloaderThread::preloadNUMAAware(
         placement_scheme->getNUMANodeForPartition(part_id);
     for (block_id curr_block_id : part_scheme.getBlocksInPartition(part_id)) {
       try {
-        BlockReference current_block = storage_manager_->getBlock(
-            curr_block_id, relation, partition_numa_node_id);
+        const CatalogRelationSchema &relation_schema =
+            database_for_schemas_ ? database_for_schemas_->getRelationSchemaById(relation.getID())
+                                  : relation;
+
+        const BlockReference current_block = storage_manager_->getBlock(
+            curr_block_id, relation_schema, partition_numa_node_id);
       } catch (...) {
         LOG(ERROR) << "Error while preloading: After loading total "
                    << blocks_loaded + num_previously_loaded_blocks

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4a95738d/storage/PreloaderThread.hpp
----------------------------------------------------------------------
diff --git a/storage/PreloaderThread.hpp b/storage/PreloaderThread.hpp
index d9853b8..af3f6ba 100644
--- a/storage/PreloaderThread.hpp
+++ b/storage/PreloaderThread.hpp
@@ -29,6 +29,7 @@
 namespace quickstep {
 
 class CatalogDatabase;
+class CatalogDatabaseLite;
 class CatalogRelation;
 class StorageManager;
 
@@ -55,13 +56,18 @@ class PreloaderThread : public Thread {
    *        evicted and the database will not be fully preloaded.
    * @param cpu_id The ID of the CPU to affinitize this thread to, or -1 to
    *        indicate no affinity.
+   * @param database_for_schemas All non-temporary relations in this database
+   *        will be used for referencing the schema in a block for the
+   *        distributed version.
    **/
   PreloaderThread(const CatalogDatabase &database,
                   StorageManager *storage_manager,
-                  const int cpu_id = -1)
+                  const int cpu_id = -1,
+                  const CatalogDatabaseLite *database_for_schemas = nullptr)
       : database_(database),
         storage_manager_(storage_manager),
-        cpu_id_(cpu_id) {
+        cpu_id_(cpu_id),
+        database_for_schemas_(database_for_schemas) {
   }
 
   ~PreloaderThread() override {
@@ -102,6 +108,8 @@ class PreloaderThread : public Thread {
 
   const int cpu_id_;
 
+  const CatalogDatabaseLite *database_for_schemas_;
+
   DISALLOW_COPY_AND_ASSIGN(PreloaderThread);
 };
 


Mime
View raw message