quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jign...@apache.org
Subject [3/5] incubator-quickstep git commit: Set CatalogDatabaseCache proto in optimizer. (#159)
Date Thu, 21 Apr 2016 15:01:07 GMT
Set CatalogDatabaseCache proto in optimizer. (#159)

Link: https://github.com/pivotalsoftware/quickstep/pull/159

Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/390310fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/390310fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/390310fe

Branch: refs/heads/master
Commit: 390310feefade09e26a5b73feaa8ea9a38428479
Parents: ef383ef
Author: Zuyu ZHANG <zuyu@users.noreply.github.com>
Authored: Wed Apr 20 21:20:48 2016 -0500
Committer: Harshad Deshmukh <d.harshad17@gmail.com>
Committed: Wed Apr 20 21:20:48 2016 -0500

----------------------------------------------------------------------
 CMakeLists.txt                            |  4 ++-
 query_optimizer/CMakeLists.txt            | 14 +++++++++
 query_optimizer/ExecutionGenerator.cpp    | 40 +++++++++++++++++++++++++-
 query_optimizer/ExecutionGenerator.hpp    | 33 +++++++++++++++++----
 query_optimizer/QueryHandle.hpp           | 22 ++++++++++++--
 query_optimizer/QueryOptimizerConfig.h.in | 17 +++++++++++
 6 files changed, 121 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/390310fe/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 642925f..c9578b0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 #   Copyright 2011-2015 Quickstep Technologies LLC.
-#   Copyright 2015 Pivotal Software, Inc.
+#   Copyright 2015-2016 Pivotal Software, Inc.
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -137,6 +137,8 @@ if (ENABLE_VECTOR_PREDICATE_SHORT_CIRCUIT)
   )
 endif()
 
+option(ENABLE_DISTRIBUTED "Use the distributed version of Quickstep" OFF)
+
 # Turn on the QUICKSTEP_DEBUG flag in the source if this is a debug build.
 if (CMAKE_MAJOR_VERSION GREATER 2)
   cmake_policy(SET CMP0043 NEW)

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/390310fe/query_optimizer/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt
index 44c8750..2d09bee 100644
--- a/query_optimizer/CMakeLists.txt
+++ b/query_optimizer/CMakeLists.txt
@@ -15,6 +15,15 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+if (ENABLE_DISTRIBUTED)
+  set(QUICKSTEP_DISTRIBUTED TRUE)
+endif()
+
+configure_file (
+  "${CMAKE_CURRENT_SOURCE_DIR}/QueryOptimizerConfig.h.in"
+  "${CMAKE_CURRENT_BINARY_DIR}/QueryOptimizerConfig.h"
+)
+
 add_subdirectory(cost_model)
 add_subdirectory(expressions)
 add_subdirectory(logical)
@@ -126,6 +135,10 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
                       quickstep_types_containers_Tuple_proto
                       quickstep_utility_Macros
                       quickstep_utility_SqlError)
+if (ENABLE_DISTRIBUTED)
+  target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator
+                        quickstep_catalog_Catalog_proto)
+endif()
 target_link_libraries(quickstep_queryoptimizer_LogicalGenerator
                       glog
                       quickstep_parser_ParseStatement
@@ -171,6 +184,7 @@ target_link_libraries(quickstep_queryoptimizer_PhysicalGenerator
                       quickstep_queryoptimizer_Validator
                       quickstep_utility_Macros)
 target_link_libraries(quickstep_queryoptimizer_QueryHandle
+                      quickstep_catalog_Catalog_proto
                       quickstep_queryexecution_QueryContext_proto
                       quickstep_queryoptimizer_QueryPlan
                       quickstep_utility_Macros)

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/390310fe/query_optimizer/ExecutionGenerator.cpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp
index 7a8e996..abcdd6c 100644
--- a/query_optimizer/ExecutionGenerator.cpp
+++ b/query_optimizer/ExecutionGenerator.cpp
@@ -25,9 +25,18 @@
 #include <string>
 #include <type_traits>
 #include <unordered_map>
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include <unordered_set>
+#endif
+
 #include <utility>
 #include <vector>
 
+#ifdef QUICKSTEP_DISTRIBUTED
+#include "catalog/Catalog.pb.h"
+#endif
+
 #include "catalog/CatalogAttribute.hpp"
 #include "catalog/CatalogDatabase.hpp"
 #include "catalog/CatalogRelation.hpp"
@@ -185,6 +194,20 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan)
{
         drop_table_index,
         temporary_relation_info.producer_operator_index);
   }
+
+#ifdef QUICKSTEP_DISTRIBUTED
+  catalog_database_cache_proto_->set_name(optimizer_context_->catalog_database()->getName());
+
+  LOG(INFO) << "CatalogDatabaseCache proto has " << referenced_relation_ids_.size()
<< " relation(s)";
+  for (const relation_id rel_id : referenced_relation_ids_) {
+    const CatalogRelationSchema &relation =
+        optimizer_context_->catalog_database()->getRelationSchemaById(rel_id);
+    LOG(INFO) << "RelationSchema " << rel_id
+              << ", name: " << relation.getName()
+              << ", " << relation.size()  << " attribute(s)";
+    catalog_database_cache_proto_->add_relations()->MergeFrom(relation.getProto());
+  }
+#endif
 }
 
 void ExecutionGenerator::generatePlanInternal(
@@ -289,6 +312,10 @@ void ExecutionGenerator::createTemporaryCatalogRelation(
   const relation_id output_rel_id = optimizer_context_->catalog_database()->addRelation(
       catalog_relation.release());
 
+#ifdef QUICKSTEP_DISTRIBUTED
+  referenced_relation_ids_.insert(output_rel_id);
+#endif
+
   insert_destination_proto->set_insert_destination_type(S::InsertDestinationType::BLOCK_POOL);
   insert_destination_proto->set_relation_id(output_rel_id);
 }
@@ -335,6 +362,11 @@ void ExecutionGenerator::convertTableReference(
   // parent (e.g. the substitution map from an AttributeReference
   // to a CatalogAttribute).
   const CatalogRelation *catalog_relation = physical_table_reference->relation();
+
+#ifdef QUICKSTEP_DISTRIBUTED
+  referenced_relation_ids_.insert(catalog_relation->getID());
+#endif
+
   const std::vector<E::AttributeReferencePtr> &attribute_references =
       physical_table_reference->attribute_list();
   DCHECK_EQ(attribute_references.size(), catalog_relation->size());
@@ -994,8 +1026,14 @@ void ExecutionGenerator::convertDeleteTuples(
 void ExecutionGenerator::convertDropTable(
     const P::DropTablePtr &physical_plan) {
   // DropTable is converted to a DropTable operator.
+  const CatalogRelation &catalog_relation = *physical_plan->catalog_relation();
+
+#ifdef QUICKSTEP_DISTRIBUTED
+  referenced_relation_ids_.insert(catalog_relation.getID());
+#endif
+
   execution_plan_->addRelationalOperator(
-      new DropTableOperator(*physical_plan->catalog_relation(),
+      new DropTableOperator(catalog_relation,
                             optimizer_context_->catalog_database()));
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/390310fe/query_optimizer/ExecutionGenerator.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp
index 11de106..df47b31 100644
--- a/query_optimizer/ExecutionGenerator.hpp
+++ b/query_optimizer/ExecutionGenerator.hpp
@@ -1,6 +1,6 @@
 /**
  *   Copyright 2011-2015 Quickstep Technologies LLC.
- *   Copyright 2015 Pivotal Software, Inc.
+ *   Copyright 2015-2016 Pivotal Software, Inc.
  *
  *   Licensed under the Apache License, Version 2.0 (the "License");
  *   you may not use this file except in compliance with the License.
@@ -21,6 +21,11 @@
 #include <memory>
 #include <string>
 #include <unordered_map>
+
+#ifdef QUICKSTEP_DISTRIBUTED
+#include <unordered_set>
+#endif
+
 #include <vector>
 
 #include "catalog/CatalogTypedefs.hpp"
@@ -54,6 +59,8 @@
 #include "query_optimizer/physical/UpdateTable.hpp"
 #include "utility/Macros.hpp"
 
+#include "glog/logging.h"
+
 namespace quickstep {
 
 class CatalogAttribute;
@@ -61,6 +68,11 @@ class CatalogRelation;
 class Predicate;
 
 namespace serialization {
+
+#ifdef QUICKSTEP_DISTRIBUTED
+class CatalogDatabase;
+#endif
+
 class InsertDestination;
 }  // namespace serialization
 
@@ -85,10 +97,14 @@ class ExecutionGenerator {
    */
   ExecutionGenerator(OptimizerContext *optimizer_context,
                      QueryHandle *query_handle)
-      : optimizer_context_(optimizer_context),
-        query_handle_(query_handle),
-        execution_plan_(query_handle->getQueryPlanMutable()),
-        query_context_proto_(query_handle->getQueryContextProtoMutable()) {
+      : optimizer_context_(DCHECK_NOTNULL(optimizer_context)),
+        query_handle_(DCHECK_NOTNULL(query_handle)),
+        execution_plan_(DCHECK_NOTNULL(query_handle->getQueryPlanMutable())),
+        query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable()))
{
+#ifdef QUICKSTEP_DISTRIBUTED
+    catalog_database_cache_proto_ = DCHECK_NOTNULL(query_handle->getCatalogDatabaseCacheProtoMutable());
+#endif
+
     setupCostModel();
   }
 
@@ -368,6 +384,13 @@ class ExecutionGenerator {
   QueryPlan *execution_plan_;  // A part of QueryHandle.
   serialization::QueryContext *query_context_proto_;  // A part of QueryHandle.
 
+#ifdef QUICKSTEP_DISTRIBUTED
+  serialization::CatalogDatabase *catalog_database_cache_proto_;  // A part of QueryHandle.
+
+  // Used to bookkeep relation ids for 'catalog_database_cache_proto_'.
+  std::unordered_set<relation_id> referenced_relation_ids_;
+#endif
+
   /**
    * @brief Used to generate distinct relation names for temporary relations.
    */

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/390310fe/query_optimizer/QueryHandle.hpp
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryHandle.hpp b/query_optimizer/QueryHandle.hpp
index 8819cc3..a17d3e8 100644
--- a/query_optimizer/QueryHandle.hpp
+++ b/query_optimizer/QueryHandle.hpp
@@ -1,5 +1,5 @@
 /**
- *   Copyright 2015 Pivotal Software, Inc.
+ *   Copyright 2015-2016 Pivotal Software, Inc.
  *
  *   Licensed under the Apache License, Version 2.0 (the "License");
  *   you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@
 #include <memory>
 #include <utility>
 
+#include "catalog/Catalog.pb.h"
 #include "query_execution/QueryContext.pb.h"
 #include "query_optimizer/QueryPlan.hpp"
 #include "utility/Macros.hpp"
@@ -41,7 +42,7 @@ class QueryHandle {
   /**
    * @brief Constructor.
    *
-   * @param The given query id.
+   * @param query_id The given query id.
    */
   explicit QueryHandle(const std::size_t query_id)
       : query_id_(query_id),
@@ -81,6 +82,20 @@ class QueryHandle {
   }
 
   /**
+    * @return The catalog database cache in the protobuf format.
+    */
+  const serialization::CatalogDatabase& getCatalogDatabaseCacheProto() const {
+    return catalog_database_cache_proto_;
+  }
+
+  /**
+   * @return The mutable catalog database cache in the protobuf format.
+   */
+  serialization::CatalogDatabase* getCatalogDatabaseCacheProtoMutable() {
+    return &catalog_database_cache_proto_;
+  }
+
+  /**
    * @brief Get the query result relation.
    */
   const CatalogRelation* getQueryResultRelation() const {
@@ -101,6 +116,9 @@ class QueryHandle {
 
   serialization::QueryContext query_context_proto_;
 
+  // TODO(quickstep-team): Use Catalog to support multiple databases.
+  serialization::CatalogDatabase catalog_database_cache_proto_;
+
   // NOTE(zuyu): The relation gets created by the optimizer,
   //             and deleted by the Cli shell.
   const CatalogRelation *query_result_relation_;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/390310fe/query_optimizer/QueryOptimizerConfig.h.in
----------------------------------------------------------------------
diff --git a/query_optimizer/QueryOptimizerConfig.h.in b/query_optimizer/QueryOptimizerConfig.h.in
new file mode 100644
index 0000000..da0fa18
--- /dev/null
+++ b/query_optimizer/QueryOptimizerConfig.h.in
@@ -0,0 +1,17 @@
+/**
+ *   Copyright 2016 Pivotal Software, Inc.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License");
+ *   you may not use this file except in compliance with the License.
+ *   You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ **/
+
+#cmakedefine QUICKSTEP_DISTRIBUTED


Mime
View raw message