quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jianq...@apache.org
Subject [06/12] incubator-quickstep git commit: Added output_num_partitions in RelationalOperators.
Date Sun, 27 Aug 2017 08:03:31 GMT
Added output_num_partitions in RelationalOperators.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/9afad881
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/9afad881
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/9afad881

Branch: refs/heads/copy-to
Commit: 9afad881da17dd3a8488a88922ab06b8ee96fbb9
Parents: 5f066fa
Author: Zuyu Zhang <zuyu@cs.wisc.edu>
Authored: Wed Aug 23 12:57:38 2017 -0500
Committer: Zuyu Zhang <zuyu@cs.wisc.edu>
Committed: Sun Aug 27 01:14:24 2017 -0500

----------------------------------------------------------------------
 relational_operators/DeleteOperator.hpp            |  3 ++-
 .../FinalizeAggregationOperator.hpp                |  6 ++++--
 relational_operators/HashJoinOperator.hpp          |  3 ++-
 relational_operators/InsertOperator.hpp            |  2 +-
 relational_operators/NestedLoopsJoinOperator.hpp   |  2 +-
 relational_operators/RelationalOperator.hpp        | 17 +++++++++++++++--
 relational_operators/SelectOperator.hpp            |  8 ++++++--
 relational_operators/SortMergeRunOperator.hpp      |  5 ++++-
 relational_operators/SortRunGenerationOperator.hpp |  7 +++++--
 relational_operators/TableGeneratorOperator.hpp    |  3 ++-
 relational_operators/TextScanOperator.hpp          |  3 ++-
 relational_operators/UpdateOperator.hpp            |  3 ++-
 relational_operators/WindowAggregationOperator.hpp |  5 ++++-
 13 files changed, 50 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/DeleteOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/DeleteOperator.hpp b/relational_operators/DeleteOperator.hpp
index b084560..cc6f6cf 100644
--- a/relational_operators/DeleteOperator.hpp
+++ b/relational_operators/DeleteOperator.hpp
@@ -74,7 +74,8 @@ class DeleteOperator : public RelationalOperator {
                  const CatalogRelation &relation,
                  const QueryContext::predicate_id predicate_index,
                  const bool relation_is_stored)
-      : RelationalOperator(query_id, relation.getNumPartitions()),
+      : RelationalOperator(query_id, relation.getNumPartitions(), false /* has_repartition
*/,
+                           relation.getNumPartitions()),
         relation_(relation),
         predicate_index_(predicate_index),
         relation_is_stored_(relation_is_stored),

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/FinalizeAggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/FinalizeAggregationOperator.hpp b/relational_operators/FinalizeAggregationOperator.hpp
index b344c13..5931ca2 100644
--- a/relational_operators/FinalizeAggregationOperator.hpp
+++ b/relational_operators/FinalizeAggregationOperator.hpp
@@ -74,12 +74,14 @@ class FinalizeAggregationOperator : public RelationalOperator {
       const std::size_t aggr_state_num_partitions,
       const CatalogRelation &output_relation,
       const QueryContext::insert_destination_id output_destination_index)
-      : RelationalOperator(query_id, num_partitions, has_repartition),
+      : RelationalOperator(query_id, num_partitions, has_repartition, output_relation.getNumPartitions()),
         aggr_state_index_(aggr_state_index),
         aggr_state_num_partitions_(aggr_state_num_partitions),
         output_relation_(output_relation),
         output_destination_index_(output_destination_index),
-        started_(false) {}
+        started_(false) {
+    DCHECK(has_repartition || num_partitions == output_num_partitions_);
+  }
 
   ~FinalizeAggregationOperator() override {}
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/HashJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp
index a5e6eb4..306875e 100644
--- a/relational_operators/HashJoinOperator.hpp
+++ b/relational_operators/HashJoinOperator.hpp
@@ -139,7 +139,7 @@ class HashJoinOperator : public RelationalOperator {
       const QueryContext::scalar_group_id selection_index,
       const std::vector<bool> *is_selection_on_build = nullptr,
       const JoinType join_type = JoinType::kInnerJoin)
-      : RelationalOperator(query_id, num_partitions, has_repartition),
+      : RelationalOperator(query_id, num_partitions, has_repartition, output_relation.getNumPartitions()),
         build_relation_(build_relation),
         probe_relation_(probe_relation),
         probe_relation_is_stored_(probe_relation_is_stored),
@@ -157,6 +157,7 @@ class HashJoinOperator : public RelationalOperator {
         probe_relation_block_ids_(num_partitions),
         num_workorders_generated_(num_partitions),
         started_(false) {
+    DCHECK(has_repartition || num_partitions == output_num_partitions_);
     DCHECK(join_type != JoinType::kLeftOuterJoin ||
                (is_selection_on_build != nullptr &&
                 residual_predicate_index == QueryContext::kInvalidPredicateId));

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/InsertOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/InsertOperator.hpp b/relational_operators/InsertOperator.hpp
index b0bdf87..b103538 100644
--- a/relational_operators/InsertOperator.hpp
+++ b/relational_operators/InsertOperator.hpp
@@ -68,7 +68,7 @@ class InsertOperator : public RelationalOperator {
       const CatalogRelation &output_relation,
       const QueryContext::insert_destination_id output_destination_index,
       const QueryContext::tuple_id tuple_index)
-      : RelationalOperator(query_id),
+      : RelationalOperator(query_id, 1u, false, output_relation.getNumPartitions()),
         output_relation_(output_relation),
         output_destination_index_(output_destination_index),
         tuple_index_(tuple_index),

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/NestedLoopsJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.hpp b/relational_operators/NestedLoopsJoinOperator.hpp
index 777a737..daca91f 100644
--- a/relational_operators/NestedLoopsJoinOperator.hpp
+++ b/relational_operators/NestedLoopsJoinOperator.hpp
@@ -102,7 +102,7 @@ class NestedLoopsJoinOperator : public RelationalOperator {
       const QueryContext::scalar_group_id selection_index,
       const bool left_relation_is_stored,
       const bool right_relation_is_stored)
-      : RelationalOperator(query_id, num_partitions, has_repartition),
+      : RelationalOperator(query_id, num_partitions, has_repartition, output_relation.getNumPartitions()),
         nested_loops_join_index_(nested_loops_join_index),
         left_input_relation_(left_input_relation),
         right_input_relation_(right_input_relation),

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/RelationalOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/RelationalOperator.hpp b/relational_operators/RelationalOperator.hpp
index 0541bbd..9307b5a 100644
--- a/relational_operators/RelationalOperator.hpp
+++ b/relational_operators/RelationalOperator.hpp
@@ -278,6 +278,15 @@ class RelationalOperator {
   }
 
   /**
+   * @brief Get the number of partitions of the output relation in the operator.
+   *
+   * @return The number of partitions of the output relation.
+   */
+  std::size_t getOutputNumPartitions() const {
+    return output_num_partitions_;
+  }
+
+  /**
    * @brief Deploy a group of LIPFilters to this operator.
    */
   void deployLIPFilters(const QueryContext::lip_deployment_id lip_deployment_index,
@@ -294,18 +303,22 @@ class RelationalOperator {
    * @param num_partitions The number of partitions of the input relation.
    *        If create table / index, return zero. If no partitions, return one.
    * @param has_repartition Whether this operator does repartition.
+   * @param output_num_partitions The number of partitions of the output
+   *        relation. If no output, return zero. If no partitions, return one.
    **/
   explicit RelationalOperator(const std::size_t query_id,
                               const std::size_t num_partitions = 1u,
-                              const bool has_repartition = false)
+                              const bool has_repartition = false,
+                              const std::size_t output_num_partitions = 0u)
       : query_id_(query_id),
         num_partitions_(num_partitions),
+        output_num_partitions_(output_num_partitions),
         has_repartition_(has_repartition),
         done_feeding_input_relation_(false),
         lip_deployment_index_(QueryContext::kInvalidLIPDeploymentId) {}
 
   const std::size_t query_id_;
-  const std::size_t num_partitions_;
+  const std::size_t num_partitions_, output_num_partitions_;
   const bool has_repartition_;
 
   bool done_feeding_input_relation_;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/SelectOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp
index 71dcfca..e7cdcd0 100644
--- a/relational_operators/SelectOperator.hpp
+++ b/relational_operators/SelectOperator.hpp
@@ -96,7 +96,8 @@ class SelectOperator : public RelationalOperator {
       const QueryContext::predicate_id predicate_index,
       const QueryContext::scalar_group_id selection_index,
       const bool input_relation_is_stored)
-      : RelationalOperator(query_id, input_relation.getNumPartitions(), has_repartition),
+      : RelationalOperator(query_id, input_relation.getNumPartitions(), has_repartition,
+                           output_relation.getNumPartitions()),
         input_relation_(input_relation),
         output_relation_(output_relation),
         output_destination_index_(output_destination_index),
@@ -107,6 +108,7 @@ class SelectOperator : public RelationalOperator {
         simple_projection_(false),
         input_relation_is_stored_(input_relation_is_stored),
         started_(false) {
+    DCHECK(has_repartition || num_partitions_ == output_num_partitions_);
 #ifdef QUICKSTEP_HAVE_LIBNUMA
     placement_scheme_ = input_relation.getNUMAPlacementSchemePtr();
 #endif
@@ -153,7 +155,8 @@ class SelectOperator : public RelationalOperator {
       const QueryContext::predicate_id predicate_index,
       std::vector<attribute_id> &&selection,
       const bool input_relation_is_stored)
-      : RelationalOperator(query_id, input_relation.getNumPartitions(), has_repartition),
+      : RelationalOperator(query_id, input_relation.getNumPartitions(), has_repartition,
+                           output_relation.getNumPartitions()),
         input_relation_(input_relation),
         output_relation_(output_relation),
         output_destination_index_(output_destination_index),
@@ -165,6 +168,7 @@ class SelectOperator : public RelationalOperator {
         simple_projection_(true),
         input_relation_is_stored_(input_relation_is_stored),
         started_(false) {
+    DCHECK(has_repartition || num_partitions_ == output_num_partitions_);
 #ifdef QUICKSTEP_HAVE_LIBNUMA
     placement_scheme_ = input_relation.getNUMAPlacementSchemePtr();
 #endif

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/SortMergeRunOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortMergeRunOperator.hpp b/relational_operators/SortMergeRunOperator.hpp
index a8ecb88..6287068 100644
--- a/relational_operators/SortMergeRunOperator.hpp
+++ b/relational_operators/SortMergeRunOperator.hpp
@@ -107,7 +107,8 @@ class SortMergeRunOperator : public RelationalOperator {
       const std::size_t merge_factor,
       const std::size_t top_k,
       const bool input_relation_is_stored)
-      : RelationalOperator(query_id),
+      : RelationalOperator(query_id, 1u /* input_num_partitions */, false /* has_repartition
*/,
+                           1u /* output_num_partition */),
         input_relation_(input_relation),
         output_relation_(output_relation),
         output_destination_index_(output_destination_index),
@@ -124,6 +125,8 @@ class SortMergeRunOperator : public RelationalOperator {
         input_relation_is_stored_(input_relation_is_stored),
         input_stream_done_(input_relation_is_stored),
         started_(false) {
+    DCHECK_EQ(1u, input_relation.getNumPartitions());
+    DCHECK_EQ(1u, output_relation.getNumPartitions());
     DCHECK_GT(merge_factor_, 1u);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/SortRunGenerationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/SortRunGenerationOperator.hpp b/relational_operators/SortRunGenerationOperator.hpp
index 716d4f7..aaac1c3 100644
--- a/relational_operators/SortRunGenerationOperator.hpp
+++ b/relational_operators/SortRunGenerationOperator.hpp
@@ -98,7 +98,8 @@ class SortRunGenerationOperator : public RelationalOperator {
       const QueryContext::insert_destination_id output_destination_index,
       const QueryContext::sort_config_id sort_config_index,
       bool input_relation_is_stored)
-      : RelationalOperator(query_id),
+      : RelationalOperator(query_id, 1u /* input_num_partitions */, false /* has_repartition
*/,
+                           1u /* output_num_partition */),
         input_relation_(input_relation),
         output_relation_(output_relation),
         output_destination_index_(output_destination_index),
@@ -108,7 +109,9 @@ class SortRunGenerationOperator : public RelationalOperator {
                                       : std::vector<block_id>()),
         num_workorders_generated_(0),
         started_(false),
-        input_relation_is_stored_(input_relation_is_stored) {}
+        input_relation_is_stored_(input_relation_is_stored) {
+    DCHECK_EQ(1u, output_relation.getNumPartitions());
+  }
 
   ~SortRunGenerationOperator() {}
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/TableGeneratorOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/TableGeneratorOperator.hpp b/relational_operators/TableGeneratorOperator.hpp
index adfe2bb..c6a6b7d 100644
--- a/relational_operators/TableGeneratorOperator.hpp
+++ b/relational_operators/TableGeneratorOperator.hpp
@@ -70,7 +70,8 @@ class TableGeneratorOperator : public RelationalOperator {
       const CatalogRelation &output_relation,
       const QueryContext::insert_destination_id output_destination_index,
       const QueryContext::generator_function_id generator_function_index)
-      : RelationalOperator(query_id),
+      : RelationalOperator(query_id, 1u, output_relation.getNumPartitions() != 1u /* has_repartition
*/,
+                           output_relation.getNumPartitions()),
         output_relation_(output_relation),
         output_destination_index_(output_destination_index),
         generator_function_index_(generator_function_index),

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/TextScanOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/TextScanOperator.hpp b/relational_operators/TextScanOperator.hpp
index f6be8c8..4dbeb92 100644
--- a/relational_operators/TextScanOperator.hpp
+++ b/relational_operators/TextScanOperator.hpp
@@ -125,7 +125,8 @@ class TextScanOperator : public RelationalOperator {
                    const bool process_escape_sequences,
                    const CatalogRelation &output_relation,
                    const QueryContext::insert_destination_id output_destination_index)
-      : RelationalOperator(query_id),
+      : RelationalOperator(query_id, 1u, output_relation.getNumPartitions() != 1u /* has_repartition
*/,
+                           output_relation.getNumPartitions()),
         file_pattern_(file_pattern),
         field_terminator_(field_terminator),
         process_escape_sequences_(process_escape_sequences),

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/UpdateOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/UpdateOperator.hpp b/relational_operators/UpdateOperator.hpp
index fd2096b..7b7fad6 100644
--- a/relational_operators/UpdateOperator.hpp
+++ b/relational_operators/UpdateOperator.hpp
@@ -88,7 +88,8 @@ class UpdateOperator : public RelationalOperator {
       const QueryContext::insert_destination_id relocation_destination_index,
       const QueryContext::predicate_id predicate_index,
       const QueryContext::update_group_id update_group_index)
-      : RelationalOperator(query_id, relation.getNumPartitions()),
+      : RelationalOperator(query_id, relation.getNumPartitions(), false /* has_repartition
*/,
+                           relation.getNumPartitions()),
         relation_(relation),
         relocation_destination_index_(relocation_destination_index),
         predicate_index_(predicate_index),

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/9afad881/relational_operators/WindowAggregationOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/WindowAggregationOperator.hpp b/relational_operators/WindowAggregationOperator.hpp
index 10546b4..beaf1a3 100644
--- a/relational_operators/WindowAggregationOperator.hpp
+++ b/relational_operators/WindowAggregationOperator.hpp
@@ -70,7 +70,10 @@ class WindowAggregationOperator : public RelationalOperator {
                             const CatalogRelation &output_relation,
                             const QueryContext::window_aggregation_state_id window_aggregation_state_index,
                             const QueryContext::insert_destination_id output_destination_index)
-      : RelationalOperator(query_id),
+      : RelationalOperator(query_id, input_relation.getNumPartitions(),
+                           input_relation.getNumPartitions() !=
+                               output_relation.getNumPartitions() /* has_repartition */,
+                           output_relation.getNumPartitions()),
         input_relation_(input_relation),
         output_relation_(output_relation),
         window_aggregation_state_index_(window_aggregation_state_index),


Mime
View raw message