quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jianq...@apache.org
Subject [6/8] incubator-quickstep git commit: Fixed a bug in partitioned NLJ.
Date Wed, 20 Sep 2017 01:24:10 GMT
Fixed a bug in partitioned NLJ.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/d85f7a9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/d85f7a9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/d85f7a9a

Branch: refs/heads/fix-iwyu
Commit: d85f7a9a3b285c60892f40b1ba0a8b32ba629885
Parents: 56b3436
Author: Zuyu Zhang <zuyu@cs.wisc.edu>
Authored: Wed Sep 13 21:21:35 2017 -0500
Committer: Zuyu Zhang <zuyu@cs.wisc.edu>
Committed: Thu Sep 14 11:32:14 2017 -0500

----------------------------------------------------------------------
 .../tests/execution_generator/Partition.test    | 37 +++++++++++++++-
 .../NestedLoopsJoinOperator.cpp                 | 44 ++++++++++----------
 .../NestedLoopsJoinOperator.hpp                 | 25 ++---------
 3 files changed, 62 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d85f7a9a/query_optimizer/tests/execution_generator/Partition.test
----------------------------------------------------------------------
diff --git a/query_optimizer/tests/execution_generator/Partition.test b/query_optimizer/tests/execution_generator/Partition.test
index 5d44164..da9b6b8 100644
--- a/query_optimizer/tests/execution_generator/Partition.test
+++ b/query_optimizer/tests/execution_generator/Partition.test
@@ -226,7 +226,42 @@ GROUP BY fact.id;
 +-----------+------------------------+
 ==
 
-# Partitioned NestedLoopsJoin
+# Partitioned NestedLoopsJoin w/ both stored relations.
+SELECT COUNT(*)
+FROM dim_4_hash_partitions, fact;
+--
++--------------------+
+|COUNT(*)            |
++--------------------+
+|                 220|
++--------------------+
+==
+
+# Partitioned NestedLoopsJoin w/ one stored relation.
+SELECT COUNT(*)
+FROM dim_4_hash_partitions, fact
+WHERE dim_4_hash_partitions.id > 20;
+--
++--------------------+
+|COUNT(*)            |
++--------------------+
+|                  20|
++--------------------+
+==
+
+# Partitioned NestedLoopsJoin w/ the other stored relation.
+SELECT COUNT(*)
+FROM dim_4_hash_partitions, fact
+WHERE fact.id > 0;
+--
++--------------------+
+|COUNT(*)            |
++--------------------+
+|                 220|
++--------------------+
+==
+
+# Partitioned NestedLoopsJoin w/ both non-stored relations.
 SELECT dim_4_hash_partitions.id as dim_id, fact.id as fact_id
 FROM dim_4_hash_partitions, fact
 WHERE dim_4_hash_partitions.id > 20 AND fact.id > 0;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d85f7a9a/relational_operators/NestedLoopsJoinOperator.cpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.cpp b/relational_operators/NestedLoopsJoinOperator.cpp
index 121b1c3..658f84b 100644
--- a/relational_operators/NestedLoopsJoinOperator.cpp
+++ b/relational_operators/NestedLoopsJoinOperator.cpp
@@ -63,7 +63,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
 
     for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
       for (const block_id left_block_id : left_relation_block_ids_[part_id]) {
-        for (const block_id right_block_id : right_relation_block_ids_[part_id]) {
+        for (const block_id right_block_id : right_relation_block_ids_) {
           container->addNormalWorkOrder(
               new NestedLoopsJoinWorkOrder(
                   query_id_,
@@ -89,7 +89,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
       std::vector<block_id>::size_type new_left_blocks
           = left_relation_block_ids_[part_id].size() - num_left_workorders_generated_[part_id];
       std::vector<block_id>::size_type new_right_blocks
-          = right_relation_block_ids_[part_id].size() - num_right_workorders_generated_[part_id];
+          = right_relation_block_ids_.size() - num_right_workorders_generated_[part_id];
 
       std::size_t new_workorders = 0;
       if (new_left_blocks > 0 && new_right_blocks > 0) {
@@ -102,7 +102,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
                                                              0,
                                                              left_relation_block_ids_[part_id].size(),
                                                              num_right_workorders_generated_[part_id],
-                                                             right_relation_block_ids_[part_id].size());
+                                                             right_relation_block_ids_.size());
 
         // Now generate new_left_blocks * (right).
         new_workorders += getAllWorkOrdersHelperBothNotStored(container,
@@ -122,7 +122,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
                                                              0,
                                                              left_relation_block_ids_[part_id].size(),
                                                              num_right_workorders_generated_[part_id],
-                                                             right_relation_block_ids_[part_id].size());
+                                                             right_relation_block_ids_.size());
       } else if (new_left_blocks > 0 && new_right_blocks == 0) {
         // Generate new_left_blocks * right
         new_workorders = getAllWorkOrdersHelperBothNotStored(container,
@@ -132,11 +132,11 @@ bool NestedLoopsJoinOperator::getAllWorkOrders(
                                                              num_left_workorders_generated_[part_id],
                                                              left_relation_block_ids_[part_id].size(),
                                                              0,
-                                                             right_relation_block_ids_[part_id].size());
+                                                             right_relation_block_ids_.size());
       }
       if (new_workorders > 0) {
         num_left_workorders_generated_[part_id] = left_relation_block_ids_[part_id].size();
-        num_right_workorders_generated_[part_id] = right_relation_block_ids_[part_id].size();
+        num_right_workorders_generated_[part_id] = right_relation_block_ids_.size();
       }
     }
     return done_feeding_left_relation_ && done_feeding_right_relation_;
@@ -155,7 +155,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer
*co
 
     for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
       for (const block_id left_block_id : left_relation_block_ids_[part_id]) {
-        for (const block_id right_block_id : right_relation_block_ids_[part_id]) {
+        for (const block_id right_block_id : right_relation_block_ids_) {
           container->addWorkOrderProto(createWorkOrderProto(part_id, left_block_id, right_block_id),
                                        op_index_);
         }
@@ -169,7 +169,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer
*co
       const std::vector<block_id>::size_type new_left_blocks
           = left_relation_block_ids_[part_id].size() - num_left_workorders_generated_[part_id];
       const std::vector<block_id>::size_type new_right_blocks
-          = right_relation_block_ids_[part_id].size() - num_right_workorders_generated_[part_id];
+          = right_relation_block_ids_.size() - num_right_workorders_generated_[part_id];
 
       std::size_t new_workorders = 0;
       if (new_left_blocks > 0 && new_right_blocks > 0) {
@@ -181,7 +181,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer
*co
                                                      0,
                                                      left_relation_block_ids_[part_id].size(),
                                                      num_right_workorders_generated_[part_id],
-                                                     right_relation_block_ids_[part_id].size());
+                                                     right_relation_block_ids_.size());
 
         // Now generate new_left_blocks * (right).
         new_workorders +=
@@ -199,7 +199,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer
*co
                                                      0,
                                                      left_relation_block_ids_[part_id].size(),
                                                      num_right_workorders_generated_[part_id],
-                                                     right_relation_block_ids_[part_id].size());
+                                                     right_relation_block_ids_.size());
       } else if (new_left_blocks > 0 && new_right_blocks == 0) {
         // Generate new_left_blocks * right
         new_workorders =
@@ -208,11 +208,11 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtos(WorkOrderProtosContainer
*co
                                                      num_left_workorders_generated_[part_id],
                                                      left_relation_block_ids_[part_id].size(),
                                                      0,
-                                                     right_relation_block_ids_[part_id].size());
+                                                     right_relation_block_ids_.size());
       }
       if (new_workorders > 0) {
         num_left_workorders_generated_[part_id] = left_relation_block_ids_[part_id].size();
-        num_right_workorders_generated_[part_id] = right_relation_block_ids_[part_id].size();
+        num_right_workorders_generated_[part_id] = right_relation_block_ids_.size();
       }
     }
     return done_feeding_left_relation_ && done_feeding_right_relation_;
@@ -247,7 +247,7 @@ std::size_t NestedLoopsJoinOperator::getAllWorkOrdersHelperBothNotStored(WorkOrd
               right_input_relation_,
               part_id,
               left_relation_block_ids_[part_id][left_index],
-              right_relation_block_ids_[part_id][right_index],
+              right_relation_block_ids_[right_index],
               query_context->getPredicate(join_predicate_index_),
               query_context->getScalarGroup(selection_index_),
               query_context->getInsertDestination(output_destination_index_),
@@ -274,7 +274,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
   if (left_relation_is_stored_) {
     for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
       for (std::vector<block_id>::size_type right_index = num_right_workorders_generated_[part_id];
-           right_index < right_relation_block_ids_[part_id].size();
+           right_index < right_relation_block_ids_.size();
            ++right_index) {
         for (const block_id left_block_id : left_relation_block_ids_[part_id]) {
           container->addNormalWorkOrder(
@@ -284,7 +284,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
                   right_input_relation_,
                   part_id,
                   left_block_id,
-                  right_relation_block_ids_[part_id][right_index],
+                  right_relation_block_ids_[right_index],
                   join_predicate,
                   selection,
                   output_destination,
@@ -292,7 +292,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
               op_index_);
         }
       }
-      num_right_workorders_generated_[part_id] = right_relation_block_ids_[part_id].size();
+      num_right_workorders_generated_[part_id] = right_relation_block_ids_.size();
     }
     return done_feeding_right_relation_;
   } else {
@@ -300,7 +300,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrdersHelperOneStored(WorkOrdersContaine
       for (std::vector<block_id>::size_type left_index = num_left_workorders_generated_[part_id];
            left_index < left_relation_block_ids_[part_id].size();
            ++left_index) {
-        for (const block_id right_block_id : right_relation_block_ids_[part_id]) {
+        for (const block_id right_block_id : right_relation_block_ids_) {
           container->addNormalWorkOrder(
               new NestedLoopsJoinWorkOrder(query_id_,
                                            left_input_relation_,
@@ -340,7 +340,7 @@ std::size_t NestedLoopsJoinOperator::getAllWorkOrderProtosHelperBothNotStored(
          ++right_index) {
       container->addWorkOrderProto(
           createWorkOrderProto(part_id, left_relation_block_ids_[part_id][left_index],
-                               right_relation_block_ids_[part_id][right_index]),
+                               right_relation_block_ids_[right_index]),
           op_index_);
     }
   }
@@ -354,15 +354,15 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtosHelperOneStored(WorkOrderProt
   if (left_relation_is_stored_) {
     for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
       for (std::vector<block_id>::size_type right_index = num_right_workorders_generated_[part_id];
-           right_index < right_relation_block_ids_[part_id].size();
+           right_index < right_relation_block_ids_.size();
            ++right_index) {
         for (const block_id left_block_id : left_relation_block_ids_[part_id]) {
           container->addWorkOrderProto(
-              createWorkOrderProto(part_id, left_block_id, right_relation_block_ids_[part_id][right_index]),
+              createWorkOrderProto(part_id, left_block_id, right_relation_block_ids_[right_index]),
               op_index_);
         }
       }
-      num_right_workorders_generated_[part_id] = right_relation_block_ids_[part_id].size();
+      num_right_workorders_generated_[part_id] = right_relation_block_ids_.size();
     }
     return done_feeding_right_relation_;
   } else {
@@ -370,7 +370,7 @@ bool NestedLoopsJoinOperator::getAllWorkOrderProtosHelperOneStored(WorkOrderProt
       for (std::vector<block_id>::size_type left_index = num_left_workorders_generated_[part_id];
            left_index < left_relation_block_ids_[part_id].size();
            ++left_index) {
-        for (const block_id right_block_id : right_relation_block_ids_[part_id]) {
+        for (const block_id right_block_id : right_relation_block_ids_) {
           container->addWorkOrderProto(
               createWorkOrderProto(part_id, left_relation_block_ids_[part_id][left_index],
right_block_id),
               op_index_);

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/d85f7a9a/relational_operators/NestedLoopsJoinOperator.hpp
----------------------------------------------------------------------
diff --git a/relational_operators/NestedLoopsJoinOperator.hpp b/relational_operators/NestedLoopsJoinOperator.hpp
index 5960b81..a379c17 100644
--- a/relational_operators/NestedLoopsJoinOperator.hpp
+++ b/relational_operators/NestedLoopsJoinOperator.hpp
@@ -113,7 +113,8 @@ class NestedLoopsJoinOperator : public RelationalOperator {
         left_relation_is_stored_(left_relation_is_stored),
         right_relation_is_stored_(right_relation_is_stored),
         left_relation_block_ids_(num_partitions),
-        right_relation_block_ids_(num_partitions),
+        right_relation_block_ids_(right_relation_is_stored ? right_input_relation_.getBlocksSnapshot()
+                                                           : std::vector<block_id>()),
         num_left_workorders_generated_(num_partitions),
         num_right_workorders_generated_(num_partitions),
         done_feeding_left_relation_(false),
@@ -133,21 +134,6 @@ class NestedLoopsJoinOperator : public RelationalOperator {
         left_relation_block_ids_[0] = left_input_relation_.getBlocksSnapshot();
       }
     }
-
-    if (right_relation_is_stored) {
-      if (right_input_relation_.hasPartitionScheme()) {
-        const PartitionScheme &part_scheme = *right_input_relation_.getPartitionScheme();
-        DCHECK_EQ(num_partitions_, part_scheme.getPartitionSchemeHeader().getNumPartitions());
-        for (std::size_t part_id = 0; part_id < num_partitions_; ++part_id) {
-          right_relation_block_ids_[part_id] = part_scheme.getBlocksInPartition(part_id);
-        }
-      } else {
-        // Broadcast right (smaller) side upon partitioned nlj.
-        for (partition_id part_id = 0; part_id < num_partitions_; ++part_id) {
-          right_relation_block_ids_[part_id] = right_input_relation_.getBlocksSnapshot();
-        }
-      }
-    }
   }
 
   ~NestedLoopsJoinOperator() override {}
@@ -183,10 +169,7 @@ class NestedLoopsJoinOperator : public RelationalOperator {
     if (input_relation_id == left_input_relation_.getID()) {
       left_relation_block_ids_[part_id].push_back(input_block_id);
     } else if (input_relation_id == right_input_relation_.getID()) {
-      // Broadcast right (smaller) side upon partitioned nlj.
-      for (partition_id input_part_id = 0; input_part_id < num_partitions_; ++input_part_id)
{
-        right_relation_block_ids_[input_part_id].push_back(input_block_id);
-      }
+      right_relation_block_ids_.push_back(input_block_id);
     } else {
       LOG(FATAL) << "The input block sent to the NestedLoopsJoinOperator belongs "
                  << "to a different relation than the left and right relations";
@@ -315,7 +298,7 @@ class NestedLoopsJoinOperator : public RelationalOperator {
   const bool right_relation_is_stored_;
 
   std::vector<BlocksInPartition> left_relation_block_ids_;
-  std::vector<BlocksInPartition> right_relation_block_ids_;
+  std::vector<block_id> right_relation_block_ids_;
 
   // At a given point of time, we have paired num_left_workorders_generated[part_id]
   // number of blocks from the left relation with num_right_workorders_generated[part_id]


Mime
View raw message