Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8DD30200C11 for ; Sat, 21 Jan 2017 00:26:49 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 8C5E0160B55; Fri, 20 Jan 2017 23:26:49 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 85773160B48 for ; Sat, 21 Jan 2017 00:26:48 +0100 (CET) Received: (qmail 90874 invoked by uid 500); 20 Jan 2017 23:26:47 -0000 Mailing-List: contact commits-help@quickstep.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@quickstep.incubator.apache.org Delivered-To: mailing list commits@quickstep.incubator.apache.org Received: (qmail 90865 invoked by uid 99); 20 Jan 2017 23:26:47 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 20 Jan 2017 23:26:47 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 5298CC0036 for ; Fri, 20 Jan 2017 23:26:47 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -6.219 X-Spam-Level: X-Spam-Status: No, score=-6.219 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id l7urkcZLM_OE for ; Fri, 20 Jan 2017 23:26:46 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id D03465FAF3 for ; Fri, 20 Jan 2017 23:26:44 +0000 (UTC) Received: (qmail 90858 invoked by uid 99); 20 Jan 2017 23:26:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 20 Jan 2017 23:26:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E7701DFFA8; Fri, 20 Jan 2017 23:26:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zuyuz@apache.org To: commits@quickstep.incubator.apache.org Date: Fri, 20 Jan 2017 23:26:43 -0000 Message-Id: <1a9cb7c6ffe44491a142909b4e0333e0@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] incubator-quickstep git commit: Added Operator support for Partitioned HashJoin. [Forced Update!] archived-at: Fri, 20 Jan 2017 23:26:49 -0000 Repository: incubator-quickstep Updated Branches: refs/heads/partitioned_join_hash_table 5af8904d5 -> 300284c2c (forced update) http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/300284c2/relational_operators/tests/HashJoinOperator_unittest.cpp ---------------------------------------------------------------------- diff --git a/relational_operators/tests/HashJoinOperator_unittest.cpp b/relational_operators/tests/HashJoinOperator_unittest.cpp index 239547f..2370c92 100644 --- a/relational_operators/tests/HashJoinOperator_unittest.cpp +++ b/relational_operators/tests/HashJoinOperator_unittest.cpp @@ -97,6 +97,8 @@ constexpr tuple_id kBlockSize = 10; constexpr std::size_t kQueryId = 0; constexpr int kOpIndex = 0; +constexpr std::size_t kNumPartitions = 1; + } // namespace class HashJoinOperatorTest : public ::testing::TestWithParam { @@ -192,6 +194,8 @@ class HashJoinOperatorTest : public ::testing::TestWithParam } storage_block->rebuild(); } + + num_partitions_ = kNumPartitions; } virtual void TearDown() { @@ -291,6 +295,8 @@ class HashJoinOperatorTest : public ::testing::TestWithParam unique_ptr db_; // The following CatalogRelations are owned by db_. CatalogRelation *dim_table_, *fact_table_; + + std::size_t num_partitions_; }; TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) { @@ -302,7 +308,7 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) { query_context_proto.join_hash_tables_size(); serialization::HashTable *hash_table_proto = - query_context_proto.add_join_hash_tables(); + query_context_proto.add_join_hash_tables()->mutable_join_hash_table(); switch (GetParam()) { case HashTableImplType::kLinearOpenAddressing: hash_table_proto->set_hash_table_impl_type( @@ -341,6 +347,7 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) { true /* is_stored */, std::vector(1, dim_col_long.getID()), dim_col_long.getType().isNullable(), + num_partitions_, join_hash_table_index)); // Create the prober operator with one selection attribute. @@ -370,6 +377,7 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) { true /* is_stored */, std::vector(1, fact_col_long.getID()), fact_col_long.getType().isNullable(), + num_partitions_, *result_table, output_destination_index, join_hash_table_index, @@ -427,7 +435,7 @@ TEST_P(HashJoinOperatorTest, LongKeyHashJoinTest) { } // Create cleaner operator. - unique_ptr cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index)); + unique_ptr cleaner(new DestroyHashOperator(kQueryId, num_partitions_, join_hash_table_index)); cleaner->informAllBlockingDependenciesMet(); fetchAndExecuteWorkOrders(cleaner.get()); @@ -443,7 +451,7 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) { query_context_proto.join_hash_tables_size(); serialization::HashTable *hash_table_proto = - query_context_proto.add_join_hash_tables(); + query_context_proto.add_join_hash_tables()->mutable_join_hash_table(); switch (GetParam()) { case HashTableImplType::kLinearOpenAddressing: hash_table_proto->set_hash_table_impl_type( @@ -485,6 +493,7 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) { true /* is_stored */, std::vector(1, dim_col_int.getID()), dim_col_int.getType().isNullable(), + num_partitions_, join_hash_table_index)); // Create the prober operator with two selection attributes. @@ -519,6 +528,7 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) { true /* is_stored */, std::vector(1, fact_col_int.getID()), fact_col_int.getType().isNullable(), + num_partitions_, *result_table, output_destination_index, join_hash_table_index, @@ -598,7 +608,7 @@ TEST_P(HashJoinOperatorTest, IntDuplicateKeyHashJoinTest) { } // Create cleaner operator. - unique_ptr cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index)); + unique_ptr cleaner(new DestroyHashOperator(kQueryId, num_partitions_, join_hash_table_index)); cleaner->informAllBlockingDependenciesMet(); fetchAndExecuteWorkOrders(cleaner.get()); @@ -614,7 +624,7 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) { query_context_proto.join_hash_tables_size(); serialization::HashTable *hash_table_proto = - query_context_proto.add_join_hash_tables(); + query_context_proto.add_join_hash_tables()->mutable_join_hash_table(); switch (GetParam()) { case HashTableImplType::kLinearOpenAddressing: hash_table_proto->set_hash_table_impl_type( @@ -648,6 +658,7 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) { true /* is_stored */, std::vector(1, dim_col_char.getID()), dim_col_char.getType().isNullable(), + num_partitions_, join_hash_table_index)); // Create prober operator with one selection attribute. @@ -677,6 +688,7 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) { true /* is_stored */, std::vector(1, fact_col_char.getID()), fact_col_char.getType().isNullable(), + num_partitions_, *result_table, output_destination_index, join_hash_table_index, @@ -734,7 +746,7 @@ TEST_P(HashJoinOperatorTest, CharKeyCartesianProductHashJoinTest) { } // Create cleaner operator. - unique_ptr cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index)); + unique_ptr cleaner(new DestroyHashOperator(kQueryId, num_partitions_, join_hash_table_index)); cleaner->informAllBlockingDependenciesMet(); fetchAndExecuteWorkOrders(cleaner.get()); @@ -750,7 +762,7 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) { query_context_proto.join_hash_tables_size(); serialization::HashTable *hash_table_proto = - query_context_proto.add_join_hash_tables(); + query_context_proto.add_join_hash_tables()->mutable_join_hash_table(); switch (GetParam()) { case HashTableImplType::kLinearOpenAddressing: hash_table_proto->set_hash_table_impl_type( @@ -785,6 +797,7 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) { true /* is_stored */, std::vector(1, dim_col_varchar.getID()), dim_col_varchar.getType().isNullable(), + num_partitions_, join_hash_table_index)); // Create prober operator with two selection attributes. @@ -819,6 +832,7 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) { true /* is_stored */, std::vector(1, fact_col_varchar.getID()), fact_col_varchar.getType().isNullable(), + num_partitions_, *result_table, output_destination_index, join_hash_table_index, @@ -902,7 +916,7 @@ TEST_P(HashJoinOperatorTest, VarCharDuplicateKeyHashJoinTest) { } // Create the cleaner operator. - unique_ptr cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index)); + unique_ptr cleaner(new DestroyHashOperator(kQueryId, num_partitions_, join_hash_table_index)); cleaner->informAllBlockingDependenciesMet(); fetchAndExecuteWorkOrders(cleaner.get()); @@ -918,7 +932,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) { query_context_proto.join_hash_tables_size(); serialization::HashTable *hash_table_proto = - query_context_proto.add_join_hash_tables(); + query_context_proto.add_join_hash_tables()->mutable_join_hash_table(); switch (GetParam()) { case HashTableImplType::kLinearOpenAddressing: hash_table_proto->set_hash_table_impl_type( @@ -958,6 +972,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) { true /* is_stored */, dim_key_attrs, dim_col_long.getType().isNullable() || dim_col_varchar.getType().isNullable(), + num_partitions_, join_hash_table_index)); // Create the prober operator with two selection attributes. @@ -997,6 +1012,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) { fact_key_attrs, fact_col_long.getType().isNullable() || fact_col_varchar.getType().isNullable(), + num_partitions_, *result_table, output_destination_index, join_hash_table_index, @@ -1079,7 +1095,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinTest) { } // Create cleaner operator. - unique_ptr cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index)); + unique_ptr cleaner(new DestroyHashOperator(kQueryId, num_partitions_, join_hash_table_index)); cleaner->informAllBlockingDependenciesMet(); fetchAndExecuteWorkOrders(cleaner.get()); @@ -1096,7 +1112,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) { query_context_proto.join_hash_tables_size(); serialization::HashTable *hash_table_proto = - query_context_proto.add_join_hash_tables(); + query_context_proto.add_join_hash_tables()->mutable_join_hash_table(); switch (GetParam()) { case HashTableImplType::kLinearOpenAddressing: hash_table_proto->set_hash_table_impl_type( @@ -1136,6 +1152,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) { true /* is_stored */, dim_key_attrs, dim_col_long.getType().isNullable() || dim_col_varchar.getType().isNullable(), + num_partitions_, join_hash_table_index)); // Create prober operator with two selection attributes. @@ -1185,6 +1202,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) { fact_key_attrs, fact_col_long.getType().isNullable() || fact_col_varchar.getType().isNullable(), + num_partitions_, *result_table, output_destination_index, join_hash_table_index, @@ -1267,7 +1285,7 @@ TEST_P(HashJoinOperatorTest, CompositeKeyHashJoinWithResidualPredicateTest) { } // Create cleaner operator. - unique_ptr cleaner(new DestroyHashOperator(kQueryId, join_hash_table_index)); + unique_ptr cleaner(new DestroyHashOperator(kQueryId, num_partitions_, join_hash_table_index)); cleaner->informAllBlockingDependenciesMet(); fetchAndExecuteWorkOrders(cleaner.get()); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/300284c2/storage/StorageBlockInfo.hpp ---------------------------------------------------------------------- diff --git a/storage/StorageBlockInfo.hpp b/storage/StorageBlockInfo.hpp index 361648f..ad5795b 100644 --- a/storage/StorageBlockInfo.hpp +++ b/storage/StorageBlockInfo.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include "utility/Macros.hpp" @@ -49,6 +50,8 @@ static constexpr int kBlockIdDomainLengthInDigits = std::numeric_limits::digits10; static constexpr block_id_domain kMaxDomain = static_cast(0xFFFF); +typedef std::vector BlocksInPartition; + /** * @brief All-static object that provides access to block_id. **/