Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 50756200B6B for ; Fri, 26 Aug 2016 00:25:04 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 4EB8A160ABD; Thu, 25 Aug 2016 22:25:04 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 481E1160AA5 for ; Fri, 26 Aug 2016 00:25:03 +0200 (CEST) Received: (qmail 55575 invoked by uid 500); 25 Aug 2016 22:25:02 -0000 Mailing-List: contact commits-help@kudu.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kudu.apache.org Delivered-To: mailing list commits@kudu.apache.org Received: (qmail 55566 invoked by uid 99); 25 Aug 2016 22:25:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 25 Aug 2016 22:25:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5EB85E0243; Thu, 25 Aug 2016 22:25:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: adar@apache.org To: commits@kudu.apache.org Message-Id: <5a1dd835c32246b1ac093eb8a9f1ea90@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: kudu git commit: KUDU-687: add replication factor to KuduTable Date: Thu, 25 Aug 2016 22:25:02 +0000 (UTC) archived-at: Thu, 25 Aug 2016 22:25:04 -0000 Repository: kudu Updated Branches: refs/heads/master 86afadc7f -> ebe4d78bf KUDU-687: add replication factor to KuduTable This is generally useful, and necessary if ksck is to use the C++ client. While I was at it, I reduced the use of a second table in client-test to only those tests that actually needed it. Change-Id: If76ec6c3001e78a31517991d7432f79d4645fccc Reviewed-on: http://gerrit.cloudera.org:8080/4123 Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/ebe4d78b Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/ebe4d78b Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/ebe4d78b Branch: refs/heads/master Commit: ebe4d78bf4bb9402a717493d051a304eb1bf3086 Parents: 86afadc Author: Adar Dembo Authored: Thu Aug 25 00:18:29 2016 -0700 Committer: Adar Dembo Committed: Thu Aug 25 22:24:34 2016 +0000 ---------------------------------------------------------------------- src/kudu/client/client-internal.cc | 21 +++++++++++++++------ src/kudu/client/client-internal.h | 3 ++- src/kudu/client/client-test.cc | 29 +++++++++++++++++++++-------- src/kudu/client/client.cc | 24 ++++++++++++++++-------- src/kudu/client/client.h | 6 +++++- src/kudu/client/table-internal.cc | 2 ++ src/kudu/client/table-internal.h | 6 ++++-- 7 files changed, 65 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/ebe4d78b/src/kudu/client/client-internal.cc ---------------------------------------------------------------------- diff --git a/src/kudu/client/client-internal.cc b/src/kudu/client/client-internal.cc index c126ca5..25ce831 100644 --- a/src/kudu/client/client-internal.cc +++ b/src/kudu/client/client-internal.cc @@ -542,7 +542,8 @@ Status KuduClient::Data::GetTableSchema(KuduClient* client, const MonoTime& deadline, KuduSchema* schema, PartitionSchema* partition_schema, - string* table_id) { + string* table_id, + int* num_replicas) { GetTableSchemaRequestPB req; GetTableSchemaResponsePB resp; @@ -565,11 +566,19 @@ Status KuduClient::Data::GetTableSchema(KuduClient* client, *new_schema.get(), &new_partition_schema)); - // Parsing was successful; release the schemas to the user. - delete schema->schema_; - schema->schema_ = new_schema.release(); - *partition_schema = std::move(new_partition_schema); - *table_id = resp.table_id(); + if (schema) { + delete schema->schema_; + schema->schema_ = new_schema.release(); + } + if (partition_schema) { + *partition_schema = std::move(new_partition_schema); + } + if (table_id) { + *table_id = resp.table_id(); + } + if (num_replicas) { + *num_replicas = resp.num_replicas(); + } return Status::OK(); } http://git-wip-us.apache.org/repos/asf/kudu/blob/ebe4d78b/src/kudu/client/client-internal.h ---------------------------------------------------------------------- diff --git a/src/kudu/client/client-internal.h b/src/kudu/client/client-internal.h index 86e253a..e614b62 100644 --- a/src/kudu/client/client-internal.h +++ b/src/kudu/client/client-internal.h @@ -108,7 +108,8 @@ class KuduClient::Data { const MonoTime& deadline, KuduSchema* schema, PartitionSchema* partition_schema, - std::string* table_id); + std::string* table_id, + int* num_replicas); Status InitLocalHostNames(); http://git-wip-us.apache.org/repos/asf/kudu/blob/ebe4d78b/src/kudu/client/client-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/client/client-test.cc b/src/kudu/client/client-test.cc index 7420250..26ba4c3 100644 --- a/src/kudu/client/client-test.cc +++ b/src/kudu/client/client-test.cc @@ -91,6 +91,7 @@ using std::string; using std::unique_ptr; using std::vector; using std::map; +using strings::Substitute; namespace kudu { namespace client { @@ -138,7 +139,6 @@ class ClientTest : public KuduTest { .Build(&client_)); ASSERT_NO_FATAL_FAILURE(CreateTable(kTableName, 1, GenerateSplitRows(), {}, &client_table_)); - ASSERT_NO_FATAL_FAILURE(CreateTable(kTable2Name, 1, {}, {}, &client_table2_)); } // Looks up the remote tablet entry for a given partition key in the meta cache. @@ -179,7 +179,6 @@ class ClientTest : public KuduTest { protected: static const char *kTableName; - static const char *kTable2Name; static const int32_t kNoBound; string GetFirstTabletId(KuduTable* table) { @@ -491,7 +490,7 @@ class ClientTest : public KuduTest { } } if (!ts_found) { - return Status::InvalidArgument(strings::Substitute("Could not find tablet server $1", uuid)); + return Status::InvalidArgument(Substitute("Could not find tablet server $1", uuid)); } return Status::OK(); @@ -522,14 +521,16 @@ class ClientTest : public KuduTest { gscoped_ptr cluster_; shared_ptr client_; shared_ptr client_table_; - shared_ptr client_table2_; }; const char *ClientTest::kTableName = "client-testtb"; -const char *ClientTest::kTable2Name = "client-testtb2"; const int32_t ClientTest::kNoBound = kint32max; TEST_F(ClientTest, TestListTables) { + const char* kTable2Name = "client-testtb2"; + shared_ptr second_table; + NO_FATALS(CreateTable(kTable2Name, 1, {}, {}, &second_table)); + vector tables; ASSERT_OK(client_->ListTables(&tables)); std::sort(tables.begin(), tables.end()); @@ -1916,6 +1917,9 @@ TEST_F(ClientTest, TestSessionClose) { // Test which sends multiple batches through the same session, each of which // contains multiple rows spread across multiple tablets. TEST_F(ClientTest, TestMultipleMultiRowManualBatches) { + shared_ptr second_table; + NO_FATALS(CreateTable("second table", 1, {}, {}, &second_table)); + shared_ptr session = client_->NewSession(); ASSERT_OK(session->SetFlushMode(KuduSession::MANUAL_FLUSH)); @@ -1928,7 +1932,7 @@ TEST_F(ClientTest, TestMultipleMultiRowManualBatches) { for (int i = 0; i < kRowsPerBatch; i++) { ASSERT_OK(ApplyInsertToSession( session.get(), - (row_key % 2 == 0) ? client_table_ : client_table2_, + (row_key % 2 == 0) ? client_table_ : second_table, row_key, row_key * 10, "hello world")); row_key++; } @@ -1939,7 +1943,7 @@ TEST_F(ClientTest, TestMultipleMultiRowManualBatches) { const int kNumRowsPerTablet = kNumBatches * kRowsPerBatch / 2; ASSERT_EQ(kNumRowsPerTablet, CountRowsFromClient(client_table_.get())); - ASSERT_EQ(kNumRowsPerTablet, CountRowsFromClient(client_table2_.get())); + ASSERT_EQ(kNumRowsPerTablet, CountRowsFromClient(second_table.get())); // Verify the data looks right. vector rows; @@ -3102,7 +3106,7 @@ TEST_F(ClientTest, TestServerTooBusyRetry) { int t = 0; while (!stop) { scoped_refptr thread; - ASSERT_OK(kudu::Thread::Create("test", strings::Substitute("t$0", t++), + ASSERT_OK(kudu::Thread::Create("test", Substitute("t$0", t++), &ClientTest::CheckRowCount, this, client_table_.get(), kNumRows, &thread)); threads.push_back(thread); @@ -3254,5 +3258,14 @@ TEST_F(ClientTest, TestBatchScanConstIterator) { } } +TEST_F(ClientTest, TestTableNumReplicas) { + for (int i : { 1, 3, 5, 7, 9 }) { + shared_ptr table; + NO_FATALS(CreateTable(Substitute("table_with_$0_replicas", i), + i, {}, {}, &table)); + ASSERT_EQ(i, table->num_replicas()); + } +} + } // namespace client } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/ebe4d78b/src/kudu/client/client.cc ---------------------------------------------------------------------- diff --git a/src/kudu/client/client.cc b/src/kudu/client/client.cc index 11cdee1..e7275a0 100644 --- a/src/kudu/client/client.cc +++ b/src/kudu/client/client.cc @@ -284,14 +284,13 @@ Status KuduClient::IsAlterTableInProgress(const string& table_name, Status KuduClient::GetTableSchema(const string& table_name, KuduSchema* schema) { MonoTime deadline = MonoTime::Now() + default_admin_operation_timeout(); - string table_id_ignored; - PartitionSchema partition_schema; return data_->GetTableSchema(this, table_name, deadline, schema, - &partition_schema, - &table_id_ignored); + nullptr, // partition schema + nullptr, // table id + nullptr); // number of replicas } Status KuduClient::ListTabletServers(vector* tablet_servers) { @@ -367,6 +366,7 @@ Status KuduClient::OpenTable(const string& table_name, shared_ptr* table) { KuduSchema schema; string table_id; + int num_replicas; PartitionSchema partition_schema; MonoTime deadline = MonoTime::Now() + default_admin_operation_timeout(); RETURN_NOT_OK(data_->GetTableSchema(this, @@ -374,11 +374,13 @@ Status KuduClient::OpenTable(const string& table_name, deadline, &schema, &partition_schema, - &table_id)); + &table_id, + &num_replicas)); // TODO: in the future, probably will look up the table in some map to reuse // KuduTable instances. - table->reset(new KuduTable(shared_from_this(), table_name, table_id, + table->reset(new KuduTable(shared_from_this(), + table_name, table_id, num_replicas, schema, partition_schema)); return Status::OK(); } @@ -586,10 +588,12 @@ Status KuduTableCreator::Create() { KuduTable::KuduTable(const shared_ptr& client, const string& name, - const string& table_id, + const string& id, + int num_replicas, const KuduSchema& schema, const PartitionSchema& partition_schema) - : data_(new KuduTable::Data(client, name, table_id, schema, partition_schema)) { + : data_(new KuduTable::Data(client, name, id, num_replicas, + schema, partition_schema)) { } KuduTable::~KuduTable() { @@ -608,6 +612,10 @@ const KuduSchema& KuduTable::schema() const { return data_->schema_; } +int KuduTable::num_replicas() const { + return data_->num_replicas_; +} + KuduInsert* KuduTable::NewInsert() { return new KuduInsert(shared_from_this()); } http://git-wip-us.apache.org/repos/asf/kudu/blob/ebe4d78b/src/kudu/client/client.h ---------------------------------------------------------------------- diff --git a/src/kudu/client/client.h b/src/kudu/client/client.h index 89e5a40..f551765 100644 --- a/src/kudu/client/client.h +++ b/src/kudu/client/client.h @@ -664,6 +664,9 @@ class KUDU_EXPORT KuduTable : public sp::enable_shared_from_this { /// @return Reference to the table's schema object. const KuduSchema& schema() const; + /// @return Replication factor of the table. + int num_replicas() const; + /// @return New @c INSERT operation for this table. It is the caller's /// responsibility to free the result, unless it is passed to /// KuduSession::Apply(). @@ -725,7 +728,8 @@ class KUDU_EXPORT KuduTable : public sp::enable_shared_from_this { KuduTable(const sp::shared_ptr& client, const std::string& name, - const std::string& table_id, + const std::string& id, + int num_replicas, const KuduSchema& schema, const PartitionSchema& partition_schema); http://git-wip-us.apache.org/repos/asf/kudu/blob/ebe4d78b/src/kudu/client/table-internal.cc ---------------------------------------------------------------------- diff --git a/src/kudu/client/table-internal.cc b/src/kudu/client/table-internal.cc index ec0df68..b3f3258 100644 --- a/src/kudu/client/table-internal.cc +++ b/src/kudu/client/table-internal.cc @@ -27,11 +27,13 @@ using sp::shared_ptr; KuduTable::Data::Data(shared_ptr client, string name, string id, + int num_replicas, const KuduSchema& schema, PartitionSchema partition_schema) : client_(std::move(client)), name_(std::move(name)), id_(std::move(id)), + num_replicas_(num_replicas), schema_(schema), partition_schema_(std::move(partition_schema)) { } http://git-wip-us.apache.org/repos/asf/kudu/blob/ebe4d78b/src/kudu/client/table-internal.h ---------------------------------------------------------------------- diff --git a/src/kudu/client/table-internal.h b/src/kudu/client/table-internal.h index 17c90f2..a29c146 100644 --- a/src/kudu/client/table-internal.h +++ b/src/kudu/client/table-internal.h @@ -30,15 +30,17 @@ class KuduTable::Data { public: Data(sp::shared_ptr client, std::string name, - std::string table_id, + std::string id, + int num_replicas, const KuduSchema& schema, PartitionSchema partition_schema); ~Data(); sp::shared_ptr client_; - std::string name_; + const std::string name_; const std::string id_; + const int num_replicas_; // TODO: figure out how we deal with a schema change from the client perspective. // Do we make them call a RefreshSchema() method? Or maybe reopen the table and get