From commits-return-6691-archive-asf-public=cust-asf.ponee.io@kudu.apache.org Tue Oct 30 00:09:34 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 313AF180677 for ; Tue, 30 Oct 2018 00:09:34 +0100 (CET) Received: (qmail 3249 invoked by uid 500); 29 Oct 2018 23:09:33 -0000 Mailing-List: contact commits-help@kudu.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@kudu.apache.org Delivered-To: mailing list commits@kudu.apache.org Received: (qmail 3235 invoked by uid 99); 29 Oct 2018 23:09:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 29 Oct 2018 23:09:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 718A1E048B; Mon, 29 Oct 2018 23:09:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: alexey@apache.org To: commits@kudu.apache.org Date: Mon, 29 Oct 2018 23:09:33 -0000 Message-Id: <46ea4ab93e394460bb1227c79851d0fb@git.apache.org> In-Reply-To: <3ca904a308ac4804b40d432746e3d2c8@git.apache.org> References: <3ca904a308ac4804b40d432746e3d2c8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/4] kudu git commit: [rebalancer] location-aware rebalancer (part 2/n) [rebalancer] location-aware rebalancer (part 2/n) Small refactoring on tools::Rebalancer: renamed Rebalancer::ResetKsck() into Rebalancer::RefreshKsckResults() and made the actual ksck utility run as a part of the renamed method. Change-Id: Ia1bdd4ee13d0522e49467b8b6e4e3a76ec560b26 Reviewed-on: http://gerrit.cloudera.org:8080/11743 Tested-by: Alexey Serbin Reviewed-by: Will Berkeley Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/fd507235 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/fd507235 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/fd507235 Branch: refs/heads/master Commit: fd507235b531419f6bc2447263278eee315177ab Parents: 81d0109 Author: Alexey Serbin Authored: Fri Oct 19 15:52:33 2018 -0700 Committer: Alexey Serbin Committed: Mon Oct 29 23:05:31 2018 +0000 ---------------------------------------------------------------------- src/kudu/tools/rebalancer.cc | 23 +++++++++++------------ src/kudu/tools/rebalancer.h | 11 ++--------- 2 files changed, 13 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/fd507235/src/kudu/tools/rebalancer.cc ---------------------------------------------------------------------- diff --git a/src/kudu/tools/rebalancer.cc b/src/kudu/tools/rebalancer.cc index 9dae39c..09af4bd 100644 --- a/src/kudu/tools/rebalancer.cc +++ b/src/kudu/tools/rebalancer.cc @@ -100,8 +100,7 @@ Rebalancer::Rebalancer(const Config& config) Status Rebalancer::PrintStats(std::ostream& out) { // First, report on the current balance state of the cluster. - RETURN_NOT_OK(ResetKsck()); - ignore_result(ksck_->Run()); + RETURN_NOT_OK(RefreshKsckResults()); const KsckResults& results = ksck_->results(); ClusterBalanceInfo cbi; @@ -314,7 +313,7 @@ Status Rebalancer::Run(RunStatus* result_status, size_t* moves_count) { // moves recommended at prior steps are still in progress. Status Rebalancer::KsckResultsToClusterBalanceInfo( const KsckResults& ksck_info, - const MovesInProgress& pending_moves, + const MovesInProgress& moves_in_progress, ClusterBalanceInfo* cbi) const { DCHECK(cbi); @@ -360,7 +359,7 @@ Status Rebalancer::KsckResultsToClusterBalanceInfo( // Check if it's one of the tablets which are currently being rebalanced. // If so, interpret the move as successfully completed, updating the // replica counts correspondingly. - const auto it_pending_moves = pending_moves.find(tablet.id); + const auto it_pending_moves = moves_in_progress.find(tablet.id); for (const auto& ri : tablet.replicas) { // Increment total count of replicas at the tablet server. @@ -375,7 +374,7 @@ Status Rebalancer::KsckResultsToClusterBalanceInfo( continue; } bool do_count_replica = true; - if (it_pending_moves != pending_moves.end() && + if (it_pending_moves != moves_in_progress.end() && tablet.result == KsckCheckResult::RECOVERING) { const auto& move_info = it_pending_moves->second; bool is_target_replica_present = false; @@ -453,10 +452,9 @@ Status Rebalancer::KsckResultsToClusterBalanceInfo( // Run one step of the rebalancer. Due to the inherent restrictions of the // rebalancing engine, no more than one replica per tablet is moved during // one step of the rebalancing. -Status Rebalancer::GetNextMoves(const MovesInProgress& pending_moves, +Status Rebalancer::GetNextMoves(const MovesInProgress& moves_in_progress, vector* replica_moves) { - RETURN_NOT_OK(ResetKsck()); - ignore_result(ksck_->Run()); + RETURN_NOT_OK(RefreshKsckResults()); const auto& ksck_info = ksck_->results(); // For simplicity, allow to run the rebalancing only when all tablet servers @@ -480,8 +478,8 @@ Status Rebalancer::GetNextMoves(const MovesInProgress& pending_moves, vector moves; { ClusterBalanceInfo cbi; - RETURN_NOT_OK(KsckResultsToClusterBalanceInfo(ksck_info, pending_moves, &cbi)); - RETURN_NOT_OK(algo_.GetNextMoves(std::move(cbi), max_moves, &moves)); + RETURN_NOT_OK(KsckResultsToClusterBalanceInfo(ksck_info, moves_in_progress, &cbi)); + RETURN_NOT_OK(algo_.GetNextMoves(cbi, max_moves, &moves)); } if (moves.empty()) { // No suitable moves were found: the cluster described by 'cbi' is balanced, @@ -489,7 +487,7 @@ Status Rebalancer::GetNextMoves(const MovesInProgress& pending_moves, return Status::OK(); } unordered_set tablets_in_move; - std::transform(pending_moves.begin(), pending_moves.end(), + std::transform(moves_in_progress.begin(), moves_in_progress.end(), inserter(tablets_in_move, tablets_in_move.begin()), [](const MovesInProgress::value_type& elem) { return elem.first; @@ -618,13 +616,14 @@ Status Rebalancer::FindReplicas(const TableReplicaMove& move, return Status::OK(); } -Status Rebalancer::ResetKsck() { +Status Rebalancer::RefreshKsckResults() { shared_ptr cluster; RETURN_NOT_OK_PREPEND( RemoteKsckCluster::Build(config_.master_addresses, &cluster), "unable to build KsckCluster"); ksck_.reset(new Ksck(cluster)); ksck_->set_table_filters(config_.table_filters); + ignore_result(ksck_->Run()); return Status::OK(); } http://git-wip-us.apache.org/repos/asf/kudu/blob/fd507235/src/kudu/tools/rebalancer.h ---------------------------------------------------------------------- diff --git a/src/kudu/tools/rebalancer.h b/src/kudu/tools/rebalancer.h index a6e4caf..3c038aa 100644 --- a/src/kudu/tools/rebalancer.h +++ b/src/kudu/tools/rebalancer.h @@ -36,12 +36,6 @@ #include "kudu/util/status.h" namespace kudu { -namespace tools { -struct TabletsPlacementInfo; -} // namespace tools -} // namespace kudu - -namespace kudu { namespace client { class KuduClient; @@ -50,7 +44,6 @@ class KuduClient; namespace tools { class Ksck; -struct KsckResults; // Sub-set of fields from KsckResult which are relevant to the rebalancing. struct ClusterRawInfo { @@ -281,8 +274,8 @@ class Rebalancer { const KsckResults& ksck_info, std::vector* tablet_ids) const; - // Reset ksck-related fields, preparing for a fresh ksck run. - Status ResetKsck(); + // Reset ksck-related fields and run ksck against the cluster. + Status RefreshKsckResults(); // Filter out move operations at the tablets which already have operations // in progress. The 'replica_moves' cannot be null.