kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ale...@apache.org
Subject [2/2] kudu git commit: [tests] location assignment for ExternaMiniCluster
Date Tue, 09 Oct 2018 17:43:28 GMT
[tests] location assignment for ExternaMiniCluster

Introduced location assignment rules for tablet servers running as
part of ExternalMiniCluster.  With this changelist, it's possible
to have tablet servers of ExternalMiniCluster to be spread among
locations in accordance with simple number-of-servers-at-location
mapping rules.

For example, the following is a self-descriptive example of such
mapping rules set to spread six tablet servers in a cluster among
three locations:

  L0:1  L1:2  L2:3

Added an integration test to verify the location assignment
rules for ExternalMiniCluster work as intended.

Change-Id: I63309804cf2fdd8a620b50abdd7133af6a033c30
Reviewed-on: http://gerrit.cloudera.org:8080/11606
Tested-by: Kudu Jenkins
Reviewed-by: Will Berkeley <wdberkeley@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b193e378
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b193e378
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b193e378

Branch: refs/heads/master
Commit: b193e3786861686590b6bf67bc507d8d4d18d8e3
Parents: 91faf2a
Author: Alexey Serbin <aserbin@cloudera.com>
Authored: Fri Oct 5 22:12:41 2018 -0700
Committer: Alexey Serbin <aserbin@cloudera.com>
Committed: Tue Oct 9 17:42:50 2018 +0000

----------------------------------------------------------------------
 src/kudu/integration-tests/CMakeLists.txt       |   2 +
 .../integration-tests/cluster_itest_util.cc     |   1 +
 src/kudu/integration-tests/cluster_itest_util.h |   1 +
 .../scripts/assign-location.py                  | 244 +++++++++++++++++++
 .../ts_location_assignment-itest.cc             | 132 ++++++++++
 src/kudu/master/ts_descriptor.cc                |  20 +-
 src/kudu/mini-cluster/external_mini_cluster.cc  |  24 +-
 src/kudu/mini-cluster/external_mini_cluster.h   |  10 +
 8 files changed, 428 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/integration-tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/CMakeLists.txt b/src/kudu/integration-tests/CMakeLists.txt
index 512a454..7ecb697 100644
--- a/src/kudu/integration-tests/CMakeLists.txt
+++ b/src/kudu/integration-tests/CMakeLists.txt
@@ -114,6 +114,8 @@ ADD_KUDU_TEST(tombstoned_voting-imc-itest)
 ADD_KUDU_TEST(tombstoned_voting-itest)
 ADD_KUDU_TEST(tombstoned_voting-stress-test RUN_SERIAL true)
 ADD_KUDU_TEST(token_signer-itest)
+ADD_KUDU_TEST(ts_location_assignment-itest
+  DATA_FILES scripts/assign-location.py)
 ADD_KUDU_TEST(ts_recovery-itest PROCESSORS 4)
 ADD_KUDU_TEST(ts_tablet_manager-itest)
 ADD_KUDU_TEST(update_scan_delta_compact-test RUN_SERIAL true)

http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/integration-tests/cluster_itest_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/cluster_itest_util.cc b/src/kudu/integration-tests/cluster_itest_util.cc
index 7231094..186cf01 100644
--- a/src/kudu/integration-tests/cluster_itest_util.cc
+++ b/src/kudu/integration-tests/cluster_itest_util.cc
@@ -308,6 +308,7 @@ Status CreateTabletServerMap(const shared_ptr<MasterServiceProxy>&
master_proxy,
     unique_ptr<TServerDetails> peer(new TServerDetails);
     peer->instance_id.CopyFrom(entry.instance_id());
     peer->registration.CopyFrom(entry.registration());
+    peer->location = entry.location();
 
     CreateTsClientProxies(addresses[0],
                           messenger,

http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/integration-tests/cluster_itest_util.h
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/cluster_itest_util.h b/src/kudu/integration-tests/cluster_itest_util.h
index c5ba53a..1768880 100644
--- a/src/kudu/integration-tests/cluster_itest_util.h
+++ b/src/kudu/integration-tests/cluster_itest_util.h
@@ -79,6 +79,7 @@ struct TServerDetails {
   std::unique_ptr<tserver::TabletServerAdminServiceProxy> tserver_admin_proxy;
   std::unique_ptr<consensus::ConsensusServiceProxy> consensus_proxy;
   std::unique_ptr<server::GenericServiceProxy> generic_proxy;
+  std::string location;
 
   // Convenience function to get the UUID from the instance_id struct.
   const std::string& uuid() const;

http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/integration-tests/scripts/assign-location.py
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/scripts/assign-location.py b/src/kudu/integration-tests/scripts/assign-location.py
new file mode 100644
index 0000000..5714772
--- /dev/null
+++ b/src/kudu/integration-tests/scripts/assign-location.py
@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import errno
+import fcntl
+import json
+import time
+import random
+
+# This is a simple sequencer to be run as a location assignment script
+# by a Kudu master. The script can be used in location-aware test scenarios
+# and other cases when location assignment rules are specified simply as the
+# distribution of tablet servers among locations: i.e. how many tablet
+# servers should be in every specified location (see below for an example).
+#
+# The script takes as input location mapping rules and an identifier.
+# On success, the script prints the location assigned to the specified
+# identifier to stdout. The identifier might be any string uniquely identifying
+# a tablet server.
+#
+# Locations are assigned based on:
+#   a) Location mapping rules specified in the command line and sequencer's
+#      offset persistently stored in a state file.
+#   b) Previously established and persisted { id, location } mappings in the
+#      state file.
+#
+# Once assigned, the location for the specified identifier is recorded and
+# output again upon next call of the script for the same identifier.
+#
+# It's safe to run multiple instances of the script concurrently with the
+# same set of parameters. The access to the sequencer's state file is
+# serialized and the scripts produces consistent results for all concurrent
+# callers.
+#
+# A location mapping rule is specified as a pair 'loc:num', where the 'num'
+# stands for the number of servers to assign to the location 'loc'. Location
+# mapping rules are provided to the script by --map 'loc:num' command line
+# arguments.
+#
+# Below is an example of invocation of the script for location mapping rules
+# specifying that location 'l0' should have one tablet server, location 'l1'
+# should have  two, and location 'l2' should have three. The script is run
+# to assign a location for a tablet server running at IP address 127.1.2.3.
+#
+#   assign-location.py --map l0:1 --map l1:2 --map l2:3 127.1.2.3
+#
+
+class LocationAssignmentRule(object):
+  def __init__(self, location_mapping_rules):
+    # Convert the input location information into an auxiliary array of
+    # location strings.
+    self.location_mapping_rules = location_mapping_rules
+    if self.location_mapping_rules is None:
+      self.location_mapping_rules = []
+    self.locations = []
+    self.total_count = 0
+
+    seen_locations = []
+    for info in self.location_mapping_rules:
+      location, server_num_str = info.split(':')
+      seen_locations.append(location)
+      server_num = int(server_num_str)
+      for i in range(0, server_num):
+        self.total_count += 1
+        self.locations.append(location)
+    assert (len(set(seen_locations)) == len(seen_locations)), \
+        'duplicate locations specified: {}'.format(seen_locations)
+
+  def get_location(self, idx):
+    """
+    Get location for the specified index.
+    """
+    if self.locations:
+      return self.locations[idx % len(self.locations)]
+    else:
+      return ""
+
+
+def acquire_advisory_lock(fpath):
+  """
+  Acquire a lock on a special .lock file. Don't block while trying: return
+  if failed to acquire a lock in 30 seconds.
+  """
+  timeout_seconds = 30
+  now = time.clock()
+  deadline = now + timeout_seconds
+  random.seed(int(now))
+  fpath_lock_file = fpath + ".lock"
+  # Open the lock file; create the file if doesn't exist.
+  lock_file = open(fpath_lock_file, 'w+')
+  got_lock = False
+  while time.clock() < deadline:
+    try:
+      fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
+      got_lock = True
+      break
+    except IOError as e:
+      if e.errno != errno.EAGAIN:
+        raise
+      else:
+        time.sleep(random.uniform(0.001, 0.100))
+
+  if not got_lock:
+    raise Exception('could not obtain exclusive lock for {} in {} seconds',
+        fpath_lock_file, timeout_seconds)
+
+  return lock_file
+
+
+def get_location(fpath, rule, uid, relaxed):
+  """
+  Return location for the specified identifier 'uid'. To do that, use the
+  specified location mapping rules and the information stored
+  in the sequencer's state file.
+
+  * Obtain advisory lock for the state file (using additional .lock file)
+  * If the sequencer's state file exists:
+      1. Open the state file in read-only mode.
+      2. Read the information from the state file and search for location
+         assigned to the server with the specified identifier.
+           a. If already assigned location found:
+                -- Return the location.
+           b. If location assigned to the identifier is not found:
+                -- Use current sequence number 'seq' to assign next location
+                   by calling LocationAssignmentRule.get_location(seq).
+                -- Add the newly generated location assignment into the
+                   sequencer's state.
+                -- Increment the sequence number.
+                -- Reopen the state file for writing (if file exists)
+                -- Rewrite the file with the new state of the sequencer.
+                -- Return the newly assigned location.
+  * If the sequencer's state file does not exist:
+      1. Set sequence number 'seq' to 0.
+      2. Use current sequence number 'seq' to assign next location
+         by calling LocationAssignmentRule.get_location(seq).
+      3. Update the sequencer's state accordingly.
+      3. Rewrite the file with the new state of the sequencer.
+      4. Return the newly assigned location.
+  """
+  lock_file = acquire_advisory_lock(fpath)
+  state_file = None
+  try:
+    state_file = open(fpath)
+  except IOError as e:
+    if e.errno != errno.ENOENT:
+      raise
+
+  new_assignment = False
+  if state_file is None:
+    seq = 0
+    state = {}
+    state['seq'] = seq
+    state['mapping_rules'] = rule.location_mapping_rules
+    state['mappings'] = {}
+    mappings = state['mappings']
+    new_assignment = True
+  else:
+    # If the file exists, it must have proper content.
+    state = json.load(state_file)
+    seq = state.get('seq')
+    mapping_rules = state.get('mapping_rules')
+    # Make sure the stored mapping rule corresponds to the specified in args.
+    rule_stored = json.dumps(mapping_rules)
+    rule_specified = json.dumps(rule.location_mapping_rules)
+    if rule_stored != rule_specified:
+      raise Exception('stored and specified mapping rules mismatch: '
+                      '{} vs {}'.format(rule_stored, rule_specified))
+    mappings = state['mappings']
+    location = mappings.get(uid, None)
+    if location is None:
+      seq += 1
+      state['seq'] = seq
+      new_assignment = True
+
+  if not new_assignment:
+    return location
+
+  if not relaxed and rule.total_count != 0 and rule.total_count <= seq:
+    raise Exception('too many unique identifiers ({}) to assign next location '
+                    'using mapping rules {}'.format(
+                        seq + 1, rule.location_mapping_rules))
+
+  if relaxed and rule.total_count <= seq:
+    return ""
+
+  # Get next location and add the { uid, location} binding into the mappings.
+  location = rule.get_location(seq)
+  mappings[uid] = location
+
+  # Rewrite the file with the updated state information.
+  if state_file is not None:
+    state_file.close()
+  state_file = open(fpath, 'w+')
+  json.dump(state, state_file)
+  state_file.close()
+  lock_file.close()
+  return location
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--state_store",
+      nargs="?",
+      default="/tmp/location-sequencer-state",
+      help="path to a file to store the sequencer's state")
+  parser.add_argument("--map", "-m",
+      action="append",
+      dest="location_mapping_rules",
+      metavar="RULE",
+      help="location mapping rule: number of tablet servers per specified "
+      "location in form <location>:<number>; this option may be specified "
+      "multiple times")
+  parser.add_argument("--relaxed",
+      action="store_true",
+      help="whether to allow more location assignments than specified "
+      "by the specified mapping rules")
+  parser.add_argument("uid",
+      help="hostname, IP address, or any other unique identifier")
+  args = parser.parse_args()
+
+  location = get_location(args.state_store,
+      LocationAssignmentRule(args.location_mapping_rules), args.uid, args.relaxed)
+  print(location)
+
+
+if __name__ == "__main__":
+  main()

http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/integration-tests/ts_location_assignment-itest.cc
----------------------------------------------------------------------
diff --git a/src/kudu/integration-tests/ts_location_assignment-itest.cc b/src/kudu/integration-tests/ts_location_assignment-itest.cc
new file mode 100644
index 0000000..96b3dcd
--- /dev/null
+++ b/src/kudu/integration-tests/ts_location_assignment-itest.cc
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <memory>
+#include <ostream>
+#include <string>
+#include <unordered_map>
+#include <utility>
+
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "kudu/gutil/map-util.h"
+#include "kudu/gutil/stl_util.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/integration-tests/cluster_itest_util.h"
+#include "kudu/mini-cluster/external_mini_cluster.h"
+#include "kudu/util/random.h"
+#include "kudu/util/status.h"
+#include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
+
+using kudu::cluster::ExternalMiniCluster;
+using kudu::cluster::ExternalMiniClusterOptions;
+using kudu::cluster::LocationInfo;
+using kudu::itest::TServerDetails;
+using std::string;
+using std::unique_ptr;
+using std::unordered_map;
+using strings::Substitute;
+
+namespace kudu {
+
+class TsLocationAssignmentITest :
+    public KuduTest,
+    public ::testing::WithParamInterface<std::tuple<int, int>> {
+ public:
+  TsLocationAssignmentITest()
+      : rng_(SeedRandom()) {
+    const auto& param = GetParam();
+    opts_.num_masters = std::get<0>(param);
+    opts_.num_tablet_servers = std::get<1>(param);
+  }
+
+  virtual ~TsLocationAssignmentITest() = default;
+
+ protected:
+  void StartCluster() {
+    // Generate random location mapping.
+    LocationInfo info;
+    int num_mappings_left = opts_.num_tablet_servers;
+    int loc_idx = 0;
+    while (true) {
+      auto location = Substitute("/L$0", loc_idx);
+      if (num_mappings_left <= 1) {
+        EmplaceOrDie(&info, std::move(location), 1);
+        break;
+      }
+      const int num = static_cast<int>(rng_.Uniform(num_mappings_left));
+      if (num == 0) {
+        continue;
+      }
+      EmplaceOrDie(&info, std::move(location), num);
+
+      num_mappings_left -= num;
+      ++loc_idx;
+    }
+
+    opts_.location_info = std::move(info);
+    cluster_.reset(new ExternalMiniCluster(opts_));
+    ASSERT_OK(cluster_->Start());
+  }
+
+  void CheckLocationInfo() {
+    unordered_map<string, itest::TServerDetails*> ts_map;
+    ASSERT_OK(itest::CreateTabletServerMap(cluster_->master_proxy(0),
+                                           cluster_->messenger(),
+                                           &ts_map));
+    ValueDeleter deleter(&ts_map);
+
+    LocationInfo location_info;
+    for (const auto& desc : ts_map) {
+      ++LookupOrEmplace(&location_info, desc.second->location, 0);
+    }
+    ASSERT_EQ(opts_.location_info, location_info);
+  }
+
+  ThreadSafeRandom rng_;
+  ExternalMiniClusterOptions opts_;
+  unique_ptr<cluster::ExternalMiniCluster> cluster_;
+};
+
+// Verify that the location assignment works as expected for tablet servers
+// run as part of ExternalMiniCluster. Also verify that every tablet server
+// is assigned the same location after restart once the location assignment
+// script is kept the same between restarts.
+TEST_P(TsLocationAssignmentITest, Basic) {
+  if (!AllowSlowTests()) {
+    LOG(WARNING) << "test is skipped; set KUDU_ALLOW_SLOW_TESTS=1 to run";
+    return;
+  }
+
+  NO_FATALS(StartCluster());
+  NO_FATALS(CheckLocationInfo());
+  NO_FATALS(cluster_->AssertNoCrashes());
+
+  cluster_->Shutdown();
+
+  ASSERT_OK(cluster_->Restart());
+  NO_FATALS(CheckLocationInfo());
+  NO_FATALS(cluster_->AssertNoCrashes());
+}
+
+INSTANTIATE_TEST_CASE_P(, TsLocationAssignmentITest,
+    ::testing::Combine(::testing::Values(1, 3),
+                       ::testing::Values(1, 8, 16, 32)));
+
+} // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/master/ts_descriptor.cc
----------------------------------------------------------------------
diff --git a/src/kudu/master/ts_descriptor.cc b/src/kudu/master/ts_descriptor.cc
index e3c7d76..ae5398a 100644
--- a/src/kudu/master/ts_descriptor.cc
+++ b/src/kudu/master/ts_descriptor.cc
@@ -60,9 +60,14 @@ DEFINE_string(location_mapping_cmd, "",
               "using location awareness features this flag should not be set.");
 TAG_FLAG(location_mapping_cmd, evolving);
 
+DEFINE_bool(location_mapping_by_uuid, false,
+            "Whether the location command is given tablet server identifier "
+            "instead of hostname/IP address (for tests only).");
+TAG_FLAG(location_mapping_by_uuid, hidden);
+TAG_FLAG(location_mapping_by_uuid, unsafe);
+
 using kudu::pb_util::SecureDebugString;
 using kudu::pb_util::SecureShortDebugString;
-using std::make_shared;
 using std::shared_ptr;
 using std::string;
 using std::vector;
@@ -223,13 +228,18 @@ Status TSDescriptor::Register(const NodeInstancePB& instance,
   // mapping script.
   const string& location_mapping_cmd = FLAGS_location_mapping_cmd;
   if (!location_mapping_cmd.empty()) {
-    const auto& host = registration_->rpc_addresses(0).host();
+    // In some test scenarios the location is assigned per tablet server UUID.
+    // That's the case when multiple (or even all) tablet servers have the same
+    // IP address for their RPC endpoint.
+    const auto& cmd_arg = FLAGS_location_mapping_by_uuid
+        ? permanent_uuid() : registration_->rpc_addresses(0).host();
+    TRACE(Substitute("tablet server $0: assigning location", permanent_uuid()));
     string location;
-    TRACE("Assigning location");
     Status s = GetLocationFromLocationMappingCmd(location_mapping_cmd,
-                                                 host,
+                                                 cmd_arg,
                                                  &location);
-    TRACE("Assigned location");
+    TRACE(Substitute(
+        "tablet server $0: assigned location '$1'", permanent_uuid(), location));
 
     // Assign the location under the lock if location resolution succeeds. If
     // it fails, log the error.

http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/mini-cluster/external_mini_cluster.cc
----------------------------------------------------------------------
diff --git a/src/kudu/mini-cluster/external_mini_cluster.cc b/src/kudu/mini-cluster/external_mini_cluster.cc
index 414956d..f86294c 100644
--- a/src/kudu/mini-cluster/external_mini_cluster.cc
+++ b/src/kudu/mini-cluster/external_mini_cluster.cc
@@ -223,7 +223,6 @@ Status ExternalMiniCluster::Start() {
   return Status::OK();
 }
 
-
 void ExternalMiniCluster::ShutdownNodes(ClusterNodes nodes) {
   if (nodes == ClusterNodes::ALL || nodes == ClusterNodes::TS_ONLY) {
     for (const scoped_refptr<ExternalTabletServer>& ts : tablet_servers_) {
@@ -377,6 +376,29 @@ Status ExternalMiniCluster::StartMasters() {
     flags.emplace_back(Substitute("--master_addresses=$0",
                                   HostPort::ToCommaSeparatedString(master_rpc_addrs)));
   }
+  if (!opts_.location_info.empty()) {
+    string bin_path;
+    RETURN_NOT_OK(DeduceBinRoot(&bin_path));
+    const auto mapping_script_path =
+        JoinPathSegments(bin_path, "scripts/assign-location.py");
+    const auto state_store_fpath =
+        JoinPathSegments(opts_.cluster_root, "location-assignment.state");
+    auto location_cmd = Substitute("$0 --state_store=$1",
+                                   mapping_script_path, state_store_fpath);
+    for (const auto& elem : opts_.location_info) {
+      // Per-location mapping rule specified as a pair 'location:num_servers',
+      // where 'location' is the location string and 'num_servers' is the number
+      // of tablet servers to be assigned the location.
+      location_cmd += Substitute(" --map $0:$1", elem.first, elem.second);
+    }
+    flags.emplace_back(Substitute("--location_mapping_cmd=$0", location_cmd));
+#   if defined(__APPLE__)
+    // On macOS, it's not possible to have unique loopback interfaces. To make
+    // location mapping working, a tablet server is identified by its UUID
+    // instead of IP address of its RPC end-point.
+    flags.emplace_back("--location_mapping_by_uuid");
+#   endif
+  }
   string exe = GetBinaryPath(kMasterBinaryName);
 
   // Start the masters.

http://git-wip-us.apache.org/repos/asf/kudu/blob/b193e378/src/kudu/mini-cluster/external_mini_cluster.h
----------------------------------------------------------------------
diff --git a/src/kudu/mini-cluster/external_mini_cluster.h b/src/kudu/mini-cluster/external_mini_cluster.h
index b3f3a4b..6244887 100644
--- a/src/kudu/mini-cluster/external_mini_cluster.h
+++ b/src/kudu/mini-cluster/external_mini_cluster.h
@@ -76,6 +76,9 @@ class ExternalDaemon;
 class ExternalMaster;
 class ExternalTabletServer;
 
+// Location --> number of tablet servers in location.
+typedef std::map<std::string, int> LocationInfo;
+
 struct ExternalMiniClusterOptions {
   ExternalMiniClusterOptions();
 
@@ -160,6 +163,13 @@ struct ExternalMiniClusterOptions {
   //
   // Default: 3 seconds.
   MonoDelta rpc_negotiation_timeout;
+
+  // Parameter to specify the layout of tablet servers across cluster locations
+  // in form of pairs { location, num_tablet_servers }. The empty container
+  // means no locations are configured for the cluster.
+  //
+  // Default: empty
+  LocationInfo location_info;
 };
 
 // A mini-cluster made up of subprocesses running each of the daemons


Mime
View raw message