cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sn...@apache.org
Subject [3/6] cassandra git commit: token-generator - generated tokens too long
Date Thu, 14 May 2015 08:41:58 GMT
token-generator - generated tokens too long

Patch by Robert Stupp; Reviewed by Stefania for CASSANDRA-9300


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/1611ef3d
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/1611ef3d
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/1611ef3d

Branch: refs/heads/trunk
Commit: 1611ef3d1995337998b9d7514c7ea5c35fa59895
Parents: 418deaf
Author: Robert Stupp <snazy@snazy.de>
Authored: Thu May 14 10:35:35 2015 +0200
Committer: Robert Stupp <snazy@snazy.de>
Committed: Thu May 14 10:35:35 2015 +0200

----------------------------------------------------------------------
 CHANGES.txt                   |  3 +++
 tools/bin/token-generator     | 53 +++++++++++++++++++++++++++-----------
 tools/bin/token-generator.bat | 34 ++++++++++++++++++++++++
 3 files changed, 75 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/1611ef3d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d7d01cf..cee28bc 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,6 @@
+2.0.16:
+ * token-generator - generated tokens too long (CASSANDRA-9300)
+
 2.0.15:
  * Fix counting of tombstones for TombstoneOverwhelmingException (CASSANDRA-9299)
  * Fix ReconnectableSnitch reconnecting to peers during upgrade (CASSANDRA-6702)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1611ef3d/tools/bin/token-generator
----------------------------------------------------------------------
diff --git a/tools/bin/token-generator b/tools/bin/token-generator
index c958529..b70866d 100755
--- a/tools/bin/token-generator
+++ b/tools/bin/token-generator
@@ -30,24 +30,38 @@ from tempfile import NamedTemporaryFile
 
 description = '''Given a list of numbers indicating the number of nodes
 in each separate datacenter, outputs a recommended list of tokens to use
-with RandomPartitioner: one for each node in each datacenter.
+with Murmur3Partitioner (by default) or with RandomPartitioner.
+The list contains one token for each node in each datacenter.
 '''
 
 usage = "%prog <nodes_in_dc1> [<nodes_in_dc2> [...]]"
 
 parser = optparse.OptionParser(description=description, usage=usage)
+
+def part_murmur3(option, opt, value, parser):
+    parser.values.ringoffset=-(1<<63)
+    parser.values.ringrange=(1<<64)
+    return
+def part_random(option, opt, value, parser):
+    parser.values.ringoffset=0
+    parser.values.ringrange=(1<<127)
+    return
+parser.add_option('--murmur3', action='callback', callback=part_murmur3,
+                  help='Generate tokens for Murmur3Partitioner (default).')
+parser.add_option('--random', action='callback', callback=part_random,
+                  help='Generate tokens for RandomPartitioner.')
+parser.add_option('--ringoffset', type='int',
+                  help=optparse.SUPPRESS_HELP)
 parser.add_option('--ringrange', type='int',
-                  help='Specify a numeric maximum token value for your ring, '
-                       'different from the default value of 2^127.')
+                  help=optparse.SUPPRESS_HELP)
+
 parser.add_option('--graph', action='store_true',
                   help='Show a rendering of the generated tokens as line '
                        'segments in a circle, colored according to datacenter')
 parser.add_option('-n', '--nts', action='store_const', dest='strat', const='nts',
-                  help='Optimize multi-cluster distribution for '
-                       'NetworkTopologyStrategy [default]')
+                  help=optparse.SUPPRESS_HELP)
 parser.add_option('-o', '--onts', action='store_const', dest='strat', const='onts',
-                  help='Optimize multi-cluster distribution for '
-                       'OldNetworkTopologyStrategy')
+                  help=optparse.SUPPRESS_HELP)
 
 parser.add_option('--test', action='store_true',
                   help='Run in test mode, outputting an HTML file to display '
@@ -58,8 +72,11 @@ parser.add_option('--browser-wait-time', type='float', help=optparse.SUPPRESS_HE
 parser.add_option('--test-colors', help=optparse.SUPPRESS_HELP)
 parser.add_option('--test-graphsize', type='int', help=optparse.SUPPRESS_HELP)
 
+
 parser.set_defaults(
-    ringrange=(1<<127),
+    # default is Murmur3
+    ringoffset=-(1<<63),
+    ringrange=(1<<64),
 
     # whether to create (and try to display) graph output
     graph=False,
@@ -94,7 +111,8 @@ class Ring:
     MIN_DC_OFFSET_DIVIDER = 235
     offset_spacer = 2
 
-    def __init__(self, dc_counts, ringrange, strategy='nts'):
+    def __init__(self, dc_counts, ringoffset, ringrange, strategy='nts'):
+        self.ringoffset = ringoffset
         self.ringrange = ringrange
         self.dc_counts = dc_counts
         self.calculate_offset_tokens = getattr(self, 'calc_offset_tokens_' + strategy)
@@ -110,13 +128,18 @@ class Ring:
         division = max(lowest_division, self.MIN_DC_OFFSET_DIVIDER)
         return -self.ringrange // division
 
+    def bound_token(self, tok):
+        if tok < self.ringoffset:
+            tok += self.ringrange
+        return tok
+
     def calc_offset_tokens_nts(self):
         dc_offset = self.best_per_dc_offset()
         dcs = []
         for (dcnum, dccount) in enumerate(self.dc_counts):
             offset = dcnum * dc_offset
             arcsize = self.ringrange // (dccount or 1)
-            dcs.append([(n * arcsize + offset) % self.ringrange for n in xrange(dccount)])
+            dcs.append(sorted([self.bound_token((n * arcsize + offset) - self.ringoffset
% self.ringrange) for n in xrange(dccount)]))
         return dcs
 
     def calc_offset_tokens_onts(self):
@@ -127,7 +150,7 @@ class Ring:
 
         final = [[] for x in dcs_by_count]
         for pos, dc in enumerate(layout):
-            final[dc].append(pos * self.ringrange // len(layout))
+            final[dc].append(self.ringoffset + pos * self.ringrange // len(layout))
         return final
 
 
@@ -139,8 +162,8 @@ def print_tokens(tokens, tokenwidth, indent=0):
         for tnum, tok in enumerate(toklist):
             print "%s  Node #%0*d: % *d" % (indentstr, nwidth, tnum + 1, tokenwidth, tok)
 
-def calculate_ideal_tokens(datacenters, ringrange, strategy):
-    return Ring(datacenters, ringrange, strategy).calculate_offset_tokens()
+def calculate_ideal_tokens(datacenters, ringoffset, ringrange, strategy):
+    return Ring(datacenters, ringoffset, ringrange, strategy).calculate_offset_tokens()
 
 def file_to_url(path):
     path = os.path.abspath(path)
@@ -248,7 +271,7 @@ def run_tests(opts):
     tokensets = []
     for test in tests:
         print "Test %r" % (test,)
-        tokens = calculate_ideal_tokens(test, opts.ringrange, opts.strat)
+        tokens = calculate_ideal_tokens(test, opts.ringoffset, opts.ringrange, opts.strat)
         print_tokens(tokens, len(str(opts.ringrange)) + 1, indent=2)
         tokensets.append(tokens)
     return tokensets
@@ -315,7 +338,7 @@ def main(opts, args):
             parser.error('Arguments should be integers.')
         renderer = RingRenderer(ringrange=opts.ringrange, graphsize=opts.graphsize,
                                 colors=opts.colorlist)
-        tokens = calculate_ideal_tokens(datacenters, opts.ringrange, opts.strat)
+        tokens = calculate_ideal_tokens(datacenters, opts.ringoffset, opts.ringrange, opts.strat)
         print_tokens(tokens, len(str(opts.ringrange)) + 1)
         tokensets = [tokens]
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/1611ef3d/tools/bin/token-generator.bat
----------------------------------------------------------------------
diff --git a/tools/bin/token-generator.bat b/tools/bin/token-generator.bat
new file mode 100644
index 0000000..a7188db
--- /dev/null
+++ b/tools/bin/token-generator.bat
@@ -0,0 +1,34 @@
+@ECHO OFF
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one or more
+@REM contributor license agreements. See the NOTICE file distributed with
+@REM this work for additional information regarding copyright ownership.
+@REM The ASF licenses this file to You under the Apache License, Version 2.0
+@REM (the "License"); you may not use this file except in compliance with
+@REM the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing, software
+@REM distributed under the License is distributed on an "AS IS" BASIS,
+@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@REM See the License for the specific language governing permissions and
+@REM limitations under the License.
+
+@echo off
+
+if "%OS%" == "Windows_NT" setlocal
+
+python -V >nul 2>&1
+if ERRORLEVEL 1 goto err
+
+python "%~dp0\token-generator" %*
+goto finally
+
+:err
+echo Can't detect Python version!
+
+:finally
+
+ENDLOCAL
+


Mime
View raw message