Return-Path: X-Original-To: apmail-cassandra-commits-archive@www.apache.org Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 05B221888A for ; Wed, 6 Jan 2016 17:03:01 +0000 (UTC) Received: (qmail 42914 invoked by uid 500); 6 Jan 2016 17:02:59 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 42681 invoked by uid 500); 6 Jan 2016 17:02:59 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 42114 invoked by uid 99); 6 Jan 2016 17:02:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 06 Jan 2016 17:02:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 849A8E109B; Wed, 6 Jan 2016 17:02:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: slebresne@apache.org To: commits@cassandra.apache.org Date: Wed, 06 Jan 2016 17:03:08 -0000 Message-Id: In-Reply-To: <31cb1e462271489c843cdd71bfa2af1e@git.apache.org> References: <31cb1e462271489c843cdd71bfa2af1e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [11/50] [abbrv] cassandra git commit: Match cassandra-loader options in COPY FROM (2.2 version) http://git-wip-us.apache.org/repos/asf/cassandra/blob/202cf9b0/pylib/cqlshlib/formatting.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/formatting.py b/pylib/cqlshlib/formatting.py index 62ecd10..2e219c8 100644 --- a/pylib/cqlshlib/formatting.py +++ b/pylib/cqlshlib/formatting.py @@ -60,7 +60,8 @@ empty_colormap = defaultdict(lambda: '') def format_by_type(cqltype, val, encoding, colormap=None, addcolor=False, - nullval=None, date_time_format=None, float_precision=None): + nullval=None, date_time_format=None, float_precision=None, + decimal_sep=None, thousands_sep=None, boolean_styles=None): if nullval is None: nullval = default_null_placeholder if val is None: @@ -75,7 +76,8 @@ def format_by_type(cqltype, val, encoding, colormap=None, addcolor=False, float_precision = default_float_precision return format_value(cqltype, val, encoding=encoding, colormap=colormap, date_time_format=date_time_format, float_precision=float_precision, - nullval=nullval) + nullval=nullval, decimal_sep=decimal_sep, thousands_sep=thousands_sep, + boolean_styles=boolean_styles) def color_text(bval, colormap, displaywidth=None): @@ -155,7 +157,9 @@ def format_python_formatted_type(val, colormap, color, quote=False): @formatter_for('Decimal') -def format_value_decimal(val, colormap, **_): +def format_value_decimal(val, float_precision, colormap, decimal_sep=None, thousands_sep=None, **_): + if (decimal_sep and decimal_sep != '.') or thousands_sep: + return format_floating_point_type(val, colormap, float_precision, decimal_sep, thousands_sep) return format_python_formatted_type(val, colormap, 'decimal') @@ -170,34 +174,60 @@ def formatter_value_inet(val, colormap, quote=False, **_): @formatter_for('bool') -def format_value_boolean(val, colormap, **_): +def format_value_boolean(val, colormap, boolean_styles=None, **_): + if boolean_styles: + val = boolean_styles[0] if val else boolean_styles[1] return format_python_formatted_type(val, colormap, 'boolean') -def format_floating_point_type(val, colormap, float_precision, **_): +def format_floating_point_type(val, colormap, float_precision, decimal_sep=None, thousands_sep=None, **_): if math.isnan(val): bval = 'NaN' elif math.isinf(val): bval = 'Infinity' if val > 0 else '-Infinity' else: - exponent = int(math.log10(abs(val))) if abs(val) > sys.float_info.epsilon else -sys.maxsize - 1 - if -4 <= exponent < float_precision: - # when this is true %g will not use scientific notation, - # increasing precision should not change this decision - # so we increase the precision to take into account the - # digits to the left of the decimal point - float_precision = float_precision + exponent + 1 - bval = '%.*g' % (float_precision, val) + if thousands_sep: + dpart, ipart = math.modf(val) + bval = format_integer_with_thousands_sep(ipart, thousands_sep) + dpart_str = ('%.*f' % (float_precision, math.fabs(dpart)))[2:].rstrip('0') + if dpart_str: + bval += '%s%s' % ('.' if not decimal_sep else decimal_sep, dpart_str) + else: + exponent = int(math.log10(abs(val))) if abs(val) > sys.float_info.epsilon else -sys.maxsize - 1 + if -4 <= exponent < float_precision: + # when this is true %g will not use scientific notation, + # increasing precision should not change this decision + # so we increase the precision to take into account the + # digits to the left of the decimal point + float_precision = float_precision + exponent + 1 + bval = '%.*g' % (float_precision, val) + if decimal_sep: + bval = bval.replace('.', decimal_sep) + return colorme(bval, colormap, 'float') formatter_for('float')(format_floating_point_type) -def format_integer_type(val, colormap, **_): +def format_integer_type(val, colormap, thousands_sep=None, **_): # base-10 only for now; support others? - bval = str(val) + bval = format_integer_with_thousands_sep(val, thousands_sep) if thousands_sep else str(val) return colorme(bval, colormap, 'int') +# We can get rid of this in cassandra-2.2 +if sys.version_info >= (2, 7): + def format_integer_with_thousands_sep(val, thousands_sep=','): + return "{:,.0f}".format(val).replace(',', thousands_sep) +else: + def format_integer_with_thousands_sep(val, thousands_sep=','): + if val < 0: + return '-' + format_integer_with_thousands_sep(-val, thousands_sep) + result = '' + while val >= 1000: + val, r = divmod(val, 1000) + result = "%s%03d%s" % (thousands_sep, r, result) + return "%d%s" % (val, result) + formatter_for('long')(format_integer_type) formatter_for('int')(format_integer_type) @@ -242,10 +272,12 @@ formatter_for('unicode')(format_value_text) def format_simple_collection(val, lbracket, rbracket, encoding, - colormap, date_time_format, float_precision, nullval): + colormap, date_time_format, float_precision, nullval, + decimal_sep, thousands_sep, boolean_styles): subs = [format_value(type(sval), sval, encoding=encoding, colormap=colormap, date_time_format=date_time_format, float_precision=float_precision, - nullval=nullval, quote=True) + nullval=nullval, quote=True, decimal_sep=decimal_sep, + thousands_sep=thousands_sep, boolean_styles=boolean_styles) for sval in val] bval = lbracket + ', '.join(get_str(sval) for sval in subs) + rbracket if colormap is NO_COLOR_MAP: @@ -259,32 +291,40 @@ def format_simple_collection(val, lbracket, rbracket, encoding, @formatter_for('list') -def format_value_list(val, encoding, colormap, date_time_format, float_precision, nullval, **_): +def format_value_list(val, encoding, colormap, date_time_format, float_precision, nullval, + decimal_sep, thousands_sep, boolean_styles, **_): return format_simple_collection(val, '[', ']', encoding, colormap, - date_time_format, float_precision, nullval) + date_time_format, float_precision, nullval, + decimal_sep, thousands_sep, boolean_styles) @formatter_for('tuple') -def format_value_tuple(val, encoding, colormap, date_time_format, float_precision, nullval, **_): +def format_value_tuple(val, encoding, colormap, date_time_format, float_precision, nullval, + decimal_sep, thousands_sep, boolean_styles, **_): return format_simple_collection(val, '(', ')', encoding, colormap, - date_time_format, float_precision, nullval) + date_time_format, float_precision, nullval, + decimal_sep, thousands_sep, boolean_styles) @formatter_for('set') -def format_value_set(val, encoding, colormap, date_time_format, float_precision, nullval, **_): +def format_value_set(val, encoding, colormap, date_time_format, float_precision, nullval, + decimal_sep, thousands_sep, boolean_styles, **_): return format_simple_collection(sorted(val), '{', '}', encoding, colormap, - date_time_format, float_precision, nullval) + date_time_format, float_precision, nullval, + decimal_sep, thousands_sep, boolean_styles) formatter_for('frozenset')(format_value_set) formatter_for('sortedset')(format_value_set) formatter_for('SortedSet')(format_value_set) @formatter_for('dict') -def format_value_map(val, encoding, colormap, date_time_format, float_precision, nullval, **_): +def format_value_map(val, encoding, colormap, date_time_format, float_precision, nullval, + decimal_sep, thousands_sep, boolean_styles, **_): def subformat(v): return format_value(type(v), v, encoding=encoding, colormap=colormap, date_time_format=date_time_format, float_precision=float_precision, - nullval=nullval, quote=True) + nullval=nullval, quote=True, decimal_sep=decimal_sep, + thousands_sep=thousands_sep, boolean_styles=boolean_styles) subs = [(subformat(k), subformat(v)) for (k, v) in sorted(val.items())] bval = '{' + ', '.join(get_str(k) + ': ' + get_str(v) for (k, v) in subs) + '}' @@ -303,13 +343,15 @@ formatter_for('OrderedMap')(format_value_map) formatter_for('OrderedMapSerializedKey')(format_value_map) -def format_value_utype(val, encoding, colormap, date_time_format, float_precision, nullval, **_): +def format_value_utype(val, encoding, colormap, date_time_format, float_precision, nullval, + decimal_sep, thousands_sep, boolean_styles, **_): def format_field_value(v): if v is None: return colorme(nullval, colormap, 'error') return format_value(type(v), v, encoding=encoding, colormap=colormap, date_time_format=date_time_format, float_precision=float_precision, - nullval=nullval, quote=True) + nullval=nullval, quote=True, decimal_sep=decimal_sep, + thousands_sep=thousands_sep, boolean_styles=boolean_styles) def format_field_name(name): return format_value_text(name, encoding=encoding, colormap=colormap, quote=False) http://git-wip-us.apache.org/repos/asf/cassandra/blob/202cf9b0/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java index 46dfda5..43a80bb 100644 --- a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java @@ -33,12 +33,14 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.*; import org.apache.cassandra.db.*; import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.service.QueryState; import org.apache.cassandra.service.StorageProxy; -import org.apache.cassandra.thrift.Column; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.messages.ResultMessage; import org.apache.cassandra.utils.NoSpamLogger; @@ -49,7 +51,7 @@ import org.apache.cassandra.utils.Pair; */ public class BatchStatement implements CQLStatement { - public static enum Type + public enum Type { LOGGED, UNLOGGED, COUNTER } @@ -282,13 +284,23 @@ public class BatchStatement implements CQLStatement Set ksCfPairs = new HashSet<>(); Set keySet = new HashSet<>(); + Map>> localTokensByKs = new HashMap<>(); + boolean localMutationsOnly = true; + for (IMutation im : mutations) { keySet.add(im.key()); for (ColumnFamily cf : im.getColumnFamilies()) ksCfPairs.add(String.format("%s.%s", cf.metadata().ksName, cf.metadata().cfName)); + + if (localMutationsOnly) + localMutationsOnly &= isMutationLocal(localTokensByKs, im); } + // CASSANDRA-9303: If we only have local mutations we do not warn + if (localMutationsOnly) + return; + NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, unloggedBatchWarning, keySet.size(), keySet.size() == 1 ? "" : "s", ksCfPairs.size() == 1 ? "" : "s", ksCfPairs); @@ -299,6 +311,18 @@ public class BatchStatement implements CQLStatement } } + private boolean isMutationLocal(Map>> localTokensByKs, IMutation mutation) + { + Collection> localRanges = localTokensByKs.get(mutation.getKeyspaceName()); + if (localRanges == null) + { + localRanges = StorageService.instance.getLocalRanges(mutation.getKeyspaceName()); + localTokensByKs.put(mutation.getKeyspaceName(), localRanges); + } + + return Range.isInRanges(StorageService.getPartitioner().getToken(mutation.key()), localRanges); + } + public ResultMessage execute(QueryState queryState, QueryOptions options) throws RequestExecutionException, RequestValidationException { return execute(queryState, BatchQueryOptions.withoutPerStatementVariables(options)); http://git-wip-us.apache.org/repos/asf/cassandra/blob/202cf9b0/test/unit/org/apache/cassandra/service/ClientWarningsTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/service/ClientWarningsTest.java b/test/unit/org/apache/cassandra/service/ClientWarningsTest.java index 363fc7d..cfd5f7a 100644 --- a/test/unit/org/apache/cassandra/service/ClientWarningsTest.java +++ b/test/unit/org/apache/cassandra/service/ClientWarningsTest.java @@ -55,12 +55,11 @@ public class ClientWarningsTest extends CQLTester QueryMessage query = new QueryMessage(createBatchStatement2(1), QueryOptions.DEFAULT); Message.Response resp = client.execute(query); - assertEquals(1, resp.getWarnings().size()); + assertNull(resp.getWarnings()); query = new QueryMessage(createBatchStatement2(DatabaseDescriptor.getBatchSizeWarnThreshold()), QueryOptions.DEFAULT); resp = client.execute(query); - assertEquals(2, resp.getWarnings().size()); - + assertEquals(1, resp.getWarnings().size()); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/202cf9b0/tools/bin/cassandra-stress.bat ---------------------------------------------------------------------- diff --git a/tools/bin/cassandra-stress.bat b/tools/bin/cassandra-stress.bat index f1bbcc9..5c2ecfa 100644 --- a/tools/bin/cassandra-stress.bat +++ b/tools/bin/cassandra-stress.bat @@ -19,4 +19,4 @@ if "%OS%" == "Windows_NT" setlocal pushd "%~dp0" call cassandra.in.bat if NOT DEFINED STRESS_HOME set STRESS_HOME=%CD%\.. -"%JAVA_HOME%\bin\java" -cp %CASSANDRA_CLASSPATH% org.apache.cassandra.stress.Stress %* +"%JAVA_HOME%\bin\java" %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% org.apache.cassandra.stress.Stress %*