cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stefa...@apache.org
Subject [10/10] cassandra git commit: Merge branch 'cassandra-3.0' into trunk
Date Wed, 27 Apr 2016 01:12:25 GMT
Merge branch 'cassandra-3.0' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e6505a60
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e6505a60
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e6505a60

Branch: refs/heads/trunk
Commit: e6505a606f98707d3aab436e0713f5dab004a5ab
Parents: 0b81f72 f2afd04
Author: Stefania Alborghetti <stefania.alborghetti@datastax.com>
Authored: Wed Apr 27 09:09:34 2016 +0800
Committer: Stefania Alborghetti <stefania.alborghetti@datastax.com>
Committed: Wed Apr 27 09:10:17 2016 +0800

----------------------------------------------------------------------
 CHANGES.txt                  |  1 +
 pylib/cqlshlib/copyutil.py   | 39 +++++++++++++++++++++++++++++----------
 pylib/cqlshlib/formatting.py | 22 +++++++++++++++++++++-
 3 files changed, 51 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/e6505a60/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e6505a60/pylib/cqlshlib/copyutil.py
----------------------------------------------------------------------
diff --cc pylib/cqlshlib/copyutil.py
index 36c951c,b17bdfe..46c7eb6
--- a/pylib/cqlshlib/copyutil.py
+++ b/pylib/cqlshlib/copyutil.py
@@@ -306,9 -306,8 +306,10 @@@ class CopyTask(object)
          copy_options['pagetimeout'] = int(opts.pop('pagetimeout', max(10, 10 * (copy_options['pagesize']
/ 1000))))
          copy_options['maxattempts'] = int(opts.pop('maxattempts', 5))
          copy_options['dtformats'] = DateTimeFormat(opts.pop('datetimeformat', shell.display_timestamp_format),
--                                                   shell.display_date_format, shell.display_nanotime_format)
 -        copy_options['float_precision'] = shell.display_float_precision
++                                                   shell.display_date_format, shell.display_nanotime_format,
++                                                   milliseconds_only=True)
 +        copy_options['floatprecision'] = int(opts.pop('floatprecision', '5'))
 +        copy_options['doubleprecision'] = int(opts.pop('doubleprecision', '12'))
          copy_options['chunksize'] = int(opts.pop('chunksize', 5000))
          copy_options['ingestrate'] = int(opts.pop('ingestrate', 100000))
          copy_options['maxbatchsize'] = int(opts.pop('maxbatchsize', 20))

http://git-wip-us.apache.org/repos/asf/cassandra/blob/e6505a60/pylib/cqlshlib/formatting.py
----------------------------------------------------------------------
diff --cc pylib/cqlshlib/formatting.py
index aa50204,dcd08da..5364c18
--- a/pylib/cqlshlib/formatting.py
+++ b/pylib/cqlshlib/formatting.py
@@@ -101,101 -99,20 +101,102 @@@ def color_text(bval, colormap, displayw
  
  DEFAULT_NANOTIME_FORMAT = '%H:%M:%S.%N'
  DEFAULT_DATE_FORMAT = '%Y-%m-%d'
 -DEFAULT_TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S%z'
  
 -if platform.system() == 'Windows':
 -    DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
 +DEFAULT_TIMESTAMP_FORMAT = os.environ.get('CQLSH_DEFAULT_TIMESTAMP_FORMAT', '')
 +if not DEFAULT_TIMESTAMP_FORMAT:
 +    DEFAULT_TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S.%f%z'
  
  
 -class DateTimeFormat():
 +class DateTimeFormat:
  
      def __init__(self, timestamp_format=DEFAULT_TIMESTAMP_FORMAT, date_format=DEFAULT_DATE_FORMAT,
--                 nanotime_format=DEFAULT_NANOTIME_FORMAT, timezone=None):
++                 nanotime_format=DEFAULT_NANOTIME_FORMAT, timezone=None, milliseconds_only=False):
          self.timestamp_format = timestamp_format
          self.date_format = date_format
          self.nanotime_format = nanotime_format
          self.timezone = timezone
++        self.milliseconds_only = milliseconds_only  # the microseconds part, .NNNNNN, wil
be rounded to .NNN
 +
 +
 +class CqlType(object):
 +    """
 +    A class for converting a string into a cql type name that can match a formatter
 +    and a list of its sub-types, if any.
 +    """
 +    pattern = re.compile('^([^<]*)<(.*)>$')  # *<*>
 +
 +    def __init__(self, typestring, ksmeta=None):
 +        self.type_name, self.sub_types, self.formatter = self.parse(typestring, ksmeta)
 +
 +    def __str__(self):
 +        return "%s%s" % (self.type_name, self.sub_types or '')
 +
 +    __repr__ = __str__
 +
 +    def get_n_sub_types(self, num):
 +        """
 +        Return the sub-types if the requested number matches the length of the sub-types
(tuples)
 +        or the first sub-type times the number requested if the length of the sub-types
is one (list, set),
 +        otherwise raise an exception
 +        """
 +        if len(self.sub_types) == num:
 +            return self.sub_types
 +        elif len(self.sub_types) == 1:
 +            return [self.sub_types[0]] * num
 +        else:
 +            raise Exception("Unexpected number of subtypes %d - %s" % (num, self.sub_types))
 +
 +    def parse(self, typestring, ksmeta):
 +        """
 +        Parse the typestring by looking at this pattern: *<*>. If there is no match
then the type
 +        is either a simple type or a user type, otherwise it must be a composite type
 +        for which we need to look-up the sub-types. For user types the sub types can be
extracted
 +        from the keyspace metadata.
 +        """
 +        while True:
 +            m = self.pattern.match(typestring)
 +            if not m:  # no match, either a simple or a user type
 +                name = typestring
 +                if ksmeta and name in ksmeta.user_types:  # a user type, look at ks meta
for sub types
 +                    sub_types = [CqlType(t, ksmeta) for t in ksmeta.user_types[name].field_types]
 +                    return name, sub_types, format_value_utype
 +                else:
 +                    return name, [], self._get_formatter(name)
 +            else:
 +                if m.group(1) == 'frozen':  # ignore frozen<>
 +                    typestring = m.group(2)
 +                    continue
 +
 +                name = m.group(1)  # a composite type, parse sub types
 +                return name, self.parse_sub_types(m.group(2), ksmeta), self._get_formatter(name)
 +
 +    @staticmethod
 +    def _get_formatter(name):
 +        return _formatters.get(name.lower())
 +
 +    @staticmethod
 +    def parse_sub_types(val, ksmeta):
 +        """
 +        Split val into sub-strings separated by commas but only if not within a <>
pair
 +        Return a list of CqlType instances where each instance is initialized with the sub-strings
 +        that were found.
 +        """
 +        last = 0
 +        level = 0
 +        ret = []
 +        for i, c in enumerate(val):
 +            if c == '<':
 +                level += 1
 +            elif c == '>':
 +                level -= 1
 +            elif c == ',' and level == 0:
 +                ret.append(val[last:i].strip())
 +                last = i + 1
 +
 +        if last < len(val) - 1:
 +            ret.append(val[last:].strip())
 +
 +        return [CqlType(r, ksmeta) for r in ret]
  
  
  def format_value_default(val, colormap, **_):
@@@ -331,14 -236,7 +332,16 @@@ formatter_for('varint')(format_integer_
  
  @formatter_for('datetime')
  def format_value_timestamp(val, colormap, date_time_format, quote=False, **_):
 -    bval = strftime(date_time_format.timestamp_format, calendar.timegm(val.utctimetuple()),
timezone=date_time_format.timezone)
 +    if isinstance(val, datetime.datetime):
 +        bval = strftime(date_time_format.timestamp_format,
 +                        calendar.timegm(val.utctimetuple()),
 +                        microseconds=val.microsecond,
 +                        timezone=date_time_format.timezone)
++        if date_time_format.milliseconds_only:
++            bval = round_microseconds(bval)
 +    else:
 +        bval = str(val)
 +
      if quote:
          bval = "'%s'" % bval
      return colorme(bval, colormap, 'timestamp')
@@@ -353,6 -248,6 +356,23 @@@ def strftime(time_format, seconds, micr
          ret_dt = ret_dt.astimezone(timezone)
      return ret_dt.strftime(time_format)
  
++microseconds_regex = re.compile("(.*)(?:\.(\d{1,6}))(.*)")
++
++
++def round_microseconds(val):
++    """
++    For COPY TO, we need to round microsecond to milliseconds because server side
++    TimestampSerializer.dateStringPatterns only parses milliseconds. If we keep microseconds,
++    users may try to import with COPY FROM a file generated with COPY TO and have problems
if
++    prepared statements are disabled, see CASSANDRA-11631.
++    """
++    m = microseconds_regex.match(val)
++    if not m:
++        return val
++
++    milliseconds = int(m.group(2)) * pow(10, 3 - len(m.group(2)))
++    return '%s.%03d%s' % (m.group(1), milliseconds, '' if not m.group(3) else m.group(3))
++
  
  @formatter_for('Date')
  def format_value_date(val, colormap, **_):


Mime
View raw message