cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brandonwilli...@apache.org
Subject [3/7] cqlsh: update recognized syntax for cql3 Patch by paul cannon, reviewed by brandonwilliams for CASSANDRA-4198
Date Fri, 18 May 2012 15:33:25 GMT
http://git-wip-us.apache.org/repos/asf/cassandra/blob/38748b43/pylib/cqlshlib/cqlhandling.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/cqlhandling.py b/pylib/cqlshlib/cqlhandling.py
index 09a2980..3866f3c 100644
--- a/pylib/cqlshlib/cqlhandling.py
+++ b/pylib/cqlshlib/cqlhandling.py
@@ -18,191 +18,435 @@
 # i.e., stuff that's not necessarily cqlsh-specific
 
 import re
-from . import pylexotron
-from itertools import izip
+import traceback
+from . import pylexotron, util
 
 Hint = pylexotron.Hint
 
-keywords = set((
-    'select', 'from', 'where', 'and', 'key', 'insert', 'update', 'with',
-    'limit', 'using', 'consistency', 'one', 'quorum', 'all', 'any',
-    'local_quorum', 'each_quorum', 'two', 'three', 'use', 'count', 'set',
-    'begin', 'apply', 'batch', 'truncate', 'delete', 'in', 'create',
-    'keyspace', 'schema', 'columnfamily', 'table', 'index', 'on', 'drop',
-    'primary', 'into', 'values', 'timestamp', 'ttl', 'alter', 'add', 'type',
-    'first', 'reversed'
-))
-
-columnfamily_options = (
-    # (CQL option name, Thrift option name (or None if same))
-    ('comment', None),
-    ('comparator', 'comparator_type'),
-    ('read_repair_chance', None),
-    ('gc_grace_seconds', None),
-    ('default_validation', 'default_validation_class'),
-    ('min_compaction_threshold', None),
-    ('max_compaction_threshold', None),
-    ('replicate_on_write', None),
-    ('compaction_strategy_class', 'compaction_strategy'),
-)
-
-obsolete_cf_options = (
-    ('key_cache_size', None),
-    ('row_cache_size', None),
-    ('row_cache_save_period_in_seconds', None),
-    ('key_cache_save_period_in_seconds', None),
-    ('memtable_throughput_in_mb', None),
-    ('memtable_operations_in_millions', None),
-    ('memtable_flush_after_mins', None),
-    ('row_cache_provider', None),
-)
-
-all_columnfamily_options = columnfamily_options + obsolete_cf_options
-
-columnfamily_map_options = (
-    ('compaction_strategy_options', None,
-        ()),
-    ('compression_parameters', 'compression_options',
-        ('sstable_compression', 'chunk_length_kb', 'crc_check_chance')),
-)
-
-available_compression_classes = (
-    'DeflateCompressor',
-    'SnappyCompressor',
-)
-
-available_compaction_classes = (
-    'LeveledCompactionStrategy',
-    'SizeTieredCompactionStrategy'
-)
-
-cql_type_to_apache_class = {
-    'ascii': 'AsciiType',
-    'bigint': 'LongType',
-    'blob': 'BytesType',
-    'boolean': 'BooleanType',
-    'counter': 'CounterColumnType',
-    'decimal': 'DecimalType',
-    'double': 'DoubleType',
-    'float': 'FloatType',
-    'int': 'Int32Type',
-    'text': 'UTF8Type',
-    'timestamp': 'DateType',
-    'uuid': 'UUIDType',
-    'varchar': 'UTF8Type',
-    'varint': 'IntegerType'
-}
-
-apache_class_to_cql_type = dict((v,k) for (k,v) in cql_type_to_apache_class.items())
-
-cql_types = sorted(cql_type_to_apache_class.keys())
-
-def find_validator_class(cqlname):
-    return cql_type_to_apache_class[cqlname]
-
-replication_strategies = (
-    'SimpleStrategy',
-    'OldNetworkTopologyStrategy',
-    'NetworkTopologyStrategy'
-)
-
-consistency_levels = (
-    'ANY',
-    'ONE',
-    'QUORUM',
-    'ALL',
-    'LOCAL_QUORUM',
-    'EACH_QUORUM'
-)
-
-# if a term matches this, it shouldn't need to be quoted to be valid cql
-valid_cql_word_re = re.compile(r"^(?:[a-z][a-z0-9_]*|-?[0-9][0-9.]*)$", re.I)
-
-def is_valid_cql_word(s):
-    return valid_cql_word_re.match(s) is not None and s not in keywords
-
-def tokenize_cql(cql_text):
-    return CqlLexotron.scan(cql_text)[0]
-
-def cql_extract_orig(toklist, srcstr):
-    # low end of span for first token, to high end of span for last token
-    return srcstr[toklist[0][2][0]:toklist[-1][2][1]]
-
-# note: commands_end_with_newline may be extended by an importing module.
-commands_end_with_newline = set()
-
-def token_dequote(tok):
-    if tok[0] == 'stringLiteral':
-        # strip quotes
-        return tok[1][1:-1].replace("''", "'")
-    if tok[0] == 'unclosedString':
-        # strip one quote
-        return tok[1][1:].replace("''", "'")
-    if tok[0] == 'unclosedComment':
-        return ''
-    return tok[1]
-
-def cql_dequote(cqlword):
-    cqlword = cqlword.strip()
-    if cqlword == '':
+class CqlParsingRuleSet(pylexotron.ParsingRuleSet):
+    keywords = set((
+        'select', 'from', 'where', 'and', 'key', 'insert', 'update', 'with',
+        'limit', 'using', 'consistency', 'one', 'quorum', 'all', 'any',
+        'local_quorum', 'each_quorum', 'two', 'three', 'use', 'count', 'set',
+        'begin', 'apply', 'batch', 'truncate', 'delete', 'in', 'create',
+        'keyspace', 'schema', 'columnfamily', 'table', 'index', 'on', 'drop',
+        'primary', 'into', 'values', 'timestamp', 'ttl', 'alter', 'add', 'type',
+        'first', 'reversed'
+    ))
+
+    columnfamily_options = (
+        # (CQL option name, Thrift option name (or None if same))
+        ('comment', None),
+        ('comparator', 'comparator_type'),
+        ('read_repair_chance', None),
+        ('gc_grace_seconds', None),
+        ('default_validation', 'default_validation_class'),
+        ('min_compaction_threshold', None),
+        ('max_compaction_threshold', None),
+        ('replicate_on_write', None),
+        ('compaction_strategy_class', 'compaction_strategy'),
+    )
+
+    obsolete_cf_options = (
+        ('key_cache_size', None),
+        ('row_cache_size', None),
+        ('row_cache_save_period_in_seconds', None),
+        ('key_cache_save_period_in_seconds', None),
+        ('memtable_throughput_in_mb', None),
+        ('memtable_operations_in_millions', None),
+        ('memtable_flush_after_mins', None),
+        ('row_cache_provider', None),
+    )
+
+    all_columnfamily_options = columnfamily_options + obsolete_cf_options
+
+    columnfamily_map_options = (
+        ('compaction_strategy_options', None,
+            ()),
+        ('compression_parameters', 'compression_options',
+            ('sstable_compression', 'chunk_length_kb', 'crc_check_chance')),
+    )
+
+    available_compression_classes = (
+        'DeflateCompressor',
+        'SnappyCompressor',
+    )
+
+    available_compaction_classes = (
+        'LeveledCompactionStrategy',
+        'SizeTieredCompactionStrategy'
+    )
+
+    cql_type_to_apache_class = {
+        'ascii': 'AsciiType',
+        'bigint': 'LongType',
+        'blob': 'BytesType',
+        'boolean': 'BooleanType',
+        'counter': 'CounterColumnType',
+        'decimal': 'DecimalType',
+        'double': 'DoubleType',
+        'float': 'FloatType',
+        'int': 'Int32Type',
+        'text': 'UTF8Type',
+        'timestamp': 'DateType',
+        'uuid': 'UUIDType',
+        'varchar': 'UTF8Type',
+        'varint': 'IntegerType'
+    }
+
+    apache_class_to_cql_type = dict((v,k) for (k,v) in cql_type_to_apache_class.items())
+
+    cql_types = sorted(cql_type_to_apache_class.keys())
+
+    replication_strategies = (
+        'SimpleStrategy',
+        'OldNetworkTopologyStrategy',
+        'NetworkTopologyStrategy'
+    )
+
+    consistency_levels = (
+        'ANY',
+        'ONE',
+        'TWO',
+        'THREE',
+        'QUORUM',
+        'ALL',
+        'LOCAL_QUORUM',
+        'EACH_QUORUM'
+    )
+
+    # if a term matches this, it shouldn't need to be quoted to be valid cql
+    valid_cql_word_re = re.compile(r"^(?:[a-z][a-z0-9_]*|-?[0-9][0-9.]*)$", re.I)
+
+    def __init__(self, *args, **kwargs):
+        pylexotron.ParsingRuleSet.__init__(self, *args, **kwargs)
+
+        # note: commands_end_with_newline may be extended by callers.
+        self.commands_end_with_newline = set()
+        self.set_keywords_as_syntax()
+
+    def completer_for(self, rulename, symname):
+        def registrator(f):
+            def completerwrapper(ctxt):
+                cass = ctxt.get_binding('cassandra_conn', None)
+                if cass is None:
+                    return ()
+                return f(ctxt, cass)
+            completerwrapper.func_name = 'completerwrapper_on_' + f.func_name
+            self.register_completer(completerwrapper, rulename, symname)
+            return completerwrapper
+        return registrator
+
+    def explain_completion(self, rulename, symname, explanation=None):
+        if explanation is None:
+            explanation = '<%s>' % (symname,)
+        @self.completer_for(rulename, symname)
+        def explainer(ctxt, cass):
+            return [Hint(explanation)]
+        return explainer
+
+    def set_keywords_as_syntax(self):
+        syntax = []
+        for k in self.keywords:
+            syntax.append('<K_%s> ::= "%s" ;' % (k.upper(), k))
+        self.append_rules('\n'.join(syntax))
+
+    def cql_massage_tokens(self, toklist):
+        curstmt = []
+        output = []
+
+        term_on_nl = False
+
+        for t in toklist:
+            if t[0] == 'endline':
+                if term_on_nl:
+                    t = ('endtoken',) + t[1:]
+                else:
+                    # don't put any 'endline' tokens in output
+                    continue
+            curstmt.append(t)
+            if t[0] == 'endtoken':
+                term_on_nl = False
+                output.extend(curstmt)
+                curstmt = []
+            else:
+                if len(curstmt) == 1:
+                    # first token in statement; command word
+                    cmd = t[1].lower()
+                    term_on_nl = bool(cmd in self.commands_end_with_newline)
+
+        output.extend(curstmt)
+        return output
+
+    def cql_parse(self, text, startsymbol='Start'):
+        tokens = self.lex(text)
+        tokens = self.cql_massage_tokens(tokens)
+        return self.parse(startsymbol, tokens, init_bindings={'*SRC*': text})
+
+    def cql_whole_parse_tokens(self, toklist, srcstr=None, startsymbol='Start'):
+        return self.whole_match(startsymbol, toklist, srcstr=srcstr)
+
+    def cql_split_statements(self, text):
+        tokens = self.lex(text)
+        tokens = self.cql_massage_tokens(tokens)
+        stmts = util.split_list(tokens, lambda t: t[0] == 'endtoken')
+        output = []
+        in_batch = False
+        for stmt in stmts:
+            if in_batch:
+                output[-1].extend(stmt)
+            else:
+                output.append(stmt)
+            if len(stmt) > 1 \
+            and stmt[0][0] == 'identifier' and stmt[1][0] == 'identifier' \
+            and stmt[1][1].lower() == 'batch':
+                if stmt[0][1].lower() == 'begin':
+                    in_batch = True
+                elif stmt[0][1].lower() == 'apply':
+                    in_batch = False
+        return output, in_batch
+
+    def cql_complete_single(self, text, partial, init_bindings={}, ignore_case=True,
+                            startsymbol='Start'):
+        tokens = (self.cql_split_statements(text)[0] or [[]])[-1]
+        bindings = init_bindings.copy()
+
+        # handle some different completion scenarios- in particular, completing
+        # inside a string literal
+        prefix = None
+        dequoter = util.identity
+        if tokens:
+            if tokens[-1][0] == 'unclosedString':
+                prefix = self.token_dequote(tokens[-1])
+                tokens = tokens[:-1]
+                partial = prefix + partial
+                dequoter = self.dequote_value
+                requoter = self.escape_value
+            elif tokens[-1][0] == 'unclosedName':
+                prefix = self.token_dequote(tokens[-1])
+                tokens = tokens[:-1]
+                partial = prefix + partial
+                dequoter = self.dequote_name
+                requoter = self.escape_name
+            elif tokens[-1][0] == 'unclosedComment':
+                return []
+        bindings['partial'] = partial
+        bindings['*SRC*'] = text
+
+        # find completions for the position
+        completions = self.complete(startsymbol, tokens, bindings)
+
+        hints, strcompletes = util.list_bifilter(pylexotron.is_hint, completions)
+
+        # it's possible to get a newline token from completion; of course, we
+        # don't want to actually have that be a candidate, we just want to hint
+        if '\n' in strcompletes:
+            strcompletes.remove('\n')
+            if partial == '':
+                hints.append(Hint('<enter>'))
+
+        # find matches with the partial word under completion
+        if ignore_case:
+            partial = partial.lower()
+            f = lambda s: s and dequoter(s).lower().startswith(partial)
+        else:
+            f = lambda s: s and dequoter(s).startswith(partial)
+        candidates = filter(f, strcompletes)
+
+        if prefix is not None:
+            # dequote, re-escape, strip quotes: gets us the right quoted text
+            # for completion. the opening quote is already there on the command
+            # line and not part of the word under completion, and readline
+            # fills in the closing quote for us.
+            candidates = [requoter(dequoter(c))[len(prefix)+1:-1] for c in candidates]
+
+            # the above process can result in an empty string; this doesn't help for
+            # completions
+            candidates = filter(None, candidates)
+
+        # prefix a space when desirable for pleasant cql formatting
+        if tokens:
+            newcandidates = []
+            for c in candidates:
+                if self.want_space_between(tokens[-1], c) \
+                and prefix is None \
+                and not text[-1].isspace() \
+                and not c[0].isspace():
+                    c = ' ' + c
+                newcandidates.append(c)
+            candidates = newcandidates
+
+        # append a space for single, complete identifiers
+        if len(candidates) == 1 and candidates[0][-1].isalnum():
+            candidates[0] += ' '
+        return candidates, hints
+
+    @staticmethod
+    def want_space_between(tok, following):
+        if following in (',', ')', ':'):
+            return False
+        if tok[0] == 'op' and tok[1] in (',', ')', '='):
+            return True
+        if tok[0] == 'stringLiteral' and following[0] != ';':
+            return True
+        if tok[0] == 'star' and following[0] != ')':
+            return True
+        if tok[0] == 'endtoken':
+            return True
+        if tok[1][-1].isalnum() and following[0] != ',':
+            return True
+        return False
+
+    def cql_complete(self, text, partial, cassandra_conn=None, ignore_case=True, debug=False,
+                     startsymbol='Start'):
+        init_bindings = {'cassandra_conn': cassandra_conn}
+        if debug:
+            init_bindings['*DEBUG*'] = True
+
+        completions, hints = self.cql_complete_single(text, partial, init_bindings,
+                                                      startsymbol=startsymbol)
+
+        if hints:
+            hints = [h.text for h in hints]
+            hints.append('')
+
+        if len(completions) == 1 and len(hints) == 0:
+            c = completions[0]
+            if debug:
+                print "** Got one completion: %r. Checking for further matches...\n" % (c,)
+            if not c.isspace():
+                new_c = self.cql_complete_multiple(text, c, init_bindings, startsymbol=startsymbol)
+                completions = [new_c]
+            if debug:
+                print "** New list of completions: %r" % (completions,)
+
+        return hints + completions
+
+    def cql_complete_multiple(self, text, first, init_bindings, startsymbol='Start'):
+        debug = init_bindings.get('*DEBUG*', False)
+        try:
+            completions, hints = self.cql_complete_single(text + first, '', init_bindings,
+                                                          startsymbol=startsymbol)
+        except Exception:
+            if debug:
+                print "** completion expansion had a problem:"
+                traceback.print_exc()
+            return first
+        if hints:
+            if not first[-1].isspace():
+                first += ' '
+            if debug:
+                print "** completion expansion found hints: %r" % (hints,)
+            return first
+        if len(completions) == 1 and completions[0] != '':
+            if debug:
+                print "** Got another completion: %r." % (completions[0],)
+            if completions[0][0] in (',', ')', ':') and first[-1] == ' ':
+                first = first[:-1]
+            first += completions[0]
+        else:
+            common_prefix = util.find_common_prefix(completions)
+            if common_prefix == '':
+                return first
+            if common_prefix[0] in (',', ')', ':') and first[-1] == ' ':
+                first = first[:-1]
+            if debug:
+                print "** Got a partial completion: %r." % (common_prefix,)
+            first += common_prefix
+        if debug:
+            print "** New total completion: %r. Checking for further matches...\n" % (first,)
+        return self.cql_complete_multiple(text, first, init_bindings, startsymbol=startsymbol)
+
+    @classmethod
+    def cql_typename(cls, classname):
+        fq_classname = 'org.apache.cassandra.db.marshal.'
+        if classname.startswith(fq_classname):
+            classname = classname[len(fq_classname):]
+        try:
+            return cls.apache_class_to_cql_type[classname]
+        except KeyError:
+            return cls.escape_value(classname)
+
+    @classmethod
+    def find_validator_class(cls, cqlname):
+        return cls.cql_type_to_apache_class[cqlname]
+
+    @classmethod
+    def is_valid_cql_word(cls, s):
+        return cls.valid_cql_word_re.match(s) is not None and s.lower() not in cls.keywords
+
+    @staticmethod
+    def cql_extract_orig(toklist, srcstr):
+        # low end of span for first token, to high end of span for last token
+        return srcstr[toklist[0][2][0]:toklist[-1][2][1]]
+
+    @staticmethod
+    def token_dequote(tok):
+        if tok[0] == 'stringLiteral':
+            # strip quotes
+            return tok[1][1:-1].replace("''", "'")
+        if tok[0] == 'unclosedString':
+            # strip one quote
+            return tok[1][1:].replace("''", "'")
+        if tok[0] == 'unclosedComment':
+            return ''
+        return tok[1]
+
+    @staticmethod
+    def token_is_word(tok):
+        return tok[0] == 'identifier'
+
+    @classmethod
+    def cql2_maybe_escape_name(cls, name):
+        if cls.is_valid_cql_word(name):
+            return name
+        return cls.cql2_escape_name(name)
+
+    # XXX: this doesn't really belong here.
+    @classmethod
+    def is_counter_col(cls, cfdef, colname):
+        col_info = [cm for cm in cfdef.column_metadata if cm.name == colname]
+        return bool(col_info and cls.cql_typename(col_info[0].validation_class) == 'counter')
+
+    @staticmethod
+    def cql2_dequote_value(cqlword):
+        cqlword = cqlword.strip()
+        if cqlword == '':
+            return cqlword
+        if cqlword[0] == "'":
+            cqlword = cqlword[1:-1].replace("''", "'")
         return cqlword
-    if cqlword[0] == "'":
-        cqlword = cqlword[1:-1].replace("''", "'")
-    return cqlword
-
-def token_is_word(tok):
-    return tok[0] == 'identifier'
-
-def cql_escape(value):
-    if value is None:
-        return 'NULL' # this totally won't work
-    if isinstance(value, bool):
-        value = str(value).lower()
-    elif isinstance(value, float):
-        return '%f' % value
-    elif isinstance(value, int):
-        return str(value)
-    return "'%s'" % value.replace("'", "''")
-
-def maybe_cql_escape(value):
-    if is_valid_cql_word(value):
-        return value
-    return cql_escape(value)
-
-def cql_typename(classname):
-    fq_classname = 'org.apache.cassandra.db.marshal.'
-    if classname.startswith(fq_classname):
-        classname = classname[len(fq_classname):]
-    try:
-        return apache_class_to_cql_type[classname]
-    except KeyError:
-        return cql_escape(classname)
-
-special_completers = []
-
-def completer_for(rulename, symname):
-    def registrator(f):
-        def completerwrapper(ctxt):
-            cass = ctxt.get_binding('cassandra_conn', None)
-            if cass is None:
-                return ()
-            return f(ctxt, cass)
-        completerwrapper.func_name = 'completerwrapper_on_' + f.func_name
-        special_completers.append((rulename, symname, completerwrapper))
-        return completerwrapper
-    return registrator
-
-def explain_completion(rulename, symname, explanation=None):
-    if explanation is None:
-        explanation = '<%s>' % (symname,)
-    @completer_for(rulename, symname)
-    def explainer(ctxt, cass):
-        return [Hint(explanation)]
-    return explainer
-
-def is_counter_col(cfdef, colname):
-    col_info = [cm for cm in cfdef.column_metadata if cm.name == colname]
-    return bool(col_info and cql_typename(col_info[0].validation_class) == 'counter')
+
+    @staticmethod
+    def cql2_escape_value(value):
+        if value is None:
+            return 'NULL' # this totally won't work
+        if isinstance(value, bool):
+            value = str(value).lower()
+        elif isinstance(value, float):
+            return '%f' % value
+        elif isinstance(value, int):
+            return str(value)
+        return "'%s'" % value.replace("'", "''")
+
+    # use _name for keyspace, cf, and column names, and _value otherwise.
+    # also use the cql2_ prefix when dealing with cql2, or leave it off to
+    # get whatever behavior is default for this CqlParsingRuleSet.
+    cql2_dequote_name = dequote_name = dequote_value = cql2_dequote_value
+    cql2_escape_name = escape_name = escape_value = cql2_escape_value
+    maybe_escape_name = cql2_maybe_escape_name
+    dequote_any = cql2_dequote_value
+
+CqlRuleSet = CqlParsingRuleSet()
+
+# convenience for remainder of module
+shorthands = ('completer_for', 'explain_completion',
+              'dequote_value', 'dequote_name',
+              'escape_value', 'escape_name',
+              'maybe_escape_name')
+
+for shorthand in shorthands:
+    globals()[shorthand] = getattr(CqlRuleSet, shorthand)
 
 
 
@@ -221,7 +465,6 @@ syntax_rules = r'''
 JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ;
 
 <stringLiteral> ::= /'([^']|'')*'/ ;
-<dquoteLiteral> ::= /"([^"]|"")*"/ ;
 <float> ::=         /-?[0-9]+\.[0-9]+/ ;
 <integer> ::=       /-?[0-9]+/ ;
 <uuid> ::=          /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/ ;
@@ -243,7 +486,6 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ;
            ;
 <name> ::= <identifier>
          | <stringLiteral>
-         | <dquoteLiteral>
          | <integer>
          ;
 <term> ::= <stringLiteral>
@@ -279,29 +521,52 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ;
 
 <consistencylevel> ::= cl=<identifier> ;
 
-<storageType> ::= typename=( <identifier> | <stringLiteral> );
+<storageType> ::= typename=( <identifier> | <stringLiteral> ) ;
+
+<keyspaceName> ::= ksname=<name> ;
+
+<columnFamilyName> ::= ( ksname=<name> "." )? cfname=<name> ;
 '''
 
 @completer_for('consistencylevel', 'cl')
 def cl_completer(ctxt, cass):
-    return consistency_levels
+    return CqlRuleSet.consistency_levels
 
 @completer_for('storageType', 'typename')
 def storagetype_completer(ctxt, cass):
-    return cql_types
+    return CqlRuleSet.cql_types
 
-syntax_rules += r'''
-<useStatement> ::= "USE" ksname=<name>
-                 ;
-'''
+@completer_for('keyspaceName', 'ksname')
+def ks_name_completer(ctxt, cass):
+    return map(maybe_escape_name, cass.get_keyspace_names())
 
-@completer_for('useStatement', 'ksname')
-def use_ks_completer(ctxt, cass):
-    return map(maybe_cql_escape, cass.get_keyspace_names())
+@completer_for('columnFamilyName', 'ksname')
+def cf_ks_name_completer(ctxt, cass):
+    return [maybe_escape_name(ks) + '.' for ks in cass.get_keyspace_names()]
+
+@completer_for('columnFamilyName', 'cfname')
+def cf_name_completer(ctxt, cass):
+    ks = ctxt.get_binding('ksname', None)
+    if ks is not None:
+        ks = dequote_name(ks)
+    try:
+        cfnames = cass.get_columnfamily_names(ks)
+    except Exception:
+        if ks is None:
+            return ()
+        raise
+    return map(maybe_escape_name, cfnames)
+
+def get_cfdef(ctxt, cass):
+    ks = ctxt.get_binding('ksname', None)
+    cf = ctxt.get_binding('cfname')
+    return cass.get_columnfamily(cf, ksname=ks)
 
 syntax_rules += r'''
+<useStatement> ::= "USE" ksname=<keyspaceName>
+                 ;
 <selectStatement> ::= "SELECT" <whatToSelect>
-                        "FROM" ( selectks=<name> "." )? selectsource=<name>
+                        "FROM" cf=<columnFamilyName>
                           ("USING" "CONSISTENCY" <consistencylevel>)?
                           ("WHERE" <selectWhereClause>)?
                           ("LIMIT" <integer>)?
@@ -318,39 +583,15 @@ syntax_rules += r'''
                  ;
 '''
 
-@completer_for('selectStatement', 'selectsource')
-def select_source_completer(ctxt, cass):
-    ks = ctxt.get_binding('selectks', None)
-    if ks is not None:
-        ks = cql_dequote(ks)
-    try:
-        cfnames = cass.get_columnfamily_names(ks)
-    except Exception:
-        if ks is None:
-            return ()
-        raise
-    return map(maybe_cql_escape, cfnames)
-
-@completer_for('selectStatement', 'selectks')
-def select_keyspace_completer(ctxt, cass):
-    return [maybe_cql_escape(ks) + '.' for ks in cass.get_keyspace_names()]
-
 @completer_for('selectWhereClause', 'keyname')
 def select_where_keyname_completer(ctxt, cass):
-    ksname = ctxt.get_binding('selectks')
-    if ksname is not None:
-        ksname = cql_dequote(ksname)
-    selectsource = cql_dequote(ctxt.get_binding('selectsource'))
-    cfdef = cass.get_columnfamily(selectsource, ksname=ksname)
+    cfdef = get_cfdef(ctxt, cass)
     return [cfdef.key_alias if cfdef.key_alias is not None else 'KEY']
 
 @completer_for('relation', 'rel_lhs')
 def select_relation_lhs_completer(ctxt, cass):
-    ksname = ctxt.get_binding('selectks')
-    if ksname is not None:
-        ksname = cql_dequote(ksname)
-    selectsource = cql_dequote(ctxt.get_binding('selectsource'))
-    return map(maybe_cql_escape, cass.filterable_column_names(selectsource, ksname=ksname))
+    cfdef = get_cfdef(ctxt, cass)
+    return map(maybe_escape_name, cass.filterable_column_names(cfdef))
 
 @completer_for('whatToSelect', 'countparens')
 def select_count_parens_completer(ctxt, cass):
@@ -361,7 +602,7 @@ explain_completion('whatToSelect', 'rangestart', '<range_start>')
 explain_completion('whatToSelect', 'rangeend', '<range_end>')
 
 syntax_rules += r'''
-<insertStatement> ::= "INSERT" "INTO" ( insertks=<name> "." )? insertcf=<name>
+<insertStatement> ::= "INSERT" "INTO" cf=<columnFamilyName>
                                "(" keyname=<colname> ","
                                    [colname]=<colname> ( "," [colname]=<colname> )* ")"
                       "VALUES" "(" <term> "," <term> ( "," <term> )* ")"
@@ -374,27 +615,9 @@ syntax_rules += r'''
                 ;
 '''
 
-@completer_for('insertStatement', 'insertks')
-def insert_ks_completer(ctxt, cass):
-    return [maybe_cql_escape(ks) + '.' for ks in cass.get_keyspace_names()]
-
-@completer_for('insertStatement', 'insertcf')
-def insert_cf_completer(ctxt, cass):
-    ks = ctxt.get_binding('insertks', None)
-    if ks is not None:
-        ks = cql_dequote(ks)
-    try:
-        cfnames = cass.get_columnfamily_names(ks)
-    except Exception:
-        if ks is None:
-            return ()
-        raise
-    return map(maybe_cql_escape, cfnames)
-
 @completer_for('insertStatement', 'keyname')
 def insert_keyname_completer(ctxt, cass):
-    insertcf = ctxt.get_binding('insertcf')
-    cfdef = cass.get_columnfamily(cql_dequote(insertcf))
+    cfdef = get_cfdef(ctxt, cass)
     return [cfdef.key_alias if cfdef.key_alias is not None else 'KEY']
 
 explain_completion('insertStatement', 'colname')
@@ -407,7 +630,7 @@ def insert_option_completer(ctxt, cass):
     return opts
 
 syntax_rules += r'''
-<updateStatement> ::= "UPDATE" ( updateks=<name> "." )? updatecf=<name>
+<updateStatement> ::= "UPDATE" cf=<columnFamilyName>
                         ( "USING" [updateopt]=<usingOption>
                                   ( "AND" [updateopt]=<usingOption> )* )?
                         "SET" <assignment> ( "," <assignment> )*
@@ -421,23 +644,6 @@ syntax_rules += r'''
                       ;
 '''
 
-@completer_for('updateStatement', 'updateks')
-def update_cf_completer(ctxt, cass):
-    return [maybe_cql_escape(ks) + '.' for ks in cass.get_keyspace_names()]
-
-@completer_for('updateStatement', 'updatecf')
-def update_cf_completer(ctxt, cass):
-    ks = ctxt.get_binding('updateks', None)
-    if ks is not None:
-        ks = cql_dequote(ks)
-    try:
-        cfnames = cass.get_columnfamily_names(ks)
-    except Exception:
-        if ks is None:
-            return ()
-        raise
-    return map(maybe_cql_escape, cfnames)
-
 @completer_for('updateStatement', 'updateopt')
 def insert_option_completer(ctxt, cass):
     opts = set('CONSISTENCY TIMESTAMP TTL'.split())
@@ -447,41 +653,41 @@ def insert_option_completer(ctxt, cass):
 
 @completer_for('assignment', 'updatecol')
 def update_col_completer(ctxt, cass):
-    cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf')))
-    colnames = map(maybe_cql_escape, [cm.name for cm in cfdef.column_metadata])
+    cfdef = get_cfdef(ctxt, cass)
+    colnames = map(maybe_escape_name, [cm.name for cm in cfdef.column_metadata])
     return colnames + [Hint('<colname>')]
 
 @completer_for('assignment', 'update_rhs')
 def update_countername_completer(ctxt, cass):
-    cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf')))
-    curcol = cql_dequote(ctxt.get_binding('updatecol', ''))
-    return [maybe_cql_escape(curcol)] if is_counter_col(cfdef, curcol) else [Hint('<term>')]
+    cfdef = get_cfdef(ctxt, cass)
+    curcol = dequote_name(ctxt.get_binding('updatecol', ''))
+    return [maybe_escape_name(curcol)] if CqlRuleSet.is_counter_col(cfdef, curcol) else [Hint('<term>')]
 
 @completer_for('assignment', 'counterop')
 def update_counterop_completer(ctxt, cass):
-    cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf')))
-    curcol = cql_dequote(ctxt.get_binding('updatecol', ''))
-    return ['+', '-'] if is_counter_col(cfdef, curcol) else []
+    cfdef = get_cfdef(ctxt, cass)
+    curcol = dequote_name(ctxt.get_binding('updatecol', ''))
+    return ['+', '-'] if CqlRuleSet.is_counter_col(cfdef, curcol) else []
 
 @completer_for('updateWhereClause', 'updatefiltercol')
 def update_filtercol_completer(ctxt, cass):
-    cfname = cql_dequote(ctxt.get_binding('cf'))
-    return map(maybe_cql_escape, cass.filterable_column_names(cfname))
+    cfdef = get_cfdef(ctxt, cass)
+    return map(maybe_escape_name, cass.filterable_column_names(cfdef))
 
 @completer_for('updateWhereClause', 'updatefilterkey')
 def update_filterkey_completer(ctxt, cass):
-    cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf')))
+    cfdef = get_cfdef(ctxt, cass)
     return [cfdef.key_alias if cfdef.key_alias is not None else 'KEY']
 
 @completer_for('updateWhereClause', 'filter_in')
 def update_filter_in_completer(ctxt, cass):
-    cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf')))
+    cfdef = get_cfdef(ctxt, cass)
     fk = ctxt.get_binding('updatefilterkey')
     return ['IN'] if fk in ('KEY', cfdef.key_alias) else []
 
 syntax_rules += r'''
 <deleteStatement> ::= "DELETE" ( [delcol]=<colname> ( "," [delcol]=<colname> )* )?
-                        "FROM" ( deleteks=<name> "." )? deletecf=<name>
+                        "FROM" cf=<columnFamilyName>
                         ( "USING" [delopt]=<deleteOption> ( "AND" [delopt]=<deleteOption> )* )?
                         "WHERE" <updateWhereClause>
                     ;
@@ -490,23 +696,6 @@ syntax_rules += r'''
                  ;
 '''
 
-@completer_for('deleteStatement', 'deleteks')
-def update_cf_completer(ctxt, cass):
-    return [maybe_cql_escape(ks) + '.' for ks in cass.get_keyspace_names()]
-
-@completer_for('deleteStatement', 'deletecf')
-def delete_cf_completer(ctxt, cass):
-    ks = ctxt.get_binding('deleteks', None)
-    if ks is not None:
-        ks = cql_dequote(ks)
-    try:
-        cfnames = cass.get_columnfamily_names(ks)
-    except Exception:
-        if ks is None:
-            return ()
-        raise
-    return map(maybe_cql_escape, cfnames)
-
 @completer_for('deleteStatement', 'delopt')
 def delete_opt_completer(ctxt, cass):
     opts = set('CONSISTENCY TIMESTAMP'.split())
@@ -538,27 +727,10 @@ def batch_opt_completer(ctxt, cass):
     return opts
 
 syntax_rules += r'''
-<truncateStatement> ::= "TRUNCATE" ( truncateks=<name> "." )? truncatecf=<name>
+<truncateStatement> ::= "TRUNCATE" cf=<columnFamilyName>
                       ;
 '''
 
-@completer_for('truncateStatement', 'truncateks')
-def update_cf_completer(ctxt, cass):
-    return [maybe_cql_escape(ks) + '.' for ks in cass.get_keyspace_names()]
-
-@completer_for('truncateStatement', 'truncatecf')
-def truncate_cf_completer(ctxt, cass):
-    ks = ctxt.get_binding('truncateks', None)
-    if ks is not None:
-        ks = cql_dequote(ks)
-    try:
-        cfnames = cass.get_columnfamily_names(ks)
-    except Exception:
-        if ks is None:
-            return ()
-        raise
-    return map(maybe_cql_escape, cfnames)
-
 syntax_rules += r'''
 <createKeyspaceStatement> ::= "CREATE" "KEYSPACE" ksname=<name>
                                  "WITH" [optname]=<optionName> "=" [optval]=<optionVal>
@@ -582,7 +754,7 @@ def create_ks_opt_completer(ctxt, cass):
     except ValueError:
         return ['strategy_class =']
     vals = ctxt.get_binding('optval')
-    stratclass = cql_dequote(vals[stratopt])
+    stratclass = dequote_value(vals[stratopt])
     if stratclass in ('SimpleStrategy', 'OldNetworkTopologyStrategy'):
         return ['strategy_options:replication_factor =']
     return [Hint('<strategy_option_name>')]
@@ -591,11 +763,11 @@ def create_ks_opt_completer(ctxt, cass):
 def create_ks_optval_completer(ctxt, cass):
     exist_opts = ctxt.get_binding('optname', (None,))
     if exist_opts[-1] == 'strategy_class':
-        return map(cql_escape, replication_strategies)
+        return map(escape_value, CqlRuleSet.replication_strategies)
     return [Hint('<option_value>')]
 
 syntax_rules += r'''
-<createColumnFamilyStatement> ::= "CREATE" "COLUMNFAMILY" cf=<name>
+<createColumnFamilyStatement> ::= "CREATE" ( "COLUMNFAMILY" | "TABLE" ) cf=<name>
                                     "(" keyalias=<colname> <storageType> "PRIMARY" "KEY"
                                         ( "," colname=<colname> <storageType> )* ")"
                                    ( "WITH" [cfopt]=<cfOptionName> "=" [optval]=<cfOptionVal>
@@ -612,19 +784,19 @@ syntax_rules += r'''
                 ;
 '''
 
-explain_completion('createColumnFamilyStatement', 'keyalias', '<new_key_alias>')
-explain_completion('createColumnFamilyStatement', 'cf', '<new_columnfamily_name>')
+explain_completion('createColumnFamilyStatement', 'keyalias', '<new_key_name>')
+explain_completion('createColumnFamilyStatement', 'cf', '<new_table_name>')
 explain_completion('createColumnFamilyStatement', 'colname', '<new_column_name>')
 
 @completer_for('cfOptionName', 'cfoptname')
 def create_cf_option_completer(ctxt, cass):
-    return [c[0] for c in columnfamily_options] + \
-           [c[0] + ':' for c in columnfamily_map_options]
+    return [c[0] for c in CqlRuleSet.columnfamily_options] + \
+           [c[0] + ':' for c in CqlRuleSet.columnfamily_map_options]
 
 @completer_for('cfOptionName', 'cfoptsep')
 def create_cf_suboption_separator(ctxt, cass):
     opt = ctxt.get_binding('cfoptname')
-    if any(opt == c[0] for c in columnfamily_map_options):
+    if any(opt == c[0] for c in CqlRuleSet.columnfamily_map_options):
         return [':']
     return ()
 
@@ -637,7 +809,7 @@ def create_cf_suboption_completer(ctxt, cass):
         prevvals = ctxt.get_binding('optval', ())
         for prevopt, prevval in zip(prevopts, prevvals):
             if prevopt == 'compaction_strategy_class':
-                csc = cql_dequote(prevval)
+                csc = dequote_value(prevval)
                 break
         else:
             cf = ctxt.get_binding('cf')
@@ -650,7 +822,7 @@ def create_cf_suboption_completer(ctxt, cass):
             return ['min_sstable_size']
         elif csc == 'LeveledCompactionStrategy':
             return ['sstable_size_in_mb']
-    for optname, _, subopts in columnfamily_map_options:
+    for optname, _, subopts in CqlRuleSet.columnfamily_map_options:
         if opt == optname:
             return subopts
     return ()
@@ -659,13 +831,13 @@ def create_cf_option_val_completer(ctxt, cass):
     exist_opts = ctxt.get_binding('cfopt')
     this_opt = exist_opts[-1]
     if this_opt == 'compression_parameters:sstable_compression':
-        return map(cql_escape, available_compression_classes)
+        return map(escape_value, CqlRuleSet.available_compression_classes)
     if this_opt == 'compaction_strategy_class':
-        return map(cql_escape, available_compaction_classes)
-    if any(this_opt == opt[0] for opt in obsolete_cf_options):
+        return map(escape_value, CqlRuleSet.available_compaction_classes)
+    if any(this_opt == opt[0] for opt in CqlRuleSet.obsolete_cf_options):
         return ["'<obsolete_option>'"]
     if this_opt in ('comparator', 'default_validation'):
-        return cql_types
+        return CqlRuleSet.cql_types
     if this_opt == 'read_repair_chance':
         return [Hint('<float_between_0_and_1>')]
     if this_opt == 'replicate_on_write':
@@ -687,43 +859,43 @@ explain_completion('createIndexStatement', 'indexname', '<new_index_name>')
 
 @completer_for('createIndexStatement', 'cf')
 def create_index_cf_completer(ctxt, cass):
-    return map(maybe_cql_escape, cass.get_columnfamily_names())
+    return map(maybe_escape_name, cass.get_columnfamily_names())
 
 @completer_for('createIndexStatement', 'col')
 def create_index_col_completer(ctxt, cass):
-    cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf')))
+    cfdef = cass.get_columnfamily(dequote_name(ctxt.get_binding('cf')))
     colnames = [md.name for md in cfdef.column_metadata if md.index_name is None]
-    return map(maybe_cql_escape, colnames)
+    return map(maybe_escape_name, colnames)
 
 syntax_rules += r'''
-<dropKeyspaceStatement> ::= "DROP" "KEYSPACE" ksname=<name>
+<dropKeyspaceStatement> ::= "DROP" "KEYSPACE" ksname=<keyspaceName>
                           ;
 '''
 
 @completer_for('dropKeyspaceStatement', 'ksname')
 def drop_ks_completer(ctxt, cass):
-    return map(maybe_cql_escape, cass.get_keyspace_names())
+    return map(maybe_escape_name, cass.get_keyspace_names())
 
 syntax_rules += r'''
-<dropColumnFamilyStatement> ::= "DROP" "COLUMNFAMILY" cf=<name>
+<dropColumnFamilyStatement> ::= "DROP" ( "COLUMNFAMILY" | "TABLE" ) cf=<name>
                               ;
 '''
 
 @completer_for('dropColumnFamilyStatement', 'cf')
 def drop_cf_completer(ctxt, cass):
-    return map(maybe_cql_escape, cass.get_columnfamily_names())
+    return map(maybe_escape_name, cass.get_columnfamily_names())
 
 syntax_rules += r'''
 <dropIndexStatement> ::= "DROP" "INDEX" indexname=<name>
                        ;
 '''
 
-@completer_for('dropIndexStatement', 'cf')
+@completer_for('dropIndexStatement', 'indexname')
 def drop_index_completer(ctxt, cass):
-    return map(maybe_cql_escape, cass.get_index_names())
+    return map(maybe_escape_name, cass.get_index_names())
 
 syntax_rules += r'''
-<alterTableStatement> ::= "ALTER" "COLUMNFAMILY" cf=<name> <alterInstructions>
+<alterTableStatement> ::= "ALTER" ( "COLUMNFAMILY" | "TABLE" ) cf=<name> <alterInstructions>
                         ;
 <alterInstructions> ::= "ALTER" existcol=<name> "TYPE" <storageType>
                       | "ADD" newcol=<name> <storageType>
@@ -735,15 +907,15 @@ syntax_rules += r'''
 
 @completer_for('alterTableStatement', 'cf')
 def alter_table_cf_completer(ctxt, cass):
-    return map(maybe_cql_escape, cass.get_columnfamily_names())
+    return map(maybe_escape_name, cass.get_columnfamily_names())
 
 @completer_for('alterInstructions', 'existcol')
 def alter_table_col_completer(ctxt, cass):
-    cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf')))
+    cfdef = cass.get_columnfamily(dequote_name(ctxt.get_binding('cf')))
     cols = [md.name for md in cfdef.column_metadata]
     if cfdef.key_alias is not None:
         cols.append(cfdef.key_alias)
-    return map(maybe_cql_escape, cols)
+    return map(maybe_escape_name, cols)
 
 explain_completion('alterInstructions', 'newcol', '<new_column_name>')
 
@@ -752,211 +924,4 @@ completer_for('alterInstructions', 'optval') \
 
 # END SYNTAX/COMPLETION RULE DEFINITIONS
 
-
-
-CqlRuleSet = pylexotron.ParsingRuleSet.from_rule_defs(syntax_rules)
-for rulename, symname, compf in special_completers:
-    CqlRuleSet.register_completer(compf, rulename, symname)
-
-def cql_add_completer(rulename, symname):
-    registrator = completer_for(rulename, symname)
-    def more_registration(f):
-        f = registrator(f)
-        CqlRuleSet.register_completer(f, rulename, symname)
-        return f
-    return more_registration
-
-def cql_parse(text, startsymbol='Start'):
-    tokens = CqlRuleSet.lex(text)
-    tokens = cql_massage_tokens(tokens)
-    return CqlRuleSet.parse(startsymbol, tokens, init_bindings={'*SRC*': text})
-
-def cql_whole_parse_tokens(toklist, srcstr=None, startsymbol='Start'):
-    return CqlRuleSet.whole_match(startsymbol, toklist, srcstr=srcstr)
-
-def cql_massage_tokens(toklist):
-    curstmt = []
-    output = []
-
-    term_on_nl = False
-
-    for t in toklist:
-        if t[0] == 'endline':
-            if term_on_nl:
-                t = ('endtoken',) + t[1:]
-            else:
-                # don't put any 'endline' tokens in output
-                continue
-        curstmt.append(t)
-        if t[0] == 'endtoken':
-            term_on_nl = False
-            output.extend(curstmt)
-            curstmt = []
-        else:
-            if len(curstmt) == 1:
-                # first token in statement; command word
-                cmd = t[1].lower()
-                term_on_nl = bool(cmd in commands_end_with_newline)
-
-    output.extend(curstmt)
-    return output
-
-def split_list(items, pred):
-    thisresult = []
-    results = [thisresult]
-    for i in items:
-        thisresult.append(i)
-        if pred(i):
-            thisresult = []
-            results.append(thisresult)
-    return results
-
-def cql_split_statements(text):
-    tokens = CqlRuleSet.lex(text)
-    tokens = cql_massage_tokens(tokens)
-    stmts = split_list(tokens, lambda t: t[0] == 'endtoken')
-    output = []
-    in_batch = False
-    for stmt in stmts:
-        if in_batch:
-            output[-1].extend(stmt)
-        else:
-            output.append(stmt)
-        if len(stmt) > 1 \
-        and stmt[0][0] == 'identifier' and stmt[1][0] == 'identifier' \
-        and stmt[1][1].lower() == 'batch':
-            if stmt[0][1].lower() == 'begin':
-                in_batch = True
-            elif stmt[0][1].lower() == 'apply':
-                in_batch = False
-    return output, in_batch
-
-def want_space_between(tok, following):
-    if tok[0] == 'op' and tok[1] in (',', ')', '='):
-        return True
-    if tok[0] == 'stringLiteral' and following[0] != ';':
-        return True
-    if tok[0] == 'star' and following[0] != ')':
-        return True
-    if tok[0] == 'endtoken':
-        return True
-    if tok[1][-1].isalnum() and following[0] != ',':
-        return True
-    return False
-
-def find_common_prefix(strs):
-    common = []
-    for cgroup in izip(*strs):
-        if all(x == cgroup[0] for x in cgroup[1:]):
-            common.append(cgroup[0])
-        else:
-            break
-    return ''.join(common)
-
-def list_bifilter(pred, iterable):
-    yes_s = []
-    no_s = []
-    for i in iterable:
-        (yes_s if pred(i) else no_s).append(i)
-    return yes_s, no_s
-
-def cql_complete_single(text, partial, init_bindings={}, ignore_case=True, startsymbol='Start'):
-    tokens = (cql_split_statements(text)[0] or [[]])[-1]
-    bindings = init_bindings.copy()
-
-    # handle some different completion scenarios- in particular, completing
-    # inside a string literal
-    prefix = None
-    if tokens and tokens[-1][0] == 'unclosedString':
-        prefix = token_dequote(tokens[-1])
-        tokens = tokens[:-1]
-        partial = prefix + partial
-    if tokens and tokens[-1][0] == 'unclosedComment':
-        return []
-    bindings['partial'] = partial
-    bindings['*SRC*'] = text
-
-    # find completions for the position
-    completions = CqlRuleSet.complete(startsymbol, tokens, bindings)
-
-    hints, strcompletes = list_bifilter(pylexotron.is_hint, completions)
-
-    # it's possible to get a newline token from completion; of course, we
-    # don't want to actually have that be a candidate, we just want to hint
-    if '\n' in strcompletes:
-        strcompletes.remove('\n')
-        if partial == '':
-            hints.append(Hint('<enter>'))
-
-    # find matches with the partial word under completion
-    if ignore_case:
-        partial = partial.lower()
-        f = lambda s: s and cql_dequote(s).lower().startswith(partial)
-    else:
-        f = lambda s: s and cql_dequote(s).startswith(partial)
-    candidates = filter(f, strcompletes)
-
-    if prefix is not None:
-        # dequote, re-escape, strip quotes: gets us the right quoted text
-        # for completion. the opening quote is already there on the command
-        # line and not part of the word under completion, and readline
-        # fills in the closing quote for us.
-        candidates = [cql_escape(cql_dequote(c))[len(prefix)+1:-1] for c in candidates]
-
-        # the above process can result in an empty string; this doesn't help for
-        # completions
-        candidates = filter(None, candidates)
-
-    # prefix a space when desirable for pleasant cql formatting
-    if tokens:
-        newcandidates = []
-        for c in candidates:
-            if want_space_between(tokens[-1], c) \
-            and prefix is None \
-            and not text[-1].isspace() \
-            and not c[0].isspace():
-                c = ' ' + c
-            newcandidates.append(c)
-        candidates = newcandidates
-
-    return candidates, hints
-
-def cql_complete(text, partial, cassandra_conn=None, ignore_case=True, debug=False,
-                 startsymbol='Start'):
-    init_bindings = {'cassandra_conn': cassandra_conn}
-    if debug:
-        init_bindings['*DEBUG*'] = True
-
-    completions, hints = cql_complete_single(text, partial, init_bindings, startsymbol=startsymbol)
-
-    if hints:
-        hints = [h.text for h in hints]
-        hints.append('')
-
-    if len(completions) == 1 and len(hints) == 0:
-        c = completions[0]
-        if not c.isspace():
-            new_c = cql_complete_multiple(text, c, init_bindings, startsymbol=startsymbol)
-            completions = [new_c]
-
-    return hints + completions
-
-def cql_complete_multiple(text, first, init_bindings, startsymbol='Start'):
-    try:
-        completions, hints = cql_complete_single(text + first, '', init_bindings,
-                                                 startsymbol=startsymbol)
-    except Exception:
-        return first
-    if hints:
-        if not first[-1].isspace():
-            first += ' '
-        return first
-    if len(completions) == 1 and completions[0] != '':
-        first += completions[0]
-    else:
-        common_prefix = find_common_prefix(completions)
-        if common_prefix != '':
-            first += common_prefix
-        else:
-            return first
-    return cql_complete_multiple(text, first, init_bindings, startsymbol=startsymbol)
+CqlRuleSet.append_rules(syntax_rules)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/38748b43/pylib/cqlshlib/pylexotron.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/pylexotron.py b/pylib/cqlshlib/pylexotron.py
index 7482aba..0534c87 100644
--- a/pylib/cqlshlib/pylexotron.py
+++ b/pylib/cqlshlib/pylexotron.py
@@ -269,6 +269,28 @@ class case_match(text_match):
     def pattern(self):
         return re.escape(self.arg)
 
+class word_match(text_match):
+    def pattern(self):
+        return r'\b' + text_match.pattern(self) + r'\b'
+
+class case_word_match(case_match):
+    def pattern(self):
+        return r'\b' + case_match.pattern(self) + r'\b'
+
+class terminal_type_matcher(matcher):
+    def __init__(self, tokentype, submatcher):
+        matcher.__init__(self, tokentype)
+        self.tokentype = tokentype
+        self.submatcher = submatcher
+
+    def match(self, ctxt, completions):
+        if ctxt.remainder:
+            if ctxt.remainder[0][0] == self.tokentype:
+                return [ctxt.with_match(1)]
+        elif completions is not None:
+            self.submatcher.match(ctxt, completions)
+        return []
+
 class ParsingRuleSet:
     RuleSpecScanner = SaferScanner([
         (r'::=', lambda s,t: t),
@@ -309,9 +331,10 @@ class ParsingRuleSet:
                     raise ValueError('Unexpected token %r; expected "::="' % (assign,))
                 name = t[1]
                 production = cls.read_rule_tokens_until(';', tokeniter)
-                rules[name] = production
                 if isinstance(production, terminal_matcher):
                     terminals.append((name, production))
+                    production = terminal_type_matcher(name, production)
+                rules[name] = production
             else:
                 raise ValueError('Unexpected token %r; expected name' % (t,))
         return rules, terminals
@@ -345,7 +368,10 @@ class ParsingRuleSet:
                 if t[0] == 'reference':
                     t = rule_reference(t[1])
                 elif t[0] == 'litstring':
-                    t = text_match(t[1])
+                    if t[1][1].isalnum() or t[1][1] == '_':
+                        t = word_match(t[1])
+                    else:
+                        t = text_match(t[1])
                 elif t[0] == 'regex':
                     t = regex_rule(t[1])
                 elif t[0] == 'named_collector':

http://git-wip-us.apache.org/repos/asf/cassandra/blob/38748b43/pylib/cqlshlib/util.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/util.py b/pylib/cqlshlib/util.py
new file mode 100644
index 0000000..ea0fbf4
--- /dev/null
+++ b/pylib/cqlshlib/util.py
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from itertools import izip
+
+def split_list(items, pred):
+    """
+    Split up a list (or other iterable) on the elements which satisfy the
+    given predicate 'pred'. Elements for which 'pred' returns true start a new
+    sublist for subsequent elements, which will accumulate in the new sublist
+    until the next satisfying element.
+
+    >>> split_list([0, 1, 2, 5, 99, 8], lambda n: (n % 2) == 0)
+    [[0], [1, 2], [5, 99, 8], []]
+    """
+
+    thisresult = []
+    results = [thisresult]
+    for i in items:
+        thisresult.append(i)
+        if pred(i):
+            thisresult = []
+            results.append(thisresult)
+    return results
+
+def find_common_prefix(strs):
+    """
+    Given a list (iterable) of strings, return the longest common prefix.
+
+    >>> find_common_prefix(['abracadabra', 'abracadero', 'abranch'])
+    'abra'
+    >>> find_common_prefix(['abracadabra', 'abracadero', 'mt. fuji'])
+    ''
+    """
+
+    common = []
+    for cgroup in izip(*strs):
+        if all(x == cgroup[0] for x in cgroup[1:]):
+            common.append(cgroup[0])
+        else:
+            break
+    return ''.join(common)
+
+def list_bifilter(pred, iterable):
+    """
+    Filter an iterable into two output lists: the first containing all
+    elements of the iterable for which 'pred' returns true, and the second
+    containing all others. Order of the elements is otherwise retained.
+
+    >>> list_bifilter(lambda x: isinstance(x, int), (4, 'bingo', 1.2, 6, True))
+    ([4, 6], ['bingo', 1.2, True])
+    """
+
+    yes_s = []
+    no_s = []
+    for i in iterable:
+        (yes_s if pred(i) else no_s).append(i)
+    return yes_s, no_s
+
+def identity(x):
+    return x


Mime
View raw message