cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alek...@apache.org
Subject [2/6] cassandra git commit: Fix cqlshlib tests on Windows
Date Fri, 18 Dec 2015 22:49:41 GMT
Fix cqlshlib tests on Windows

patch by Paulo Motta; reviewed by Jim Witschey for CASSANDRA-10541


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9dd2b5ea
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9dd2b5ea
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9dd2b5ea

Branch: refs/heads/cassandra-3.0
Commit: 9dd2b5ea0f842eb465ad38ac98f9dd987ba6b7d2
Parents: b03ce9f
Author: Paulo Motta <pauloricardomg@gmail.com>
Authored: Mon Nov 16 15:48:20 2015 -0200
Committer: Aleksey Yeschenko <aleksey@apache.org>
Committed: Fri Dec 18 22:46:31 2015 +0000

----------------------------------------------------------------------
 bin/cqlsh.py                                 |   4 +-
 pylib/cqlshlib/test/run_cqlsh.py             | 108 +++++++++++++++-------
 pylib/cqlshlib/test/test_cqlsh_completion.py |   3 +
 pylib/cqlshlib/test/test_cqlsh_output.py     |  22 +++--
 pylib/cqlshlib/test/winpty.py                |  50 ++++++++++
 5 files changed, 144 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/9dd2b5ea/bin/cqlsh.py
----------------------------------------------------------------------
diff --git a/bin/cqlsh.py b/bin/cqlsh.py
index 42c2923..1119289 100644
--- a/bin/cqlsh.py
+++ b/bin/cqlsh.py
@@ -210,6 +210,8 @@ parser.add_option('--cqlversion', default=DEFAULT_CQLVER,
 parser.add_option("-e", "--execute", help='Execute the statement and quit.')
 parser.add_option("--connect-timeout", default=DEFAULT_CONNECT_TIMEOUT_SECONDS, dest='connect_timeout',
                   help='Specify the connection timeout in seconds (default: %default seconds).')
+parser.add_option("-t", "--tty", action='store_true', dest='tty',
+                  help='Force tty mode (command prompt).')
 
 optvalues = optparse.Values()
 (options, arguments) = parser.parse_args(sys.argv[1:], values=optvalues)
@@ -2328,7 +2330,7 @@ def read_options(cmdlineargs, environment):
     optvalues.ssl = False
     optvalues.encoding = None
 
-    optvalues.tty = sys.stdin.isatty()
+    optvalues.tty = option_with_default(configs.getboolean, 'ui', 'tty', sys.stdin.isatty())
     optvalues.cqlversion = option_with_default(configs.get, 'cql', 'version', DEFAULT_CQLVER)
     optvalues.connect_timeout = option_with_default(configs.getint, 'connection', 'timeout',
DEFAULT_CONNECT_TIMEOUT_SECONDS)
     optvalues.execute = None

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9dd2b5ea/pylib/cqlshlib/test/run_cqlsh.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/test/run_cqlsh.py b/pylib/cqlshlib/test/run_cqlsh.py
index 08e9e41..b011df4 100644
--- a/pylib/cqlshlib/test/run_cqlsh.py
+++ b/pylib/cqlshlib/test/run_cqlsh.py
@@ -17,17 +17,28 @@
 # NOTE: this testing tool is *nix specific
 
 import os
+import sys
 import re
-import pty
-import fcntl
 import contextlib
 import subprocess
 import signal
 import math
 from time import time
 from . import basecase
+from os.path import join, normpath
 
-DEFAULT_CQLSH_PROMPT = os.linesep + '(\S+@)?cqlsh(:\S+)?> '
+
+def is_win():
+    return sys.platform in ("cygwin", "win32")
+
+if is_win():
+    from winpty import WinPty
+    DEFAULT_PREFIX = ''
+else:
+    import pty
+    DEFAULT_PREFIX = os.linesep
+
+DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + '(\S+@)?cqlsh(:\S+)?> '
 DEFAULT_CQLSH_TERM = 'xterm'
 
 cqlshlog = basecase.cqlshlog
@@ -41,10 +52,6 @@ def set_controlling_pty(master, slave):
         os.close(slave)
     os.close(os.open(os.ttyname(1), os.O_RDWR))
 
-def set_nonblocking(fd):
-    flags = fcntl.fcntl(fd, fcntl.F_GETFL)
-    fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
 @contextlib.contextmanager
 def raising_signal(signum, exc):
     """
@@ -93,12 +100,21 @@ def timing_out_alarm(seconds):
         finally:
             signal.alarm(0)
 
-# setitimer is new in 2.6, but it's still worth supporting, for potentially
-# faster tests because of sub-second resolution on timeouts.
-if hasattr(signal, 'setitimer'):
-    timing_out = timing_out_itimer
+if is_win():
+    try:
+        import eventlet
+    except ImportError, e:
+        sys.exit("evenlet library required to run cqlshlib tests on Windows")
+
+    def timing_out(seconds):
+        return eventlet.Timeout(seconds, TimeoutError)
 else:
-    timing_out = timing_out_alarm
+    # setitimer is new in 2.6, but it's still worth supporting, for potentially
+    # faster tests because of sub-second resolution on timeouts.
+    if hasattr(signal, 'setitimer'):
+        timing_out = timing_out_itimer
+    else:
+        timing_out = timing_out_alarm
 
 def noop(*a):
     pass
@@ -108,6 +124,7 @@ class ProcRunner:
         self.exe_path = path
         self.args = args
         self.tty = bool(tty)
+        self.realtty = self.tty and not is_win()
         if env is None:
             env = {}
         self.env = env
@@ -118,30 +135,35 @@ class ProcRunner:
     def start_proc(self):
         preexec = noop
         stdin = stdout = stderr = None
-        if self.tty:
-            masterfd, slavefd = pty.openpty()
-            preexec = lambda: set_controlling_pty(masterfd, slavefd)
-        else:
-            stdin = stdout = subprocess.PIPE
-            stderr = subprocess.STDOUT
         cqlshlog.info("Spawning %r subprocess with args: %r and env: %r"
                       % (self.exe_path, self.args, self.env))
-        self.proc = subprocess.Popen(('python', self.exe_path,) + tuple(self.args),
-                                     env=self.env, preexec_fn=preexec,
-                                     stdin=stdin, stdout=stdout, stderr=stderr,
-                                     close_fds=False)
-        if self.tty:
+        if self.realtty:
+            masterfd, slavefd = pty.openpty()
+            preexec = (lambda: set_controlling_pty(masterfd, slavefd))
+            self.proc = subprocess.Popen((self.exe_path,) + tuple(self.args),
+                                         env=self.env, preexec_fn=preexec,
+                                         stdin=stdin, stdout=stdout, stderr=stderr,
+                                         close_fds=False)
             os.close(slavefd)
             self.childpty = masterfd
             self.send = self.send_tty
             self.read = self.read_tty
         else:
+            stdin = stdout = subprocess.PIPE
+            stderr = subprocess.STDOUT
+            self.proc = subprocess.Popen((self.exe_path,) + tuple(self.args),
+                                         env=self.env, stdin=stdin, stdout=stdout,
+                                         stderr=stderr, bufsize=0, close_fds=False)
             self.send = self.send_pipe
-            self.read = self.read_pipe
+            if self.tty:
+                self.winpty = WinPty(self.proc.stdout)
+                self.read = self.read_winpty
+            else:
+                self.read = self.read_pipe
 
     def close(self):
         cqlshlog.info("Closing %r subprocess." % (self.exe_path,))
-        if self.tty:
+        if self.realtty:
             os.close(self.childpty)
         else:
             self.proc.stdin.close()
@@ -154,20 +176,24 @@ class ProcRunner:
     def send_pipe(self, data):
         self.proc.stdin.write(data)
 
-    def read_tty(self, blksize):
+    def read_tty(self, blksize, timeout=None):
         return os.read(self.childpty, blksize)
 
-    def read_pipe(self, blksize):
+    def read_pipe(self, blksize, timeout=None):
         return self.proc.stdout.read(blksize)
 
-    def read_until(self, until, blksize=4096, timeout=None, flags=0):
+    def read_winpty(self, blksize, timeout=None):
+        return self.winpty.read(blksize, timeout)
+
+    def read_until(self, until, blksize=4096, timeout=None,
+                   flags=0, ptty_timeout=None):
         if not isinstance(until, re._pattern_type):
             until = re.compile(until, flags)
         got = self.readbuf
         self.readbuf = ''
         with timing_out(timeout):
             while True:
-                val = self.read(blksize)
+                val = self.read(blksize, ptty_timeout)
                 cqlshlog.debug("read %r from subproc" % (val,))
                 if val == '':
                     raise EOFError("'until' pattern %r not found" % (until.pattern,))
@@ -205,15 +231,22 @@ class ProcRunner:
 
 class CqlshRunner(ProcRunner):
     def __init__(self, path=None, host=None, port=None, keyspace=None, cqlver=None,
-                 args=(), prompt=DEFAULT_CQLSH_PROMPT, env=None, **kwargs):
+                 args=(), prompt=DEFAULT_CQLSH_PROMPT, env=None,
+                 win_force_colors=True, tty=True, **kwargs):
         if path is None:
-            path = basecase.path_to_cqlsh
+            cqlsh_bin = 'cqlsh'
+            if is_win():
+                cqlsh_bin = 'cqlsh.bat'
+            path = normpath(join(basecase.cqlshdir, cqlsh_bin))
         if host is None:
             host = basecase.TEST_HOST
         if port is None:
             port = basecase.TEST_PORT
         if env is None:
             env = {}
+        if is_win():
+            env['PYTHONUNBUFFERED'] = '1'
+            env.update(os.environ.copy())
         env.setdefault('TERM', 'xterm')
         env.setdefault('CQLSH_NO_BUNDLED', os.environ.get('CQLSH_NO_BUNDLED', ''))
         env.setdefault('PYTHONPATH', os.environ.get('PYTHONPATH', ''))
@@ -222,8 +255,13 @@ class CqlshRunner(ProcRunner):
             args += ('--cqlversion', str(cqlver))
         if keyspace is not None:
             args += ('--keyspace', keyspace)
+        if tty and is_win():
+            args += ('--tty',)
+            args += ('--encoding', 'utf-8')
+            if win_force_colors:
+                args += ('--color',)
         self.keyspace = keyspace
-        ProcRunner.__init__(self, path, args=args, env=env, **kwargs)
+        ProcRunner.__init__(self, path, tty=tty, args=args, env=env, **kwargs)
         self.prompt = prompt
         if self.prompt is None:
             self.output_header = ''
@@ -231,7 +269,7 @@ class CqlshRunner(ProcRunner):
             self.output_header = self.read_to_next_prompt()
 
     def read_to_next_prompt(self):
-        return self.read_until(self.prompt, timeout=10.0)
+        return self.read_until(self.prompt, timeout=10.0, ptty_timeout=3)
 
     def read_up_to_timeout(self, timeout, blksize=4096):
         output = ProcRunner.read_up_to_timeout(self, timeout, blksize=blksize)
@@ -247,7 +285,7 @@ class CqlshRunner(ProcRunner):
         output = output.replace(' \r', '')
         output = output.replace('\r', '')
         output = output.replace(' \b', '')
-        if self.tty:
+        if self.realtty:
             echo, output = output.split('\n', 1)
             assert echo == cmd, "unexpected echo %r instead of %r" % (echo, cmd)
         try:
@@ -255,7 +293,7 @@ class CqlshRunner(ProcRunner):
         except ValueError:
             promptline = output
             output = ''
-        assert re.match(self.prompt, '\n' + promptline), \
+        assert re.match(self.prompt, DEFAULT_PREFIX + promptline), \
                 'last line of output %r does not match %r?' % (promptline, self.prompt)
         return output + '\n'
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9dd2b5ea/pylib/cqlshlib/test/test_cqlsh_completion.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py
index e5eb9e1..e67fefe 100644
--- a/pylib/cqlshlib/test/test_cqlsh_completion.py
+++ b/pylib/cqlshlib/test/test_cqlsh_completion.py
@@ -22,6 +22,8 @@ from __future__ import with_statement
 import re
 from .basecase import BaseTestCase, cqlsh
 from .cassconnect import testrun_cqlsh
+import unittest
+import sys
 
 BEL = '\x07'  # the terminal-bell character
 CTRL_C = '\x03'
@@ -36,6 +38,7 @@ COMPLETION_RESPONSE_TIME = 0.5
 completion_separation_re = re.compile(r'\s+')
 
 
+@unittest.skipIf(sys.platform == "win32", 'Tab completion tests not supported on Windows')
 class CqlshCompletionCase(BaseTestCase):
 
     def setUp(self):

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9dd2b5ea/pylib/cqlshlib/test/test_cqlsh_output.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py
index 7a2fc86..c62ed69 100644
--- a/pylib/cqlshlib/test/test_cqlsh_output.py
+++ b/pylib/cqlshlib/test/test_cqlsh_output.py
@@ -27,11 +27,12 @@ from .cassconnect import (get_test_keyspace, testrun_cqlsh, testcall_cqlsh,
                           cassandra_cursor, split_cql_commands, quote_name)
 from .ansi_colors import (ColoredText, lookup_colorcode, lookup_colorname,
                           lookup_colorletter, ansi_seq)
+import unittest
+import sys
 
 CONTROL_C = '\x03'
 CONTROL_D = '\x04'
 
-
 class TestCqlshOutput(BaseTestCase):
 
     def setUp(self):
@@ -92,7 +93,8 @@ class TestCqlshOutput(BaseTestCase):
     def test_no_color_output(self):
         for termname in ('', 'dumb', 'vt100'):
             cqlshlog.debug('TERM=%r' % termname)
-            with testrun_cqlsh(tty=True, env={'TERM': termname}) as c:
+            with testrun_cqlsh(tty=True, env={'TERM': termname},
+                               win_force_colors=False) as c:
                 c.send('select * from has_all_types;\n')
                 self.assertNoHasColors(c.read_to_next_prompt())
                 c.send('select count(*) from has_all_types;\n')
@@ -526,9 +528,13 @@ class TestCqlshOutput(BaseTestCase):
             c.send('use NONEXISTENTKEYSPACE;\n')
             outputlines = c.read_to_next_prompt().splitlines()
 
-            self.assertEqual(outputlines[0], 'use NONEXISTENTKEYSPACE;')
-            self.assertTrue(outputlines[2].endswith('cqlsh:system> '))
-            midline = ColoredText(outputlines[1])
+            start_index = 0
+            if c.realtty:
+                self.assertEqual(outputlines[start_index], 'use NONEXISTENTKEYSPACE;')
+                start_index = 1
+
+            self.assertTrue(outputlines[start_index+1].endswith('cqlsh:system> '))
+            midline = ColoredText(outputlines[start_index])
             self.assertEqual(midline.plain(),
                              'InvalidRequest: code=2200 [Invalid query] message="Keyspace
\'nonexistentkeyspace\' does not exist"')
             self.assertColorFromTags(midline,
@@ -716,6 +722,7 @@ class TestCqlshOutput(BaseTestCase):
             self.assertRegexpMatches(output, '^Connected to .* at %s:%d\.$'
                                              % (re.escape(TEST_HOST), TEST_PORT))
 
+    @unittest.skipIf(sys.platform == "win32", 'EOF signaling not supported on Windows')
     def test_eof_prints_newline(self):
         with testrun_cqlsh(tty=True) as c:
             c.send(CONTROL_D)
@@ -730,8 +737,9 @@ class TestCqlshOutput(BaseTestCase):
             with testrun_cqlsh(tty=True) as c:
                 cmd = 'exit%s\n' % semicolon
                 c.send(cmd)
-                out = c.read_lines(1)[0].replace('\r', '')
-                self.assertEqual(out, cmd)
+                if c.realtty:
+                    out = c.read_lines(1)[0].replace('\r', '')
+                    self.assertEqual(out, cmd)
                 with self.assertRaises(BaseException) as cm:
                     c.read_lines(1)
                 self.assertIn(type(cm.exception), (EOFError, OSError))

http://git-wip-us.apache.org/repos/asf/cassandra/blob/9dd2b5ea/pylib/cqlshlib/test/winpty.py
----------------------------------------------------------------------
diff --git a/pylib/cqlshlib/test/winpty.py b/pylib/cqlshlib/test/winpty.py
new file mode 100644
index 0000000..0db9ec3
--- /dev/null
+++ b/pylib/cqlshlib/test/winpty.py
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from threading import Thread
+from cStringIO import StringIO
+from Queue import Queue, Empty
+
+
+class WinPty:
+
+    def __init__(self, stdin):
+        self._s = stdin
+        self._q = Queue()
+
+        def _read_next_char(stdin, queue):
+            while True:
+                char = stdin.read(1)  # potentially blocking read
+                if char:
+                    queue.put(char)
+                else:
+                    break
+
+        self._t = Thread(target=_read_next_char, args=(self._s, self._q))
+        self._t.daemon = True
+        self._t.start()  # read characters asynchronously from stdin
+
+    def read(self, blksize=-1, timeout=1):
+        buf = StringIO()
+        count = 0
+        try:
+            while count < blksize or blksize == -1:
+                next = self._q.get(block=timeout is not None, timeout=timeout)
+                buf.write(next)
+                count = count + 1
+        except Empty:
+            pass
+        return buf.getvalue()


Mime
View raw message