Fix cqlshlib tests on Windows patch by Paulo Motta; reviewed by Jim Witschey for CASSANDRA-10541
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9dd2b5ea Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9dd2b5ea Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9dd2b5ea Branch: refs/heads/trunk Commit: 9dd2b5ea0f842eb465ad38ac98f9dd987ba6b7d2 Parents: b03ce9f Author: Paulo Motta <pauloricard...@gmail.com> Authored: Mon Nov 16 15:48:20 2015 -0200 Committer: Aleksey Yeschenko <alek...@apache.org> Committed: Fri Dec 18 22:46:31 2015 +0000 ---------------------------------------------------------------------- bin/cqlsh.py | 4 +- pylib/cqlshlib/test/run_cqlsh.py | 108 +++++++++++++++------- pylib/cqlshlib/test/test_cqlsh_completion.py | 3 + pylib/cqlshlib/test/test_cqlsh_output.py | 22 +++-- pylib/cqlshlib/test/winpty.py | 50 ++++++++++ 5 files changed, 144 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/9dd2b5ea/bin/cqlsh.py ---------------------------------------------------------------------- diff --git a/bin/cqlsh.py b/bin/cqlsh.py index 42c2923..1119289 100644 --- a/bin/cqlsh.py +++ b/bin/cqlsh.py @@ -210,6 +210,8 @@ parser.add_option('--cqlversion', default=DEFAULT_CQLVER, parser.add_option("-e", "--execute", help='Execute the statement and quit.') parser.add_option("--connect-timeout", default=DEFAULT_CONNECT_TIMEOUT_SECONDS, dest='connect_timeout', help='Specify the connection timeout in seconds (default: %default seconds).') +parser.add_option("-t", "--tty", action='store_true', dest='tty', + help='Force tty mode (command prompt).') optvalues = optparse.Values() (options, arguments) = parser.parse_args(sys.argv[1:], values=optvalues) @@ -2328,7 +2330,7 @@ def read_options(cmdlineargs, environment): optvalues.ssl = False optvalues.encoding = None - optvalues.tty = sys.stdin.isatty() + optvalues.tty = option_with_default(configs.getboolean, 'ui', 'tty', sys.stdin.isatty()) optvalues.cqlversion = option_with_default(configs.get, 'cql', 'version', DEFAULT_CQLVER) optvalues.connect_timeout = option_with_default(configs.getint, 'connection', 'timeout', DEFAULT_CONNECT_TIMEOUT_SECONDS) optvalues.execute = None http://git-wip-us.apache.org/repos/asf/cassandra/blob/9dd2b5ea/pylib/cqlshlib/test/run_cqlsh.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/test/run_cqlsh.py b/pylib/cqlshlib/test/run_cqlsh.py index 08e9e41..b011df4 100644 --- a/pylib/cqlshlib/test/run_cqlsh.py +++ b/pylib/cqlshlib/test/run_cqlsh.py @@ -17,17 +17,28 @@ # NOTE: this testing tool is *nix specific import os +import sys import re -import pty -import fcntl import contextlib import subprocess import signal import math from time import time from . import basecase +from os.path import join, normpath -DEFAULT_CQLSH_PROMPT = os.linesep + '(\S+@)?cqlsh(:\S+)?> ' + +def is_win(): + return sys.platform in ("cygwin", "win32") + +if is_win(): + from winpty import WinPty + DEFAULT_PREFIX = '' +else: + import pty + DEFAULT_PREFIX = os.linesep + +DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + '(\S+@)?cqlsh(:\S+)?> ' DEFAULT_CQLSH_TERM = 'xterm' cqlshlog = basecase.cqlshlog @@ -41,10 +52,6 @@ def set_controlling_pty(master, slave): os.close(slave) os.close(os.open(os.ttyname(1), os.O_RDWR)) -def set_nonblocking(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) - @contextlib.contextmanager def raising_signal(signum, exc): """ @@ -93,12 +100,21 @@ def timing_out_alarm(seconds): finally: signal.alarm(0) -# setitimer is new in 2.6, but it's still worth supporting, for potentially -# faster tests because of sub-second resolution on timeouts. -if hasattr(signal, 'setitimer'): - timing_out = timing_out_itimer +if is_win(): + try: + import eventlet + except ImportError, e: + sys.exit("evenlet library required to run cqlshlib tests on Windows") + + def timing_out(seconds): + return eventlet.Timeout(seconds, TimeoutError) else: - timing_out = timing_out_alarm + # setitimer is new in 2.6, but it's still worth supporting, for potentially + # faster tests because of sub-second resolution on timeouts. + if hasattr(signal, 'setitimer'): + timing_out = timing_out_itimer + else: + timing_out = timing_out_alarm def noop(*a): pass @@ -108,6 +124,7 @@ class ProcRunner: self.exe_path = path self.args = args self.tty = bool(tty) + self.realtty = self.tty and not is_win() if env is None: env = {} self.env = env @@ -118,30 +135,35 @@ class ProcRunner: def start_proc(self): preexec = noop stdin = stdout = stderr = None - if self.tty: - masterfd, slavefd = pty.openpty() - preexec = lambda: set_controlling_pty(masterfd, slavefd) - else: - stdin = stdout = subprocess.PIPE - stderr = subprocess.STDOUT cqlshlog.info("Spawning %r subprocess with args: %r and env: %r" % (self.exe_path, self.args, self.env)) - self.proc = subprocess.Popen(('python', self.exe_path,) + tuple(self.args), - env=self.env, preexec_fn=preexec, - stdin=stdin, stdout=stdout, stderr=stderr, - close_fds=False) - if self.tty: + if self.realtty: + masterfd, slavefd = pty.openpty() + preexec = (lambda: set_controlling_pty(masterfd, slavefd)) + self.proc = subprocess.Popen((self.exe_path,) + tuple(self.args), + env=self.env, preexec_fn=preexec, + stdin=stdin, stdout=stdout, stderr=stderr, + close_fds=False) os.close(slavefd) self.childpty = masterfd self.send = self.send_tty self.read = self.read_tty else: + stdin = stdout = subprocess.PIPE + stderr = subprocess.STDOUT + self.proc = subprocess.Popen((self.exe_path,) + tuple(self.args), + env=self.env, stdin=stdin, stdout=stdout, + stderr=stderr, bufsize=0, close_fds=False) self.send = self.send_pipe - self.read = self.read_pipe + if self.tty: + self.winpty = WinPty(self.proc.stdout) + self.read = self.read_winpty + else: + self.read = self.read_pipe def close(self): cqlshlog.info("Closing %r subprocess." % (self.exe_path,)) - if self.tty: + if self.realtty: os.close(self.childpty) else: self.proc.stdin.close() @@ -154,20 +176,24 @@ class ProcRunner: def send_pipe(self, data): self.proc.stdin.write(data) - def read_tty(self, blksize): + def read_tty(self, blksize, timeout=None): return os.read(self.childpty, blksize) - def read_pipe(self, blksize): + def read_pipe(self, blksize, timeout=None): return self.proc.stdout.read(blksize) - def read_until(self, until, blksize=4096, timeout=None, flags=0): + def read_winpty(self, blksize, timeout=None): + return self.winpty.read(blksize, timeout) + + def read_until(self, until, blksize=4096, timeout=None, + flags=0, ptty_timeout=None): if not isinstance(until, re._pattern_type): until = re.compile(until, flags) got = self.readbuf self.readbuf = '' with timing_out(timeout): while True: - val = self.read(blksize) + val = self.read(blksize, ptty_timeout) cqlshlog.debug("read %r from subproc" % (val,)) if val == '': raise EOFError("'until' pattern %r not found" % (until.pattern,)) @@ -205,15 +231,22 @@ class ProcRunner: class CqlshRunner(ProcRunner): def __init__(self, path=None, host=None, port=None, keyspace=None, cqlver=None, - args=(), prompt=DEFAULT_CQLSH_PROMPT, env=None, **kwargs): + args=(), prompt=DEFAULT_CQLSH_PROMPT, env=None, + win_force_colors=True, tty=True, **kwargs): if path is None: - path = basecase.path_to_cqlsh + cqlsh_bin = 'cqlsh' + if is_win(): + cqlsh_bin = 'cqlsh.bat' + path = normpath(join(basecase.cqlshdir, cqlsh_bin)) if host is None: host = basecase.TEST_HOST if port is None: port = basecase.TEST_PORT if env is None: env = {} + if is_win(): + env['PYTHONUNBUFFERED'] = '1' + env.update(os.environ.copy()) env.setdefault('TERM', 'xterm') env.setdefault('CQLSH_NO_BUNDLED', os.environ.get('CQLSH_NO_BUNDLED', '')) env.setdefault('PYTHONPATH', os.environ.get('PYTHONPATH', '')) @@ -222,8 +255,13 @@ class CqlshRunner(ProcRunner): args += ('--cqlversion', str(cqlver)) if keyspace is not None: args += ('--keyspace', keyspace) + if tty and is_win(): + args += ('--tty',) + args += ('--encoding', 'utf-8') + if win_force_colors: + args += ('--color',) self.keyspace = keyspace - ProcRunner.__init__(self, path, args=args, env=env, **kwargs) + ProcRunner.__init__(self, path, tty=tty, args=args, env=env, **kwargs) self.prompt = prompt if self.prompt is None: self.output_header = '' @@ -231,7 +269,7 @@ class CqlshRunner(ProcRunner): self.output_header = self.read_to_next_prompt() def read_to_next_prompt(self): - return self.read_until(self.prompt, timeout=10.0) + return self.read_until(self.prompt, timeout=10.0, ptty_timeout=3) def read_up_to_timeout(self, timeout, blksize=4096): output = ProcRunner.read_up_to_timeout(self, timeout, blksize=blksize) @@ -247,7 +285,7 @@ class CqlshRunner(ProcRunner): output = output.replace(' \r', '') output = output.replace('\r', '') output = output.replace(' \b', '') - if self.tty: + if self.realtty: echo, output = output.split('\n', 1) assert echo == cmd, "unexpected echo %r instead of %r" % (echo, cmd) try: @@ -255,7 +293,7 @@ class CqlshRunner(ProcRunner): except ValueError: promptline = output output = '' - assert re.match(self.prompt, '\n' + promptline), \ + assert re.match(self.prompt, DEFAULT_PREFIX + promptline), \ 'last line of output %r does not match %r?' % (promptline, self.prompt) return output + '\n' http://git-wip-us.apache.org/repos/asf/cassandra/blob/9dd2b5ea/pylib/cqlshlib/test/test_cqlsh_completion.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py index e5eb9e1..e67fefe 100644 --- a/pylib/cqlshlib/test/test_cqlsh_completion.py +++ b/pylib/cqlshlib/test/test_cqlsh_completion.py @@ -22,6 +22,8 @@ from __future__ import with_statement import re from .basecase import BaseTestCase, cqlsh from .cassconnect import testrun_cqlsh +import unittest +import sys BEL = '\x07' # the terminal-bell character CTRL_C = '\x03' @@ -36,6 +38,7 @@ COMPLETION_RESPONSE_TIME = 0.5 completion_separation_re = re.compile(r'\s+') +@unittest.skipIf(sys.platform == "win32", 'Tab completion tests not supported on Windows') class CqlshCompletionCase(BaseTestCase): def setUp(self): http://git-wip-us.apache.org/repos/asf/cassandra/blob/9dd2b5ea/pylib/cqlshlib/test/test_cqlsh_output.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index 7a2fc86..c62ed69 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -27,11 +27,12 @@ from .cassconnect import (get_test_keyspace, testrun_cqlsh, testcall_cqlsh, cassandra_cursor, split_cql_commands, quote_name) from .ansi_colors import (ColoredText, lookup_colorcode, lookup_colorname, lookup_colorletter, ansi_seq) +import unittest +import sys CONTROL_C = '\x03' CONTROL_D = '\x04' - class TestCqlshOutput(BaseTestCase): def setUp(self): @@ -92,7 +93,8 @@ class TestCqlshOutput(BaseTestCase): def test_no_color_output(self): for termname in ('', 'dumb', 'vt100'): cqlshlog.debug('TERM=%r' % termname) - with testrun_cqlsh(tty=True, env={'TERM': termname}) as c: + with testrun_cqlsh(tty=True, env={'TERM': termname}, + win_force_colors=False) as c: c.send('select * from has_all_types;\n') self.assertNoHasColors(c.read_to_next_prompt()) c.send('select count(*) from has_all_types;\n') @@ -526,9 +528,13 @@ class TestCqlshOutput(BaseTestCase): c.send('use NONEXISTENTKEYSPACE;\n') outputlines = c.read_to_next_prompt().splitlines() - self.assertEqual(outputlines[0], 'use NONEXISTENTKEYSPACE;') - self.assertTrue(outputlines[2].endswith('cqlsh:system> ')) - midline = ColoredText(outputlines[1]) + start_index = 0 + if c.realtty: + self.assertEqual(outputlines[start_index], 'use NONEXISTENTKEYSPACE;') + start_index = 1 + + self.assertTrue(outputlines[start_index+1].endswith('cqlsh:system> ')) + midline = ColoredText(outputlines[start_index]) self.assertEqual(midline.plain(), 'InvalidRequest: code=2200 [Invalid query] message="Keyspace \'nonexistentkeyspace\' does not exist"') self.assertColorFromTags(midline, @@ -716,6 +722,7 @@ class TestCqlshOutput(BaseTestCase): self.assertRegexpMatches(output, '^Connected to .* at %s:%d\.$' % (re.escape(TEST_HOST), TEST_PORT)) + @unittest.skipIf(sys.platform == "win32", 'EOF signaling not supported on Windows') def test_eof_prints_newline(self): with testrun_cqlsh(tty=True) as c: c.send(CONTROL_D) @@ -730,8 +737,9 @@ class TestCqlshOutput(BaseTestCase): with testrun_cqlsh(tty=True) as c: cmd = 'exit%s\n' % semicolon c.send(cmd) - out = c.read_lines(1)[0].replace('\r', '') - self.assertEqual(out, cmd) + if c.realtty: + out = c.read_lines(1)[0].replace('\r', '') + self.assertEqual(out, cmd) with self.assertRaises(BaseException) as cm: c.read_lines(1) self.assertIn(type(cm.exception), (EOFError, OSError)) http://git-wip-us.apache.org/repos/asf/cassandra/blob/9dd2b5ea/pylib/cqlshlib/test/winpty.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/test/winpty.py b/pylib/cqlshlib/test/winpty.py new file mode 100644 index 0000000..0db9ec3 --- /dev/null +++ b/pylib/cqlshlib/test/winpty.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from threading import Thread +from cStringIO import StringIO +from Queue import Queue, Empty + + +class WinPty: + + def __init__(self, stdin): + self._s = stdin + self._q = Queue() + + def _read_next_char(stdin, queue): + while True: + char = stdin.read(1) # potentially blocking read + if char: + queue.put(char) + else: + break + + self._t = Thread(target=_read_next_char, args=(self._s, self._q)) + self._t.daemon = True + self._t.start() # read characters asynchronously from stdin + + def read(self, blksize=-1, timeout=1): + buf = StringIO() + count = 0 + try: + while count < blksize or blksize == -1: + next = self._q.get(block=timeout is not None, timeout=timeout) + buf.write(next) + count = count + 1 + except Empty: + pass + return buf.getvalue()