This is an automated email from the ASF dual-hosted git repository. djoshi pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/cassandra-dtest.git
The following commit(s) were added to refs/heads/master by this push: new 175a083 Make cqlsh and cqlshlib Python 2 & 3 compatible 175a083 is described below commit 175a083a6f3b4d5d58f3702d31ce6920af519669 Author: Patrick Bannister <ptbannis...@gmail.com> AuthorDate: Fri Apr 13 22:18:57 2018 -0400 Make cqlsh and cqlshlib Python 2 & 3 compatible Patch by Patrick Bannister; reviewed by Dinesh Joshi, Andy Tolbert and David Capwell for CASSANDRA-10190 --- conftest.py | 11 +- cql_test.py | 2 +- cqlsh_tests/cqlsh_test_types.py | 136 ++++++++++++++++ cqlsh_tests/cqlsh_tools.py | 4 +- cqlsh_tests/test_cqlsh.py | 335 +++++++++++++++++++++++++++++++++------- cqlsh_tests/test_cqlsh_copy.py | 211 ++++++++++++------------- dtest.py | 2 +- dtest_setup.py | 2 +- requirements.txt | 1 - run_dtests.py | 6 +- tools/metadata_wrapper.py | 4 +- 11 files changed, 530 insertions(+), 184 deletions(-) diff --git a/conftest.py b/conftest.py index e680ca9..b46f8b5 100644 --- a/conftest.py +++ b/conftest.py @@ -7,8 +7,7 @@ import re import platform import copy import inspect - -from itertools import zip_longest +import subprocess from dtest import running_in_docker, cleanup_docker_environment_before_test_execution @@ -25,6 +24,9 @@ from dtest_config import DTestConfig from dtest_setup import DTestSetup from dtest_setup_overrides import DTestSetupOverrides +# Python 3 imports +from itertools import zip_longest + logger = logging.getLogger(__name__) def check_required_loopback_interfaces_available(): @@ -497,9 +499,8 @@ def pytest_collection_modifyitems(items, config): if config.getoption("use_off_heap_memtables"): deselect_test = True - # temporarily deselect tests in cqlsh_copy_tests that depend on cqlshlib, - # until cqlshlib is Python 3 compatibile - if item.get_marker("depends_cqlshlib"): + # deselect cqlsh tests that depend on fixing a driver behavior + if item.get_closest_marker("depends_driver"): deselect_test = True if deselect_test: diff --git a/cql_test.py b/cql_test.py index 84ded2b..659cbae 100644 --- a/cql_test.py +++ b/cql_test.py @@ -1105,7 +1105,7 @@ class TestCQLSlowQuery(CQLTester): @jira_ticket CASSANDRA-12403 """ cluster = self.cluster - cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 10, + cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 1, 'request_timeout_in_ms': 120000, 'read_request_timeout_in_ms': 120000, 'range_request_timeout_in_ms': 120000}) diff --git a/cqlsh_tests/cqlsh_test_types.py b/cqlsh_tests/cqlsh_test_types.py new file mode 100644 index 0000000..d78864e --- /dev/null +++ b/cqlsh_tests/cqlsh_test_types.py @@ -0,0 +1,136 @@ +import datetime +import sys + +from collections import namedtuple +from contextlib import contextmanager + +from cassandra.util import SortedSet + + +@contextmanager +def _cqlshlib(cqlshlib_path): + """ + Returns the cqlshlib module found at the specified path. + """ + # This method accomplishes its goal by manually adding the library to + # sys.path, returning the module, then restoring the old path once the + # context manager exits. This isn't great for maintainability and should + # be replaced if cqlshlib is made easier to interact with. + saved_path = list(sys.path) + + try: + sys.path = sys.path + [cqlshlib_path] + import cqlshlib + yield cqlshlib + finally: + sys.path = saved_path + + +def maybe_quote(s): + """ + Return a quoted string representation for strings, unicode and date time parameters, + otherwise return a string representation of the parameter. + """ + return "'{}'".format(s) if isinstance(s, (str, Datetime)) else str(s) + + +class Address(namedtuple('Address', ('name', 'number', 'street', 'phones'))): + __slots__ = () + + def __repr__(self): + phones_str = "{{{}}}".format(', '.join(maybe_quote(p) for p in sorted(self.phones))) + return "{{name: {}, number: {}, street: '{}', phones: {}}}".format(self.name, + self.number, + self.street, + phones_str) + + def __str__(self): + phones_str = "{{{}}}".format(', '.join(maybe_quote(p) for p in sorted(self.phones))) + return "{{name: {}, number: {}, street: '{}', phones: {}}}".format(self.name, + self.number, + self.street, + phones_str) + + +class Datetime(datetime.datetime): + """ + Extend standard datetime.datetime class with cql formatting. + This could be cleaner if this class was declared inside TestCqlshCopy, but then pickle + wouldn't have access to the class. + """ + def __new__(cls, year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, cqlshlib_path=None): + self = datetime.datetime.__new__(cls, year, month, day, hour, minute, second, microsecond, tzinfo) + if (cqlshlib_path is not None): + with _cqlshlib(cqlshlib_path) as cqlshlib: + from cqlshlib.formatting import DEFAULT_TIMESTAMP_FORMAT, round_microseconds + self.default_time_format = DEFAULT_TIMESTAMP_FORMAT + self.round_microseconds = round_microseconds + else: + self.default_time_format = '%Y-%m-%d %H:%M:%S%z' + return self + + def __repr__(self): + return self._format_for_csv() + + def __str__(self): + return self._format_for_csv() + + def _format_for_csv(self): + ret = self.strftime(self.default_time_format) + return self.round_microseconds(ret) if self.round_microseconds else ret + + +class ImmutableDict(frozenset): + """ + Immutable dictionary implementation to represent map types. + We need to pass BoundStatement.bind() a dict() because it calls iteritems(), + except we can't create a dict with another dict as the key, hence we use a class + that adds iteritems to a frozen set of tuples (which is how dict are normally made + immutable in python). + Must be declared in the top level of the module to be available for pickling. + """ + iteritems = frozenset.__iter__ + + def items(self): + for k, v in self.iteritems(): + yield k, v + + def __repr__(self): + return '{{{}}}'.format(', '.join(['{0}: {1}'.format(maybe_quote(k), maybe_quote(v)) for k, v in sorted(self.items())])) + + +class ImmutableSet(SortedSet): + + def __repr__(self): + return '{{{}}}'.format(', '.join([maybe_quote(t) for t in sorted(self._items)])) + + def __str__(self): + return '{{{}}}'.format(', '.join([maybe_quote(t) for t in sorted(self._items)])) + + def __hash__(self): + return hash(tuple([e for e in self])) + + +class Name(namedtuple('Name', ('firstname', 'lastname'))): + __slots__ = () + + def __repr__(self): + return "{{firstname: '{}', lastname: '{}'}}".format(self.firstname, self.lastname) + + def __str__(self): + return "{{firstname: '{}', lastname: '{}'}}".format(self.firstname, self.lastname) + + +class UTC(datetime.tzinfo): + """ + A utility class to specify a UTC timezone. + """ + + def utcoffset(self, dt): + return datetime.timedelta(0) + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return datetime.timedelta(0) diff --git a/cqlsh_tests/cqlsh_tools.py b/cqlsh_tests/cqlsh_tools.py index 9544d3e..0b49232 100644 --- a/cqlsh_tests/cqlsh_tools.py +++ b/cqlsh_tests/cqlsh_tools.py @@ -1,10 +1,12 @@ +from __future__ import unicode_literals + import csv import random +from typing import List import cassandra from cassandra.cluster import ResultSet -from typing import List class DummyColorMap(object): diff --git a/cqlsh_tests/test_cqlsh.py b/cqlsh_tests/test_cqlsh.py index ab905c5..c9a11e2 100644 --- a/cqlsh_tests/test_cqlsh.py +++ b/cqlsh_tests/test_cqlsh.py @@ -1,11 +1,14 @@ # coding=utf-8 +from __future__ import unicode_literals + import binascii import csv import datetime import locale import os import re +import six import subprocess import sys import logging @@ -23,8 +26,10 @@ from ccmlib import common from .cqlsh_tools import monkeypatch_driver, unmonkeypatch_driver from dtest import Tester, create_ks, create_cf +from dtest_setup_overrides import DTestSetupOverrides from tools.assertions import assert_all, assert_none from tools.data import create_c1c2_table, insert_c1c2, rows_to_list +from tools.misc import ImmutableMapping since = pytest.mark.since logger = logging.getLogger(__name__) @@ -32,6 +37,15 @@ logger = logging.getLogger(__name__) class TestCqlsh(Tester): + # override cluster options to enable user defined functions + # currently only needed for test_describe + @pytest.fixture + def fixture_dtest_setup_overrides(self): + dtest_setup_overrides = DTestSetupOverrides() + dtest_setup_overrides.cluster_options = ImmutableMapping({'enable_user_defined_functions': 'true', + 'enable_scripted_user_defined_functions': 'true'}) + return dtest_setup_overrides + @classmethod def setUpClass(cls): cls._cached_driver_methods = monkeypatch_driver() @@ -53,11 +67,11 @@ class TestCqlsh(Tester): """ @jira_ticket CASSANDRA-10066 Checks that cqlsh is compliant with pycodestyle (formally known as pep8) with the following command: - pycodestyle --ignore E501,E402,E731 pylib/cqlshlib/*.py bin/cqlsh.py + pycodestyle --ignore E501,E402,E731,W503 pylib/cqlshlib/*.py bin/cqlsh.py """ cluster = self.cluster - if cluster.version() < '2.2': + if cluster.version() < LooseVersion('2.2'): cqlsh_path = os.path.join(cluster.get_install_dir(), 'bin', 'cqlsh') else: cqlsh_path = os.path.join(cluster.get_install_dir(), 'bin', 'cqlsh.py') @@ -66,7 +80,7 @@ class TestCqlsh(Tester): cqlshlib_paths = os.listdir(cqlshlib_path) cqlshlib_paths = [os.path.join(cqlshlib_path, x) for x in cqlshlib_paths if '.py' in x and '.pyc' not in x] - cmds = ['pycodestyle', '--ignore', 'E501,E402,E731', cqlsh_path] + cqlshlib_paths + cmds = ['pycodestyle', '--ignore', 'E501,E402,E731,W503', cqlsh_path] + cqlshlib_paths logger.debug(cmds) @@ -303,7 +317,7 @@ class TestCqlsh(Tester): 'I can eat glass and it does not hurt me': 1400 }) - output, err = self.run_cqlsh(node, 'use testks; SELECT * FROM varcharmaptable', ['--encoding=utf-8']) + output, _ = self.run_cqlsh(node, 'use testks; SELECT * FROM varcharmaptable', ['--encoding=utf-8']) assert output.count('Можам да јадам стакло, а не ме штета.') == 16 assert output.count(' ⠊⠀⠉⠁⠝⠀⠑⠁⠞⠀⠛⠇⠁⠎⠎⠀⠁⠝⠙⠀⠊⠞⠀⠙⠕⠑⠎⠝⠞⠀⠓⠥⠗⠞⠀⠍⠑') == 16 @@ -316,7 +330,7 @@ class TestCqlsh(Tester): node1, = self.cluster.nodelist() - node1.run_cqlsh(cmds="""create KEYSPACE testks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}; + cmds = """create KEYSPACE testks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}; use testks; CREATE TABLE varcharmaptable ( @@ -427,7 +441,8 @@ INSERT INTO varcharmaptable (varcharkey, varcharvarintmap ) VALUES ('᚛᚛ UPDATE varcharmaptable SET varcharvarintmap = varcharvarintmap + {'Vitrum edere possum, mihi non nocet.':20000} WHERE varcharkey= '᚛᚛ᚉᚑᚅᚔᚉᚉᚔᚋ ᚔᚈᚔ ᚍᚂᚐᚅᚑ ᚅᚔᚋᚌᚓᚅᚐ᚜'; UPDATE varcharmaptable SET varcharvarintmap['Vitrum edere possum, mihi non nocet.'] = 1010010101020400204143243 WHERE varcharkey= '᚛᚛ᚉᚑᚅᚔᚉᚉᚔᚋ ᚔᚈᚔ ᚍᚂᚐᚅᚑ ᚅᚔᚋᚌᚓᚅᚐ᚜' - """) + """ + node1.run_cqlsh(cmds=cmds) self.verify_glass(node1) @@ -452,7 +467,9 @@ UPDATE varcharmaptable SET varcharvarintmap['Vitrum edere possum, mihi non nocet node1, = self.cluster.nodelist() - output, err, _ = node1.run_cqlsh(cmds="ä;") + cmds = "ä;" + _, err, _ = node1.run_cqlsh(cmds=cmds) + assert 'Invalid syntax' in err assert 'ä' in err @@ -468,7 +485,7 @@ UPDATE varcharmaptable SET varcharvarintmap['Vitrum edere possum, mihi non nocet node1, = self.cluster.nodelist() cmd = '''create keyspace "ä" WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'};''' - output, err, _ = node1.run_cqlsh(cmds=cmd, cqlsh_options=["--debug"]) + _, err, _ = node1.run_cqlsh(cmds=cmd, cqlsh_options=["--debug"]) if self.cluster.version() >= LooseVersion('4.0'): assert "Keyspace name must not be empty, more than 48 characters long, or contain non-alphanumeric-underscore characters (got 'ä')" in err @@ -484,7 +501,7 @@ UPDATE varcharmaptable SET varcharvarintmap['Vitrum edere possum, mihi non nocet node1, = self.cluster.nodelist() - node1.run_cqlsh(cmds="""create keyspace CASSANDRA_7196 WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1} ; + cmds = """create keyspace CASSANDRA_7196 WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1} ; use CASSANDRA_7196; @@ -539,9 +556,13 @@ INSERT INTO has_all_types (num, intcol, asciicol, bigintcol, blobcol, booleancol timestampcol, uuidcol, varcharcol, varintcol) VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDecimal(0x), blobAsDouble(0x), blobAsFloat(0x), '', blobAsTimestamp(0x), blobAsUuid(0x), '', - blobAsVarint(0x))""") + blobAsVarint(0x))""" + + node1.run_cqlsh(cmds=cmds) + + select_cmd = "select intcol, bigintcol, varintcol from CASSANDRA_7196.has_all_types where num in (0, 1, 2, 3, 4)" + output, err = self.run_cqlsh(node1, cmds=select_cmd) - output, err = self.run_cqlsh(node1, "select intcol, bigintcol, varintcol from CASSANDRA_7196.has_all_types where num in (0, 1, 2, 3, 4)") if common.is_win(): output = output.replace('\r', '') @@ -637,7 +658,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec conn.execute("CREATE USER user1 WITH PASSWORD 'user1'") conn.execute("GRANT ALL ON ks.t1 TO user1") - if self.cluster.version() >= '4.0': + if self.cluster.version() >= LooseVersion('4.0'): self.verify_output("LIST USERS", node1, """ name | super | datacenters -----------+-------+------------- @@ -646,7 +667,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec (2 rows) """) - elif self.cluster.version() >= '2.2': + elif self.cluster.version() >= LooseVersion('2.2'): self.verify_output("LIST USERS", node1, """ name | super -----------+------- @@ -665,7 +686,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec (2 rows) """) - if self.cluster.version() >= '2.2': + if self.cluster.version() >= LooseVersion('2.2'): self.verify_output("LIST ALL PERMISSIONS OF user1", node1, """ role | username | resource | permission -------+----------+---------------+------------ @@ -698,6 +719,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec self.cluster.populate(1) self.cluster.start(wait_for_binary_proto=True) node1, = self.cluster.nodelist() + self.execute( cql=""" CREATE KEYSPACE test WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}; @@ -715,21 +737,21 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec assert "system" in output # Describe keyspace - self.execute(cql="DESCRIBE KEYSPACE test", expected_output=self.get_keyspace_output()) - self.execute(cql="DESCRIBE test", expected_output=self.get_keyspace_output()) + self.execute(cql="DESCRIBE KEYSPACE test", expected_output=self.get_keyspace_output(), output_is_ordered=False) + self.execute(cql="DESCRIBE test", expected_output=self.get_keyspace_output(), output_is_ordered=False) self.execute(cql="DESCRIBE test2", expected_err="'test2' not found in keyspaces") - self.execute(cql="USE test; DESCRIBE KEYSPACE", expected_output=self.get_keyspace_output()) + self.execute(cql="USE test; DESCRIBE KEYSPACE", expected_output=self.get_keyspace_output(), output_is_ordered=False) # Describe table - self.execute(cql="DESCRIBE TABLE test.test", expected_output=self.get_test_table_output()) - self.execute(cql="DESCRIBE TABLE test.users", expected_output=self.get_users_table_output()) - self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output()) - self.execute(cql="DESCRIBE test.users", expected_output=self.get_users_table_output()) + self.execute(cql="DESCRIBE TABLE test.test", expected_output=self.get_test_table_output(), output_is_ordered=False) + self.execute(cql="DESCRIBE TABLE test.users", expected_output=self.get_users_table_output(), output_is_ordered=False) + self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output(), output_is_ordered=False) + self.execute(cql="DESCRIBE test.users", expected_output=self.get_users_table_output(), output_is_ordered=False) self.execute(cql="DESCRIBE test.users2", expected_err="'users2' not found in keyspace 'test'") - self.execute(cql="USE test; DESCRIBE TABLE test", expected_output=self.get_test_table_output()) - self.execute(cql="USE test; DESCRIBE TABLE users", expected_output=self.get_users_table_output()) - self.execute(cql="USE test; DESCRIBE test", expected_output=self.get_keyspace_output()) - self.execute(cql="USE test; DESCRIBE users", expected_output=self.get_users_table_output()) + self.execute(cql="USE test; DESCRIBE TABLE test", expected_output=self.get_test_table_output(), output_is_ordered=False) + self.execute(cql="USE test; DESCRIBE TABLE users", expected_output=self.get_users_table_output(), output_is_ordered=False) + self.execute(cql="USE test; DESCRIBE test", expected_output=self.get_keyspace_output(), output_is_ordered=False) + self.execute(cql="USE test; DESCRIBE users", expected_output=self.get_users_table_output(), output_is_ordered=False) self.execute(cql="USE test; DESCRIBE users2", expected_err="'users2' not found in keyspace 'test'") # Describe index @@ -757,7 +779,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec CREATE INDEX myindex ON test.users (age); CREATE INDEX "QuotedNameIndex" on test.users (firstname) """) - self.execute(cql="DESCRIBE test.users", expected_output=self.get_users_table_output()) + self.execute(cql="DESCRIBE test.users", expected_output=self.get_users_table_output(), output_is_ordered=False) self.execute(cql='DESCRIBE test.myindex', expected_output=self.get_index_output('myindex', 'test', 'users', 'age')) # Drop index and recreate @@ -774,10 +796,10 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec # Prior to 3.0 the index would have been automatically dropped, but now we need to explicitly do that. self.execute(cql='DROP INDEX test.test_val_idx') self.execute(cql='ALTER TABLE test.test DROP val') - self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output(has_val=False, has_val_idx=False)) + self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output(has_val=False, has_val_idx=False), output_is_ordered=False) self.execute(cql='DESCRIBE test.test_val_idx', expected_err="'test_val_idx' not found in keyspace 'test'") self.execute(cql='ALTER TABLE test.test ADD val text') - self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output(has_val=True, has_val_idx=False)) + self.execute(cql="DESCRIBE test.test", expected_output=self.get_test_table_output(has_val=True, has_val_idx=False), output_is_ordered=False) self.execute(cql='DESCRIBE test.test_val_idx', expected_err="'test_val_idx' not found in keyspace 'test'") def test_describe_describes_non_default_compaction_parameters(self): @@ -794,6 +816,97 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec assert "'min_threshold': '10'" in stdout assert "'max_threshold': '100'" in stdout + def test_describe_functions(self, fixture_dtest_setup_overrides): + """Test DESCRIBE statements for functions and aggregate functions""" + self.cluster.populate(1) + self.cluster.start(wait_for_binary_proto=True) + + create_ks_statement = "CREATE KEYSPACE test WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}" + create_function_statement = """ +CREATE FUNCTION test.some_function(arg int) + RETURNS NULL ON NULL INPUT + RETURNS int + LANGUAGE java + AS $$ return arg; + $$""" + create_aggregate_dependencies_statement = """ +CREATE OR REPLACE FUNCTION test.average_state(state tuple<int,bigint>, val int) + CALLED ON NULL INPUT + RETURNS tuple<int,bigint> + LANGUAGE java + AS $$ if (val != null) { + state.setInt(0, state.getInt(0)+1); + state.setLong(1, state.getLong(1)+val.intValue()); + } + return state; + $$; +CREATE OR REPLACE FUNCTION test.average_final (state tuple<int,bigint>) + CALLED ON NULL INPUT + RETURNS double + LANGUAGE java + AS $$ + double r = 0; + if (state.getInt(0) == 0) return null; + r = state.getLong(1); + r /= state.getInt(0); + return Double.valueOf(r); + $$ +""" + create_aggregate_statement = """ +CREATE OR REPLACE AGGREGATE test.average(int) + SFUNC average_state + STYPE tuple<int,bigint> + FINALFUNC average_final + INITCOND (0, 0) +""" + # the expected output of a DESCRIBE AGGREGATE statement + # does not look like a valid CREATE AGGREGATE statement + describe_aggregate_expected = """ +CREATE AGGREGATE test.average(int) + SFUNC average_state + STYPE frozen<tuple<int, bigint>> + FINALFUNC average_final + INITCOND (0, 0); +""" + + # create keyspace, scalar function, and aggregate function + self.execute(cql=create_ks_statement) + self.execute(cql=create_function_statement) + self.execute(cql=create_aggregate_dependencies_statement) + self.execute(cql=create_aggregate_statement) + # describe scalar functions + self.execute(cql='DESCRIBE FUNCTION test.some_function', expected_output='{};'.format(create_function_statement)) + # describe aggregate functions + self.execute(cql='DESCRIBE AGGREGATE test.average', expected_output=describe_aggregate_expected) + + def test_describe_types(self): + """Test DESCRIBE statements for user defined datatypes""" + self.cluster.populate(1) + self.cluster.start(wait_for_binary_proto=True) + + create_ks_statement = "CREATE KEYSPACE test WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}" + create_name_type_statement = """ +CREATE TYPE test.name_type ( + firstname text, + lastname text +)""" + create_address_type_statement = """ +CREATE TYPE test.address_type ( + name frozen<name_type>, + number int, + street text, + phones set<text> +)""" + + # create test keyspace and some user defined types + self.execute(cql=create_ks_statement) + self.execute(create_name_type_statement) + self.execute(create_address_type_statement) + + # DESCRIBE user defined types + self.execute(cql='DESCRIBE TYPE test.name_type', expected_output='{};'.format(create_name_type_statement)) + self.execute(cql='DESCRIBE TYPE test.address_type', expected_output='{};'.format(create_address_type_statement)) + def test_describe_on_non_reserved_keywords(self): """ @jira_ticket CASSANDRA-9232 @@ -842,13 +955,14 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec self.execute(cql='USE test; DESCRIBE "users_by_state"', expected_output=self.get_users_by_state_mv_output()) def get_keyspace_output(self): - return ("CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;" + - self.get_test_table_output() + - self.get_users_table_output()) + return ["CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;", + self.get_test_table_output(), + self.get_users_table_output()] def get_test_table_output(self, has_val=True, has_val_idx=True): + create_table = None if has_val: - ret = """ + create_table = """ CREATE TABLE test.test ( id int, col int, @@ -856,7 +970,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec PRIMARY KEY (id, col) """ else: - ret = """ + create_table = """ CREATE TABLE test.test ( id int, col int, @@ -864,7 +978,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec """ if self.cluster.version() >= LooseVersion('4.0'): - ret += """ + create_table += """ ) WITH CLUSTERING ORDER BY (col ASC) AND bloom_filter_fp_chance = 0.01 AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} @@ -882,7 +996,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec AND speculative_retry = '99p'; """ elif self.cluster.version() >= LooseVersion('3.9'): - ret += """ + create_table += """ ) WITH CLUSTERING ORDER BY (col ASC) AND bloom_filter_fp_chance = 0.01 AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} @@ -900,7 +1014,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec AND speculative_retry = '99PERCENTILE'; """ elif self.cluster.version() >= LooseVersion('3.0'): - ret += """ + create_table += """ ) WITH CLUSTERING ORDER BY (col ASC) AND bloom_filter_fp_chance = 0.01 AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} @@ -918,7 +1032,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec AND speculative_retry = '99PERCENTILE'; """ else: - ret += """ + create_table += """ ) WITH CLUSTERING ORDER BY (col ASC) AND bloom_filter_fp_chance = 0.01 AND caching = '{"keys":"ALL", "rows_per_partition":"NONE"}' @@ -936,22 +1050,18 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec """ col_idx_def = self.get_index_output('test_col_idx', 'test', 'test', 'col') - + expected_output = [create_table, col_idx_def] if has_val_idx: - val_idx_def = self.get_index_output('test_val_idx', 'test', 'test', 'val') - if self.cluster.version() >= LooseVersion('2.2'): - return ret + "\n" + val_idx_def + "\n" + col_idx_def - else: - return ret + "\n" + col_idx_def + "\n" + val_idx_def - else: - return ret + "\n" + col_idx_def + expected_output.append(self.get_index_output('test_val_idx', 'test', 'test', 'val')) + return expected_output def get_users_table_output(self): quoted_index_output = self.get_index_output('"QuotedNameIndex"', 'test', 'users', 'firstname') myindex_output = self.get_index_output('myindex', 'test', 'users', 'age') + create_table = None if self.cluster.version() >= LooseVersion('4.0'): - return """ + create_table = """ CREATE TABLE test.users ( userid text PRIMARY KEY, age int, @@ -971,9 +1081,9 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec AND min_index_interval = 128 AND read_repair_chance = 0.0 AND speculative_retry = '99p'; - """ + quoted_index_output + "\n" + myindex_output + """ elif self.cluster.version() >= LooseVersion('3.9'): - return """ + create_table = """ CREATE TABLE test.users ( userid text PRIMARY KEY, age int, @@ -993,9 +1103,9 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec AND min_index_interval = 128 AND read_repair_chance = 0.0 AND speculative_retry = '99PERCENTILE'; - """ + quoted_index_output + "\n" + myindex_output + """ elif self.cluster.version() >= LooseVersion('3.0'): - return """ + create_table = """ CREATE TABLE test.users ( userid text PRIMARY KEY, age int, @@ -1015,9 +1125,9 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec AND min_index_interval = 128 AND read_repair_chance = 0.0 AND speculative_retry = '99PERCENTILE'; - """ + quoted_index_output + "\n" + myindex_output + """ else: - return """ + create_table = """ CREATE TABLE test.users ( userid text PRIMARY KEY, age int, @@ -1036,8 +1146,8 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec AND min_index_interval = 128 AND read_repair_chance = 0.0 AND speculative_retry = '99.0PERCENTILE'; - """ + (quoted_index_output + "\n" + myindex_output if self.cluster.version() >= LooseVersion('2.2') else - myindex_output + "\n" + quoted_index_output) + """ + return [create_table, quoted_index_output, myindex_output] def get_index_output(self, index, ks, table, col): # a quoted index name (e.g. "FooIndex") is only correctly echoed by DESCRIBE @@ -1123,7 +1233,7 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec AND speculative_retry = '99PERCENTILE'; """ - def execute(self, cql, expected_output=None, expected_err=None, env_vars=None): + def execute(self, cql, expected_output=None, expected_err=None, env_vars=None, output_is_ordered=True, err_is_ordered=True): logger.debug(cql) node1, = self.cluster.nodelist() @@ -1132,13 +1242,19 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec if err: if expected_err: err = err[10:] # strip <stdin>:2: - self.check_response(err, expected_err) + if err_is_ordered: + self.check_response(err, expected_err) + else: + self.check_response_unordered(err, expected_err) return else: assert False, err if expected_output: - self.check_response(output, expected_output) + if output_is_ordered: + self.check_response(output, expected_output) + else: + self.check_response_unordered(output, expected_output) return output @@ -1160,6 +1276,31 @@ VALUES (4, blobAsInt(0x), '', blobAsBigint(0x), 0x, blobAsBoolean(0x), blobAsDec describe_statement = re.sub(r"WITH[\s]*;", "", describe_statement) return describe_statement + def check_response_unordered(self, response, expected_response): + """ + Assert that a response matches a concatenation of expected + strings in an arbitrary order. This is useful for features + such as DESCRIBE KEYSPACE, where output is arbitrarily + ordered. + expected_response should be a list of strings that form the + expected output. + """ + + def consume_expected(observed, expected): + unconsumed = observed + if isinstance(expected, list): + for unit in expected: + unconsumed = consume_expected(unconsumed, unit) + else: + expected_stripped = "\n".join([s.strip() for s in expected.split("\n") if s.strip()]) + assert unconsumed.find(expected_stripped) >= 0 + unconsumed = unconsumed.replace(expected_stripped, "") + return unconsumed + + stripped_response = "\n".join([s.strip() for s in response.split("\n") if s.strip()]) + unconsumed = consume_expected(stripped_response, expected_response) + assert unconsumed.replace("\n", "") == "" + def strip_read_repair_chance(self, describe_statement): """ Remove read_repair_chance and dclocal_read_repair_chance options @@ -1425,7 +1566,7 @@ CREATE TABLE int_checks.values ( val4 tinyint """) - @since('2.2') + @since('2.2', max_version='4') def test_datetime_values(self): """ Tests for CASSANDRA-9399, check tables with date and time values""" self.cluster.populate(1) @@ -1468,6 +1609,52 @@ CREATE TABLE datetime_checks.values ( PRIMARY KEY (d, t) """) + """ + Starting with 4.0, date/time format needs to conform to java.time.format.DateTimeFormatter + See CASSANDRA-15257 for more details + """ + @since('4.0') + def test_datetime_values_40(self): + self.cluster.populate(1) + self.cluster.start(wait_for_binary_proto=True) + + node1, = self.cluster.nodelist() + + stdout, stderr = self.run_cqlsh(node1, cmds=""" + CREATE KEYSPACE datetime_checks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}; + USE datetime_checks; + CREATE TABLE values (d date, t time, PRIMARY KEY (d, t)); + INSERT INTO values (d, t) VALUES ('9800-12-31', '23:59:59.999999999'); + INSERT INTO values (d, t) VALUES ('2015-05-14', '16:30:00.555555555'); + INSERT INTO values (d, t) VALUES ('1582-01-01', '00:00:00.000000000'); + INSERT INTO values (d, t) VALUES ('%04d-01-01', '00:00:00.000000000'); + INSERT INTO values (d, t) VALUES ('%04d-01-01', '01:00:00.000000000'); + INSERT INTO values (d, t) VALUES ('%02d-01-01', '02:00:00.000000000'); + INSERT INTO values (d, t) VALUES ('+%02d-01-01', '03:00:00.000000000')""" + % (datetime.MINYEAR - 1, datetime.MINYEAR, datetime.MAXYEAR, datetime.MAXYEAR+1)) + + assert 0 == len(stderr), "Failed to execute cqlsh: {}".format(stderr) + + self.verify_output("select * from datetime_checks.values", node1, """ + d | t +------------+-------------------- + -719528 | 00:00:00.000000000 + 9800-12-31 | 23:59:59.999999999 + 0001-01-01 | 01:00:00.000000000 + 1582-01-01 | 00:00:00.000000000 + 2932897 | 03:00:00.000000000 + 9999-01-01 | 02:00:00.000000000 + 2015-05-14 | 16:30:00.555555555 +""") + + self.verify_output("DESCRIBE TABLE datetime_checks.values", node1, """ +CREATE TABLE datetime_checks.values ( + d date, + t time, + PRIMARY KEY (d, t) +""") + + @since('2.2') def test_tracing(self): """ @@ -1767,7 +1954,7 @@ Tracing session:""") # Can't check escape sequence on cmd prompt. Assume no errors is good enough metric. if not common.is_win(): - assert re.search(chr(27) + r"\[[0,1,2]?J", out) + assert re.search((chr(27) + "\[[0,1,2]?J"), out) def test_batch(self): """ @@ -2088,6 +2275,34 @@ class TestCqlshSmoke(Tester): self.session.cluster.refresh_schema_metadata() return [table.name for table in list(self.session.cluster.metadata.keyspaces[keyspace].tables.values())] + def test_cjk_output(self): + """Confirm cqlsh outputs CJK text properly""" + create_ks(self.session, 'ks', 1) + create_cf(self.session, 'iroha', key_type='int', columns={'manyogana': 'text', 'modern': 'text', 'kana': 'text'}) + + self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (1, '以呂波耳本部止', '色は匂へど', 'いろはにほへと')") + self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (2, '千利奴流乎', '散りぬるを', 'ちりぬるを')") + self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (3, '和加餘多連曽', '我が世誰ぞ', 'わかよたれそ')") + self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (4, '津祢那良牟', '常ならん', 'つねならむ')") + self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (5, '有為能於久耶万', '有為の奥山', 'うゐのおくやま')") + self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (6, '計不己衣天阿', '今日越えて', 'けふこえて')") + self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (7, '佐伎喩女美之', '浅き夢見じ', 'あさきゆめみし')") + self.session.execute("INSERT INTO iroha (key, manyogana, modern, kana) VALUES (8, '恵比毛勢須', '酔ひもせず', 'ゑひもせす')") + + stdout, _, _ = self.node1.run_cqlsh('SELECT key, manyogana, modern, kana FROM ks.iroha') + stdout_lines_sorted = '\n'.join(sorted(stdout.split('\n'))) + expected = """ + 1 | 以呂波耳本部止 | 色は匂へど | いろはにほへと + 2 | 千利奴流乎 | 散りぬるを | ちりぬるを + 3 | 和加餘多連曽 | 我が世誰ぞ | わかよたれそ + 4 | 津祢那良牟 | 常ならん | つねならむ + 5 | 有為能於久耶万 | 有為の奥山 | うゐのおくやま + 6 | 計不己衣天阿 | 今日越えて | けふこえて + 7 | 佐伎喩女美之 | 浅き夢見じ | あさきゆめみし + 8 | 恵比毛勢須 | 酔ひもせず | ゑひもせす +""" + assert stdout_lines_sorted.find(expected) >= 0 + class TestCqlLogin(Tester): """ @@ -2145,7 +2360,7 @@ class TestCqlLogin(Tester): create_cf(self.session, 'ks1table') self.session.execute("CREATE USER user1 WITH PASSWORD 'changeme';") - if self.cluster.version() >= '2.2': + if self.cluster.version() >= LooseVersion('2.2'): query = ''' LOGIN user1 'changeme'; CREATE USER user2 WITH PASSWORD 'fail' SUPERUSER; diff --git a/cqlsh_tests/test_cqlsh_copy.py b/cqlsh_tests/test_cqlsh_copy.py index a170c7e..d634655 100644 --- a/cqlsh_tests/test_cqlsh_copy.py +++ b/cqlsh_tests/test_cqlsh_copy.py @@ -1,14 +1,17 @@ +# coding=utf-8 + import csv import datetime import glob +import io import json import locale +import logging import os +import pytest import re import sys import time -import pytest -import logging from collections import namedtuple from contextlib import contextmanager @@ -25,9 +28,11 @@ from cassandra.murmur3 import murmur3 from cassandra.util import SortedSet from ccmlib.common import is_win -from .cqlsh_tools import (DummyColorMap, assert_csvs_items_equal, csv_rows, - monkeypatch_driver, random_list, unmonkeypatch_driver, - write_rows_to_csv) +from .cqlsh_test_types import (Address, Datetime, ImmutableDict, + ImmutableSet, Name, UTC) +from .cqlsh_tools import (DummyColorMap, assert_csvs_items_equal, + csv_rows, monkeypatch_driver, random_list, + unmonkeypatch_driver, write_rows_to_csv) from dtest import (Tester, create_ks) from dtest import (FlakyRetryPolicy, Tester, create_ks) from tools.data import rows_to_list @@ -45,21 +50,6 @@ PARTITIONERS = { } -class UTC(datetime.tzinfo): - """ - A utility class to specify a UTC timezone. - """ - - def utcoffset(self, dt): - return datetime.timedelta(0) - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return datetime.timedelta(0) - - class TestCqlshCopy(Tester): """ Tests the COPY TO and COPY FROM features in cqlsh. @@ -205,7 +195,7 @@ class TestCqlshCopy(Tester): try: return self._default_time_format except AttributeError: - with self._cqlshlib(): + with self._cqlshlib() as cqlshlib: try: from cqlshlib.formatting import DEFAULT_TIMESTAMP_FORMAT self._default_time_format = DEFAULT_TIMESTAMP_FORMAT @@ -250,68 +240,14 @@ class TestCqlshCopy(Tester): x map<text, frozen<list<text>>> )''') - default_time_format = self.default_time_format - - try: - from cqlshlib.formatting import round_microseconds - except ImportError: - round_microseconds = None - - class Datetime(datetime.datetime): - - def _format_for_csv(self): - ret = self.strftime(default_time_format) - return round_microseconds(ret) if round_microseconds else ret - - def __str__(self): - return self._format_for_csv() - - def __repr__(self): - return self._format_for_csv() - - def maybe_quote(s): - """ - Return a quoted string representation for strings, unicode and date time parameters, - otherwise return a string representation of the parameter. - """ - return "'{}'".format(s) if isinstance(s, (str, Datetime)) else str(s) - - class ImmutableDict(frozenset): - iteritems = frozenset.__iter__ - - def __repr__(self): - return '{{{}}}'.format(', '.join(['{}: {}'.format(maybe_quote(t[0]), maybe_quote(t[1])) - for t in sorted(self)])) - - class ImmutableSet(SortedSet): - - def __repr__(self): - return '{{{}}}'.format(', '.join([maybe_quote(t) for t in sorted(self._items)])) - - def __hash__(self): - return hash(tuple([e for e in self])) - - class Name(namedtuple('Name', ('firstname', 'lastname'))): - __slots__ = () - - def __repr__(self): - return "{{firstname: '{}', lastname: '{}'}}".format(self.firstname, self.lastname) - - class Address(namedtuple('Address', ('name', 'number', 'street', 'phones'))): - __slots__ = () - - def __repr__(self): - phones_str = "{{{}}}".format(', '.join(maybe_quote(p) for p in sorted(self.phones))) - return "{{name: {}, number: {}, street: '{}', phones: {}}}".format(self.name, - self.number, - self.street, - phones_str) - self.session.cluster.register_user_type('ks', 'name_type', Name) self.session.cluster.register_user_type('ks', 'address_type', Address) - date1 = Datetime(2005, 7, 14, 12, 30, 0, 0, UTC()) - date2 = Datetime(2005, 7, 14, 13, 30, 0, 0, UTC()) + cassandra_dirpath = self.cluster.nodelist()[0].get_install_dir() + cqlshlib_dirpath = os.path.join(cassandra_dirpath, 'pylib') + + date1 = Datetime(2005, 7, 14, 12, 30, 0, 0, UTC(), cqlshlib_path=cqlshlib_dirpath) + date2 = Datetime(2005, 7, 14, 13, 30, 0, 0, UTC(), cqlshlib_path=cqlshlib_dirpath) addr1 = Address(Name('name1', 'last1'), 1, 'street 1', ImmutableSet(['1111 2222', '3333 4444'])) addr2 = Address(Name('name2', 'last2'), 2, 'street 2', ImmutableSet(['5555 6666', '7777 8888'])) @@ -388,12 +324,11 @@ class TestCqlshCopy(Tester): self.maxDiff = None - if (sort_data): - csv_results.sort() - processed_results.sort() - try: - assert csv_results == processed_results + if sort_data: + assert sorted(csv_results) == sorted(processed_results) + else: + assert csv_results == processed_results except Exception as e: if len(csv_results) != len(processed_results): logger.warning("Different # of entries. CSV: " + str(len(csv_results)) + @@ -409,15 +344,14 @@ class TestCqlshCopy(Tester): def make_csv_formatter(self, time_format, nullval): with self._cqlshlib() as cqlshlib: # noqa from cqlshlib.formatting import format_value, format_value_default - from cqlshlib.displaying import NO_COLOR_MAP - try: - from cqlshlib.formatting import DateTimeFormat - date_time_format = DateTimeFormat() - date_time_format.timestamp_format = time_format - if hasattr(date_time_format, 'milliseconds_only'): - date_time_format.milliseconds_only = True - except ImportError: - date_time_format = None + try: + from cqlshlib.formatting import DateTimeFormat + date_time_format = DateTimeFormat() + date_time_format.timestamp_format = time_format + if hasattr(date_time_format, 'milliseconds_only'): + date_time_format.milliseconds_only = True + except ImportError: + date_time_format = None encoding_name = 'utf-8' # codecs.lookup(locale.getpreferredencoding()).name color_map = DummyColorMap() @@ -439,7 +373,7 @@ class TestCqlshCopy(Tester): format_fn = format_value if val is None or val == EMPTY or val == nullval: - return format_value_default(nullval) + return format_value_default(nullval, color_map).strval # CASSANDRA-11255 increased COPY TO DOUBLE PRECISION TO 12 if cql_type_name == 'double' and self.cluster.version() >= LooseVersion('3.6'): @@ -804,7 +738,6 @@ class TestCqlshCopy(Tester): result = self.session.execute("SELECT * FROM testcounter") result_as_list = rows_to_list(result) - result_as_list.sort() assert data == sorted(result_as_list) def test_reading_counter(self): @@ -859,6 +792,7 @@ class TestCqlshCopy(Tester): result_as_list = [tuple(r) for r in rows_to_list(result)] assert [tuple(d) for d in data] == sorted(result_as_list) + @since('2.2', max_version='3.X') @pytest.mark.depends_cqlshlib def test_datetimeformat_round_trip(self): """ @@ -894,7 +828,7 @@ class TestCqlshCopy(Tester): logger.debug('Exporting to csv file: {name}'.format(name=tempfile.name)) cmds = "COPY ks.testdatetimeformat TO '{name}'".format(name=tempfile.name) cmds += " WITH DATETIMEFORMAT = '{}'".format(format) - self.run_cqlsh(cmds=cmds) + copy_to_out, copy_to_err, _ = self.run_cqlsh(cmds=cmds) with open(tempfile.name, 'r') as csvfile: csv_values = list(csv.reader(csvfile)) @@ -906,7 +840,66 @@ class TestCqlshCopy(Tester): self.session.execute("TRUNCATE testdatetimeformat") cmds = "COPY ks.testdatetimeformat FROM '{name}'".format(name=tempfile.name) cmds += " WITH DATETIMEFORMAT = '{}'".format(format) - self.run_cqlsh(cmds=cmds) + copy_from_out, copy_from_err, _ = self.run_cqlsh(cmds=cmds) + + table_meta = UpdatingTableMetadataWrapper(self.session.cluster, + ks_name=self.ks, + table_name='testdatetimeformat') + cql_type_names = [table_meta.columns[c].cql_type for c in table_meta.columns] + + imported_results = list(self.session.execute("SELECT * FROM testdatetimeformat")) + assert self.result_to_csv_rows(exported_results, cql_type_names, time_format=format) \ + == self.result_to_csv_rows(imported_results, cql_type_names, time_format=format) + + @since('4.0') + @pytest.mark.depends_cqlshlib + def test_datetimeformat_round_trip_40(self): + """ + @jira_ticket CASSANDRA-10633 + @jira_ticket CASSANDRA-9303 + + Test COPY TO and COPY FORM with the time format specified in the WITH option by: + + - creating and populating a table, + - exporting the contents of the table to a CSV file using COPY TO WITH DATETIMEFORMAT, + - checking the time format written to csv. + - importing the CSV back into the table + - comparing the table contents before and after the import + + CASSANDRA-9303 renamed TIMEFORMAT to DATETIMEFORMAT + """ + self.prepare() + self.session.execute(""" + CREATE TABLE testdatetimeformat ( + a int primary key, + b timestamp + )""") + insert_statement = self.session.prepare("INSERT INTO testdatetimeformat (a, b) VALUES (?, ?)") + args = [(1, datetime.datetime(2015, 1, 1, 0o7, 00, 0, 0, UTC())), + (2, datetime.datetime(2015, 6, 10, 12, 30, 30, 500, UTC())), + (3, datetime.datetime(2015, 12, 31, 23, 59, 59, 999, UTC()))] + execute_concurrent_with_args(self.session, insert_statement, args) + exported_results = list(self.session.execute("SELECT * FROM testdatetimeformat")) + + format = '%Y-%m-%d %H:%M:%S%z' + + tempfile = self.get_temp_file() + logger.debug('Exporting to csv file: {name}'.format(name=tempfile.name)) + cmds = "COPY ks.testdatetimeformat TO '{name}'".format(name=tempfile.name) + cmds += " WITH DATETIMEFORMAT = '{}' AND NUMPROCESSES=1".format(format) + copy_to_out, copy_to_err, _ = self.run_cqlsh(cmds=cmds) + + with open(tempfile.name, 'r') as csvfile: + csv_values = list(csv.reader(csvfile)) + + assert sorted(csv_values) == [['1', '2015-01-01 07:00:00+0000'], + ['2', '2015-06-10 12:30:30+0000'], + ['3', '2015-12-31 23:59:59+0000']] + + self.session.execute("TRUNCATE testdatetimeformat") + cmds = "COPY ks.testdatetimeformat FROM '{name}'".format(name=tempfile.name) + cmds += " WITH DATETIMEFORMAT = '{}' AND NUMPROCESSES=1".format(format) + copy_from_out, copy_from_err, _ = self.run_cqlsh(cmds=cmds) table_meta = UpdatingTableMetadataWrapper(self.session.cluster, ks_name=self.ks, @@ -1310,10 +1303,9 @@ class TestCqlshCopy(Tester): cmd += " AND ERRFILE='{}'".format(err_file.name) self.run_cqlsh(cmds=cmd) - logger.debug('Sorting') - results = sorted(rows_to_list(self.session.execute("SELECT * FROM ks.testparseerrors"))) + results = rows_to_list(self.session.execute("SELECT * FROM ks.testparseerrors")) logger.debug('Checking valid rows') - assert valid_rows == results + assert sorted(valid_rows) == sorted(results) logger.debug('Checking invalid rows') self.assertCsvResultEqual(err_file_name, invalid_rows, cql_type_names=['text', 'int', 'text']) @@ -1368,10 +1360,9 @@ class TestCqlshCopy(Tester): cmd = "COPY ks.testwrongnumcols FROM '{}' WITH ERRFILE='{}'".format(tempfile.name, err_file.name) self.run_cqlsh(cmds=cmd) - logger.debug('Sorting') - results = sorted(rows_to_list(self.session.execute("SELECT * FROM ks.testwrongnumcols"))) + results = rows_to_list(self.session.execute("SELECT * FROM ks.testwrongnumcols")) logger.debug('Checking valid rows') - assert valid_rows == results + assert sorted(valid_rows) == sorted(results) logger.debug('Checking invalid rows') self.assertCsvResultEqual(err_file.name, invalid_rows, 'testwrongnumcols', columns=['a', 'b', 'e']) @@ -1802,7 +1793,7 @@ class TestCqlshCopy(Tester): def _test(prepared_statements): logger.debug('Importing from csv file: {name}'.format(name=tempfile.name)) - self.run_cqlsh(cmds="COPY ks.testdatatype FROM '{}' WITH PREPAREDSTATEMENTS = {}" + out, err, _ = self.run_cqlsh(cmds="COPY ks.testdatatype FROM '{}' WITH PREPAREDSTATEMENTS = {}" .format(tempfile.name, prepared_statements)) results = list(self.session.execute("SELECT * FROM testdatatype")) @@ -2043,7 +2034,7 @@ class TestCqlshCopy(Tester): exported_results = list(self.session.execute("SELECT * FROM testnumberseps")) self.maxDiff = None - assert expected_vals == sorted(list(csv_rows(tempfile.name))) + assert sorted(expected_vals) == sorted(list(csv_rows(tempfile.name))) logger.debug('Importing from csv file: {} with thousands_sep {} and decimal_sep {}' .format(tempfile.name, thousands_sep, decimal_sep)) @@ -2060,8 +2051,8 @@ class TestCqlshCopy(Tester): cql_type_names = [table_meta.columns[c].cql_type for c in table_meta.columns] # we format as if we were comparing to csv to overcome loss of precision in the import - assert self.result_to_csv_rows(exported_results == cql_type_names, - self.result_to_csv_rows(imported_results, cql_type_names)) + assert self.result_to_csv_rows(exported_results, cql_type_names) \ + == self.result_to_csv_rows(imported_results, cql_type_names) do_test(expected_vals_usual, ',', '.') do_test(expected_vals_inverted, '.', ',') @@ -2532,7 +2523,8 @@ class TestCqlshCopy(Tester): run_copy_to(tempfile1) # check all records generated were exported - assert num_records == sum(1 for _ in open(tempfile1.name)) + with io.open(tempfile1.name, encoding="utf-8", newline='') as csvfile: + assert num_records == sum(1 for _ in csv.reader(csvfile, quotechar='"', escapechar='\\')) # import records from the first csv file logger.debug('Truncating {}...'.format(stress_table)) @@ -3266,6 +3258,7 @@ class TestCqlshCopy(Tester): _test(False) @pytest.mark.depends_cqlshlib + @pytest.mark.depends_driver @since('3.0') def test_unusual_dates(self): """ diff --git a/dtest.py b/dtest.py index d6aff2d..e144a58 100644 --- a/dtest.py +++ b/dtest.py @@ -227,7 +227,7 @@ def test_failure_due_to_timeout(err, *args): @flaky(rerun_filter=test_failure_due_to_timeout) -class Tester: +class Tester(object): def __getattribute__(self, name): try: diff --git a/dtest_setup.py b/dtest_setup.py index 4b2ece9..107028d 100644 --- a/dtest_setup.py +++ b/dtest_setup.py @@ -45,7 +45,7 @@ def retry_till_success(fun, *args, **kwargs): time.sleep(0.25) -class DTestSetup: +class DTestSetup(object): def __init__(self, dtest_config=None, setup_overrides=None, cluster_name="test"): self.dtest_config = dtest_config self.setup_overrides = setup_overrides diff --git a/requirements.txt b/requirements.txt index 3b17b82..139bfc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ -e git+https://github.com/datastax/python-driver.git@cassandra-test#egg=cassandra-driver # Used ccm version is tracked by cassandra-test branch in ccm repo. Please create a PR there for fixes or upgrades to new releases. -e git+https://github.com/riptano/ccm.git@cassandra-test#egg=ccm -cqlsh decorator docopt enum34 diff --git a/run_dtests.py b/run_dtests.py index 378029d..698977b 100755 --- a/run_dtests.py +++ b/run_dtests.py @@ -109,9 +109,9 @@ class RunDTests(): logger.debug('Generating configurations from the following matrix:\n\t{}'.format(args)) args_to_invoke_pytest = [] - if args.pytest_options: - for arg in args.pytest_options.split(" "): - args_to_invoke_pytest.append("'{the_arg}'".format(the_arg=arg)) + # if args.pytest_options: + # for arg in args.pytest_options.split(" "): + # args_to_invoke_pytest.append("'{the_arg}'".format(the_arg=arg)) for arg in argv: if arg.startswith("--pytest-options") or arg.startswith("--dtest-"): diff --git a/tools/metadata_wrapper.py b/tools/metadata_wrapper.py index 43bdbfb..c166283 100644 --- a/tools/metadata_wrapper.py +++ b/tools/metadata_wrapper.py @@ -1,7 +1,7 @@ from abc import ABCMeta, abstractproperty +from six import with_metaclass - -class UpdatingMetadataWrapperBase(object, metaclass=ABCMeta): +class UpdatingMetadataWrapperBase(with_metaclass(ABCMeta, object)): @abstractproperty def _wrapped(self): pass --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org