This is an automated email from the ASF dual-hosted git repository. xiazcy pushed a commit to branch python-http in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
The following commit(s) were added to refs/heads/python-http by this push: new d232c6ad26 Added marker class, updated tests to use http, disabled ws tests for future removal d232c6ad26 is described below commit d232c6ad26127f065a6c3c41efba744eeecdc53c Author: Yang Xia <55853655+xia...@users.noreply.github.com> AuthorDate: Mon Jun 17 09:42:45 2024 -0700 Added marker class, updated tests to use http, disabled ws tests for future removal --- .../gremlin_python/driver/aiohttp/transport.py | 6 +- .../main/python/gremlin_python/driver/client.py | 7 +- .../python/gremlin_python/driver/connection.py | 20 +- .../main/python/gremlin_python/driver/protocol.py | 1 + .../python/gremlin_python/driver/serializer.py | 43 +-- .../gremlin_python/structure/io/graphbinaryV1.py | 1 + .../gremlin_python/structure/io/graphbinaryV4.py | 34 +-- .../gremlin_python/structure/io/graphsonV2d0.py | 1 + .../gremlin_python/structure/io/graphsonV3d0.py | 1 + .../python/gremlin_python/structure/io/util.py | 38 +++ gremlin-python/src/main/python/tests/conftest.py | 125 ++++++-- .../src/main/python/tests/driver/test_client.py | 322 ++++++++++----------- .../tests/driver/test_driver_remote_connection.py | 4 + .../driver/test_driver_remote_connection_http.py | 36 +-- .../test_driver_remote_connection_threaded.py | 18 +- .../main/python/tests/driver/test_serializer.py | 24 -- .../driver/test_web_socket_client_behavior.py | 4 + .../tests/structure/io/test_functionalityio.py | 115 ++++---- .../tests/structure/io/test_graphbinaryV1.py | 7 +- ...test_graphbinaryV1.py => test_graphbinaryV4.py} | 14 +- .../python/tests/structure/io/test_graphsonV2d0.py | 1 + .../python/tests/structure/io/test_graphsonV3d0.py | 1 + 22 files changed, 448 insertions(+), 375 deletions(-) diff --git a/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py b/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py index c43ded6f58..661ef1a759 100644 --- a/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py +++ b/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py @@ -26,6 +26,7 @@ from gremlin_python.driver.transport import AbstractBaseTransport __author__ = 'Lyndon Bauto (lynd...@bitquilltech.com)' +# TODO: remove WS transport & refactor class AiohttpTransport(AbstractBaseTransport): nest_asyncio_applied = False @@ -195,8 +196,8 @@ class AiohttpHTTPTransport(AbstractBaseTransport): async def async_write(): basic_auth = None # basic password authentication for https connections - # if message['auth']: - # basic_auth = aiohttp.BasicAuth(message['auth']['username'], message['auth']['password']) + if message['auth']: + basic_auth = aiohttp.BasicAuth(message['auth']['username'], message['auth']['password']) async with async_timeout.timeout(self._write_timeout): self._http_req_resp = await self._client_session.post(url="/gremlin", auth=basic_auth, @@ -210,6 +211,7 @@ class AiohttpHTTPTransport(AbstractBaseTransport): def read(self): # Inner function to perform async read. async def async_read(): + # TODO: set-up chunked reading buffer = b"" # async for data, end_of_http_chunk in self._http_req_resp.content.iter_chunks(): # buffer += data diff --git a/gremlin-python/src/main/python/gremlin_python/driver/client.py b/gremlin-python/src/main/python/gremlin_python/driver/client.py index 624f55b973..fc1ef95bcd 100644 --- a/gremlin-python/src/main/python/gremlin_python/driver/client.py +++ b/gremlin-python/src/main/python/gremlin_python/driver/client.py @@ -38,6 +38,7 @@ except ImportError: __author__ = 'David M. Brown (davebs...@gmail.com), Lyndon Bauto (lynd...@bitquilltech.com)' +# TODO: remove session, update connection pooling, etc. class Client: def __init__(self, url, traversal_source, protocol_factory=None, @@ -58,7 +59,7 @@ class Client: if not self._use_http and "max_content_length" not in transport_kwargs: transport_kwargs["max_content_length"] = 10 * 1024 * 1024 if message_serializer is None: - message_serializer = serializer.GraphBinarySerializersV1() + message_serializer = serializer.GraphBinarySerializersV4() self._message_serializer = message_serializer self._username = username @@ -180,7 +181,6 @@ class Client: raise Exception("Client is closed") log.debug("message '%s'", str(message)) - # args = {'gremlin': message, 'fields': {'g': self._traversal_source}} fields = {'g': self._traversal_source} if isinstance(message, traversal.Bytecode): fields['gremlinType'] = 'bytecode' @@ -196,5 +196,6 @@ class Client: conn = self._pool.get(True) if request_options: - message.args.update(request_options) + message.fields.update(request_options) + return conn.write(message) diff --git a/gremlin-python/src/main/python/gremlin_python/driver/connection.py b/gremlin-python/src/main/python/gremlin_python/driver/connection.py index 789de181d0..e1f57c6650 100644 --- a/gremlin-python/src/main/python/gremlin_python/driver/connection.py +++ b/gremlin-python/src/main/python/gremlin_python/driver/connection.py @@ -35,7 +35,7 @@ class Connection: self._executor = executor self._transport = None self._pool = pool - self._results = {} + self._result_set = None self._inited = False self._enable_user_agent_on_connect = enable_user_agent_on_connect if self._enable_user_agent_on_connect: @@ -56,19 +56,15 @@ class Connection: self._transport.close() def write(self, request_message): + print('\n===conn===') + print(request_message) if not self._inited: self.connect() - if request_message.args.get("requestId"): - request_id = str(request_message.args.get("requestId")) - uuid.UUID(request_id) # Checks for proper UUID or else server will return an error. - else: - request_id = str(uuid.uuid4()) - result_set = resultset.ResultSet(queue.Queue(), request_id) - self._results[request_id] = result_set + self._result_set = resultset.ResultSet(queue.Queue()) # Create write task future = Future() future_write = self._executor.submit( - self._protocol.write, request_id, request_message) + self._protocol.write, request_message) def cb(f): try: @@ -79,8 +75,8 @@ class Connection: else: # Start receive task done = self._executor.submit(self._receive) - result_set.done = done - future.set_result(result_set) + self._result_set.done = done + future.set_result(self._result_set) future_write.add_done_callback(cb) return future @@ -89,7 +85,7 @@ class Connection: try: while True: data = self._transport.read() - status_code = self._protocol.data_received(data, self._results) + status_code = self._protocol.data_received(data, self._result_set) if status_code != 206: break finally: diff --git a/gremlin-python/src/main/python/gremlin_python/driver/protocol.py b/gremlin-python/src/main/python/gremlin_python/driver/protocol.py index b0bbfba9d3..ab691c0e12 100644 --- a/gremlin-python/src/main/python/gremlin_python/driver/protocol.py +++ b/gremlin-python/src/main/python/gremlin_python/driver/protocol.py @@ -65,6 +65,7 @@ class AbstractBaseProtocol(metaclass=abc.ABCMeta): pass +# TODO: remove WS protocol & refactor class GremlinServerWSProtocol(AbstractBaseProtocol): QOP_AUTH_BIT = 1 _kerberos_context = None diff --git a/gremlin-python/src/main/python/gremlin_python/driver/serializer.py b/gremlin-python/src/main/python/gremlin_python/driver/serializer.py index 65423b5ecb..254ce50589 100644 --- a/gremlin-python/src/main/python/gremlin_python/driver/serializer.py +++ b/gremlin-python/src/main/python/gremlin_python/driver/serializer.py @@ -33,10 +33,12 @@ from gremlin_python.structure.io import graphbinaryV1 from gremlin_python.structure.io import graphbinaryV4 from gremlin_python.structure.io import graphsonV2d0 from gremlin_python.structure.io import graphsonV3d0 +from gremlin_python.structure.io.util import Marker __author__ = 'David M. Brown (davebs...@gmail.com), Lyndon Bauto (lynd...@bitquilltech.com)' +# TODO: remove all classes except for GraphBinarySerializersV4 class Processor: """Base class for OpProcessor serialization system.""" @@ -294,6 +296,10 @@ class GraphBinarySerializersV1(object): return msg + +""" +GraphBinaryV4 +""" class GraphBinarySerializersV4(object): DEFAULT_READER_CLASS = graphbinaryV4.GraphBinaryReader DEFAULT_WRITER_CLASS = graphbinaryV4.GraphBinaryWriter @@ -315,21 +321,12 @@ class GraphBinarySerializersV4(object): if not writer: writer = self.DEFAULT_WRITER_CLASS() self._graphbinary_writer = writer - self.standard = Standard(writer) - self.traversal = Traversal(writer) - self.session = Session(writer) @property def version(self): """Read only property""" return self._version - def get_processor(self, processor): - processor = getattr(self, processor, None) - if not processor: - raise Exception("Unknown processor") - return processor - def serialize_message(self, request_message): message = self.build_message(request_message.fields, request_message.gremlin) return message @@ -345,7 +342,6 @@ class GraphBinarySerializersV4(object): ba = bytearray() ba.extend(graphbinaryV4.uint8_pack(0x81)) - print(bytes(ba)) fields = message["fields"] ba.extend(self.int_pack(len(fields))) for k, v in fields.items(): @@ -353,12 +349,11 @@ class GraphBinarySerializersV4(object): self._graphbinary_writer.to_dict(v, ba) gremlin = message['gremlin'] - if isinstance(gremlin, bytearray): - ba.extend(gremlin) - else: - self._graphbinary_writer.to_dict(gremlin, ba) - - print(bytes(ba)) + # TODO: hack to remove type code from gremlin value for V4 message format, writer doesn't seem to have a way to + # write value directly by passing serializer types, check back when removing bytecode + gremlin_ba = bytearray() + self._graphbinary_writer.to_dict(gremlin, gremlin_ba) + ba.extend(gremlin_ba[2:]) return bytes(ba) @@ -368,13 +363,11 @@ class GraphBinarySerializersV4(object): b.read(1) # version - result = self._graphbinary_reader.to_object(b) # data - # marker = self._graphbinary_reader.to_object(b, graphbinaryV4.DataType.marker, nullable=True) - marker = b.read(3) # need to create marker class & serializer + result = self.read_payload(b) status_code = self.int32_unpack(b.read(4))[0] # status code status_msg = self._graphbinary_reader.to_object(b, graphbinaryV4.DataType.string, nullable=True) status_ex = self._graphbinary_reader.to_object(b, graphbinaryV4.DataType.string, nullable=True) - # meta_attrs = self._graphbinary_reader.to_object(b, graphbinaryV1.DataType.map, nullable=False) + # meta_attrs = self._graphbinary_reader.to_object(b, graphbinaryV4.DataType.map, nullable=False) b.close() @@ -385,3 +378,13 @@ class GraphBinarySerializersV4(object): 'data': result}} return msg + + def read_payload(self, buffer): + results = [] + while buffer.readable(): + data = self._graphbinary_reader.to_object(buffer) + if data == Marker.end_of_stream(): + break + results.append(data) + + return results diff --git a/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV1.py b/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV1.py index cb9284f5d2..1cc487bc3e 100644 --- a/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV1.py +++ b/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV1.py @@ -49,6 +49,7 @@ _serializers = OrderedDict() _deserializers = {} +# TODO: to be removed class DataType(Enum): null = 0xfe int = 0x01 diff --git a/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV4.py b/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV4.py index d443fb0ec1..ef86445458 100644 --- a/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV4.py +++ b/gremlin-python/src/main/python/gremlin_python/structure/io/graphbinaryV4.py @@ -38,7 +38,7 @@ from gremlin_python.process.traversal import Barrier, Binding, Bytecode, Cardina TraversalStrategy, T from gremlin_python.process.graph_traversal import GraphTraversal from gremlin_python.structure.graph import Graph, Edge, Property, Vertex, VertexProperty, Path -from gremlin_python.structure.io.util import HashableDict, SymbolUtil +from gremlin_python.structure.io.util import HashableDict, SymbolUtil, Marker log = logging.getLogger(__name__) @@ -803,7 +803,7 @@ class BytecodeIO(_GraphBinaryTypeIO): @classmethod def dictify(cls, obj, writer, to_extend, as_value=False, nullable=True): - # cls.prefix_bytes(cls.graphbinary_type, as_value, nullable, to_extend) + cls.prefix_bytes(cls.graphbinary_type, as_value, nullable, to_extend) bc = obj.bytecode if isinstance(obj, Traversal) else obj to_extend.extend(int32_pack(len(bc.step_instructions))) for inst in bc.step_instructions: @@ -1161,18 +1161,18 @@ class DurationIO(_GraphBinaryTypeIO): return timedelta(seconds=seconds, microseconds=nanos / 1000) -# class MarkerIO(_GraphBinaryTypeIO): -# python_type = SingleByte -# graphbinary_type = DataType.marker -# -# @classmethod -# def dictify(cls, obj, writer, to_extend, as_value=True, nullable=True): -# cls.prefix_bytes(cls.graphbinary_type, as_value, nullable, to_extend) -# to_extend.extend(int8_pack(obj)) -# return to_extend -# -# @classmethod -# def objectify(cls, buff, reader, nullable=True): -# return cls.is_null(buff, reader, -# lambda b, r: int.__new__(SingleByte, int8_unpack(b.read(1))), -# nullable) +class MarkerIO(_GraphBinaryTypeIO): + python_type = Marker + graphbinary_type = DataType.marker + + @classmethod + def dictify(cls, obj, writer, to_extend, as_value=False, nullable=True): + cls.prefix_bytes(cls.graphbinary_type, as_value, nullable, to_extend) + to_extend.extend(int8_pack(obj.get_value())) + return to_extend + + @classmethod + def objectify(cls, buff, reader, nullable=True): + return cls.is_null(buff, reader, + lambda b, r: Marker.of(int8_unpack(b.read(1))), + nullable) \ No newline at end of file diff --git a/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV2d0.py b/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV2d0.py index d4138b6c9d..3e18ec6ac1 100644 --- a/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV2d0.py +++ b/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV2d0.py @@ -42,6 +42,7 @@ _serializers = OrderedDict() _deserializers = {} +# TODO: to be removed class GraphSONTypeType(type): def __new__(mcs, name, bases, dct): cls = super(GraphSONTypeType, mcs).__new__(mcs, name, bases, dct) diff --git a/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV3d0.py b/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV3d0.py index 692e4d27b0..932d17088a 100644 --- a/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV3d0.py +++ b/gremlin-python/src/main/python/gremlin_python/structure/io/graphsonV3d0.py @@ -43,6 +43,7 @@ _serializers = OrderedDict() _deserializers = {} +# TODO: to be removed class GraphSONTypeType(type): def __new__(mcs, name, bases, dct): cls = super(GraphSONTypeType, mcs).__new__(mcs, name, bases, dct) diff --git a/gremlin-python/src/main/python/gremlin_python/structure/io/util.py b/gremlin-python/src/main/python/gremlin_python/structure/io/util.py index 2660e9aa97..d0af103971 100644 --- a/gremlin-python/src/main/python/gremlin_python/structure/io/util.py +++ b/gremlin-python/src/main/python/gremlin_python/structure/io/util.py @@ -16,6 +16,9 @@ # under the License. import re +from threading import Lock + +from gremlin_python.statics import SingleByte class SymbolUtil: @@ -58,3 +61,38 @@ class HashableDict(dict): new_o[k] = cls.of(v) return new_o + +# referenced python singleton from https://refactoring.guru/design-patterns/singleton/python/example#example-1--main-py, +# might need some refactoring +class SingletonMeta(type): + + _instances = {} + _lock: Lock = Lock() + + def __call__(cls, *args, **kwargs): + with cls._lock: + if cls not in cls._instances: + instance = super().__call__(*args, **kwargs) + cls._instances[cls] = instance + return cls._instances[cls] + + +class Marker(metaclass=SingletonMeta): + _value: SingleByte + + def __init__(self, value: SingleByte) -> None: + self._value = value + + @staticmethod + def end_of_stream(): + return Marker(SingleByte(0)) + + def get_value(self): + return self._value + + @classmethod + def of(cls, val): + if val != 0: + raise ValueError + return cls.end_of_stream() + diff --git a/gremlin-python/src/main/python/tests/conftest.py b/gremlin-python/src/main/python/tests/conftest.py index 284c859015..711391c7a3 100644 --- a/gremlin-python/src/main/python/tests/conftest.py +++ b/gremlin-python/src/main/python/tests/conftest.py @@ -32,12 +32,14 @@ from gremlin_python.driver.connection import Connection from gremlin_python.driver import serializer from gremlin_python.driver.driver_remote_connection import ( DriverRemoteConnection) -from gremlin_python.driver.protocol import GremlinServerWSProtocol +from gremlin_python.driver.protocol import GremlinServerWSProtocol, GremlinServerHTTPProtocol from gremlin_python.driver.serializer import ( GraphSONMessageSerializer, GraphSONSerializersV2d0, GraphSONSerializersV3d0, GraphBinarySerializersV1, GraphBinarySerializersV4) from gremlin_python.driver.aiohttp.transport import AiohttpTransport, AiohttpHTTPTransport + +# TODO: update or remove once all implementations are finalized gremlin_server_url = os.environ.get('GREMLIN_SERVER_URL', 'ws://localhost:{}/gremlin') gremlin_basic_auth_url = os.environ.get('GREMLIN_SERVER_BASIC_AUTH_URL', 'wss://localhost:{}/gremlin') gremlin_socket_server_url = os.environ.get('GREMLIN_SOCKET_SERVER_URL', 'ws://localhost:{}/gremlin') @@ -48,12 +50,14 @@ kerberos_hostname = os.environ.get('KRB_HOSTNAME', socket.gethostname()) anonymous_url = gremlin_server_url.format(45940) basic_url = gremlin_basic_auth_url.format(45941) kerberos_url = gremlin_server_url.format(45942) - kerberized_service = 'test-service@{}'.format(kerberos_hostname) + +"""HTTP server testing""" gremlin_server_url_http = os.environ.get('GREMLIN_SERVER_URL_HTTP', 'http://localhost:{}/') gremlin_basic_auth_url_http = os.environ.get('GREMLIN_SERVER_BASIC_AUTH_URL_HTTP', 'https://localhost:{}/') anonymous_url_http = gremlin_server_url_http.format(45940) basic_url_http = gremlin_basic_auth_url_http.format(45941) + verbose_logging = False logging.basicConfig(format='%(asctime)s [%(levelname)8s] [%(filename)15s:%(lineno)d - %(funcName)10s()] - %(message)s', @@ -172,7 +176,7 @@ def authenticated_client(request): def remote_connection(request): try: if request.param == 'graphbinaryv1': - remote_conn = DriverRemoteConnection(anonymous_url, 'gmodern', + remote_conn = DriverRemoteConnection(anonymos_url, 'gmodern', message_serializer=serializer.GraphBinarySerializersV1()) elif request.param == 'graphsonv2': remote_conn = DriverRemoteConnection(anonymous_url, 'gmodern', @@ -281,11 +285,6 @@ def remote_connection_graphsonV2(request): return remote_conn -@pytest.fixture -def graphbinary_serializer_v4(request): - return GraphBinarySerializersV4() - - @pytest.fixture def graphson_serializer_v2(request): return GraphSONSerializersV2d0() @@ -301,18 +300,76 @@ def graphbinary_serializer_v1(request): return GraphBinarySerializersV1() -@pytest.fixture(params=['graphsonv2', 'graphsonv3', 'graphbinaryv1']) +""" +Tests below are for the HTTP server with GraphBinaryV4 +""" +@pytest.fixture +def connection_http(request): + protocol = GremlinServerHTTPProtocol( + GraphBinarySerializersV4(), + username='stephen', password='password') + executor = concurrent.futures.ThreadPoolExecutor(5) + pool = queue.Queue() + try: + conn = Connection(anonymous_url_http, 'gmodern', protocol, + lambda: AiohttpHTTPTransport(), executor, pool) + except OSError: + executor.shutdown() + pytest.skip('Gremlin Server is not running') + else: + def fin(): + executor.shutdown() + conn.close() + + request.addfinalizer(fin) + return conn + + +@pytest.fixture +def client_http(request): + try: + client = Client(anonymous_url_http, 'gmodern') + except OSError: + pytest.skip('Gremlin Server is not running') + else: + def fin(): + client.close() + + request.addfinalizer(fin) + return client + + +@pytest.fixture(params=['basic']) +def authenticated_client_http(request): + try: + if request.param == 'basic': + # turn off certificate verification for testing purposes only + ssl_opts = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + ssl_opts.verify_mode = ssl.CERT_NONE + client = Client(basic_url_http, 'gmodern', username='stephen', password='password', + transport_factory=lambda: AiohttpHTTPTransport(ssl_options=ssl_opts)) + else: + raise ValueError("Invalid authentication option - " + request.param) + except OSError: + pytest.skip('Gremlin Server is not running') + else: + def fin(): + client.close() + + request.addfinalizer(fin) + return client + + +@pytest.fixture +def graphbinary_serializer_v4(request): + return GraphBinarySerializersV4() + + +@pytest.fixture(params=['graphbinaryv4']) def remote_connection_http(request): try: - if request.param == 'graphbinaryv1': - remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern', - message_serializer=serializer.GraphBinarySerializersV1()) - elif request.param == 'graphsonv2': - remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern', - message_serializer=serializer.GraphSONSerializersV2d0()) - elif request.param == 'graphsonv3': - remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern', - message_serializer=serializer.GraphSONSerializersV3d0()) + if request.param == 'graphbinaryv4': + remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern') else: raise ValueError("Invalid serializer option - " + request.param) except OSError: @@ -325,10 +382,24 @@ def remote_connection_http(request): return remote_conn -""" -# The WsAndHttpChannelizer somehow does not distinguish the ssl handlers so authenticated https remote connection will -# only work with HttpChannelizer that is currently not in the testing set up, thus this is commented out for now +@pytest.fixture(params=['graphbinaryv4']) +def remote_connection_http_crew(request): + try: + if request.param == 'graphbinaryv4': + remote_conn = DriverRemoteConnection(anonymous_url_http, 'gcrew') + else: + raise ValueError("Invalid serializer option - " + request.param) + except OSError: + pytest.skip('Gremlin Server is not running') + else: + def fin(): + remote_conn.close() + + request.addfinalizer(fin) + return remote_conn + +# TODO: revisit once auth is updated @pytest.fixture(params=['basic']) def remote_connection_http_authenticated(request): try: @@ -338,7 +409,6 @@ def remote_connection_http_authenticated(request): ssl_opts.verify_mode = ssl.CERT_NONE remote_conn = DriverRemoteConnection(basic_url_http, 'gmodern', username='stephen', password='password', - message_serializer=serializer.GraphSONSerializersV2d0(), transport_factory=lambda: AiohttpHTTPTransport(ssl_options=ssl_opts)) else: raise ValueError("Invalid authentication option - " + request.param) @@ -350,18 +420,13 @@ def remote_connection_http_authenticated(request): request.addfinalizer(fin) return remote_conn -""" -@pytest.fixture(params=['graphsonv3', 'graphbinaryv1']) +@pytest.fixture(params=['graphbinaryv4']) def invalid_alias_remote_connection_http(request): try: - if request.param == 'graphbinaryv1': - remote_conn = DriverRemoteConnection(anonymous_url_http, 'does_not_exist', - message_serializer=serializer.GraphBinarySerializersV1()) - elif request.param == 'graphsonv3': - remote_conn = DriverRemoteConnection(anonymous_url_http, 'does_not_exist', - message_serializer=serializer.GraphSONSerializersV3d0()) + if request.param == 'graphbinaryv4': + remote_conn = DriverRemoteConnection(anonymous_url_http, 'does_not_exist') else: raise ValueError("Invalid serializer option - " + request.param) except OSError: diff --git a/gremlin-python/src/main/python/tests/driver/test_client.py b/gremlin-python/src/main/python/tests/driver/test_client.py index 2a34895e15..2b66e87c85 100644 --- a/gremlin-python/src/main/python/tests/driver/test_client.py +++ b/gremlin-python/src/main/python/tests/driver/test_client.py @@ -21,29 +21,35 @@ import os import threading import uuid +import pytest from gremlin_python.driver import serializer from gremlin_python.driver.client import Client from gremlin_python.driver.protocol import GremlinServerError -from gremlin_python.driver.request import RequestMessage +from gremlin_python.driver.request import RequestMessageV4 from gremlin_python.process.graph_traversal import __, GraphTraversalSource from gremlin_python.process.traversal import TraversalStrategies from gremlin_python.process.strategies import OptionsStrategy from gremlin_python.structure.graph import Graph, Vertex -from gremlin_python.driver.aiohttp.transport import AiohttpTransport +from gremlin_python.driver.aiohttp.transport import AiohttpTransport, AiohttpHTTPTransport from gremlin_python.statics import * from asyncio import TimeoutError __author__ = 'David M. Brown (davebs...@gmail.com)' -gremlin_server_url = os.environ.get('GREMLIN_SERVER_URL', 'ws://localhost:{}/gremlin') -test_no_auth_url = gremlin_server_url.format(45940) +gremlin_server_url_http = os.environ.get('GREMLIN_SERVER_URL_HTTP', 'http://localhost:{}/') +test_no_auth_http_url = gremlin_server_url_http.format(45940) -def test_connection(connection): +def create_basic_request_message(traversal, source='gmodern', type='bytecode'): + return RequestMessageV4(fields={'g': source, 'gremlinType': type}, gremlin=traversal.bytecode) + + +@pytest.mark.skip(reason="needs additional updates") +def test_connection(connection_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) t = g.V() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - results_set = connection.write(message).result() + message = create_basic_request_message(t) + results_set = connection_http.write(message).result() future = results_set.all() results = future.result() assert len(results) == 6 @@ -52,37 +58,39 @@ def test_connection(connection): assert 'host' in results_set.status_attributes -def test_client_message_too_big(client): +@pytest.mark.skip(reason="needs additional updates") +def test_client_message_too_big(client_http): try: - client = Client(test_no_auth_url, 'g', max_content_length=4096) - client.submit("\" \"*8000").all().result() + client_http = Client(test_no_auth_http_url, 'g', max_content_length=4096) + client_http.submit("\" \"*8000").all().result() assert False except Exception as ex: assert ex.args[0].startswith("Received error on read: 'Message size") \ and ex.args[0].endswith("exceeds limit 4096'") # confirm the client instance is still usable and not closed - assert ["test"] == client.submit("'test'").all().result() + assert ["test"] == client_http.submit("'test'").all().result() finally: - client.close() + client_http.close() -def test_client_simple_eval(client): - assert client.submit('1 + 1').all().result()[0] == 2 +def test_client_simple_eval(client_http): + assert client_http.submit('1 + 1').all().result()[0] == 2 -def test_client_simple_eval_bindings(client): - assert client.submit('x + x', {'x': 2}).all().result()[0] == 4 +def test_client_simple_eval_bindings(client_http): + assert client_http.submit('x + x', {'x': 2}).all().result()[0] == 4 -def test_client_eval_traversal(client): - assert len(client.submit('g.V()').all().result()) == 6 +def test_client_eval_traversal(client_http): + assert len(client_http.submit('g.V()').all().result()) == 6 -def test_client_error(client): +@pytest.mark.skip(reason="needs additional updates") +def test_client_error(client_http): try: # should fire an exception - client.submit('1/0').all().result() + client_http.submit('1/0').all().result() assert False except GremlinServerError as ex: assert 'exceptions' in ex.status_attributes @@ -90,65 +98,66 @@ def test_client_error(client): assert str(ex) == f"{ex.status_code}: {ex.status_message}" # still can submit after failure - assert client.submit('x + x', {'x': 2}).all().result()[0] == 4 + assert client_http.submit('x + x', {'x': 2}).all().result()[0] == 4 -def test_client_connection_pool_after_error(client): +@pytest.mark.skip(reason="needs additional updates") +def test_client_connection_pool_after_error(client_http): # Overwrite fixture with pool_size=1 client - client = Client(test_no_auth_url, 'gmodern', pool_size=1) + client_http = Client(test_no_auth_http_url, 'gmodern', pool_size=1) try: # should fire an exception - client.submit('1/0').all().result() + client_http.submit('1/0').all().result() assert False except GremlinServerError as gse: # expecting the pool size to be 1 again after query returned assert gse.status_code == 597 - assert client.available_pool_size == 1 + assert client_http.available_pool_size == 1 # still can submit after failure - assert client.submit('x + x', {'x': 2}).all().result()[0] == 4 + assert client_http.submit('x + x', {'x': 2}).all().result()[0] == 4 -def test_client_no_hang_if_submit_on_closed(client): - assert client.submit('1 + 1').all().result()[0] == 2 - client.close() +def test_client_no_hang_if_submit_on_closed(client_http): + assert client_http.submit('1 + 1').all().result()[0] == 2 + client_http.close() try: # should fail since not hang if closed - client.submit('1 + 1').all().result() + client_http.submit('1 + 1').all().result() assert False except Exception as ex: assert True -def test_client_close_all_connection_in_pool(client): - client = Client(test_no_auth_url, 'g', pool_size=1, session="75e9620e-da98-41e3-9378-0336db803de0") - assert client.available_pool_size == 1 - client.submit('2+2').all().result() - client.close() - assert client.available_pool_size == 0 +def test_client_close_all_connection_in_pool(client_http): + client_http = Client(test_no_auth_http_url, 'g', pool_size=1) + assert client_http.available_pool_size == 1 + client_http.submit('2+2').all().result() + client_http.close() + assert client_http.available_pool_size == 0 -def test_client_side_timeout_set_for_aiohttp(client): - client = Client(test_no_auth_url, 'gmodern', - transport_factory=lambda: AiohttpTransport(read_timeout=1, write_timeout=1)) +def test_client_side_timeout_set_for_aiohttp(client_http): + client_http = Client(test_no_auth_http_url, 'gmodern', + transport_factory=lambda: AiohttpHTTPTransport(read_timeout=1, write_timeout=1)) try: # should fire an exception - client.submit('Thread.sleep(2000);1').all().result() + client_http.submit('Thread.sleep(2000);1').all().result() assert False except TimeoutError as err: # asyncio TimeoutError has no message. assert str(err) == "" # still can submit after failure - assert client.submit('x + x', {'x': 2}).all().result()[0] == 4 + assert client_http.submit('x + x', {'x': 2}).all().result()[0] == 4 async def async_connect(enable): try: - transport = AiohttpTransport(call_from_event_loop=enable) - transport.connect(test_no_auth_url) + transport = AiohttpHTTPTransport(call_from_event_loop=enable) + transport.connect(test_no_auth_http_url) transport.close() return True except RuntimeError: @@ -160,8 +169,8 @@ def test_from_event_loop(): assert asyncio.get_event_loop().run_until_complete(async_connect(True)) -def test_client_gremlin(client): - result_set = client.submit('g.V(1)') +def test_client_gremlin(client_http): + result_set = client_http.submit('g.V(1)') result = result_set.all().result() assert 1 == len(result) vertex = result[0] @@ -171,7 +180,7 @@ def test_client_gremlin(client): assert 'name' == vertex.properties[0].key assert 'marko' == vertex.properties[0].value ## - result_set = client.submit('g.with("materializeProperties", "tokens").V(1)') + result_set = client_http.submit('g.with("materializeProperties", "tokens").V(1)') result = result_set.all().result() assert 1 == len(result) vertex = result[0] @@ -179,58 +188,58 @@ def test_client_gremlin(client): assert 0 == len(vertex.properties) -def test_client_bytecode(client): +def test_client_bytecode(client_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) t = g.V() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - result_set = client.submit(message) + message = create_basic_request_message(t) + result_set = client_http.submit(message) assert len(result_set.all().result()) == 6 -def test_client_bytecode_options(client): +def test_client_bytecode_options(client_http): # smoke test to validate serialization of OptionsStrategy. no way to really validate this from an integration # test perspective because there's no way to access the internals of the strategy via bytecode g = GraphTraversalSource(Graph(), TraversalStrategies()) - t = g.withStrategies(OptionsStrategy(options={"x": "test", "y": True})).V() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - result_set = client.submit(message) + t = g.with_strategies(OptionsStrategy(options={"x": "test", "y": True})).V() + message = create_basic_request_message(t) + result_set = client_http.submit(message) assert len(result_set.all().result()) == 6 ## t = g.with_("x", "test").with_("y", True).V() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - result_set = client.submit(message) + message = create_basic_request_message(t) + result_set = client_http.submit(message) assert len(result_set.all().result()) == 6 -def test_iterate_result_set(client): +def test_iterate_result_set(client_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) t = g.V() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - result_set = client.submit(message) + message = create_basic_request_message(t) + result_set = client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 6 -def test_client_async(client): +def test_client_async(client_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) t = g.V() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - future = client.submit_async(message) + message = create_basic_request_message(t) + future = client_http.submit_async(message) result_set = future.result() assert len(result_set.all().result()) == 6 -def test_connection_share(client): +def test_connection_share(client_http): # Overwrite fixture with pool_size=1 client - client = Client(test_no_auth_url, 'gmodern', pool_size=1) + client_http = Client(test_no_auth_http_url, 'gmodern', pool_size=1) g = GraphTraversalSource(Graph(), TraversalStrategies()) t = g.V() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - message2 = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - future = client.submit_async(message) - future2 = client.submit_async(message2) + message = create_basic_request_message(t) + message2 = create_basic_request_message(t) + future = client_http.submit_async(message) + future2 = client_http.submit_async(message2) result_set2 = future2.result() assert len(result_set2.all().result()) == 6 @@ -241,14 +250,14 @@ def test_connection_share(client): assert len(result_set.all().result()) == 6 -def test_multi_conn_pool(client): +def test_multi_conn_pool(client_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) t = g.V() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - message2 = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - client = Client(test_no_auth_url, 'g', pool_size=1) - future = client.submit_async(message) - future2 = client.submit_async(message2) + message = create_basic_request_message(t) + message2 = create_basic_request_message(t) + client_http = Client(test_no_auth_http_url, 'g', pool_size=1) + future = client_http.submit_async(message) + future2 = client_http.submit_async(message2) result_set2 = future2.result() assert len(result_set2.all().result()) == 6 @@ -258,7 +267,7 @@ def test_multi_conn_pool(client): assert len(result_set.all().result()) == 6 -def test_multi_thread_pool(client): +def test_multi_thread_pool(client_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) traversals = [g.V(), g.V().count(), @@ -272,10 +281,10 @@ def test_multi_thread_pool(client): condition = threading.Condition() def thread_run(tr, result_list): - message = RequestMessage('traversal', 'bytecode', {'gremlin': tr.bytecode, 'aliases': {'g': 'gmodern'}}) + message = create_basic_request_message(tr) with condition: condition.wait(5) - result_set = client.submit(message) + result_set = client_http.submit(message) for result in result_set: result_list.append(result) @@ -298,207 +307,184 @@ def test_multi_thread_pool(client): assert len(results[2][0]) == 6 assert results[3][0][0].object == 6 -def test_client_bytecode_with_short(client): + +def test_client_bytecode_with_short(client_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) t = g.V().has('age', short(16)).count() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - result_set = client.submit(message) + message = create_basic_request_message(t) + result_set = client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 1 -def test_client_bytecode_with_long(client): + +def test_client_bytecode_with_long(client_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) t = g.V().has('age', long(851401972585122)).count() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - result_set = client.submit(message) + message = create_basic_request_message(t) + result_set = client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 1 -def test_client_bytecode_with_bigint(client): +def test_client_bytecode_with_bigint(client_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) t = g.V().has('age', bigint(0x1000_0000_0000_0000_0000)).count() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'gmodern'}}) - result_set = client.submit(message) + message = create_basic_request_message(t) + result_set = client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 1 -def test_multi_request_in_session(client): - # Overwrite fixture with session client - session_id = str(uuid.uuid4()) - client = Client(test_no_auth_url, 'g', session=session_id) - - assert client.submit('x = 1').all().result()[0] == 1 - assert client.submit('x + 2').all().result()[0] == 3 - - client.close() - - # attempt reconnect to session and make sure "x" is no longer a thing - client = Client(test_no_auth_url, 'g', session=session_id) - try: - # should fire an exception - client.submit('x').all().result() - assert False - except Exception: - assert True - - -def test_client_pool_in_session(client): - # Overwrite fixture with pool_size=2 client - try: - # should fire an exception - client = Client(test_no_auth_url, 'g', session=str(uuid.uuid4()), pool_size=2) - assert False - except Exception: - assert True - - -def test_big_result_set(client): +def test_big_result_set(client_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) - t = g.inject(1).repeat(__.addV('person').property('name', __.loops())).times(20000).count() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}}) - result_set = client.submit(message) + t = g.inject(1).repeat(__.add_V('person').property('name', __.loops())).times(20000).count() + message = create_basic_request_message(t, source='g') + result_set = client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 1 t = g.V().limit(10) - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}}) - result_set = client.submit(message) + message = create_basic_request_message(t, source='g') + result_set = client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 10 t = g.V().limit(100) - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}}) - result_set = client.submit(message) + message = create_basic_request_message(t, source='g') + result_set = client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 100 t = g.V().limit(1000) - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}}) - result_set = client.submit(message) + message = create_basic_request_message(t, source='g') + result_set = client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 1000 - t = g.V().limit(10000) - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}}) - result_set = client.submit(message) - results = [] - for result in result_set: - results += result - assert len(results) == 10000 + # TODO: enable after setting up docker for GitHub or cleaning up server-side locally + # t = g.V().limit(10000) + # message = create_basic_request_message(t, source='g') + # result_set = client_http.submit(message) + # results = [] + # for result in result_set: + # results += result + # assert len(results) == 10000 -def test_big_result_set_secure(authenticated_client): +@pytest.mark.skip(reason="enable after making sure authenticated testing server is set up in docker") +def test_big_result_set_secure(authenticated_client_http): g = GraphTraversalSource(Graph(), TraversalStrategies()) - t = g.inject(1).repeat(__.addV('person').property('name', __.loops())).times(20000).count() - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}}) - result_set = authenticated_client.submit(message) + t = g.inject(1).repeat(__.add_v('person').property('name', __.loops())).times(20000).count() + message = create_basic_request_message(t, source='g') + result_set = authenticated_client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 1 t = g.V().limit(10) - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}}) - result_set = authenticated_client.submit(message) + message = create_basic_request_message(t, source='g') + result_set = authenticated_client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 10 t = g.V().limit(100) - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}}) - result_set = authenticated_client.submit(message) + message = create_basic_request_message(t, source='g') + result_set = authenticated_client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 100 t = g.V().limit(1000) - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}}) - result_set = authenticated_client.submit(message) + message = create_basic_request_message(t, source='g') + result_set = authenticated_client_http.submit(message) results = [] for result in result_set: results += result assert len(results) == 1000 - t = g.V().limit(10000) - message = RequestMessage('traversal', 'bytecode', {'gremlin': t.bytecode, 'aliases': {'g': 'g'}}) - result_set = authenticated_client.submit(message) - results = [] - for result in result_set: - results += result - assert len(results) == 10000 + # TODO: enable after setting up docker for GitHub or cleaning up server-side locally + # t = g.V().limit(10000) + # message = create_basic_request_message(t, source='g') + # result_set = authenticated_client_http.submit(message) + # results = [] + # for result in result_set: + # results += result + # assert len(results) == 10000 async def asyncio_func(): return 1 -def test_asyncio(client): +def test_asyncio(client_http): try: asyncio.get_event_loop().run_until_complete(asyncio_func()) except RuntimeError: assert False -def test_client_custom_invalid_request_id_graphson_script(client): - client = Client(test_no_auth_url, 'gmodern', message_serializer=serializer.GraphSONSerializersV3d0()) +# TODO: tests pass because requestID is now generated on HTTP server and this option gets ignored, tests to be removed +# or updated depending on if we still want to use requestID or not +def test_client_custom_invalid_request_id_graphson_script(client_http): + client = Client(test_no_auth_http_url, 'gmodern') try: - client.submit('g.V()', request_options={"requestId":"malformed"}).all().result() + client.submit('g.V()', request_options={"requestId": "malformed"}).all().result() except Exception as ex: assert "badly formed hexadecimal UUID string" in str(ex) -def test_client_custom_invalid_request_id_graphbinary_script(client): - client = Client(test_no_auth_url, 'gmodern', message_serializer=serializer.GraphBinarySerializersV1()) +def test_client_custom_invalid_request_id_graphbinary_script(client_http): + client_http = Client(test_no_auth_http_url, 'gmodern') try: - client.submit('g.V()', request_options={"requestId":"malformed"}).all().result() + client_http.submit('g.V()', request_options={"requestId": "malformed"}).all().result() except Exception as ex: assert "badly formed hexadecimal UUID string" in str(ex) -def test_client_custom_valid_request_id_script_uuid(client): - assert len(client.submit('g.V()', request_options={"requestId":uuid.uuid4()}).all().result()) == 6 +def test_client_custom_valid_request_id_script_uuid(client_http): + assert len(client_http.submit('g.V()', request_options={"requestId": uuid.uuid4()}).all().result()) == 6 -def test_client_custom_valid_request_id_script_string(client): - assert len(client.submit('g.V()', request_options={"requestId":str(uuid.uuid4())}).all().result()) == 6 +def test_client_custom_valid_request_id_script_string(client_http): + assert len(client_http.submit('g.V()', request_options={"requestId": str(uuid.uuid4())}).all().result()) == 6 -def test_client_custom_invalid_request_id_graphson_bytecode(client): - client = Client(test_no_auth_url, 'gmodern', message_serializer=serializer.GraphSONSerializersV3d0()) +def test_client_custom_invalid_request_id_graphson_bytecode(client_http): + client_http = Client(test_no_auth_http_url, 'gmodern') query = GraphTraversalSource(Graph(), TraversalStrategies()).V().bytecode try: - client.submit(query, request_options={"requestId":"malformed"}).all().result() + client_http.submit(query, request_options={"requestId": "malformed"}).all().result() except Exception as ex: assert "badly formed hexadecimal UUID string" in str(ex) -def test_client_custom_invalid_request_id_graphbinary_bytecode(client): - client = Client(test_no_auth_url, 'gmodern', message_serializer=serializer.GraphBinarySerializersV1()) +def test_client_custom_invalid_request_id_graphbinary_bytecode(client_http): + client_http = Client(test_no_auth_http_url, 'gmodern') query = GraphTraversalSource(Graph(), TraversalStrategies()).V().bytecode try: - client.submit(query, request_options={"requestId":"malformed"}).all().result() + client_http.submit(query, request_options={"requestId": "malformed"}).all().result() except Exception as ex: assert "badly formed hexadecimal UUID string" in str(ex) -def test_client_custom_valid_request_id_bytecode(client): +def test_client_custom_valid_request_id_bytecode(client_http): query = GraphTraversalSource(Graph(), TraversalStrategies()).V().bytecode - assert len(client.submit(query).all().result()) == 6 + assert len(client_http.submit(query).all().result()) == 6 diff --git a/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection.py b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection.py index 56e45a770a..1e459297b8 100644 --- a/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection.py +++ b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection.py @@ -18,6 +18,7 @@ # import os +import pytest from gremlin_python import statics from gremlin_python.driver.protocol import GremlinServerError from gremlin_python.statics import long @@ -37,6 +38,9 @@ __author__ = 'Marko A. Rodriguez (http://markorodriguez.com)' gremlin_server_url = os.environ.get('GREMLIN_SERVER_URL', 'ws://localhost:{}/gremlin') test_no_auth_url = gremlin_server_url.format(45940) + +# TODO: WS tests to be removed +@pytest.mark.skip(reason="disabling all WS tests as we move to HTTP") class TestDriverRemoteConnection(object): def test_traversals(self, remote_connection): statics.load_statics(globals()) diff --git a/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py index 8d35695328..7b6d324559 100644 --- a/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py +++ b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py @@ -18,6 +18,7 @@ # import os +import pytest from gremlin_python import statics from gremlin_python.statics import long from gremlin_python.process.traversal import Traverser @@ -29,7 +30,6 @@ from gremlin_python.process.anonymous_traversal import traversal from gremlin_python.structure.graph import Vertex from gremlin_python.process.strategies import SubgraphStrategy, SeedStrategy from gremlin_python.structure.io.util import HashableDict -from gremlin_python.driver.serializer import GraphSONSerializersV2d0 from gremlin_python.driver.protocol import GremlinServerError gremlin_server_url_http = os.environ.get('GREMLIN_SERVER_URL_HTTP', 'http://localhost:{}/') @@ -73,7 +73,7 @@ class TestDriverRemoteConnectionHttp(object): assert 'dur' in results[0] # # results = g.V().has('name', 'peter').as_('a').out('created').as_('b').select('a', 'b').by( - __.valueMap()).toList() + __.value_map()).toList() assert 1 == len(results) assert 'peter' == results[0]['a']['name'][0] assert 35 == results[0]['a']['age'][0] @@ -102,15 +102,12 @@ class TestDriverRemoteConnectionHttp(object): results = g.V().has('person', 'age', Bindings.of('x', P.lt(30))).count().next() assert 2 == results # # - # test dict keys which can only work on GraphBinary and GraphSON3 which include specific serialization + # test dict keys # types for dict - if not isinstance(remote_connection_http._client._message_serializer, GraphSONSerializersV2d0): - results = g.V().has('person', 'name', 'marko').elementMap("name").groupCount().next() - assert {HashableDict.of({T.id: 1, T.label: 'person', 'name': 'marko'}): 1} == results - if not isinstance(remote_connection_http._client._message_serializer, GraphSONSerializersV2d0): - results = g.V().has('person', 'name', 'marko').both('knows').groupCount().by( - __.values('name').fold()).next() - assert {tuple(['vadas']): 1, tuple(['josh']): 1} == results + results = g.V().has('person', 'name', 'marko').elementMap("name").groupCount().next() + assert {HashableDict.of({T.id: 1, T.label: 'person', 'name': 'marko'}): 1} == results + results = g.V().has('person', 'name', 'marko').both('knows').groupCount().by(__.values('name').fold()).next() + assert {tuple(['vadas']): 1, tuple(['josh']): 1} == results def test_iteration(self, remote_connection_http): statics.load_statics(globals()) @@ -166,8 +163,8 @@ class TestDriverRemoteConnectionHttp(object): statics.load_statics(globals()) g = traversal().with_(remote_connection_http). \ withStrategies(TraversalStrategy("SubgraphStrategy", - {"vertices": __.hasLabel("person"), - "edges": __.hasLabel("created")}, + {"vertices": __.has_label("person"), + "edges": __.has_label("created")}, "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SubgraphStrategy")) assert 4 == g.V().count().next() assert 0 == g.E().count().next() @@ -176,14 +173,14 @@ class TestDriverRemoteConnectionHttp(object): assert "person" == g.V().label().dedup().next() # g = traversal().with_(remote_connection_http). \ - withStrategies(SubgraphStrategy(vertices=__.hasLabel("person"), edges=__.hasLabel("created"))) + withStrategies(SubgraphStrategy(vertices=__.has_label("person"), edges=__.has_label("created"))) assert 4 == g.V().count().next() assert 0 == g.E().count().next() assert 1 == g.V().label().dedup().count().next() assert "person" == g.V().label().dedup().next() # g = traversal().with_(remote_connection_http). \ - withStrategies(SubgraphStrategy(edges=__.hasLabel("created"))) + withStrategies(SubgraphStrategy(edges=__.has_label("created"))) assert 6 == g.V().count().next() assert 4 == g.E().count().next() assert 1 == g.E().label().dedup().count().next() @@ -214,22 +211,19 @@ class TestDriverRemoteConnectionHttp(object): assert 10 == t.clone().limit(10).count().next() def test_receive_error(self, invalid_alias_remote_connection_http): - g = traversal().withRemote(invalid_alias_remote_connection_http) + g = traversal().with_(invalid_alias_remote_connection_http) try: g.V().next() assert False except GremlinServerError as err: assert err.status_code == 400 - assert 'Could not rebind' in err.status_message + assert 'The traversal source [does_not_exist] for alias [g] is not configured on the server.' \ + in err.status_message - """ - # The WsAndHttpChannelizer somehow does not distinguish the ssl handlers so authenticated https remote connection - # only work with HttpChannelizer that is currently not in the testing set up, thus this is commented out for now - + @pytest.mark.skip(reason="enable after making sure authenticated testing server is set up in docker") def test_authenticated(self, remote_connection_http_authenticated): statics.load_statics(globals()) g = traversal().with_(remote_connection_http_authenticated) assert long(6) == g.V().count().toList()[0] - """ diff --git a/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_threaded.py b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_threaded.py index 4423e04649..ec1a118509 100644 --- a/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_threaded.py +++ b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_threaded.py @@ -17,7 +17,6 @@ # under the License. # import concurrent.futures -import os import sys import queue from threading import Thread @@ -28,10 +27,11 @@ from gremlin_python.process.anonymous_traversal import traversal __author__ = 'David M. Brown (davebs...@gmail.com)' -gremlin_server_url = os.environ.get('GREMLIN_SERVER_URL', 'ws://localhost:{}/gremlin') -test_no_auth_url = gremlin_server_url.format(45940) +gremlin_server_url_http = 'http://localhost:{}' +test_no_auth_http_url = gremlin_server_url_http.format(45940) -def test_conns_in_threads(remote_connection): + +def test_conns_in_threads(remote_connection_http): q = queue.Queue() child = Thread(target=_executor, args=(q, None)) child2 = Thread(target=_executor, args=(q, None)) @@ -44,10 +44,10 @@ def test_conns_in_threads(remote_connection): child2.join() -def test_conn_in_threads(remote_connection): +def test_conn_in_threads(remote_connection_http): q = queue.Queue() - child = Thread(target=_executor, args=(q, remote_connection)) - child2 = Thread(target=_executor, args=(q, remote_connection)) + child = Thread(target=_executor, args=(q, remote_connection_http)) + child2 = Thread(target=_executor, args=(q, remote_connection_http)) child.start() child2.start() for x in range(2): @@ -62,7 +62,7 @@ def _executor(q, conn): if not conn: # This isn't a fixture so close manually close = True - conn = DriverRemoteConnection(test_no_auth_url, 'gmodern', pool_size=4) + conn = DriverRemoteConnection(test_no_auth_http_url, 'gmodern', pool_size=4) try: g = traversal().with_(conn) future = g.V().promise() @@ -79,7 +79,7 @@ def _executor(q, conn): def handle_request(): try: - remote_connection = DriverRemoteConnection(test_no_auth_url, "g") + remote_connection = DriverRemoteConnection(test_no_auth_http_url, "gmodern") g = traversal().with_(remote_connection) g.V().limit(1).toList() remote_connection.close() diff --git a/gremlin-python/src/main/python/tests/driver/test_serializer.py b/gremlin-python/src/main/python/tests/driver/test_serializer.py index d259dc9c42..bff69c941e 100644 --- a/gremlin-python/src/main/python/tests/driver/test_serializer.py +++ b/gremlin-python/src/main/python/tests/driver/test_serializer.py @@ -16,36 +16,12 @@ # specific language governing permissions and limitations # under the License. # -from gremlin_python.structure.io import graphsonV2d0 -from gremlin_python.structure.io import graphsonV3d0 -from gremlin_python.structure.io import graphbinaryV1 from gremlin_python.structure.io import graphbinaryV4 __author__ = 'David M. Brown' -def test_graphson_serializer_v2(graphson_serializer_v2): - assert graphson_serializer_v2.version == b"application/vnd.gremlin-v2.0+json" - assert isinstance(graphson_serializer_v2._graphson_reader, graphsonV2d0.GraphSONReader) - assert isinstance(graphson_serializer_v2.standard._writer, graphsonV2d0.GraphSONWriter) - assert isinstance(graphson_serializer_v2.traversal._writer, graphsonV2d0.GraphSONWriter) - - -def test_graphson_serializer_v3(graphson_serializer_v3): - assert graphson_serializer_v3.version == b"application/vnd.gremlin-v3.0+json" - assert isinstance(graphson_serializer_v3._graphson_reader, graphsonV3d0.GraphSONReader) - assert isinstance(graphson_serializer_v3.standard._writer, graphsonV3d0.GraphSONWriter) - assert isinstance(graphson_serializer_v3.traversal._writer, graphsonV3d0.GraphSONWriter) - - -def test_graphbinary_serializer_v1(graphbinary_serializer_v1): - assert graphbinary_serializer_v1.version == b"application/vnd.graphbinary-v1.0" - assert isinstance(graphbinary_serializer_v1._graphbinary_reader, graphbinaryV1.GraphBinaryReader) - assert isinstance(graphbinary_serializer_v1.standard._writer, graphbinaryV1.GraphBinaryWriter) - assert isinstance(graphbinary_serializer_v1.traversal._writer, graphbinaryV1.GraphBinaryWriter) - - def test_graphbinary_serializer_v4(graphbinary_serializer_v4): assert graphbinary_serializer_v4.version == b"application/vnd.graphbinary-v4.0" assert isinstance(graphbinary_serializer_v4._graphbinary_reader, graphbinaryV4.GraphBinaryReader) diff --git a/gremlin-python/src/main/python/tests/driver/test_web_socket_client_behavior.py b/gremlin-python/src/main/python/tests/driver/test_web_socket_client_behavior.py index 66848bdaaa..0126229c02 100644 --- a/gremlin-python/src/main/python/tests/driver/test_web_socket_client_behavior.py +++ b/gremlin-python/src/main/python/tests/driver/test_web_socket_client_behavior.py @@ -23,14 +23,18 @@ __author__ = 'Cole Greer (c...@colegreer.ca)' import re import operator from functools import reduce + +import pytest from gremlin_python.driver import useragent +# TODO: remove or modify after implementing equivalent support in HTTP server # Note: This test demonstrates different behavior in response to a server sending a close frame than the other GLV's. # Other GLV's will respond to this by trying to reconnect. This test is also demonstrating incorrect behavior of # client.is_closed() as it appears unaware that the event loop is dead. # These differences from other GLV's are being tracked in [TINKERPOP-2846]. If this behavior is changed to resemble # other GLV's, this test should be updated to show a vertex is being received by the second request. +@pytest.mark.skip(reason="not implemented in HTTP & need to check on server side") def test_does_not_create_new_connection_if_closed_by_server(socket_server_client, socket_server_settings): try: socket_server_client.submit( diff --git a/gremlin-python/src/main/python/tests/structure/io/test_functionalityio.py b/gremlin-python/src/main/python/tests/structure/io/test_functionalityio.py index d67efb460f..38c3c3602b 100644 --- a/gremlin-python/src/main/python/tests/structure/io/test_functionalityio.py +++ b/gremlin-python/src/main/python/tests/structure/io/test_functionalityio.py @@ -20,13 +20,13 @@ under the License. import datetime import uuid -from gremlin_python.driver.serializer import GraphSONSerializersV2d0, GraphBinarySerializersV1 +from gremlin_python.driver.serializer import GraphBinarySerializersV4 from gremlin_python.process.anonymous_traversal import traversal from gremlin_python.statics import * -def test_vertex(remote_connection): - g = traversal().with_(remote_connection) +def test_vertex(remote_connection_http): + g = traversal().with_(remote_connection_http) vertex = g.V(1).next() assert vertex.id == 1 assert vertex.label == 'person' @@ -37,8 +37,8 @@ def test_vertex(remote_connection): assert vertex.properties[1].value == 29 -def test_vertex_without_properties(remote_connection): - g = traversal().with_(remote_connection) +def test_vertex_without_properties(remote_connection_http): + g = traversal().with_(remote_connection_http) vertex = g.with_('materializeProperties', 'tokens').V(1).next() assert vertex.id == 1 assert vertex.label == 'person' @@ -46,8 +46,8 @@ def test_vertex_without_properties(remote_connection): assert vertex.properties is None or len(vertex.properties) == 0 -def test_edge(remote_connection): - g = traversal().with_(remote_connection) +def test_edge(remote_connection_http): + g = traversal().with_(remote_connection_http) edge = g.E(7).next() assert edge.id == 7 assert edge.label == 'knows' @@ -56,8 +56,8 @@ def test_edge(remote_connection): assert edge.properties[0].value == 0.5 -def test_edge_without_properties(remote_connection): - g = traversal().with_(remote_connection) +def test_edge_without_properties(remote_connection_http): + g = traversal().with_(remote_connection_http) edge = g.with_('materializeProperties', 'tokens').E(7).next() assert edge.id == 7 assert edge.label == 'knows' @@ -65,8 +65,8 @@ def test_edge_without_properties(remote_connection): assert edge.properties is None or len(edge.properties) == 0 -def test_vertex_vertex_properties(remote_connection_crew): - g = traversal().with_(remote_connection_crew) +def test_vertex_vertex_properties(remote_connection_http_crew): + g = traversal().with_(remote_connection_http_crew) vertex = g.V(7).next() assert vertex.id == 7 assert vertex.label == 'person' @@ -80,8 +80,8 @@ def test_vertex_vertex_properties(remote_connection_crew): assert vertex.properties[1].properties[1].value == 2000 -def test_timestamp(remote_connection): - g = traversal().with_(remote_connection) +def test_timestamp(remote_connection_http): + g = traversal().with_(remote_connection_http) ts = timestamp(1481750076295 / 1000) resp = g.addV('test_vertex').property('ts', ts) resp = resp.toList() @@ -94,8 +94,8 @@ def test_timestamp(remote_connection): g.V(vid).drop().iterate() -def test_datetime(remote_connection): - g = traversal().with_(remote_connection) +def test_datetime(remote_connection_http): + g = traversal().with_(remote_connection_http) dt = datetime.datetime.utcfromtimestamp(1481750076295 / 1000) resp = g.addV('test_vertex').property('dt', dt).toList() vid = resp[0].id @@ -107,8 +107,8 @@ def test_datetime(remote_connection): g.V(vid).drop().iterate() -def test_uuid(remote_connection): - g = traversal().with_(remote_connection) +def test_uuid(remote_connection_http): + g = traversal().with_(remote_connection_http) uid = uuid.UUID("41d2e28a-20a4-4ab0-b379-d810dede3786") resp = g.addV('test_vertex').property('uuid', uid).toList() vid = resp[0].id @@ -120,11 +120,11 @@ def test_uuid(remote_connection): g.V(vid).drop().iterate() -def test_short(remote_connection): - if not isinstance(remote_connection._client._message_serializer, GraphBinarySerializersV1): +def test_short(remote_connection_http): + if not isinstance(remote_connection_http._client._message_serializer, GraphBinarySerializersV4): return - g = traversal().with_(remote_connection) + g = traversal().with_(remote_connection_http) num = short(1111) resp = g.addV('test_vertex').property('short', num).toList() vid = resp[0].id @@ -136,11 +136,11 @@ def test_short(remote_connection): g.V(vid).drop().iterate() -def test_bigint_positive(remote_connection): - if not isinstance(remote_connection._client._message_serializer, GraphBinarySerializersV1): +def test_bigint_positive(remote_connection_http): + if not isinstance(remote_connection_http._client._message_serializer, GraphBinarySerializersV4): return - g = traversal().with_(remote_connection) + g = traversal().with_(remote_connection_http) big = bigint(0x1000_0000_0000_0000_0000) resp = g.addV('test_vertex').property('bigint', big).toList() vid = resp[0].id @@ -152,11 +152,11 @@ def test_bigint_positive(remote_connection): g.V(vid).drop().iterate() -def test_bigint_negative(remote_connection): - if not isinstance(remote_connection._client._message_serializer, GraphBinarySerializersV1): +def test_bigint_negative(remote_connection_http): + if not isinstance(remote_connection_http._client._message_serializer, GraphBinarySerializersV4): return - g = traversal().with_(remote_connection) + g = traversal().with_(remote_connection_http) big = bigint(-0x1000_0000_0000_0000_0000) resp = g.addV('test_vertex').property('bigint', big).toList() vid = resp[0].id @@ -168,11 +168,11 @@ def test_bigint_negative(remote_connection): g.V(vid).drop().iterate() -def test_bigdecimal(remote_connection): - if not isinstance(remote_connection._client._message_serializer, GraphBinarySerializersV1): +def test_bigdecimal(remote_connection_http): + if not isinstance(remote_connection_http._client._message_serializer, GraphBinarySerializersV4): return - g = traversal().with_(remote_connection) + g = traversal().with_(remote_connection_http) bigdecimal = BigDecimal(101, 235) resp = g.addV('test_vertex').property('bigdecimal', bigdecimal).toList() vid = resp[0].id @@ -185,32 +185,31 @@ def test_bigdecimal(remote_connection): g.V(vid).drop().iterate() -def test_odd_bits(remote_connection): - if not isinstance(remote_connection._client._message_serializer, GraphSONSerializersV2d0): - g = traversal().with_(remote_connection) - char_lower = str.__new__(SingleChar, chr(78)) - resp = g.addV('test_vertex').property('char_lower', char_lower).toList() - vid = resp[0].id - try: - v = g.V(vid).values('char_lower').toList()[0] - assert v == char_lower - finally: - g.V(vid).drop().iterate() - - char_upper = str.__new__(SingleChar, chr(57344)) - resp = g.addV('test_vertex').property('char_upper', char_upper).toList() - vid = resp[0].id - try: - v = g.V(vid).values('char_upper').toList()[0] - assert v == char_upper - finally: - g.V(vid).drop().iterate() - - dur = datetime.timedelta(seconds=1000, microseconds=1000) - resp = g.addV('test_vertex').property('dur', dur).toList() - vid = resp[0].id - try: - v = g.V(vid).values('dur').toList()[0] - assert v == dur - finally: - g.V(vid).drop().iterate() +def test_odd_bits(remote_connection_http): + g = traversal().with_(remote_connection_http) + char_lower = str.__new__(SingleChar, chr(78)) + resp = g.addV('test_vertex').property('char_lower', char_lower).toList() + vid = resp[0].id + try: + v = g.V(vid).values('char_lower').toList()[0] + assert v == char_lower + finally: + g.V(vid).drop().iterate() + + char_upper = str.__new__(SingleChar, chr(57344)) + resp = g.addV('test_vertex').property('char_upper', char_upper).toList() + vid = resp[0].id + try: + v = g.V(vid).values('char_upper').toList()[0] + assert v == char_upper + finally: + g.V(vid).drop().iterate() + + dur = datetime.timedelta(seconds=1000, microseconds=1000) + resp = g.addV('test_vertex').property('dur', dur).toList() + vid = resp[0].id + try: + v = g.V(vid).values('dur').toList()[0] + assert v == dur + finally: + g.V(vid).drop().iterate() diff --git a/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py b/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py index 32201b78a9..e3f678ebab 100644 --- a/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py +++ b/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py @@ -27,11 +27,8 @@ from gremlin_python.structure.io.graphbinaryV1 import GraphBinaryWriter, GraphBi from gremlin_python.process.traversal import Barrier, Binding, Bytecode, Merge, Direction -class TestGraphBinaryReader(object): - graphbinary_reader = GraphBinaryReader() - - -class TestGraphSONWriter(object): +# TODO: to be removed +class TestGraphBinaryV1(object): graphbinary_writer = GraphBinaryWriter() graphbinary_reader = GraphBinaryReader() diff --git a/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py b/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV4.py similarity index 96% copy from gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py copy to gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV4.py index 32201b78a9..a8e970b6d5 100644 --- a/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV1.py +++ b/gremlin-python/src/main/python/tests/structure/io/test_graphbinaryV4.py @@ -23,15 +23,12 @@ import math from gremlin_python.statics import timestamp, long, bigint, BigDecimal, SingleByte, SingleChar, ByteBufferType from gremlin_python.structure.graph import Vertex, Edge, Property, VertexProperty, Path -from gremlin_python.structure.io.graphbinaryV1 import GraphBinaryWriter, GraphBinaryReader +from gremlin_python.structure.io.graphbinaryV4 import GraphBinaryWriter, GraphBinaryReader from gremlin_python.process.traversal import Barrier, Binding, Bytecode, Merge, Direction +from gremlin_python.structure.io.util import Marker -class TestGraphBinaryReader(object): - graphbinary_reader = GraphBinaryReader() - - -class TestGraphSONWriter(object): +class TestGraphBinaryV4(object): graphbinary_writer = GraphBinaryWriter() graphbinary_reader = GraphBinaryReader() @@ -236,3 +233,8 @@ class TestGraphSONWriter(object): x = datetime.timedelta(seconds=1000, microseconds=1000) output = self.graphbinary_reader.read_object(self.graphbinary_writer.write_object(x)) assert x == output + + def test_marker(self): + x = Marker.end_of_stream() + output = self.graphbinary_reader.read_object(self.graphbinary_writer.write_object(x)) + assert x == output diff --git a/gremlin-python/src/main/python/tests/structure/io/test_graphsonV2d0.py b/gremlin-python/src/main/python/tests/structure/io/test_graphsonV2d0.py index 2bc2acbb8d..013fcf8a25 100644 --- a/gremlin-python/src/main/python/tests/structure/io/test_graphsonV2d0.py +++ b/gremlin-python/src/main/python/tests/structure/io/test_graphsonV2d0.py @@ -38,6 +38,7 @@ from gremlin_python.process.graph_traversal import __ from gremlin_python.process.anonymous_traversal import traversal +# TODO: to be removed class TestGraphSONReader(object): graphson_reader = GraphSONReader() diff --git a/gremlin-python/src/main/python/tests/structure/io/test_graphsonV3d0.py b/gremlin-python/src/main/python/tests/structure/io/test_graphsonV3d0.py index 6b852fbf68..88837a0d2e 100644 --- a/gremlin-python/src/main/python/tests/structure/io/test_graphsonV3d0.py +++ b/gremlin-python/src/main/python/tests/structure/io/test_graphsonV3d0.py @@ -37,6 +37,7 @@ from gremlin_python.process.strategies import SubgraphStrategy from gremlin_python.process.graph_traversal import __ +# TODO: to be removed class TestGraphSONReader(object): graphson_reader = GraphSONReader()